diff --git a/applicationset/controllers/applicationset_controller.go b/applicationset/controllers/applicationset_controller.go index a14148a5820a2..209bf426da579 100644 --- a/applicationset/controllers/applicationset_controller.go +++ b/applicationset/controllers/applicationset_controller.go @@ -430,7 +430,7 @@ func (r *ApplicationSetReconciler) generateApplications(applicationSetInfo argop var applicationSetReason argoprojiov1alpha1.ApplicationSetReasonType for _, requestedGenerator := range applicationSetInfo.Spec.Generators { - t, err := generators.Transform(requestedGenerator, r.Generators, applicationSetInfo.Spec.Template, &applicationSetInfo, map[string]string{}) + t, err := generators.Transform(requestedGenerator, r.Generators, applicationSetInfo.Spec.Template, &applicationSetInfo, map[string]interface{}{}) if err != nil { log.WithError(err).WithField("generator", requestedGenerator). Error("error generating application from params") @@ -445,7 +445,7 @@ func (r *ApplicationSetReconciler) generateApplications(applicationSetInfo argop tmplApplication := getTempApplication(a.Template) for _, p := range a.Params { - app, err := r.Renderer.RenderTemplateParams(tmplApplication, applicationSetInfo.Spec.SyncPolicy, p) + app, err := r.Renderer.RenderTemplateParams(tmplApplication, applicationSetInfo.Spec.SyncPolicy, p, applicationSetInfo.Spec.GoTemplate) if err != nil { log.WithError(err).WithField("params", a.Params).WithField("generator", requestedGenerator). Error("error generating application from params") diff --git a/applicationset/controllers/applicationset_controller_test.go b/applicationset/controllers/applicationset_controller_test.go index 1d4f152ae8d5c..eb50cc5aa4ab2 100644 --- a/applicationset/controllers/applicationset_controller_test.go +++ b/applicationset/controllers/applicationset_controller_test.go @@ -45,10 +45,10 @@ func (g *generatorMock) GetTemplate(appSetGenerator *argoprojiov1alpha1.Applicat return args.Get(0).(*argoprojiov1alpha1.ApplicationSetTemplate) } -func (g *generatorMock) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, _ *argoprojiov1alpha1.ApplicationSet) ([]map[string]string, error) { +func (g *generatorMock) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, _ *argoprojiov1alpha1.ApplicationSet) ([]map[string]interface{}, error) { args := g.Called(appSetGenerator) - return args.Get(0).([]map[string]string), args.Error(1) + return args.Get(0).([]map[string]interface{}), args.Error(1) } type rendererMock struct { @@ -61,8 +61,8 @@ func (g *generatorMock) GetRequeueAfter(appSetGenerator *argoprojiov1alpha1.Appl return args.Get(0).(time.Duration) } -func (r *rendererMock) RenderTemplateParams(tmpl *argov1alpha1.Application, syncPolicy *argoprojiov1alpha1.ApplicationSetSyncPolicy, params map[string]string) (*argov1alpha1.Application, error) { - args := r.Called(tmpl, params) +func (r *rendererMock) RenderTemplateParams(tmpl *argov1alpha1.Application, syncPolicy *argoprojiov1alpha1.ApplicationSetSyncPolicy, params map[string]interface{}, useGoTemplate bool) (*argov1alpha1.Application, error) { + args := r.Called(tmpl, params, useGoTemplate) if args.Error(1) != nil { return nil, args.Error(1) @@ -82,7 +82,7 @@ func TestExtractApplications(t *testing.T) { for _, c := range []struct { name string - params []map[string]string + params []map[string]interface{} template argoprojiov1alpha1.ApplicationSetTemplate generateParamsError error rendererError error @@ -91,7 +91,7 @@ func TestExtractApplications(t *testing.T) { }{ { name: "Generate two applications", - params: []map[string]string{{"name": "app1"}, {"name": "app2"}}, + params: []map[string]interface{}{{"name": "app1"}, {"name": "app2"}}, template: argoprojiov1alpha1.ApplicationSetTemplate{ ApplicationSetTemplateMeta: argoprojiov1alpha1.ApplicationSetTemplateMeta{ Name: "name", @@ -110,7 +110,7 @@ func TestExtractApplications(t *testing.T) { }, { name: "Handles error from the render", - params: []map[string]string{{"name": "app1"}, {"name": "app2"}}, + params: []map[string]interface{}{{"name": "app1"}, {"name": "app2"}}, template: argoprojiov1alpha1.ApplicationSetTemplate{ ApplicationSetTemplateMeta: argoprojiov1alpha1.ApplicationSetTemplateMeta{ Name: "name", @@ -161,10 +161,10 @@ func TestExtractApplications(t *testing.T) { for _, p := range cc.params { if cc.rendererError != nil { - rendererMock.On("RenderTemplateParams", getTempApplication(cc.template), p). + rendererMock.On("RenderTemplateParams", getTempApplication(cc.template), p, false). Return(nil, cc.rendererError) } else { - rendererMock.On("RenderTemplateParams", getTempApplication(cc.template), p). + rendererMock.On("RenderTemplateParams", getTempApplication(cc.template), p, false). Return(&app, nil) expectedApps = append(expectedApps, app) } @@ -220,7 +220,7 @@ func TestMergeTemplateApplications(t *testing.T) { for _, c := range []struct { name string - params []map[string]string + params []map[string]interface{} template argoprojiov1alpha1.ApplicationSetTemplate overrideTemplate argoprojiov1alpha1.ApplicationSetTemplate expectedMerged argoprojiov1alpha1.ApplicationSetTemplate @@ -228,7 +228,7 @@ func TestMergeTemplateApplications(t *testing.T) { }{ { name: "Generate app", - params: []map[string]string{{"name": "app1"}}, + params: []map[string]interface{}{{"name": "app1"}}, template: argoprojiov1alpha1.ApplicationSetTemplate{ ApplicationSetTemplateMeta: argoprojiov1alpha1.ApplicationSetTemplateMeta{ Name: "name", @@ -281,7 +281,7 @@ func TestMergeTemplateApplications(t *testing.T) { rendererMock := rendererMock{} - rendererMock.On("RenderTemplateParams", getTempApplication(cc.expectedMerged), cc.params[0]). + rendererMock.On("RenderTemplateParams", getTempApplication(cc.expectedMerged), cc.params[0], false). Return(&cc.expectedApps[0], nil) r := ApplicationSetReconciler{ @@ -1792,6 +1792,7 @@ func TestReconcilerValidationErrorBehaviour(t *testing.T) { Namespace: "argocd", }, Spec: argoprojiov1alpha1.ApplicationSetSpec{ + GoTemplate: true, Generators: []argoprojiov1alpha1.ApplicationSetGenerator{ { List: &argoprojiov1alpha1.ListGenerator{ @@ -1805,13 +1806,13 @@ func TestReconcilerValidationErrorBehaviour(t *testing.T) { }, Template: argoprojiov1alpha1.ApplicationSetTemplate{ ApplicationSetTemplateMeta: argoprojiov1alpha1.ApplicationSetTemplateMeta{ - Name: "{{cluster}}", + Name: "{{.cluster}}", Namespace: "argocd", }, Spec: argov1alpha1.ApplicationSpec{ Source: argov1alpha1.ApplicationSource{RepoURL: "https://github.com/argoproj/argocd-example-apps", Path: "guestbook"}, Project: "default", - Destination: argov1alpha1.ApplicationDestination{Server: "{{url}}"}, + Destination: argov1alpha1.ApplicationDestination{Server: "{{.url}}"}, }, }, }, diff --git a/applicationset/examples/cluster/cluster-example-fasttemplate.yaml b/applicationset/examples/cluster/cluster-example-fasttemplate.yaml new file mode 100644 index 0000000000000..497e7657de68c --- /dev/null +++ b/applicationset/examples/cluster/cluster-example-fasttemplate.yaml @@ -0,0 +1,19 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: guestbook +spec: + generators: + - clusters: {} + template: + metadata: + name: '{{name}}-guestbook' + spec: + project: "default" + source: + repoURL: https://github.com/argoproj/argocd-example-apps/ + targetRevision: HEAD + path: guestbook + destination: + server: '{{server}}' + namespace: guestbook diff --git a/applicationset/examples/cluster/cluster-example.yaml b/applicationset/examples/cluster/cluster-example.yaml index 497e7657de68c..9714ce1952e9c 100644 --- a/applicationset/examples/cluster/cluster-example.yaml +++ b/applicationset/examples/cluster/cluster-example.yaml @@ -3,11 +3,12 @@ kind: ApplicationSet metadata: name: guestbook spec: + goTemplate: true generators: - clusters: {} template: metadata: - name: '{{name}}-guestbook' + name: '{{.name}}-guestbook' spec: project: "default" source: @@ -15,5 +16,5 @@ spec: targetRevision: HEAD path: guestbook destination: - server: '{{server}}' + server: '{{.server}}' namespace: guestbook diff --git a/applicationset/examples/clusterDecisionResource/ducktype-example-fasttemplate.yaml b/applicationset/examples/clusterDecisionResource/ducktype-example-fasttemplate.yaml new file mode 100644 index 0000000000000..1663bbb06e483 --- /dev/null +++ b/applicationset/examples/clusterDecisionResource/ducktype-example-fasttemplate.yaml @@ -0,0 +1,27 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: book-import +spec: + generators: + - clusterDecisionResource: + configMapRef: ocm-placement + name: test-placement + requeueAfterSeconds: 30 + template: + metadata: + name: '{{clusterName}}-book-import' + spec: + project: "default" + source: + repoURL: https://github.com/open-cluster-management/application-samples.git + targetRevision: HEAD + path: book-import + destination: + name: '{{clusterName}}' + namespace: bookimport + syncPolicy: + automated: + prune: true + syncOptions: + - CreateNamespace=true diff --git a/applicationset/examples/clusterDecisionResource/ducktype-example.yaml b/applicationset/examples/clusterDecisionResource/ducktype-example.yaml index 1663bbb06e483..c6058e870bbf6 100644 --- a/applicationset/examples/clusterDecisionResource/ducktype-example.yaml +++ b/applicationset/examples/clusterDecisionResource/ducktype-example.yaml @@ -3,6 +3,7 @@ kind: ApplicationSet metadata: name: book-import spec: + goTemplate: true generators: - clusterDecisionResource: configMapRef: ocm-placement @@ -10,7 +11,7 @@ spec: requeueAfterSeconds: 30 template: metadata: - name: '{{clusterName}}-book-import' + name: '{{.clusterName}}-book-import' spec: project: "default" source: @@ -18,7 +19,7 @@ spec: targetRevision: HEAD path: book-import destination: - name: '{{clusterName}}' + name: '{{.clusterName}}' namespace: bookimport syncPolicy: automated: diff --git a/applicationset/examples/design-doc/applicationset-fasttemplate.yaml b/applicationset/examples/design-doc/applicationset-fasttemplate.yaml new file mode 100644 index 0000000000000..8249b727d2dc9 --- /dev/null +++ b/applicationset/examples/design-doc/applicationset-fasttemplate.yaml @@ -0,0 +1,22 @@ +# This is an example of a typical ApplicationSet which uses the cluster generator. +# An ApplicationSet is comprised with two stanzas: +# - spec.generator - producer of a list of values supplied as arguments to an app template +# - spec.template - an application template, which has been parameterized +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: guestbook +spec: + generators: + - clusters: {} + template: + metadata: + name: '{{name}}-guestbook' + spec: + source: + repoURL: https://github.com/infra-team/cluster-deployments.git + targetRevision: HEAD + chart: guestbook + destination: + server: '{{server}}' + namespace: guestbook diff --git a/applicationset/examples/design-doc/applicationset.yaml b/applicationset/examples/design-doc/applicationset.yaml index 8249b727d2dc9..b1e49bd814d15 100644 --- a/applicationset/examples/design-doc/applicationset.yaml +++ b/applicationset/examples/design-doc/applicationset.yaml @@ -7,16 +7,17 @@ kind: ApplicationSet metadata: name: guestbook spec: + goTemplate: true generators: - clusters: {} template: metadata: - name: '{{name}}-guestbook' + name: '{{.name}}-guestbook' spec: source: repoURL: https://github.com/infra-team/cluster-deployments.git targetRevision: HEAD chart: guestbook destination: - server: '{{server}}' + server: '{{.server}}' namespace: guestbook diff --git a/applicationset/examples/design-doc/clusters.yaml b/applicationset/examples/design-doc/clusters.yaml index 966455a67981c..474d3cc7cdad5 100644 --- a/applicationset/examples/design-doc/clusters.yaml +++ b/applicationset/examples/design-doc/clusters.yaml @@ -19,15 +19,15 @@ spec: project: default template: metadata: - name: '{{name}}-guestbook' + name: '{{.name}}-guestbook' labels: - environment: '{{metadata.labels.environment}}' + environment: '{{.metadata.labels.environment}}' spec: - project: '{{values.project}}' + project: '{{.values.project}}' source: repoURL: https://github.com/infra-team/cluster-deployments.git targetRevision: HEAD chart: guestbook destination: - server: '{{server}}' + server: '{{.server}}' namespace: guestbook diff --git a/applicationset/examples/design-doc/git-directory-discovery-fasttemplate.yaml b/applicationset/examples/design-doc/git-directory-discovery-fasttemplate.yaml new file mode 100644 index 0000000000000..44a9dcf905508 --- /dev/null +++ b/applicationset/examples/design-doc/git-directory-discovery-fasttemplate.yaml @@ -0,0 +1,44 @@ +# This example demonstrates the git directory generator, which produces an items list +# based on discovery of directories in a git repo matching a specified pattern. +# Git generators automatically provide {{path}} and {{path.basename}} as available +# variables to the app template. +# +# Suppose the following git directory structure (note the use of different config tools): +# +# cluster-deployments +# └── add-ons +# ├── argo-rollouts +# │   ├── all.yaml +# │   └── kustomization.yaml +# ├── argo-workflows +# │   └── install.yaml +# ├── grafana +# │   ├── Chart.yaml +# │   └── values.yaml +# └── prometheus-operator +# ├── Chart.yaml +# └── values.yaml +# +# The following ApplicationSet would produce four applications (in different namespaces), +# using the directory basename as both the namespace and application name. +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: cluster-addons +spec: + generators: + - git: + repoURL: https://github.com/infra-team/cluster-deployments.git + directories: + - path: add-ons/* + template: + metadata: + name: '{{path.basename}}' + spec: + source: + repoURL: https://github.com/infra-team/cluster-deployments.git + targetRevision: HEAD + path: '{{path.path}}' + destination: + server: http://kubernetes.default.svc + namespace: '{{path.basename}}' diff --git a/applicationset/examples/design-doc/git-directory-discovery.yaml b/applicationset/examples/design-doc/git-directory-discovery.yaml index 7ff9bb3c053e5..2f62e33cd6ca6 100644 --- a/applicationset/examples/design-doc/git-directory-discovery.yaml +++ b/applicationset/examples/design-doc/git-directory-discovery.yaml @@ -26,6 +26,7 @@ kind: ApplicationSet metadata: name: cluster-addons spec: + goTemplate: true generators: - git: repoURL: https://github.com/infra-team/cluster-deployments.git @@ -33,12 +34,12 @@ spec: - path: add-ons/* template: metadata: - name: '{{path.basename}}' + name: '{{.path.basename}}' spec: source: repoURL: https://github.com/infra-team/cluster-deployments.git targetRevision: HEAD - path: '{{path}}' + path: '{{.path.path}}' destination: server: http://kubernetes.default.svc - namespace: '{{path.basename}}' + namespace: '{{.path.basename}}' diff --git a/applicationset/examples/design-doc/git-files-discovery-fasttemplate.yaml b/applicationset/examples/design-doc/git-files-discovery-fasttemplate.yaml new file mode 100644 index 0000000000000..7cb717d31d49a --- /dev/null +++ b/applicationset/examples/design-doc/git-files-discovery-fasttemplate.yaml @@ -0,0 +1,55 @@ +# This example demonstrates a git file generator which traverses the directory structure of a git +# repository to discover items based on a filename convention. For each file discovered, the +# contents of the discovered files themselves, act as the set of inputs to the app template. +# +# Suppose the following git directory structure: +# +# cluster-deployments +# ├── apps +# │ └── guestbook +# │ └── install.yaml +# └── cluster-config +# ├── engineering +# │ ├── dev +# │ │ └── config.json +# │ └── prod +# │ └── config.json +# └── finance +# ├── dev +# │ └── config.json +# └── prod +# └── config.json +# +# The discovered files (e.g. config.json) files can be any structured data supplied to the +# generated application. e.g.: +# { +# "aws_account": "123456", +# "asset_id": "11223344" +# "cluster": { +# "owner": "Jesse_Suen@intuit.com", +# "name": "engineering-dev", +# "address": "http://1.2.3.4" +# } +# } +# +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: guestbook +spec: + generators: + - git: + repoURL: https://github.com/infra-team/cluster-deployments.git + files: + - path: "**/config.json" + template: + metadata: + name: '{{cluster.name}}-guestbook' + spec: + source: + repoURL: https://github.com/infra-team/cluster-deployments.git + targetRevision: HEAD + path: apps/guestbook + destination: + server: '{{cluster.address}}' + namespace: guestbook diff --git a/applicationset/examples/design-doc/git-files-discovery.yaml b/applicationset/examples/design-doc/git-files-discovery.yaml index 7cb717d31d49a..3a4167886de69 100644 --- a/applicationset/examples/design-doc/git-files-discovery.yaml +++ b/applicationset/examples/design-doc/git-files-discovery.yaml @@ -37,6 +37,7 @@ kind: ApplicationSet metadata: name: guestbook spec: + goTemplate: true generators: - git: repoURL: https://github.com/infra-team/cluster-deployments.git @@ -44,12 +45,12 @@ spec: - path: "**/config.json" template: metadata: - name: '{{cluster.name}}-guestbook' + name: '{{.cluster.name}}-guestbook' spec: source: repoURL: https://github.com/infra-team/cluster-deployments.git targetRevision: HEAD path: apps/guestbook destination: - server: '{{cluster.address}}' + server: '{{.cluster.address}}' namespace: guestbook diff --git a/applicationset/examples/design-doc/git-files-literal-fasttemplate.yaml b/applicationset/examples/design-doc/git-files-literal-fasttemplate.yaml new file mode 100644 index 0000000000000..f2c52b0c220f7 --- /dev/null +++ b/applicationset/examples/design-doc/git-files-literal-fasttemplate.yaml @@ -0,0 +1,68 @@ +# This example demonstrates a git file generator which produces its items based on one or +# more files referenced in a git repo. The referenced files would contain a json/yaml list of +# arbitrary structured objects. Each item of the list would become a set of parameters to a +# generated application. +# +# Suppose the following git directory structure: +# +# cluster-deployments +# ├── apps +# │ └── guestbook +# │ ├── v1.0 +# │ │ └── install.yaml +# │ └── v2.0 +# │ └── install.yaml +# └── config +# └── clusters.json +# +# In this example, the `clusters.json` file is json list of structured data: +# [ +# { +# "account": "123456", +# "asset_id": "11223344", +# "cluster": { +# "owner": "Jesse_Suen@intuit.com", +# "name": "engineering-dev", +# "address": "http://1.2.3.4" +# }, +# "appVersions": { +# "prometheus-operator": "v0.38", +# "guestbook": "v2.0" +# } +# }, +# { +# "account": "456789", +# "asset_id": "55667788", +# "cluster": { +# "owner": "Alexander_Matyushentsev@intuit.com", +# "name": "engineering-prod", +# "address": "http://2.4.6.8" +# }, +# "appVersions": { +# "prometheus-operator": "v0.38", +# "guestbook": "v1.0" +# } +# } +# ] +# +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: guestbook +spec: + generators: + - git: + repoURL: https://github.com/infra-team/cluster-deployments.git + files: + - path: config/clusters.json + template: + metadata: + name: '{{cluster.name}}-guestbook' + spec: + source: + repoURL: https://github.com/infra-team/cluster-deployments.git + targetRevision: HEAD + path: apps/guestbook/{{appVersions.guestbook}} + destination: + server: http://kubernetes.default.svc + namespace: guestbook diff --git a/applicationset/examples/design-doc/git-files-literal.yaml b/applicationset/examples/design-doc/git-files-literal.yaml index f2c52b0c220f7..5cb9bd9553446 100644 --- a/applicationset/examples/design-doc/git-files-literal.yaml +++ b/applicationset/examples/design-doc/git-files-literal.yaml @@ -50,6 +50,7 @@ kind: ApplicationSet metadata: name: guestbook spec: + goTemplate: true generators: - git: repoURL: https://github.com/infra-team/cluster-deployments.git @@ -57,12 +58,12 @@ spec: - path: config/clusters.json template: metadata: - name: '{{cluster.name}}-guestbook' + name: '{{.cluster.name}}-guestbook' spec: source: repoURL: https://github.com/infra-team/cluster-deployments.git targetRevision: HEAD - path: apps/guestbook/{{appVersions.guestbook}} + path: apps/guestbook/{{.appVersions.guestbook}} destination: server: http://kubernetes.default.svc namespace: guestbook diff --git a/applicationset/examples/design-doc/list-fasttemplate.yaml b/applicationset/examples/design-doc/list-fasttemplate.yaml new file mode 100644 index 0000000000000..7cdbc5552442a --- /dev/null +++ b/applicationset/examples/design-doc/list-fasttemplate.yaml @@ -0,0 +1,33 @@ +# The list generator specifies a literal list of argument values to the app spec template. +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: guestbook +spec: + generators: + - list: + elements: + - cluster: engineering-dev + url: https://1.2.3.4 + values: + project: dev + - cluster: engineering-prod + url: https://2.4.6.8 + values: + project: prod + - cluster: finance-preprod + url: https://9.8.7.6 + values: + project: preprod + template: + metadata: + name: '{{cluster}}-guestbook' + spec: + project: '{{values.project}}' + source: + repoURL: https://github.com/infra-team/cluster-deployments.git + targetRevision: HEAD + path: guestbook/{{cluster}} + destination: + server: '{{url}}' + namespace: guestbook diff --git a/applicationset/examples/design-doc/list.yaml b/applicationset/examples/design-doc/list.yaml index 7cdbc5552442a..3f76526b17df5 100644 --- a/applicationset/examples/design-doc/list.yaml +++ b/applicationset/examples/design-doc/list.yaml @@ -4,6 +4,7 @@ kind: ApplicationSet metadata: name: guestbook spec: + goTemplate: true generators: - list: elements: @@ -21,13 +22,13 @@ spec: project: preprod template: metadata: - name: '{{cluster}}-guestbook' + name: '{{.cluster}}-guestbook' spec: - project: '{{values.project}}' + project: '{{.values.project}}' source: repoURL: https://github.com/infra-team/cluster-deployments.git targetRevision: HEAD - path: guestbook/{{cluster}} + path: guestbook/{{.cluster}} destination: - server: '{{url}}' + server: '{{.url}}' namespace: guestbook diff --git a/applicationset/examples/design-doc/template-override-fasttemplate.yaml b/applicationset/examples/design-doc/template-override-fasttemplate.yaml new file mode 100644 index 0000000000000..9eade199f9762 --- /dev/null +++ b/applicationset/examples/design-doc/template-override-fasttemplate.yaml @@ -0,0 +1,48 @@ +# App templates can also be defined as part of the generator's template stanza. Sometimes it is +# useful to do this in order to override the spec.template stanza, and when simple string +# parameterization are insufficient. In the below examples, the generators[].XXX.template is +# a partial definition, which overrides/patch the default template. +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: guestbook +spec: + generators: + - list: + elements: + - cluster: engineering-dev + url: https://1.2.3.4 + template: + metadata: {} + spec: + project: "project" + source: + repoURL: https://github.com/infra-team/cluster-deployments.git + path: '{{cluster}}-override' + destination: {} + + - list: + elements: + - cluster: engineering-prod + url: https://1.2.3.4 + template: + metadata: {} + spec: + project: "project2" + source: + repoURL: https://github.com/infra-team/cluster-deployments.git + path: '{{cluster}}-override2' + destination: {} + + template: + metadata: + name: '{{cluster}}-guestbook' + spec: + project: "project" + source: + repoURL: https://github.com/infra-team/cluster-deployments.git + targetRevision: HEAD + path: guestbook/{{cluster}} + destination: + server: '{{url}}' + namespace: guestbook diff --git a/applicationset/examples/design-doc/template-override.yaml b/applicationset/examples/design-doc/template-override.yaml index 9eade199f9762..be55e739e15a2 100644 --- a/applicationset/examples/design-doc/template-override.yaml +++ b/applicationset/examples/design-doc/template-override.yaml @@ -7,6 +7,7 @@ kind: ApplicationSet metadata: name: guestbook spec: + goTemplate: true generators: - list: elements: @@ -18,7 +19,7 @@ spec: project: "project" source: repoURL: https://github.com/infra-team/cluster-deployments.git - path: '{{cluster}}-override' + path: '{{.cluster}}-override' destination: {} - list: @@ -31,18 +32,18 @@ spec: project: "project2" source: repoURL: https://github.com/infra-team/cluster-deployments.git - path: '{{cluster}}-override2' + path: '{{.cluster}}-override2' destination: {} template: metadata: - name: '{{cluster}}-guestbook' + name: '{{.cluster}}-guestbook' spec: project: "project" source: repoURL: https://github.com/infra-team/cluster-deployments.git targetRevision: HEAD - path: guestbook/{{cluster}} + path: guestbook/{{.cluster}} destination: - server: '{{url}}' + server: '{{.url}}' namespace: guestbook diff --git a/applicationset/examples/git-generator-directory/excludes/git-directories-exclude-example.yaml b/applicationset/examples/git-generator-directory/excludes/git-directories-exclude-example.yaml index 46b20a3bb9ce1..a36359f289527 100644 --- a/applicationset/examples/git-generator-directory/excludes/git-directories-exclude-example.yaml +++ b/applicationset/examples/git-generator-directory/excludes/git-directories-exclude-example.yaml @@ -13,13 +13,13 @@ spec: path: applicationset/examples/git-generator-directory/excludes/cluster-addons/exclude-helm-guestbook template: metadata: - name: '{{path.basename}}' + name: '{{.path.basename}}' spec: project: default source: repoURL: https://github.com/argoproj/argo-cd.git targetRevision: HEAD - path: '{{path}}' + path: '{{.path}}' destination: server: https://kubernetes.default.svc - namespace: '{{path.basename}}' + namespace: '{{.path.basename}}' diff --git a/applicationset/examples/git-generator-directory/git-directories-example-fasttemplate.yaml b/applicationset/examples/git-generator-directory/git-directories-example-fasttemplate.yaml new file mode 100644 index 0000000000000..186acfc3e9569 --- /dev/null +++ b/applicationset/examples/git-generator-directory/git-directories-example-fasttemplate.yaml @@ -0,0 +1,23 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: cluster-addons +spec: + generators: + - git: + repoURL: https://github.com/argoproj/argo-cd.git + revision: HEAD + directories: + - path: applicationset/examples/git-generator-directory/cluster-addons/* + template: + metadata: + name: '{{path.basename}}' + spec: + project: default + source: + repoURL: https://github.com/argoproj/argo-cd.git + targetRevision: HEAD + path: '{{path}}' + destination: + server: https://kubernetes.default.svc + namespace: '{{path.basename}}' diff --git a/applicationset/examples/git-generator-directory/git-directories-example.yaml b/applicationset/examples/git-generator-directory/git-directories-example.yaml index 186acfc3e9569..f0bb77dfb50fd 100644 --- a/applicationset/examples/git-generator-directory/git-directories-example.yaml +++ b/applicationset/examples/git-generator-directory/git-directories-example.yaml @@ -3,6 +3,7 @@ kind: ApplicationSet metadata: name: cluster-addons spec: + goTemplate: true generators: - git: repoURL: https://github.com/argoproj/argo-cd.git @@ -11,13 +12,13 @@ spec: - path: applicationset/examples/git-generator-directory/cluster-addons/* template: metadata: - name: '{{path.basename}}' + name: '{{.path.basename}}' spec: project: default source: repoURL: https://github.com/argoproj/argo-cd.git targetRevision: HEAD - path: '{{path}}' + path: '{{.path.path}}' destination: server: https://kubernetes.default.svc - namespace: '{{path.basename}}' + namespace: '{{.path.basename}}' diff --git a/applicationset/examples/git-generator-files-discovery/git-generator-files-fasttemplate.yaml b/applicationset/examples/git-generator-files-discovery/git-generator-files-fasttemplate.yaml new file mode 100644 index 0000000000000..99923b72c565a --- /dev/null +++ b/applicationset/examples/git-generator-files-discovery/git-generator-files-fasttemplate.yaml @@ -0,0 +1,24 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: guestbook +spec: + generators: + - git: + repoURL: https://github.com/argoproj/argo-cd.git + revision: HEAD + files: + - path: "applicationset/examples/git-generator-files-discovery/cluster-config/**/config.json" + template: + metadata: + name: '{{cluster.name}}-guestbook' + spec: + project: default + source: + repoURL: https://github.com/argoproj/argo-cd.git + targetRevision: HEAD + path: "applicationset/examples/git-generator-files-discovery/apps/guestbook" + destination: + server: https://kubernetes.default.svc + #server: '{{cluster.address}}' + namespace: guestbook diff --git a/applicationset/examples/git-generator-files-discovery/git-generator-files.yaml b/applicationset/examples/git-generator-files-discovery/git-generator-files.yaml index 99923b72c565a..7ccd68f6c6b88 100644 --- a/applicationset/examples/git-generator-files-discovery/git-generator-files.yaml +++ b/applicationset/examples/git-generator-files-discovery/git-generator-files.yaml @@ -3,6 +3,7 @@ kind: ApplicationSet metadata: name: guestbook spec: + goTemplate: true generators: - git: repoURL: https://github.com/argoproj/argo-cd.git @@ -11,7 +12,7 @@ spec: - path: "applicationset/examples/git-generator-files-discovery/cluster-config/**/config.json" template: metadata: - name: '{{cluster.name}}-guestbook' + name: '{{.cluster.name}}-guestbook' spec: project: default source: @@ -20,5 +21,5 @@ spec: path: "applicationset/examples/git-generator-files-discovery/apps/guestbook" destination: server: https://kubernetes.default.svc - #server: '{{cluster.address}}' + #server: '{{.cluster.address}}' namespace: guestbook diff --git a/applicationset/examples/list-generator/list-example-fasttemplate.yaml b/applicationset/examples/list-generator/list-example-fasttemplate.yaml new file mode 100644 index 0000000000000..a671ee45fab02 --- /dev/null +++ b/applicationset/examples/list-generator/list-example-fasttemplate.yaml @@ -0,0 +1,24 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: guestbook +spec: + generators: + - list: + elements: + - cluster: engineering-dev + url: https://kubernetes.default.svc + - cluster: engineering-prod + url: https://kubernetes.default.svc + template: + metadata: + name: '{{cluster}}-guestbook' + spec: + project: default + source: + repoURL: https://github.com/argoproj/argo-cd.git + targetRevision: HEAD + path: applicationset/examples/list-generator/guestbook/{{cluster}} + destination: + server: '{{url}}' + namespace: guestbook diff --git a/applicationset/examples/list-generator/list-example.yaml b/applicationset/examples/list-generator/list-example.yaml index a671ee45fab02..a54fa0cfd92e1 100644 --- a/applicationset/examples/list-generator/list-example.yaml +++ b/applicationset/examples/list-generator/list-example.yaml @@ -3,6 +3,7 @@ kind: ApplicationSet metadata: name: guestbook spec: + goTemplate: true generators: - list: elements: @@ -12,13 +13,13 @@ spec: url: https://kubernetes.default.svc template: metadata: - name: '{{cluster}}-guestbook' + name: '{{.cluster}}-guestbook' spec: project: default source: repoURL: https://github.com/argoproj/argo-cd.git targetRevision: HEAD - path: applicationset/examples/list-generator/guestbook/{{cluster}} + path: applicationset/examples/list-generator/guestbook/{{.cluster}} destination: - server: '{{url}}' + server: '{{.url}}' namespace: guestbook diff --git a/applicationset/examples/matrix/cluster-and-git-fasttemplate.yaml b/applicationset/examples/matrix/cluster-and-git-fasttemplate.yaml new file mode 100644 index 0000000000000..f416ea1deffad --- /dev/null +++ b/applicationset/examples/matrix/cluster-and-git-fasttemplate.yaml @@ -0,0 +1,33 @@ +# This example demonstrates the combining of the git generator with a cluster generator +# The expected output would be an application per git directory and a cluster (application_count = git directory * clusters) +# +# +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: cluster-git +spec: + generators: + - matrix: + generators: + - git: + repoURL: https://github.com/argoproj/argo-cd.git + revision: HEAD + directories: + - path: applicationset/examples/matrix/cluster-addons/* + - clusters: + selector: + matchLabels: + argocd.argoproj.io/secret-type: cluster + template: + metadata: + name: '{{path.basename}}-{{name}}' + spec: + project: '{{metadata.labels.environment}}' + source: + repoURL: https://github.com/argoproj/argo-cd.git + targetRevision: HEAD + path: '{{path.path}}' + destination: + server: '{{server}}' + namespace: '{{path.basename}}' diff --git a/applicationset/examples/matrix/cluster-and-git.yaml b/applicationset/examples/matrix/cluster-and-git.yaml index 4c7497f71242e..a42568db821f3 100644 --- a/applicationset/examples/matrix/cluster-and-git.yaml +++ b/applicationset/examples/matrix/cluster-and-git.yaml @@ -7,6 +7,7 @@ kind: ApplicationSet metadata: name: cluster-git spec: + goTemplate: true generators: - matrix: generators: @@ -21,13 +22,13 @@ spec: argocd.argoproj.io/secret-type: cluster template: metadata: - name: '{{path.basename}}-{{name}}' + name: '{{.path.basename}}-{{.name}}' spec: - project: '{{metadata.labels.environment}}' + project: '{{.metadata.labels.environment}}' source: repoURL: https://github.com/argoproj/argo-cd.git targetRevision: HEAD - path: '{{path}}' + path: '{{.path.path}}' destination: - server: '{{server}}' - namespace: '{{path.basename}}' + server: '{{.server}}' + namespace: '{{.path.basename}}' diff --git a/applicationset/examples/matrix/list-and-git-fasttemplate.yaml b/applicationset/examples/matrix/list-and-git-fasttemplate.yaml new file mode 100644 index 0000000000000..cc46263f339ae --- /dev/null +++ b/applicationset/examples/matrix/list-and-git-fasttemplate.yaml @@ -0,0 +1,39 @@ +# This example demonstrates the combining of the git generator with a list generator +# The expected output would be an application per git directory and a list entry (application_count = git directory * list entries) +# +# +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: list-git +spec: + generators: + - matrix: + generators: + - git: + repoURL: https://github.com/argoproj/argo-cd.git + revision: HEAD + directories: + - path: applicationset/examples/matrix/cluster-addons/* + - list: + elements: + - cluster: engineering-dev + url: https://1.2.3.4 + values: + project: dev + - cluster: engineering-prod + url: https://2.4.6.8 + values: + project: prod + template: + metadata: + name: '{{path.basename}}-{{cluster}}' + spec: + project: '{{values.project}}' + source: + repoURL: https://github.com/argoproj/argo-cd.git + targetRevision: HEAD + path: '{{path.path}}' + destination: + server: '{{url}}' + namespace: '{{path.basename}}' diff --git a/applicationset/examples/matrix/list-and-git.yaml b/applicationset/examples/matrix/list-and-git.yaml index 33f5511902777..d1a2979daedfe 100644 --- a/applicationset/examples/matrix/list-and-git.yaml +++ b/applicationset/examples/matrix/list-and-git.yaml @@ -7,6 +7,7 @@ kind: ApplicationSet metadata: name: list-git spec: + goTemplate: true generators: - matrix: generators: @@ -27,13 +28,13 @@ spec: project: prod template: metadata: - name: '{{path.basename}}-{{cluster}}' + name: '{{.path.basename}}-{{.cluster}}' spec: - project: '{{values.project}}' + project: '{{.values.project}}' source: repoURL: https://github.com/argoproj/argo-cd.git targetRevision: HEAD - path: '{{path}}' + path: '{{.path.path}}' destination: - server: '{{url}}' - namespace: '{{path.basename}}' + server: '{{.url}}' + namespace: '{{.path.basename}}' diff --git a/applicationset/examples/matrix/list-and-list-fasttemplate.yaml b/applicationset/examples/matrix/list-and-list-fasttemplate.yaml new file mode 100644 index 0000000000000..33cbd21794182 --- /dev/null +++ b/applicationset/examples/matrix/list-and-list-fasttemplate.yaml @@ -0,0 +1,37 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: list-and-list + namespace: argocd +spec: + generators: + - matrix: + generators: + - list: + elements: + - cluster: engineering-dev + url: https://kubernetes.default.svc + values: + project: default + - cluster: engineering-prod + url: https://kubernetes.default.svc + values: + project: default + - list: + elements: + - values: + suffix: '1' + - values: + suffix: '2' + template: + metadata: + name: '{{cluster}}-{{values.suffix}}' + spec: + project: '{{values.project}}' + source: + repoURL: https://github.com/argoproj/argo-cd.git + targetRevision: HEAD + path: '{{path.path}}' + destination: + server: '{{url}}' + namespace: '{{path.basename}}' diff --git a/applicationset/examples/matrix/list-and-list.yaml b/applicationset/examples/matrix/list-and-list.yaml index 7e1ac1237ad29..fe5606a4b4b53 100644 --- a/applicationset/examples/matrix/list-and-list.yaml +++ b/applicationset/examples/matrix/list-and-list.yaml @@ -4,6 +4,7 @@ metadata: name: list-and-list namespace: argocd spec: + goTemplate: true generators: - matrix: generators: @@ -25,13 +26,13 @@ spec: suffix: '2' template: metadata: - name: '{{cluster}}-{{values.suffix}}' + name: '{{.cluster}}-{{.values.suffix}}' spec: - project: '{{values.project}}' + project: '{{.values.project}}' source: repoURL: https://github.com/argoproj/argo-cd.git targetRevision: HEAD - path: '{{path}}' + path: '{{.path.path}}' destination: - server: '{{url}}' - namespace: '{{path.basename}}' + server: '{{.url}}' + namespace: '{{.path.basename}}' diff --git a/applicationset/examples/matrix/matrix-and-union-in-matrix-fasttemplate.yaml b/applicationset/examples/matrix/matrix-and-union-in-matrix-fasttemplate.yaml new file mode 100644 index 0000000000000..d5f903d86aa7c --- /dev/null +++ b/applicationset/examples/matrix/matrix-and-union-in-matrix-fasttemplate.yaml @@ -0,0 +1,67 @@ +# The matrix generator can contain other combination-type generators (matrix and union). But nested matrix and union +# generators cannot contain further-nested matrix or union generators. +# +# The generators are evaluated from most-nested to least-nested. In this case: +# 1. The union generator joins two lists to make 3 parameter sets. +# 2. The inner matrix generator takes the cartesian product of the two lists to make 4 parameters sets. +# 3. The outer matrix generator takes the cartesian product of the 3 union and the 4 inner matrix parameter sets to +# make 3*4=12 final parameter sets. +# 4. The 12 final parameter sets are evaluated against the top-level template to generate 12 Applications. +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: matrix-and-union-in-matrix +spec: + generators: + - matrix: + generators: + - union: + mergeKeys: + - cluster + generators: + - list: + elements: + - cluster: engineering-dev + url: https://kubernetes.default.svc + values: + project: default + - cluster: engineering-prod + url: https://kubernetes.default.svc + values: + project: default + - list: + elements: + - cluster: engineering-dev + url: https://kubernetes.default.svc + values: + project: default + - cluster: engineering-test + url: https://kubernetes.default.svc + values: + project: default + - matrix: + generators: + - list: + elements: + - values: + suffix: '1' + - values: + suffix: '2' + - list: + elements: + - values: + prefix: 'first' + - values: + prefix: 'second' + template: + metadata: + name: '{{values.prefix}}-{{cluster}}-{{values.suffix}}' + spec: + project: '{{values.project}}' + source: + repoURL: https://github.com/argoproj/argo-cd.git + targetRevision: HEAD + path: '{{path.path}}' + destination: + server: '{{url}}' + namespace: '{{path.basename}}' diff --git a/applicationset/examples/matrix/matrix-and-union-in-matrix.yaml b/applicationset/examples/matrix/matrix-and-union-in-matrix.yaml index eb5938382942d..783b4c94b5c3a 100644 --- a/applicationset/examples/matrix/matrix-and-union-in-matrix.yaml +++ b/applicationset/examples/matrix/matrix-and-union-in-matrix.yaml @@ -12,6 +12,7 @@ kind: ApplicationSet metadata: name: matrix-and-union-in-matrix spec: + goTemplate: true generators: - matrix: generators: @@ -55,13 +56,13 @@ spec: prefix: 'second' template: metadata: - name: '{{values.prefix}}-{{cluster}}-{{values.suffix}}' + name: '{{.values.prefix}}-{{.cluster}}-{{.values.suffix}}' spec: - project: '{{values.project}}' + project: '{{.values.project}}' source: repoURL: https://github.com/argoproj/argo-cd.git targetRevision: HEAD - path: '{{path}}' + path: '{{.path.path}}' destination: - server: '{{url}}' - namespace: '{{path.basename}}' + server: '{{.url}}' + namespace: '{{.path.basename}}' diff --git a/applicationset/examples/merge/merge-clusters-and-list-fasttemplate.yaml b/applicationset/examples/merge/merge-clusters-and-list-fasttemplate.yaml new file mode 100644 index 0000000000000..5b6971238edd3 --- /dev/null +++ b/applicationset/examples/merge/merge-clusters-and-list-fasttemplate.yaml @@ -0,0 +1,44 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: merge-clusters-and-list +spec: + generators: + - merge: + mergeKeys: + - server + generators: + - clusters: + values: + kafka: 'true' + redis: 'false' + # For clusters with a specific label, enable Kafka. + - clusters: + selector: + matchLabels: + use-kafka: 'false' + values: + kafka: 'false' + # For a specific cluster, enable Redis. + - list: + elements: + - server: https://some-specific-cluster + values.redis: 'true' + template: + metadata: + name: '{{name}}' + spec: + project: default + source: + repoURL: https://github.com/argoproj/argocd-example-apps/ + targetRevision: HEAD + path: helm-guestbook + helm: + parameters: + - name: kafka + value: '{{values.kafka}}' + - name: redis + value: '{{values.redis}}' + destination: + server: '{{server}}' + namespace: default diff --git a/applicationset/examples/merge/merge-clusters-and-list.yaml b/applicationset/examples/merge/merge-clusters-and-list.yaml index 5b6971238edd3..48b35b0251ed4 100644 --- a/applicationset/examples/merge/merge-clusters-and-list.yaml +++ b/applicationset/examples/merge/merge-clusters-and-list.yaml @@ -3,6 +3,7 @@ kind: ApplicationSet metadata: name: merge-clusters-and-list spec: + goTemplate: true generators: - merge: mergeKeys: @@ -26,7 +27,7 @@ spec: values.redis: 'true' template: metadata: - name: '{{name}}' + name: '{{.name}}' spec: project: default source: @@ -36,9 +37,9 @@ spec: helm: parameters: - name: kafka - value: '{{values.kafka}}' + value: '{{.values.kafka}}' - name: redis - value: '{{values.redis}}' + value: '{{.values.redis}}' destination: - server: '{{server}}' + server: '{{.server}}' namespace: default diff --git a/applicationset/examples/merge/merge-two-matrixes-fasttemplate.yaml b/applicationset/examples/merge/merge-two-matrixes-fasttemplate.yaml new file mode 100644 index 0000000000000..f47463d7293c5 --- /dev/null +++ b/applicationset/examples/merge/merge-two-matrixes-fasttemplate.yaml @@ -0,0 +1,43 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: merge-two-matrixes +spec: + generators: + - merge: + mergeKeys: + - server + - environment + generators: + - matrix: + generators: + - clusters: + values: + replicaCount: '2' + - list: + elements: + - environment: staging + namespace: guestbook-non-prod + - environment: prod + namespace: guestbook + - list: + elements: + - server: https://kubernetes.default.svc + environment: staging + values.replicaCount: '1' + template: + metadata: + name: '{{name}}-guestbook-{{environment}}' + spec: + project: default + source: + repoURL: https://github.com/argoproj/argocd-example-apps/ + targetRevision: HEAD + path: helm-guestbook + helm: + parameters: + - name: replicaCount + value: '{{values.replicaCount}}' + destination: + server: '{{server}}' + namespace: '{{namespace}}' diff --git a/applicationset/examples/merge/merge-two-matrixes.yaml b/applicationset/examples/merge/merge-two-matrixes.yaml index f47463d7293c5..f7590fb685d9f 100644 --- a/applicationset/examples/merge/merge-two-matrixes.yaml +++ b/applicationset/examples/merge/merge-two-matrixes.yaml @@ -3,6 +3,7 @@ kind: ApplicationSet metadata: name: merge-two-matrixes spec: + goTemplate: true generators: - merge: mergeKeys: @@ -27,7 +28,7 @@ spec: values.replicaCount: '1' template: metadata: - name: '{{name}}-guestbook-{{environment}}' + name: '{{.name}}-guestbook-{{.environment}}' spec: project: default source: @@ -37,7 +38,7 @@ spec: helm: parameters: - name: replicaCount - value: '{{values.replicaCount}}' + value: '{{.values.replicaCount}}' destination: - server: '{{server}}' - namespace: '{{namespace}}' + server: '{{.server}}' + namespace: '{{.namespace}}' diff --git a/applicationset/examples/pull-request-generator/pull-request-example-fasttemplate.yaml b/applicationset/examples/pull-request-generator/pull-request-example-fasttemplate.yaml new file mode 100644 index 0000000000000..e5d2d5adc0ad8 --- /dev/null +++ b/applicationset/examples/pull-request-generator/pull-request-example-fasttemplate.yaml @@ -0,0 +1,40 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: myapp +spec: + generators: + - pullRequest: + github: + # The GitHub organization or user. + owner: myorg + # The Github repository + repo: myrepo + # For GitHub Enterprise. (optional) + api: https://git.example.com/ + # Reference to a Secret containing an access token. (optional) + tokenRef: + secretName: github-token + key: token + # Labels is used to filter the PRs that you want to target. (optional) + labels: + - preview + template: + metadata: + name: 'myapp-{{ branch }}-{{ number }}' + spec: + source: + repoURL: 'https://github.com/myorg/myrepo.git' + targetRevision: '{{ head_sha }}' + path: helm-guestbook + helm: + parameters: + - name: "image.tag" + value: "pull-{{ head_sha }}" + project: default + destination: + server: https://kubernetes.default.svc + namespace: "{{ branch }}-{{ number }}" + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/applicationset/examples/pull-request-generator/pull-request-example.yaml b/applicationset/examples/pull-request-generator/pull-request-example.yaml index e5d2d5adc0ad8..0d0580af2cab0 100644 --- a/applicationset/examples/pull-request-generator/pull-request-example.yaml +++ b/applicationset/examples/pull-request-generator/pull-request-example.yaml @@ -3,6 +3,7 @@ kind: ApplicationSet metadata: name: myapp spec: + goTemplate: true generators: - pullRequest: github: @@ -21,20 +22,20 @@ spec: - preview template: metadata: - name: 'myapp-{{ branch }}-{{ number }}' + name: 'myapp-{{ .branch }}-{{ .number }}' spec: source: repoURL: 'https://github.com/myorg/myrepo.git' - targetRevision: '{{ head_sha }}' + targetRevision: '{{ .head_sha }}' path: helm-guestbook helm: parameters: - name: "image.tag" - value: "pull-{{ head_sha }}" + value: "pull-{{ .head_sha }}" project: default destination: server: https://kubernetes.default.svc - namespace: "{{ branch }}-{{ number }}" + namespace: "{{ .branch }}-{{ .number }}" syncPolicy: syncOptions: - CreateNamespace=true diff --git a/applicationset/examples/scm-provider-generator/scm-provider-example-fasttemplate.yaml b/applicationset/examples/scm-provider-generator/scm-provider-example-fasttemplate.yaml new file mode 100644 index 0000000000000..24d8ba41c2aed --- /dev/null +++ b/applicationset/examples/scm-provider-generator/scm-provider-example-fasttemplate.yaml @@ -0,0 +1,24 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: guestbook +spec: + generators: + - scmProvider: + github: + organization: argoproj + cloneProtocol: https + filters: + - repositoryMatch: example-apps + template: + metadata: + name: '{{ repository }}-guestbook' + spec: + project: "default" + source: + repoURL: '{{ url }}' + targetRevision: '{{ branch }}' + path: guestbook + destination: + server: https://kubernetes.default.svc + namespace: guestbook diff --git a/applicationset/examples/scm-provider-generator/scm-provider-example.yaml b/applicationset/examples/scm-provider-generator/scm-provider-example.yaml index 24d8ba41c2aed..8e310d45ccda5 100644 --- a/applicationset/examples/scm-provider-generator/scm-provider-example.yaml +++ b/applicationset/examples/scm-provider-generator/scm-provider-example.yaml @@ -3,6 +3,7 @@ kind: ApplicationSet metadata: name: guestbook spec: + goTemplate: true generators: - scmProvider: github: @@ -12,12 +13,12 @@ spec: - repositoryMatch: example-apps template: metadata: - name: '{{ repository }}-guestbook' + name: '{{ .repository }}-guestbook' spec: project: "default" source: - repoURL: '{{ url }}' - targetRevision: '{{ branch }}' + repoURL: '{{ .url }}' + targetRevision: '{{ .branch }}' path: guestbook destination: server: https://kubernetes.default.svc diff --git a/applicationset/examples/template-override/template-overrides-example-fasttemplate.yaml b/applicationset/examples/template-override/template-overrides-example-fasttemplate.yaml new file mode 100644 index 0000000000000..a8fe9916b94d6 --- /dev/null +++ b/applicationset/examples/template-override/template-overrides-example-fasttemplate.yaml @@ -0,0 +1,36 @@ +# App templates can also be defined as part of the generator's template stanza. Sometimes it is +# useful to do this in order to override the spec.template stanza, and when simple string +# parameterization are insufficient. In the below examples, the generators[].XXX.template is +# a partial definition, which overrides/patch the default template. +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: guestbook +spec: + generators: + - list: + elements: + - cluster: engineering-dev + url: https://kubernetes.default.svc + template: + metadata: {} + spec: + project: "default" + source: + targetRevision: HEAD + repoURL: https://github.com/argoproj/argo-cd.git + path: 'applicationset/examples/template-override/{{cluster}}-override' + destination: {} + + template: + metadata: + name: '{{cluster}}-guestbook' + spec: + project: "default" + source: + repoURL: https://github.com/argoproj/argo-cd.git + targetRevision: HEAD + path: applicationset/examples/template-override/default + destination: + server: '{{url}}' + namespace: guestbook diff --git a/applicationset/examples/template-override/template-overrides-example.yaml b/applicationset/examples/template-override/template-overrides-example.yaml index a8fe9916b94d6..dbc19418b4716 100644 --- a/applicationset/examples/template-override/template-overrides-example.yaml +++ b/applicationset/examples/template-override/template-overrides-example.yaml @@ -7,6 +7,7 @@ kind: ApplicationSet metadata: name: guestbook spec: + goTemplate: true generators: - list: elements: @@ -19,12 +20,12 @@ spec: source: targetRevision: HEAD repoURL: https://github.com/argoproj/argo-cd.git - path: 'applicationset/examples/template-override/{{cluster}}-override' + path: 'applicationset/examples/template-override/{{.cluster}}-override' destination: {} template: metadata: - name: '{{cluster}}-guestbook' + name: '{{.cluster}}-guestbook' spec: project: "default" source: @@ -32,5 +33,5 @@ spec: targetRevision: HEAD path: applicationset/examples/template-override/default destination: - server: '{{url}}' + server: '{{.url}}' namespace: guestbook diff --git a/applicationset/generators/cluster.go b/applicationset/generators/cluster.go index cbcb877258629..0fed6ef25309f 100644 --- a/applicationset/generators/cluster.go +++ b/applicationset/generators/cluster.go @@ -3,9 +3,6 @@ package generators import ( "context" "fmt" - "github.com/valyala/fasttemplate" - "regexp" - "strings" "time" log "github.com/sirupsen/logrus" @@ -63,7 +60,7 @@ func (g *ClusterGenerator) GetTemplate(appSetGenerator *argoappsetv1alpha1.Appli } func (g *ClusterGenerator) GenerateParams( - appSetGenerator *argoappsetv1alpha1.ApplicationSetGenerator, _ *argoappsetv1alpha1.ApplicationSet) ([]map[string]string, error) { + appSetGenerator *argoappsetv1alpha1.ApplicationSetGenerator, appSet *argoappsetv1alpha1.ApplicationSet) ([]map[string]interface{}, error) { if appSetGenerator == nil { return nil, EmptyAppSetGeneratorError @@ -92,7 +89,7 @@ func (g *ClusterGenerator) GenerateParams( return nil, err } - res := []map[string]string{} + res := []map[string]interface{}{} secretsFound := []corev1.Secret{} @@ -105,12 +102,12 @@ func (g *ClusterGenerator) GenerateParams( } else if !ignoreLocalClusters { // If there is no secret for the cluster, it's the local cluster, so handle it here. - params := map[string]string{} + params := map[string]interface{}{} params["name"] = cluster.Name params["nameNormalized"] = cluster.Name params["server"] = cluster.Server - err = appendTemplatedValues(appSetGenerator.Clusters.Values, params) + err = appendTemplatedValues(appSetGenerator.Clusters.Values, params, appSet) if err != nil { return nil, err } @@ -123,19 +120,34 @@ func (g *ClusterGenerator) GenerateParams( // For each matching cluster secret (non-local clusters only) for _, cluster := range secretsFound { - params := map[string]string{} + params := map[string]interface{}{} params["name"] = string(cluster.Data["name"]) - params["nameNormalized"] = sanitizeName(string(cluster.Data["name"])) + params["nameNormalized"] = utils.SanitizeName(string(cluster.Data["name"])) params["server"] = string(cluster.Data["server"]) - for key, value := range cluster.ObjectMeta.Annotations { - params[fmt.Sprintf("metadata.annotations.%s", key)] = value - } - for key, value := range cluster.ObjectMeta.Labels { - params[fmt.Sprintf("metadata.labels.%s", key)] = value + + if appSet.Spec.GoTemplate { + meta := map[string]interface{}{} + + if len(cluster.ObjectMeta.Annotations) > 0 { + meta["annotations"] = cluster.ObjectMeta.Annotations + } + if len(cluster.ObjectMeta.Labels) > 0 { + meta["labels"] = cluster.ObjectMeta.Labels + } + + params["metadata"] = meta + } else { + for key, value := range cluster.ObjectMeta.Annotations { + params[fmt.Sprintf("metadata.annotations.%s", key)] = value + } + + for key, value := range cluster.ObjectMeta.Labels { + params[fmt.Sprintf("metadata.labels.%s", key)] = value + } } - err = appendTemplatedValues(appSetGenerator.Clusters.Values, params) + err = appendTemplatedValues(appSetGenerator.Clusters.Values, params, appSet) if err != nil { return nil, err } @@ -148,20 +160,27 @@ func (g *ClusterGenerator) GenerateParams( return res, nil } -func appendTemplatedValues(clusterValues map[string]string, params map[string]string) error { +func appendTemplatedValues(clusterValues map[string]string, params map[string]interface{}, appSet *argoappsetv1alpha1.ApplicationSet) error { // We create a local map to ensure that we do not fall victim to a billion-laughs attack. We iterate through the // cluster values map and only replace values in said map if it has already been whitelisted in the params map. // Once we iterate through all the cluster values we can then safely merge the `tmp` map into the main params map. - tmp := map[string]string{} + tmp := map[string]interface{}{} for key, value := range clusterValues { - result, err := replaceTemplatedString(value, params) + result, err := replaceTemplatedString(value, params, appSet) if err != nil { return err } - tmp[fmt.Sprintf("values.%s", key)] = result + if appSet.Spec.GoTemplate { + if tmp["values"] == nil { + tmp["values"] = map[string]string{} + } + tmp["values"].(map[string]string)[key] = result + } else { + tmp[fmt.Sprintf("values.%s", key)] = result + } } for key, value := range tmp { @@ -171,9 +190,8 @@ func appendTemplatedValues(clusterValues map[string]string, params map[string]st return nil } -func replaceTemplatedString(value string, params map[string]string) (string, error) { - fstTmpl := fasttemplate.New(value, "{{", "}}") - replacedTmplStr, err := render.Replace(fstTmpl, params, true) +func replaceTemplatedString(value string, params map[string]interface{}, appSet *argoappsetv1alpha1.ApplicationSet) (string, error) { + replacedTmplStr, err := render.Replace(value, params, appSet.Spec.GoTemplate) if err != nil { return "", err } @@ -206,20 +224,3 @@ func (g *ClusterGenerator) getSecretsByClusterName(appSetGenerator *argoappsetv1 return res, nil } - -// sanitize the name in accordance with the below rules -// 1. contain no more than 253 characters -// 2. contain only lowercase alphanumeric characters, '-' or '.' -// 3. start and end with an alphanumeric character -func sanitizeName(name string) string { - invalidDNSNameChars := regexp.MustCompile("[^-a-z0-9.]") - maxDNSNameLength := 253 - - name = strings.ToLower(name) - name = invalidDNSNameChars.ReplaceAllString(name, "-") - if len(name) > maxDNSNameLength { - name = name[:maxDNSNameLength] - } - - return strings.Trim(name, "-.") -} diff --git a/applicationset/generators/cluster_test.go b/applicationset/generators/cluster_test.go index fe1f8b99ea858..049107373cc0c 100644 --- a/applicationset/generators/cluster_test.go +++ b/applicationset/generators/cluster_test.go @@ -3,6 +3,7 @@ package generators import ( "context" "fmt" + "testing" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -10,11 +11,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "testing" - kubefake "k8s.io/client-go/kubernetes/fake" + "github.com/argoproj/argo-cd/v2/applicationset/utils" argoappsetv1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" + argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" "github.com/stretchr/testify/assert" ) @@ -86,7 +87,7 @@ func TestGenerateParams(t *testing.T) { name string selector metav1.LabelSelector values map[string]string - expected []map[string]string + expected []map[string]interface{} // clientError is true if a k8s client error should be simulated clientError bool expectedError error @@ -103,7 +104,7 @@ func TestGenerateParams(t *testing.T) { "bat": "{{ metadata.labels.environment }}", "aaa": "{{ server }}", "no-op": "{{ this-does-not-exist }}", - }, expected: []map[string]string{ + }, expected: []map[string]interface{}{ {"values.lol1": "lol", "values.lol2": "{{values.lol1}}{{values.lol1}}", "values.lol3": "{{values.lol2}}{{values.lol2}}{{values.lol2}}", "values.foo": "bar", "values.bar": "production", "values.no-op": "{{ this-does-not-exist }}", "values.bat": "production", "values.aaa": "https://production-01.example.com", "name": "production_01/west", "nameNormalized": "production-01-west", "server": "https://production-01.example.com", "metadata.labels.environment": "production", "metadata.labels.org": "bar", "metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "production"}, @@ -123,7 +124,7 @@ func TestGenerateParams(t *testing.T) { }, }, values: nil, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"name": "production_01/west", "nameNormalized": "production-01-west", "server": "https://production-01.example.com", "metadata.labels.environment": "production", "metadata.labels.org": "bar", "metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "production"}, @@ -143,7 +144,7 @@ func TestGenerateParams(t *testing.T) { values: map[string]string{ "foo": "bar", }, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"values.foo": "bar", "name": "production_01/west", "nameNormalized": "production-01-west", "server": "https://production-01.example.com", "metadata.labels.environment": "production", "metadata.labels.org": "bar", "metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "production"}, }, @@ -167,7 +168,7 @@ func TestGenerateParams(t *testing.T) { values: map[string]string{ "foo": "bar", }, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"values.foo": "bar", "name": "staging-01", "nameNormalized": "staging-01", "server": "https://staging-01.example.com", "metadata.labels.environment": "staging", "metadata.labels.org": "foo", "metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "staging"}, {"values.foo": "bar", "name": "production_01/west", "nameNormalized": "production-01-west", "server": "https://production-01.example.com", "metadata.labels.environment": "production", "metadata.labels.org": "bar", @@ -196,7 +197,7 @@ func TestGenerateParams(t *testing.T) { values: map[string]string{ "name": "baz", }, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"values.name": "baz", "name": "staging-01", "nameNormalized": "staging-01", "server": "https://staging-01.example.com", "metadata.labels.environment": "staging", "metadata.labels.org": "foo", "metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "staging"}, }, @@ -233,12 +234,395 @@ func TestGenerateParams(t *testing.T) { var clusterGenerator = NewClusterGenerator(cl, context.Background(), appClientset, "namespace") + applicationSetInfo := argoprojiov1alpha1.ApplicationSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "set", + }, + Spec: argoprojiov1alpha1.ApplicationSetSpec{}, + } + + got, err := clusterGenerator.GenerateParams(&argoappsetv1alpha1.ApplicationSetGenerator{ + Clusters: &argoappsetv1alpha1.ClusterGenerator{ + Selector: testCase.selector, + Values: testCase.values, + }, + }, &applicationSetInfo) + + if testCase.expectedError != nil { + assert.EqualError(t, err, testCase.expectedError.Error()) + } else { + assert.NoError(t, err) + assert.ElementsMatch(t, testCase.expected, got) + } + + }) + } +} + +func TestGenerateParamsGoTemplate(t *testing.T) { + clusters := []client.Object{ + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "staging-01", + Namespace: "namespace", + Labels: map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "staging", + "org": "foo", + }, + Annotations: map[string]string{ + "foo.argoproj.io": "staging", + }, + }, + Data: map[string][]byte{ + "config": []byte("{}"), + "name": []byte("staging-01"), + "server": []byte("https://staging-01.example.com"), + }, + Type: corev1.SecretType("Opaque"), + }, + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "production-01", + Namespace: "namespace", + Labels: map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "production", + "org": "bar", + }, + Annotations: map[string]string{ + "foo.argoproj.io": "production", + }, + }, + Data: map[string][]byte{ + "config": []byte("{}"), + "name": []byte("production_01/west"), + "server": []byte("https://production-01.example.com"), + }, + Type: corev1.SecretType("Opaque"), + }, + } + testCases := []struct { + name string + selector metav1.LabelSelector + values map[string]string + expected []map[string]interface{} + // clientError is true if a k8s client error should be simulated + clientError bool + expectedError error + }{ + { + name: "no label selector", + selector: metav1.LabelSelector{}, + values: map[string]string{ + "lol1": "lol", + "lol2": "{{ .values.lol1 }}{{ .values.lol1 }}", + "lol3": "{{ .values.lol2 }}{{ .values.lol2 }}{{ .values.lol2 }}", + "foo": "bar", + "bar": "{{ if not (empty .metadata) }}{{index .metadata.annotations \"foo.argoproj.io\" }}{{ end }}", + "bat": "{{ if not (empty .metadata) }}{{.metadata.labels.environment}}{{ end }}", + "aaa": "{{ .server }}", + "no-op": "{{ .thisDoesNotExist }}", + }, expected: []map[string]interface{}{ + { + "name": "production_01/west", + "nameNormalized": "production-01-west", + "server": "https://production-01.example.com", + "metadata": map[string]interface{}{ + "labels": map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "production", + "org": "bar", + }, + "annotations": map[string]string{ + "foo.argoproj.io": "production", + }, + }, + "values": map[string]string{ + "lol1": "lol", + "lol2": "", + "lol3": "", + "foo": "bar", + "bar": "production", + "bat": "production", + "aaa": "https://production-01.example.com", + "no-op": "", + }, + }, + { + "name": "staging-01", + "nameNormalized": "staging-01", + "server": "https://staging-01.example.com", + "metadata": map[string]interface{}{ + "labels": map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "staging", + "org": "foo", + }, + "annotations": map[string]string{ + "foo.argoproj.io": "staging", + }, + }, + "values": map[string]string{ + "lol1": "lol", + "lol2": "", + "lol3": "", + "foo": "bar", + "bar": "staging", + "bat": "staging", + "aaa": "https://staging-01.example.com", + "no-op": "", + }, + }, + { + "nameNormalized": "in-cluster", + "name": "in-cluster", + "server": "https://kubernetes.default.svc", + "values": map[string]string{ + "lol1": "lol", + "lol2": "", + "lol3": "", + "foo": "bar", + "bar": "", + "bat": "", + "aaa": "https://kubernetes.default.svc", + "no-op": "", + }, + }, + }, + clientError: false, + expectedError: nil, + }, + { + name: "secret type label selector", + selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + }, + }, + values: nil, + expected: []map[string]interface{}{ + { + "name": "production_01/west", + "nameNormalized": "production-01-west", + "server": "https://production-01.example.com", + "metadata": map[string]interface{}{ + "labels": map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "production", + "org": "bar", + }, + "annotations": map[string]string{ + "foo.argoproj.io": "production", + }, + }, + }, + { + "name": "staging-01", + "nameNormalized": "staging-01", + "server": "https://staging-01.example.com", + "metadata": map[string]interface{}{ + "labels": map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "staging", + "org": "foo", + }, + "annotations": map[string]string{ + "foo.argoproj.io": "staging", + }, + }, + }, + }, + clientError: false, + expectedError: nil, + }, + { + name: "production-only", + selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "environment": "production", + }, + }, + values: map[string]string{ + "foo": "bar", + }, + expected: []map[string]interface{}{ + { + "name": "production_01/west", + "nameNormalized": "production-01-west", + "server": "https://production-01.example.com", + "metadata": map[string]interface{}{ + "labels": map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "production", + "org": "bar", + }, + "annotations": map[string]string{ + "foo.argoproj.io": "production", + }, + }, + "values": map[string]string{ + "foo": "bar", + }, + }, + }, + clientError: false, + expectedError: nil, + }, + { + name: "production or staging", + selector: metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "environment", + Operator: "In", + Values: []string{ + "production", + "staging", + }, + }, + }, + }, + values: map[string]string{ + "foo": "bar", + }, + expected: []map[string]interface{}{ + { + "name": "production_01/west", + "nameNormalized": "production-01-west", + "server": "https://production-01.example.com", + "metadata": map[string]interface{}{ + "labels": map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "production", + "org": "bar", + }, + "annotations": map[string]string{ + "foo.argoproj.io": "production", + }, + }, + "values": map[string]string{ + "foo": "bar", + }, + }, + { + "name": "staging-01", + "nameNormalized": "staging-01", + "server": "https://staging-01.example.com", + "metadata": map[string]interface{}{ + "labels": map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "staging", + "org": "foo", + }, + "annotations": map[string]string{ + "foo.argoproj.io": "staging", + }, + }, + "values": map[string]string{ + "foo": "bar", + }, + }, + }, + clientError: false, + expectedError: nil, + }, + { + name: "production or staging with match labels", + selector: metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "environment", + Operator: "In", + Values: []string{ + "production", + "staging", + }, + }, + }, + MatchLabels: map[string]string{ + "org": "foo", + }, + }, + values: map[string]string{ + "name": "baz", + }, + expected: []map[string]interface{}{ + { + "name": "staging-01", + "nameNormalized": "staging-01", + "server": "https://staging-01.example.com", + "metadata": map[string]interface{}{ + "labels": map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "staging", + "org": "foo", + }, + "annotations": map[string]string{ + "foo.argoproj.io": "staging", + }, + }, + "values": map[string]string{ + "name": "baz", + }, + }, + }, + clientError: false, + expectedError: nil, + }, + { + name: "simulate client error", + selector: metav1.LabelSelector{}, + values: nil, + expected: nil, + clientError: true, + expectedError: fmt.Errorf("could not list Secrets"), + }, + } + + // convert []client.Object to []runtime.Object, for use by kubefake package + runtimeClusters := []runtime.Object{} + for _, clientCluster := range clusters { + runtimeClusters = append(runtimeClusters, clientCluster) + } + + for _, testCase := range testCases { + + t.Run(testCase.name, func(t *testing.T) { + + appClientset := kubefake.NewSimpleClientset(runtimeClusters...) + + fakeClient := fake.NewClientBuilder().WithObjects(clusters...).Build() + cl := &possiblyErroringFakeCtrlRuntimeClient{ + fakeClient, + testCase.clientError, + } + + var clusterGenerator = NewClusterGenerator(cl, context.Background(), appClientset, "namespace") + + applicationSetInfo := argoprojiov1alpha1.ApplicationSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "set", + }, + Spec: argoprojiov1alpha1.ApplicationSetSpec{ + GoTemplate: true, + }, + } + got, err := clusterGenerator.GenerateParams(&argoappsetv1alpha1.ApplicationSetGenerator{ Clusters: &argoappsetv1alpha1.ClusterGenerator{ Selector: testCase.selector, Values: testCase.values, }, - }, nil) + }, &applicationSetInfo) if testCase.expectedError != nil { assert.EqualError(t, err, testCase.expectedError.Error()) @@ -253,10 +637,10 @@ func TestGenerateParams(t *testing.T) { func TestSanitizeClusterName(t *testing.T) { t.Run("valid DNS-1123 subdomain name", func(t *testing.T) { - assert.Equal(t, "cluster-name", sanitizeName("cluster-name")) + assert.Equal(t, "cluster-name", utils.SanitizeName("cluster-name")) }) t.Run("invalid DNS-1123 subdomain name", func(t *testing.T) { invalidName := "-.--CLUSTER/name -./.-" - assert.Equal(t, "cluster-name", sanitizeName(invalidName)) + assert.Equal(t, "cluster-name", utils.SanitizeName(invalidName)) }) } diff --git a/applicationset/generators/duck_type.go b/applicationset/generators/duck_type.go index ee4383bc7cc78..0b86a42f0b36c 100644 --- a/applicationset/generators/duck_type.go +++ b/applicationset/generators/duck_type.go @@ -60,7 +60,7 @@ func (g *DuckTypeGenerator) GetTemplate(appSetGenerator *argoprojiov1alpha1.Appl return &appSetGenerator.ClusterDecisionResource.Template } -func (g *DuckTypeGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, _ *argoprojiov1alpha1.ApplicationSet) ([]map[string]string, error) { +func (g *DuckTypeGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, appSet *argoprojiov1alpha1.ApplicationSet) ([]map[string]interface{}, error) { if appSetGenerator == nil { return nil, EmptyAppSetGeneratorError @@ -152,7 +152,7 @@ func (g *DuckTypeGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.A } - res := []map[string]string{} + res := []map[string]interface{}{} clusterDecisions := []interface{}{} // Build the decision slice @@ -178,7 +178,7 @@ func (g *DuckTypeGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.A for _, cluster := range clusterDecisions { // generated instance of cluster params - params := map[string]string{} + params := map[string]interface{}{} log.Infof("cluster: %v", cluster) matchValue := cluster.(map[string]interface{})[matchKey] @@ -215,7 +215,14 @@ func (g *DuckTypeGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.A } for key, value := range appSetGenerator.ClusterDecisionResource.Values { - params[fmt.Sprintf("values.%s", key)] = value + if appSet.Spec.GoTemplate { + if params["values"] == nil { + params["values"] = map[string]string{} + } + params["values"].(map[string]string)[key] = value + } else { + params[fmt.Sprintf("values.%s", key)] = value + } } res = append(res, params) diff --git a/applicationset/generators/duck_type_test.go b/applicationset/generators/duck_type_test.go index e6eb5fe6900b0..ba887fcc2fbaa 100644 --- a/applicationset/generators/duck_type_test.go +++ b/applicationset/generators/duck_type_test.go @@ -149,7 +149,7 @@ func TestGenerateParamsForDuckType(t *testing.T) { labelSelector metav1.LabelSelector resource *unstructured.Unstructured values map[string]string - expected []map[string]string + expected []map[string]interface{} expectedError error }{ { @@ -157,7 +157,7 @@ func TestGenerateParamsForDuckType(t *testing.T) { resourceName: "", resource: duckType, values: nil, - expected: []map[string]string{}, + expected: []map[string]interface{}{}, expectedError: fmt.Errorf("There is a problem with the definition of the ClusterDecisionResource generator"), }, /*** This does not work with the FAKE runtime client, fieldSelectors are broken. @@ -175,7 +175,7 @@ func TestGenerateParamsForDuckType(t *testing.T) { resourceName: resourceName, resource: duckType, values: nil, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"clusterName": "production-01", "name": "production-01", "server": "https://production-01.example.com"}, {"clusterName": "staging-01", "name": "staging-01", "server": "https://staging-01.example.com"}, @@ -189,7 +189,7 @@ func TestGenerateParamsForDuckType(t *testing.T) { values: map[string]string{ "foo": "bar", }, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"clusterName": "production-01", "values.foo": "bar", "name": "production-01", "server": "https://production-01.example.com"}, }, expectedError: nil, @@ -217,7 +217,7 @@ func TestGenerateParamsForDuckType(t *testing.T) { labelSelector: metav1.LabelSelector{MatchLabels: map[string]string{"duck": "all-species"}}, resource: duckType, values: nil, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"clusterName": "production-01", "name": "production-01", "server": "https://production-01.example.com"}, {"clusterName": "staging-01", "name": "staging-01", "server": "https://staging-01.example.com"}, @@ -232,7 +232,7 @@ func TestGenerateParamsForDuckType(t *testing.T) { values: map[string]string{ "foo": "bar", }, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"clusterName": "production-01", "values.foo": "bar", "name": "production-01", "server": "https://production-01.example.com"}, }, expectedError: nil, @@ -249,7 +249,305 @@ func TestGenerateParamsForDuckType(t *testing.T) { }}, resource: duckType, values: nil, - expected: []map[string]string{ + expected: []map[string]interface{}{ + {"clusterName": "production-01", "name": "production-01", "server": "https://production-01.example.com"}, + + {"clusterName": "staging-01", "name": "staging-01", "server": "https://staging-01.example.com"}, + }, + expectedError: nil, + }, + { + name: "duck type generator resourceName and labelSelector.matchExpressions", + resourceName: resourceName, + labelSelector: metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "duck", + Operator: "In", + Values: []string{"all-species", "marbled"}, + }, + }}, + resource: duckType, + values: nil, + expected: nil, + expectedError: fmt.Errorf("There is a problem with the definition of the ClusterDecisionResource generator"), + }, + } + + // convert []client.Object to []runtime.Object, for use by kubefake package + runtimeClusters := []runtime.Object{} + for _, clientCluster := range clusters { + runtimeClusters = append(runtimeClusters, clientCluster) + } + + for _, testCase := range testCases { + + t.Run(testCase.name, func(t *testing.T) { + + appClientset := kubefake.NewSimpleClientset(append(runtimeClusters, configMap)...) + + gvrToListKind := map[schema.GroupVersionResource]string{{ + Group: "mallard.io", + Version: "v1", + Resource: "ducks", + }: "DuckList"} + + fakeDynClient := dynfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gvrToListKind, testCase.resource) + + var duckTypeGenerator = NewDuckTypeGenerator(context.Background(), fakeDynClient, appClientset, "namespace") + + applicationSetInfo := argoprojiov1alpha1.ApplicationSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "set", + }, + Spec: argoprojiov1alpha1.ApplicationSetSpec{}, + } + + got, err := duckTypeGenerator.GenerateParams(&argoprojiov1alpha1.ApplicationSetGenerator{ + ClusterDecisionResource: &argoprojiov1alpha1.DuckTypeGenerator{ + ConfigMapRef: "my-configmap", + Name: testCase.resourceName, + LabelSelector: testCase.labelSelector, + Values: testCase.values, + }, + }, &applicationSetInfo) + + if testCase.expectedError != nil { + assert.EqualError(t, err, testCase.expectedError.Error()) + } else { + assert.NoError(t, err) + assert.ElementsMatch(t, testCase.expected, got) + } + }) + } +} + +func TestGenerateParamsForDuckTypeGoTemplate(t *testing.T) { + clusters := []client.Object{ + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "staging-01", + Namespace: "namespace", + Labels: map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "staging", + "org": "foo", + }, + Annotations: map[string]string{ + "foo.argoproj.io": "staging", + }, + }, + Data: map[string][]byte{ + "config": []byte("{}"), + "name": []byte("staging-01"), + "server": []byte("https://staging-01.example.com"), + }, + Type: corev1.SecretType("Opaque"), + }, + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "production-01", + Namespace: "namespace", + Labels: map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "production", + "org": "bar", + }, + Annotations: map[string]string{ + "foo.argoproj.io": "production", + }, + }, + Data: map[string][]byte{ + "config": []byte("{}"), + "name": []byte("production-01"), + "server": []byte("https://production-01.example.com"), + }, + Type: corev1.SecretType("Opaque"), + }, + } + + duckType := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": resourceApiVersion, + "kind": "Duck", + "metadata": map[string]interface{}{ + "name": resourceName, + "namespace": "namespace", + "labels": map[string]interface{}{"duck": "all-species"}, + }, + "status": map[string]interface{}{ + "decisions": []interface{}{ + map[string]interface{}{ + "clusterName": "staging-01", + }, + map[string]interface{}{ + "clusterName": "production-01", + }, + }, + }, + }, + } + + duckTypeProdOnly := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": resourceApiVersion, + "kind": "Duck", + "metadata": map[string]interface{}{ + "name": resourceName, + "namespace": "namespace", + "labels": map[string]interface{}{"duck": "spotted"}, + }, + "status": map[string]interface{}{ + "decisions": []interface{}{ + map[string]interface{}{ + "clusterName": "production-01", + }, + }, + }, + }, + } + + duckTypeEmpty := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": resourceApiVersion, + "kind": "Duck", + "metadata": map[string]interface{}{ + "name": resourceName, + "namespace": "namespace", + "labels": map[string]interface{}{"duck": "canvasback"}, + }, + "status": map[string]interface{}{}, + }, + } + + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-configmap", + Namespace: "namespace", + }, + Data: map[string]string{ + "apiVersion": resourceApiVersion, + "kind": resourceKind, + "statusListKey": "decisions", + "matchKey": "clusterName", + }, + } + + testCases := []struct { + name string + configMapRef string + resourceName string + labelSelector metav1.LabelSelector + resource *unstructured.Unstructured + values map[string]string + expected []map[string]interface{} + expectedError error + }{ + { + name: "no duck resource", + resourceName: "", + resource: duckType, + values: nil, + expected: []map[string]interface{}{}, + expectedError: fmt.Errorf("There is a problem with the definition of the ClusterDecisionResource generator"), + }, + /*** This does not work with the FAKE runtime client, fieldSelectors are broken. + { + name: "invalid name for duck resource", + resourceName: resourceName + "-different", + resource: duckType, + values: nil, + expected: []map[string]string{}, + expectedError: fmt.Errorf("duck.mallard.io \"quak\" not found"), + }, + ***/ + { + name: "duck type generator resourceName", + resourceName: resourceName, + resource: duckType, + values: nil, + expected: []map[string]interface{}{ + {"clusterName": "production-01", "name": "production-01", "server": "https://production-01.example.com"}, + + {"clusterName": "staging-01", "name": "staging-01", "server": "https://staging-01.example.com"}, + }, + expectedError: nil, + }, + { + name: "production-only", + resourceName: resourceName, + resource: duckTypeProdOnly, + values: map[string]string{ + "foo": "bar", + }, + expected: []map[string]interface{}{ + {"clusterName": "production-01", "values": map[string]string{"foo": "bar"}, "name": "production-01", "server": "https://production-01.example.com"}, + }, + expectedError: nil, + }, + { + name: "duck type empty status", + resourceName: resourceName, + resource: duckTypeEmpty, + values: nil, + expected: nil, + expectedError: nil, + }, + { + name: "duck type empty status labelSelector.matchLabels", + resourceName: "", + labelSelector: metav1.LabelSelector{MatchLabels: map[string]string{"duck": "canvasback"}}, + resource: duckTypeEmpty, + values: nil, + expected: nil, + expectedError: nil, + }, + { + name: "duck type generator labelSelector.matchLabels", + resourceName: "", + labelSelector: metav1.LabelSelector{MatchLabels: map[string]string{"duck": "all-species"}}, + resource: duckType, + values: nil, + expected: []map[string]interface{}{ + {"clusterName": "production-01", "name": "production-01", "server": "https://production-01.example.com"}, + + {"clusterName": "staging-01", "name": "staging-01", "server": "https://staging-01.example.com"}, + }, + expectedError: nil, + }, + { + name: "production-only labelSelector.matchLabels", + resourceName: "", + resource: duckTypeProdOnly, + labelSelector: metav1.LabelSelector{MatchLabels: map[string]string{"duck": "spotted"}}, + values: map[string]string{ + "foo": "bar", + }, + expected: []map[string]interface{}{ + {"clusterName": "production-01", "values": map[string]string{"foo": "bar"}, "name": "production-01", "server": "https://production-01.example.com"}, + }, + expectedError: nil, + }, + { + name: "duck type generator labelSelector.matchExpressions", + resourceName: "", + labelSelector: metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "duck", + Operator: "In", + Values: []string{"all-species", "marbled"}, + }, + }}, + resource: duckType, + values: nil, + expected: []map[string]interface{}{ {"clusterName": "production-01", "name": "production-01", "server": "https://production-01.example.com"}, {"clusterName": "staging-01", "name": "staging-01", "server": "https://staging-01.example.com"}, @@ -295,6 +593,15 @@ func TestGenerateParamsForDuckType(t *testing.T) { var duckTypeGenerator = NewDuckTypeGenerator(context.Background(), fakeDynClient, appClientset, "namespace") + applicationSetInfo := argoprojiov1alpha1.ApplicationSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "set", + }, + Spec: argoprojiov1alpha1.ApplicationSetSpec{ + GoTemplate: true, + }, + } + got, err := duckTypeGenerator.GenerateParams(&argoprojiov1alpha1.ApplicationSetGenerator{ ClusterDecisionResource: &argoprojiov1alpha1.DuckTypeGenerator{ ConfigMapRef: "my-configmap", @@ -302,7 +609,7 @@ func TestGenerateParamsForDuckType(t *testing.T) { LabelSelector: testCase.labelSelector, Values: testCase.values, }, - }, nil) + }, &applicationSetInfo) if testCase.expectedError != nil { assert.EqualError(t, err, testCase.expectedError.Error()) diff --git a/applicationset/generators/generator_spec_processor.go b/applicationset/generators/generator_spec_processor.go index 3fa539c475bda..346e00be43249 100644 --- a/applicationset/generators/generator_spec_processor.go +++ b/applicationset/generators/generator_spec_processor.go @@ -4,8 +4,6 @@ import ( "encoding/json" "reflect" - "github.com/valyala/fasttemplate" - "github.com/argoproj/argo-cd/v2/applicationset/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -22,12 +20,12 @@ const ( ) type TransformResult struct { - Params []map[string]string + Params []map[string]interface{} Template argoprojiov1alpha1.ApplicationSetTemplate } //Transform a spec generator to list of paramSets and a template -func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, allGenerators map[string]Generator, baseTemplate argoprojiov1alpha1.ApplicationSetTemplate, appSet *argoprojiov1alpha1.ApplicationSet, genParams map[string]string) ([]TransformResult, error) { +func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, allGenerators map[string]Generator, baseTemplate argoprojiov1alpha1.ApplicationSetTemplate, appSet *argoprojiov1alpha1.ApplicationSet, genParams map[string]interface{}) ([]TransformResult, error) { selector, err := metav1.LabelSelectorAsSelector(requestedGenerator.Selector) if err != nil { return nil, err @@ -49,9 +47,9 @@ func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, al } continue } - var params []map[string]string + var params []map[string]interface{} if len(genParams) != 0 { - tempInterpolatedGenerator, err := interpolateGenerator(&requestedGenerator, genParams) + tempInterpolatedGenerator, err := interpolateGenerator(&requestedGenerator, genParams, appSet.Spec.GoTemplate) interpolatedGenerator = &tempInterpolatedGenerator if err != nil { log.WithError(err).WithField("genParams", genParams). @@ -71,9 +69,10 @@ func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, al } continue } - var filterParams []map[string]string + var filterParams []map[string]interface{} for _, param := range params { - if requestedGenerator.Selector != nil && !selector.Matches(labels.Set(param)) { + + if requestedGenerator.Selector != nil && !selector.Matches(labels.Set(keepOnlyStringValues(param))) { continue } filterParams = append(filterParams, param) @@ -88,6 +87,18 @@ func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, al return res, firstError } +func keepOnlyStringValues(in map[string]interface{}) map[string]string { + var out map[string]string = map[string]string{} + + for key, value := range in { + if _, ok := value.(string); ok { + out[key] = value.(string) + } + } + + return out +} + func GetRelevantGenerators(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, generators map[string]Generator) []Generator { var res []Generator @@ -123,7 +134,7 @@ func mergeGeneratorTemplate(g Generator, requestedGenerator *argoprojiov1alpha1. // Currently for Matrix Generator. Allows interpolating the matrix's 2nd child generator with values from the 1st child generator // "params" parameter is an array, where each index corresponds to a generator. Each index contains a map w/ that generator's parameters. -func interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, params map[string]string) (argoprojiov1alpha1.ApplicationSetGenerator, error) { +func interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, params map[string]interface{}, useGoTemplate bool) (argoprojiov1alpha1.ApplicationSetGenerator, error) { interpolatedGenerator := requestedGenerator.DeepCopy() tmplBytes, err := json.Marshal(interpolatedGenerator) if err != nil { @@ -132,8 +143,7 @@ func interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetG } render := utils.Render{} - fstTmpl := fasttemplate.New(string(tmplBytes), "{{", "}}") - replacedTmplStr, err := render.Replace(fstTmpl, params, true) + replacedTmplStr, err := render.Replace(string(tmplBytes), params, useGoTemplate) if err != nil { log.WithError(err).WithField("interpolatedGeneratorString", replacedTmplStr).Error("error interpolating generator with other generator's parameter") return *interpolatedGenerator, err diff --git a/applicationset/generators/generator_spec_processor_test.go b/applicationset/generators/generator_spec_processor_test.go index f3235d93ba6c8..0c34581c99c11 100644 --- a/applicationset/generators/generator_spec_processor_test.go +++ b/applicationset/generators/generator_spec_processor_test.go @@ -4,14 +4,13 @@ import ( "context" "testing" + log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" argov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" - "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" - log "github.com/sirupsen/logrus" "github.com/stretchr/testify/mock" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -27,19 +26,19 @@ func TestMatchValues(t *testing.T) { name string elements []apiextensionsv1.JSON selector *metav1.LabelSelector - expected []map[string]string + expected []map[string]interface{} }{ { name: "no filter", elements: []apiextensionsv1.JSON{{Raw: []byte(`{"cluster": "cluster","url": "url"}`)}}, selector: &metav1.LabelSelector{}, - expected: []map[string]string{{"cluster": "cluster", "url": "url"}}, + expected: []map[string]interface{}{{"cluster": "cluster", "url": "url"}}, }, { name: "nil", elements: []apiextensionsv1.JSON{{Raw: []byte(`{"cluster": "cluster","url": "url"}`)}}, selector: nil, - expected: []map[string]string{{"cluster": "cluster", "url": "url"}}, + expected: []map[string]interface{}{{"cluster": "cluster", "url": "url"}}, }, { name: "values.foo should be foo but is ignore element", @@ -49,7 +48,7 @@ func TestMatchValues(t *testing.T) { "values.foo": "foo", }, }, - expected: []map[string]string{}, + expected: []map[string]interface{}{}, }, { name: "values.foo should be bar", @@ -59,7 +58,7 @@ func TestMatchValues(t *testing.T) { "values.foo": "bar", }, }, - expected: []map[string]string{{"cluster": "cluster", "url": "url", "values.foo": "bar"}}, + expected: []map[string]interface{}{{"cluster": "cluster", "url": "url", "values.foo": "bar"}}, }, } @@ -70,15 +69,22 @@ func TestMatchValues(t *testing.T) { "List": listGenerator, } + applicationSetInfo := argoprojiov1alpha1.ApplicationSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "set", + }, + Spec: argoprojiov1alpha1.ApplicationSetSpec{}, + } + results, err := Transform(argoprojiov1alpha1.ApplicationSetGenerator{ Selector: testCase.selector, - List: &v1alpha1.ListGenerator{ + List: &argoprojiov1alpha1.ListGenerator{ Elements: testCase.elements, Template: emptyTemplate(), }}, data, emptyTemplate(), - nil, nil) + &applicationSetInfo, nil) assert.NoError(t, err) assert.ElementsMatch(t, testCase.expected, results[0].Params) @@ -86,8 +92,8 @@ func TestMatchValues(t *testing.T) { } } -func emptyTemplate() v1alpha1.ApplicationSetTemplate { - return v1alpha1.ApplicationSetTemplate{ +func emptyTemplate() argoprojiov1alpha1.ApplicationSetTemplate { + return argoprojiov1alpha1.ApplicationSetTemplate{ Spec: argov1alpha1.ApplicationSpec{ Project: "project", }, @@ -219,10 +225,14 @@ func TestInterpolateGenerator(t *testing.T) { }}, }, } - gitGeneratorParams := map[string]string{ - "path": "p1/p2/app3", "path.basename": "app3", "path[0]": "p1", "path[1]": "p2", "path.basenameNormalized": "app3", + gitGeneratorParams := map[string]interface{}{ + "path": "p1/p2/app3", + "path.basename": "app3", + "path[0]": "p1", + "path[1]": "p2", + "path.basenameNormalized": "app3", } - interpolatedGenerator, err := interpolateGenerator(requestedGenerator, gitGeneratorParams) + interpolatedGenerator, err := interpolateGenerator(requestedGenerator, gitGeneratorParams, false) if err != nil { log.WithError(err).WithField("requestedGenerator", requestedGenerator).Error("error interpolating Generator") return @@ -244,10 +254,10 @@ func TestInterpolateGenerator(t *testing.T) { Template: argoprojiov1alpha1.ApplicationSetTemplate{}, }, } - clusterGeneratorParams := map[string]string{ + clusterGeneratorParams := map[string]interface{}{ "name": "production_01/west", "server": "https://production-01.example.com", } - interpolatedGenerator, err = interpolateGenerator(requestedGenerator, clusterGeneratorParams) + interpolatedGenerator, err = interpolateGenerator(requestedGenerator, clusterGeneratorParams, true) if err != nil { log.WithError(err).WithField("requestedGenerator", requestedGenerator).Error("error interpolating Generator") return diff --git a/applicationset/generators/git.go b/applicationset/generators/git.go index d12f846f174e8..8d558a626746c 100644 --- a/applicationset/generators/git.go +++ b/applicationset/generators/git.go @@ -14,6 +14,7 @@ import ( "sigs.k8s.io/yaml" "github.com/argoproj/argo-cd/v2/applicationset/services" + "github.com/argoproj/argo-cd/v2/applicationset/utils" argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" ) @@ -45,7 +46,7 @@ func (g *GitGenerator) GetRequeueAfter(appSetGenerator *argoprojiov1alpha1.Appli return DefaultRequeueAfterSeconds } -func (g *GitGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, _ *argoprojiov1alpha1.ApplicationSet) ([]map[string]string, error) { +func (g *GitGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, appSet *argoprojiov1alpha1.ApplicationSet) ([]map[string]interface{}, error) { if appSetGenerator == nil { return nil, EmptyAppSetGeneratorError @@ -56,11 +57,11 @@ func (g *GitGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.Applic } var err error - var res []map[string]string + var res []map[string]interface{} if appSetGenerator.Git.Directories != nil { - res, err = g.generateParamsForGitDirectories(appSetGenerator) + res, err = g.generateParamsForGitDirectories(appSetGenerator, appSet.Spec.GoTemplate) } else if appSetGenerator.Git.Files != nil { - res, err = g.generateParamsForGitFiles(appSetGenerator) + res, err = g.generateParamsForGitFiles(appSetGenerator, appSet.Spec.GoTemplate) } else { return nil, EmptyAppSetGeneratorError } @@ -71,7 +72,7 @@ func (g *GitGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.Applic return res, nil } -func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator) ([]map[string]string, error) { +func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, useGoTemplate bool) ([]map[string]interface{}, error) { // Directories, not files allPaths, err := g.repos.GetDirectories(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision) @@ -88,12 +89,12 @@ func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoproj requestedApps := g.filterApps(appSetGenerator.Git.Directories, allPaths) - res := g.generateParamsFromApps(requestedApps, appSetGenerator) + res := g.generateParamsFromApps(requestedApps, appSetGenerator, useGoTemplate) return res, nil } -func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator) ([]map[string]string, error) { +func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, useGoTemplate bool) ([]map[string]interface{}, error) { // Get all files that match the requested path string, removing duplicates allFiles := make(map[string][]byte) @@ -116,11 +117,11 @@ func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1al sort.Strings(allPaths) // Generate params from each path, and return - res := []map[string]string{} + res := []map[string]interface{}{} for _, path := range allPaths { // A JSON / YAML file path can contain multiple sets of parameters (ie it is an array) - paramsArray, err := g.generateParamsFromGitFile(path, allFiles[path]) + paramsArray, err := g.generateParamsFromGitFile(path, allFiles[path], useGoTemplate) if err != nil { return nil, fmt.Errorf("unable to process file '%s': %v", path, err) } @@ -132,7 +133,7 @@ func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1al return res, nil } -func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []byte) ([]map[string]string, error) { +func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []byte, useGoTemplate bool) ([]map[string]interface{}, error) { objectsFound := []map[string]interface{}{} // First, we attempt to parse as an array @@ -147,29 +148,46 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent [] objectsFound = append(objectsFound, singleObj) } - res := []map[string]string{} + res := []map[string]interface{}{} - // Flatten all objects found, and return them for _, objectFound := range objectsFound { - flat, err := flatten.Flatten(objectFound, "", flatten.DotStyle) - if err != nil { - return nil, err - } - params := map[string]string{} - for k, v := range flat { - params[k] = fmt.Sprintf("%v", v) - } - params["path"] = path.Dir(filePath) - params["path.basename"] = path.Base(params["path"]) - params["path.filename"] = path.Base(filePath) - params["path.basenameNormalized"] = sanitizeName(path.Base(params["path"])) - params["path.filenameNormalized"] = sanitizeName(path.Base(params["path.filename"])) - for k, v := range strings.Split(params["path"], "/") { - if len(v) > 0 { - params["path["+strconv.Itoa(k)+"]"] = v + params := map[string]interface{}{} + + if useGoTemplate { + for k, v := range objectFound { + params[k] = v + } + + paramPath := map[string]interface{}{} + + paramPath["path"] = path.Dir(filePath) + paramPath["basename"] = path.Base(paramPath["path"].(string)) + paramPath["filename"] = path.Base(filePath) + paramPath["basenameNormalized"] = utils.SanitizeName(path.Base(paramPath["path"].(string))) + paramPath["filenameNormalized"] = utils.SanitizeName(path.Base(paramPath["filename"].(string))) + paramPath["segments"] = strings.Split(paramPath["path"].(string), "/") + params["path"] = paramPath + } else { + flat, err := flatten.Flatten(objectFound, "", flatten.DotStyle) + if err != nil { + return nil, err + } + for k, v := range flat { + params[k] = fmt.Sprintf("%v", v) + } + params["path"] = path.Dir(filePath) + params["path.basename"] = path.Base(params["path"].(string)) + params["path.filename"] = path.Base(filePath) + params["path.basenameNormalized"] = utils.SanitizeName(path.Base(params["path"].(string))) + params["path.filenameNormalized"] = utils.SanitizeName(path.Base(params["path.filename"].(string))) + for k, v := range strings.Split(params["path"].(string), "/") { + if len(v) > 0 { + params["path["+strconv.Itoa(k)+"]"] = v + } } } + res = append(res, params) } @@ -205,21 +223,32 @@ func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitDirectoryG return res } -func (g *GitGenerator) generateParamsFromApps(requestedApps []string, _ *argoprojiov1alpha1.ApplicationSetGenerator) []map[string]string { +func (g *GitGenerator) generateParamsFromApps(requestedApps []string, _ *argoprojiov1alpha1.ApplicationSetGenerator, useGoTemplate bool) []map[string]interface{} { // TODO: At some point, the appicationSetGenerator param should be used - res := make([]map[string]string, len(requestedApps)) + res := make([]map[string]interface{}, len(requestedApps)) for i, a := range requestedApps { - params := make(map[string]string, 2) - params["path"] = a - params["path.basename"] = path.Base(a) - params["path.basenameNormalized"] = sanitizeName(path.Base(a)) - for k, v := range strings.Split(params["path"], "/") { - if len(v) > 0 { - params["path["+strconv.Itoa(k)+"]"] = v + params := make(map[string]interface{}, 5) + + if useGoTemplate { + paramPath := map[string]interface{}{} + paramPath["path"] = a + paramPath["basename"] = path.Base(a) + paramPath["basenameNormalized"] = utils.SanitizeName(path.Base(a)) + paramPath["segments"] = strings.Split(paramPath["path"].(string), "/") + params["path"] = paramPath + } else { + params["path"] = a + params["path.basename"] = path.Base(a) + params["path.basenameNormalized"] = utils.SanitizeName(path.Base(a)) + for k, v := range strings.Split(params["path"].(string), "/") { + if len(v) > 0 { + params["path["+strconv.Itoa(k)+"]"] = v + } } } + res[i] = params } diff --git a/applicationset/generators/git_test.go b/applicationset/generators/git_test.go index 18dcd20538704..6b279d0ad907b 100644 --- a/applicationset/generators/git_test.go +++ b/applicationset/generators/git_test.go @@ -51,11 +51,11 @@ func Test_generateParamsFromGitFile(t *testing.T) { params, err := (*GitGenerator)(nil).generateParamsFromGitFile("path/dir/file_name.yaml", []byte(` foo: bar: baz -`)) +`), false) if err != nil { t.Fatal(err) } - assert.Equal(t, []map[string]string{ + assert.Equal(t, []map[string]interface{}{ { "foo.bar": "baz", "path": "path/dir", @@ -69,6 +69,34 @@ foo: }, params) } +func Test_generateParamsFromGitFileGoTemplate(t *testing.T) { + params, err := (*GitGenerator)(nil).generateParamsFromGitFile("path/dir/file_name.yaml", []byte(` +foo: + bar: baz +`), true) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, []map[string]interface{}{ + { + "foo": map[string]interface{}{ + "bar": "baz", + }, + "path": map[string]interface{}{ + "path": "path/dir", + "basename": "dir", + "filename": "file_name.yaml", + "basenameNormalized": "dir", + "filenameNormalized": "file-name.yaml", + "segments": []string{ + "path", + "dir", + }, + }, + }, + }, params) +} + func TestGitGenerateParamsFromDirectories(t *testing.T) { cases := []struct { @@ -76,7 +104,7 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) { directories []argoprojiov1alpha1.GitDirectoryGeneratorItem repoApps []string repoError error - expected []map[string]string + expected []map[string]interface{} expectedError error }{ { @@ -89,7 +117,7 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) { "p1/app4", }, repoError: nil, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1", "path[0]": "app1"}, {"path": "app2", "path.basename": "app2", "path.basenameNormalized": "app2", "path[0]": "app2"}, {"path": "app_3", "path.basename": "app_3", "path.basenameNormalized": "app-3", "path[0]": "app_3"}, @@ -106,7 +134,7 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) { "p1/p2/p3/app4", }, repoError: nil, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"path": "p1/app2", "path.basename": "app2", "path[0]": "p1", "path[1]": "app2", "path.basenameNormalized": "app2"}, {"path": "p1/p2/app3", "path.basename": "app3", "path[0]": "p1", "path[1]": "p2", "path[2]": "app3", "path.basenameNormalized": "app3"}, }, @@ -123,7 +151,7 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) { "p2/app3", }, repoError: nil, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"path": "app1", "path.basename": "app1", "path[0]": "app1", "path.basenameNormalized": "app1"}, {"path": "app2", "path.basename": "app2", "path[0]": "app2", "path.basenameNormalized": "app2"}, {"path": "p2/app3", "path.basename": "app3", "path[0]": "p2", "path[1]": "app3", "path.basenameNormalized": "app3"}, @@ -141,7 +169,7 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) { "p2/app3", }, repoError: nil, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"path": "app1", "path.basename": "app1", "path[0]": "app1", "path.basenameNormalized": "app1"}, {"path": "app2", "path.basename": "app2", "path[0]": "app2", "path.basenameNormalized": "app2"}, {"path": "p2/app3", "path.basename": "app3", "path[0]": "p2", "path[1]": "app3", "path.basenameNormalized": "app3"}, @@ -153,7 +181,249 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) { directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, repoApps: []string{}, repoError: nil, - expected: []map[string]string{}, + expected: []map[string]interface{}{}, + expectedError: nil, + }, + { + name: "handles error from repo server", + directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, + repoApps: []string{}, + repoError: fmt.Errorf("error"), + expected: []map[string]interface{}{}, + expectedError: fmt.Errorf("error"), + }, + } + + for _, testCase := range cases { + testCaseCopy := testCase + + t.Run(testCaseCopy.name, func(t *testing.T) { + t.Parallel() + + argoCDServiceMock := argoCDServiceMock{mock: &mock.Mock{}} + + argoCDServiceMock.mock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything).Return(testCaseCopy.repoApps, testCaseCopy.repoError) + + var gitGenerator = NewGitGenerator(argoCDServiceMock) + applicationSetInfo := argoprojiov1alpha1.ApplicationSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "set", + }, + Spec: argoprojiov1alpha1.ApplicationSetSpec{ + Generators: []argoprojiov1alpha1.ApplicationSetGenerator{{ + Git: &argoprojiov1alpha1.GitGenerator{ + RepoURL: "RepoURL", + Revision: "Revision", + Directories: testCaseCopy.directories, + }, + }}, + }, + } + + got, err := gitGenerator.GenerateParams(&applicationSetInfo.Spec.Generators[0], &applicationSetInfo) + + if testCaseCopy.expectedError != nil { + assert.EqualError(t, err, testCaseCopy.expectedError.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, testCaseCopy.expected, got) + } + + argoCDServiceMock.mock.AssertExpectations(t) + }) + } +} + +func TestGitGenerateParamsFromDirectoriesGoTemplate(t *testing.T) { + + cases := []struct { + name string + directories []argoprojiov1alpha1.GitDirectoryGeneratorItem + repoApps []string + repoError error + expected []map[string]interface{} + expectedError error + }{ + { + name: "happy flow - created apps", + directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, + repoApps: []string{ + "app1", + "app2", + "app_3", + "p1/app4", + }, + repoError: nil, + expected: []map[string]interface{}{ + { + "path": map[string]interface{}{ + "path": "app1", + "basename": "app1", + "basenameNormalized": "app1", + "segments": []string{ + "app1", + }, + }, + }, + { + "path": map[string]interface{}{ + "path": "app2", + "basename": "app2", + "basenameNormalized": "app2", + "segments": []string{ + "app2", + }, + }, + }, + { + "path": map[string]interface{}{ + "path": "app_3", + "basename": "app_3", + "basenameNormalized": "app-3", + "segments": []string{ + "app_3", + }, + }, + }, + }, + expectedError: nil, + }, + { + name: "It filters application according to the paths", + directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "p1/*"}, {Path: "p1/*/*"}}, + repoApps: []string{ + "app1", + "p1/app2", + "p1/p2/app3", + "p1/p2/p3/app4", + }, + repoError: nil, + expected: []map[string]interface{}{ + { + "path": map[string]interface{}{ + "path": "p1/app2", + "basename": "app2", + "basenameNormalized": "app2", + "segments": []string{ + "p1", + "app2", + }, + }, + }, + { + "path": map[string]interface{}{ + "path": "p1/p2/app3", + "basename": "app3", + "basenameNormalized": "app3", + "segments": []string{ + "p1", + "p2", + "app3", + }, + }, + }, + }, + expectedError: nil, + }, + { + name: "It filters application according to the paths with Exclude", + directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "p1/*", Exclude: true}, {Path: "*"}, {Path: "*/*"}}, + repoApps: []string{ + "app1", + "app2", + "p1/app2", + "p1/app3", + "p2/app3", + }, + repoError: nil, + expected: []map[string]interface{}{ + { + "path": map[string]interface{}{ + "path": "app1", + "basename": "app1", + "basenameNormalized": "app1", + "segments": []string{ + "app1", + }, + }, + }, + { + "path": map[string]interface{}{ + "path": "app2", + "basename": "app2", + "basenameNormalized": "app2", + "segments": []string{ + "app2", + }, + }, + }, + { + "path": map[string]interface{}{ + "path": "p2/app3", + "basename": "app3", + "basenameNormalized": "app3", + "segments": []string{ + "p2", + "app3", + }, + }, + }, + }, + expectedError: nil, + }, + { + name: "Expecting same exclude behavior with different order", + directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}, {Path: "*/*"}, {Path: "p1/*", Exclude: true}}, + repoApps: []string{ + "app1", + "app2", + "p1/app2", + "p1/app3", + "p2/app3", + }, + repoError: nil, + expected: []map[string]interface{}{ + + { + "path": map[string]interface{}{ + "path": "app1", + "basename": "app1", + "basenameNormalized": "app1", + "segments": []string{ + "app1", + }, + }, + }, + { + "path": map[string]interface{}{ + "path": "app2", + "basename": "app2", + "basenameNormalized": "app2", + "segments": []string{ + "app2", + }, + }, + }, + { + "path": map[string]interface{}{ + "path": "p2/app3", + "basename": "app3", + "basenameNormalized": "app3", + "segments": []string{ + "p2", + "app3", + }, + }, + }, + }, + expectedError: nil, + }, + { + name: "handles empty response from repo server", + directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, + repoApps: []string{}, + repoError: nil, + expected: []map[string]interface{}{}, expectedError: nil, }, { @@ -161,7 +431,7 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) { directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, repoApps: []string{}, repoError: fmt.Errorf("error"), - expected: []map[string]string{}, + expected: []map[string]interface{}{}, expectedError: fmt.Errorf("error"), }, } @@ -182,6 +452,7 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) { Name: "set", }, Spec: argoprojiov1alpha1.ApplicationSetSpec{ + GoTemplate: true, Generators: []argoprojiov1alpha1.ApplicationSetGenerator{{ Git: &argoprojiov1alpha1.GitGenerator{ RepoURL: "RepoURL", @@ -192,7 +463,7 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) { }, } - got, err := gitGenerator.GenerateParams(&applicationSetInfo.Spec.Generators[0], nil) + got, err := gitGenerator.GenerateParams(&applicationSetInfo.Spec.Generators[0], &applicationSetInfo) if testCaseCopy.expectedError != nil { assert.EqualError(t, err, testCaseCopy.expectedError.Error()) @@ -217,7 +488,7 @@ func TestGitGenerateParamsFromFiles(t *testing.T) { repoFileContents map[string][]byte // if repoPathsError is non-nil, the call to GetPaths(...) will return this error value repoPathsError error - expected []map[string]string + expected []map[string]interface{} expectedError error }{ { @@ -248,7 +519,7 @@ func TestGitGenerateParamsFromFiles(t *testing.T) { }`), }, repoPathsError: nil, - expected: []map[string]string{ + expected: []map[string]interface{}{ { "cluster.owner": "john.doe@example.com", "cluster.name": "production", @@ -285,7 +556,7 @@ func TestGitGenerateParamsFromFiles(t *testing.T) { files: []argoprojiov1alpha1.GitFileGeneratorItem{{Path: "**/config.json"}}, repoFileContents: map[string][]byte{}, repoPathsError: fmt.Errorf("paths error"), - expected: []map[string]string{}, + expected: []map[string]interface{}{}, expectedError: fmt.Errorf("paths error"), }, { @@ -295,7 +566,7 @@ func TestGitGenerateParamsFromFiles(t *testing.T) { "cluster-config/production/config.json": []byte(`invalid json file`), }, repoPathsError: nil, - expected: []map[string]string{}, + expected: []map[string]interface{}{}, expectedError: fmt.Errorf("unable to process file 'cluster-config/production/config.json': unable to parse file: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type map[string]interface {}"), }, { @@ -324,7 +595,7 @@ func TestGitGenerateParamsFromFiles(t *testing.T) { ]`), }, repoPathsError: nil, - expected: []map[string]string{ + expected: []map[string]interface{}{ { "cluster.owner": "john.doe@example.com", "cluster.name": "production", @@ -376,7 +647,7 @@ cluster: `), }, repoPathsError: nil, - expected: []map[string]string{ + expected: []map[string]interface{}{ { "cluster.owner": "john.doe@example.com", "cluster.name": "production", @@ -424,7 +695,7 @@ cluster: address: https://kubernetes.default.svc`), }, repoPathsError: nil, - expected: []map[string]string{ + expected: []map[string]interface{}{ { "cluster.owner": "john.doe@example.com", "cluster.name": "production", @@ -481,7 +752,7 @@ cluster: }, } - got, err := gitGenerator.GenerateParams(&applicationSetInfo.Spec.Generators[0], nil) + got, err := gitGenerator.GenerateParams(&applicationSetInfo.Spec.Generators[0], &applicationSetInfo) fmt.Println(got, err) if testCaseCopy.expectedError != nil { @@ -494,5 +765,354 @@ cluster: argoCDServiceMock.mock.AssertExpectations(t) }) } +} + +func TestGitGenerateParamsFromFilesGoTemplate(t *testing.T) { + cases := []struct { + name string + // files is the list of paths/globs to match + files []argoprojiov1alpha1.GitFileGeneratorItem + // repoFileContents maps repo path to the literal contents of that path + repoFileContents map[string][]byte + // if repoPathsError is non-nil, the call to GetPaths(...) will return this error value + repoPathsError error + expected []map[string]interface{} + expectedError error + }{ + { + name: "happy flow: create params from git files", + files: []argoprojiov1alpha1.GitFileGeneratorItem{{Path: "**/config.json"}}, + repoFileContents: map[string][]byte{ + "cluster-config/production/config.json": []byte(`{ + "cluster": { + "owner": "john.doe@example.com", + "name": "production", + "address": "https://kubernetes.default.svc" + }, + "key1": "val1", + "key2": { + "key2_1": "val2_1", + "key2_2": { + "key2_2_1": "val2_2_1" + } + }, + "key3": 123 +}`), + "cluster-config/staging/config.json": []byte(`{ + "cluster": { + "owner": "foo.bar@example.com", + "name": "staging", + "address": "https://kubernetes.default.svc" + } +}`), + }, + repoPathsError: nil, + expected: []map[string]interface{}{ + { + "cluster": map[string]interface{}{ + "owner": "john.doe@example.com", + "name": "production", + "address": "https://kubernetes.default.svc", + }, + "key1": "val1", + "key2": map[string]interface{}{ + "key2_1": "val2_1", + "key2_2": map[string]interface{}{ + "key2_2_1": "val2_2_1", + }, + }, + "key3": float64(123), + "path": map[string]interface{}{ + "path": "cluster-config/production", + "basename": "production", + "filename": "config.json", + "basenameNormalized": "production", + "filenameNormalized": "config.json", + "segments": []string{ + "cluster-config", + "production", + }, + }, + }, + { + "cluster": map[string]interface{}{ + "owner": "foo.bar@example.com", + "name": "staging", + "address": "https://kubernetes.default.svc", + }, + "path": map[string]interface{}{ + "path": "cluster-config/staging", + "basename": "staging", + "filename": "config.json", + "basenameNormalized": "staging", + "filenameNormalized": "config.json", + "segments": []string{ + "cluster-config", + "staging", + }, + }, + }, + }, + expectedError: nil, + }, + { + name: "handles error during getting repo paths", + files: []argoprojiov1alpha1.GitFileGeneratorItem{{Path: "**/config.json"}}, + repoFileContents: map[string][]byte{}, + repoPathsError: fmt.Errorf("paths error"), + expected: []map[string]interface{}{}, + expectedError: fmt.Errorf("paths error"), + }, + { + name: "test invalid JSON file returns error", + files: []argoprojiov1alpha1.GitFileGeneratorItem{{Path: "**/config.json"}}, + repoFileContents: map[string][]byte{ + "cluster-config/production/config.json": []byte(`invalid json file`), + }, + repoPathsError: nil, + expected: []map[string]interface{}{}, + expectedError: fmt.Errorf("unable to process file 'cluster-config/production/config.json': unable to parse file: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type map[string]interface {}"), + }, + { + name: "test JSON array", + files: []argoprojiov1alpha1.GitFileGeneratorItem{{Path: "**/config.json"}}, + repoFileContents: map[string][]byte{ + "cluster-config/production/config.json": []byte(` +[ + { + "cluster": { + "owner": "john.doe@example.com", + "name": "production", + "address": "https://kubernetes.default.svc", + "inner": { + "one" : "two" + } + } + }, + { + "cluster": { + "owner": "john.doe@example.com", + "name": "staging", + "address": "https://kubernetes.default.svc" + } + } +]`), + }, + repoPathsError: nil, + expected: []map[string]interface{}{ + { + "cluster": map[string]interface{}{ + "owner": "john.doe@example.com", + "name": "production", + "address": "https://kubernetes.default.svc", + "inner": map[string]interface{}{ + "one": "two", + }, + }, + "path": map[string]interface{}{ + "path": "cluster-config/production", + "basename": "production", + "filename": "config.json", + "basenameNormalized": "production", + "filenameNormalized": "config.json", + "segments": []string{ + "cluster-config", + "production", + }, + }, + }, + { + "cluster": map[string]interface{}{ + "owner": "john.doe@example.com", + "name": "staging", + "address": "https://kubernetes.default.svc", + }, + "path": map[string]interface{}{ + "path": "cluster-config/production", + "basename": "production", + "filename": "config.json", + "basenameNormalized": "production", + "filenameNormalized": "config.json", + "segments": []string{ + "cluster-config", + "production", + }, + }, + }, + }, + expectedError: nil, + }, + { + name: "Test YAML flow", + files: []argoprojiov1alpha1.GitFileGeneratorItem{{Path: "**/config.yaml"}}, + repoFileContents: map[string][]byte{ + "cluster-config/production/config.yaml": []byte(` +cluster: + owner: john.doe@example.com + name: production + address: https://kubernetes.default.svc +key1: val1 +key2: + key2_1: val2_1 + key2_2: + key2_2_1: val2_2_1 +`), + "cluster-config/staging/config.yaml": []byte(` +cluster: + owner: foo.bar@example.com + name: staging + address: https://kubernetes.default.svc +`), + }, + repoPathsError: nil, + expected: []map[string]interface{}{ + { + "cluster": map[string]interface{}{ + "owner": "john.doe@example.com", + "name": "production", + "address": "https://kubernetes.default.svc", + }, + "key1": "val1", + "key2": map[string]interface{}{ + "key2_1": "val2_1", + "key2_2": map[string]interface{}{ + "key2_2_1": "val2_2_1", + }, + }, + "path": map[string]interface{}{ + "path": "cluster-config/production", + "basename": "production", + "filename": "config.yaml", + "basenameNormalized": "production", + "filenameNormalized": "config.yaml", + "segments": []string{ + "cluster-config", + "production", + }, + }, + }, + { + "cluster": map[string]interface{}{ + "owner": "foo.bar@example.com", + "name": "staging", + "address": "https://kubernetes.default.svc", + }, + "path": map[string]interface{}{ + "path": "cluster-config/staging", + "basename": "staging", + "filename": "config.yaml", + "basenameNormalized": "staging", + "filenameNormalized": "config.yaml", + "segments": []string{ + "cluster-config", + "staging", + }, + }, + }, + }, + expectedError: nil, + }, + { + name: "test YAML array", + files: []argoprojiov1alpha1.GitFileGeneratorItem{{Path: "**/config.yaml"}}, + repoFileContents: map[string][]byte{ + "cluster-config/production/config.yaml": []byte(` +- cluster: + owner: john.doe@example.com + name: production + address: https://kubernetes.default.svc + inner: + one: two +- cluster: + owner: john.doe@example.com + name: staging + address: https://kubernetes.default.svc`), + }, + repoPathsError: nil, + expected: []map[string]interface{}{ + { + "cluster": map[string]interface{}{ + "owner": "john.doe@example.com", + "name": "production", + "address": "https://kubernetes.default.svc", + "inner": map[string]interface{}{ + "one": "two", + }, + }, + "path": map[string]interface{}{ + "path": "cluster-config/production", + "basename": "production", + "filename": "config.yaml", + "basenameNormalized": "production", + "filenameNormalized": "config.yaml", + "segments": []string{ + "cluster-config", + "production", + }, + }, + }, + { + "cluster": map[string]interface{}{ + "owner": "john.doe@example.com", + "name": "staging", + "address": "https://kubernetes.default.svc", + }, + "path": map[string]interface{}{ + "path": "cluster-config/production", + "basename": "production", + "filename": "config.yaml", + "basenameNormalized": "production", + "filenameNormalized": "config.yaml", + "segments": []string{ + "cluster-config", + "production", + }, + }, + }, + }, + expectedError: nil, + }, + } + + for _, testCase := range cases { + testCaseCopy := testCase + + t.Run(testCaseCopy.name, func(t *testing.T) { + t.Parallel() + + argoCDServiceMock := argoCDServiceMock{mock: &mock.Mock{}} + argoCDServiceMock.mock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(testCaseCopy.repoFileContents, testCaseCopy.repoPathsError) + + var gitGenerator = NewGitGenerator(argoCDServiceMock) + applicationSetInfo := argoprojiov1alpha1.ApplicationSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "set", + }, + Spec: argoprojiov1alpha1.ApplicationSetSpec{ + GoTemplate: true, + Generators: []argoprojiov1alpha1.ApplicationSetGenerator{{ + Git: &argoprojiov1alpha1.GitGenerator{ + RepoURL: "RepoURL", + Revision: "Revision", + Files: testCaseCopy.files, + }, + }}, + }, + } + + got, err := gitGenerator.GenerateParams(&applicationSetInfo.Spec.Generators[0], &applicationSetInfo) + fmt.Println(got, err) + + if testCaseCopy.expectedError != nil { + assert.EqualError(t, err, testCaseCopy.expectedError.Error()) + } else { + assert.NoError(t, err) + assert.ElementsMatch(t, testCaseCopy.expected, got) + } + + argoCDServiceMock.mock.AssertExpectations(t) + }) + } } diff --git a/applicationset/generators/interface.go b/applicationset/generators/interface.go index 1711c4054db47..94a6b05577049 100644 --- a/applicationset/generators/interface.go +++ b/applicationset/generators/interface.go @@ -12,7 +12,7 @@ type Generator interface { // GenerateParams interprets the ApplicationSet and generates all relevant parameters for the application template. // The expected / desired list of parameters is returned, it then will be render and reconciled // against the current state of the Applications in the cluster. - GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, applicationSetInfo *argoprojiov1alpha1.ApplicationSet) ([]map[string]string, error) + GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, applicationSetInfo *argoprojiov1alpha1.ApplicationSet) ([]map[string]interface{}, error) // GetRequeueAfter is the the generator can controller the next reconciled loop // In case there is more then one generator the time will be the minimum of the times. diff --git a/applicationset/generators/list.go b/applicationset/generators/list.go index c6def48d44045..edd0a13271e0c 100644 --- a/applicationset/generators/list.go +++ b/applicationset/generators/list.go @@ -26,7 +26,7 @@ func (g *ListGenerator) GetTemplate(appSetGenerator *argoprojiov1alpha1.Applicat return &appSetGenerator.List.Template } -func (g *ListGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, _ *argoprojiov1alpha1.ApplicationSet) ([]map[string]string, error) { +func (g *ListGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, appSet *argoprojiov1alpha1.ApplicationSet) ([]map[string]interface{}, error) { if appSetGenerator == nil { return nil, EmptyAppSetGeneratorError } @@ -35,39 +35,42 @@ func (g *ListGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.Appli return nil, EmptyAppSetGeneratorError } - res := make([]map[string]string, len(appSetGenerator.List.Elements)) + res := make([]map[string]interface{}, len(appSetGenerator.List.Elements)) for i, tmpItem := range appSetGenerator.List.Elements { - params := map[string]string{} + params := map[string]interface{}{} var element map[string]interface{} err := json.Unmarshal(tmpItem.Raw, &element) if err != nil { return nil, fmt.Errorf("error unmarshling list element %v", err) } - for key, value := range element { - if key == "values" { - values, ok := (value).(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("error parsing values map") - } - for k, v := range values { - value, ok := v.(string) + if appSet.Spec.GoTemplate { + res[i] = element + } else { + for key, value := range element { + if key == "values" { + values, ok := (value).(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("error parsing values map") + } + for k, v := range values { + value, ok := v.(string) + if !ok { + return nil, fmt.Errorf("error parsing value as string %v", err) + } + params[fmt.Sprintf("values.%s", k)] = value + } + } else { + v, ok := value.(string) if !ok { return nil, fmt.Errorf("error parsing value as string %v", err) } - params[fmt.Sprintf("values.%s", k)] = value - } - } else { - v, ok := value.(string) - if !ok { - return nil, fmt.Errorf("error parsing value as string %v", err) + params[key] = v } - params[key] = v + res[i] = params } } - - res[i] = params } return res, nil diff --git a/applicationset/generators/list_test.go b/applicationset/generators/list_test.go index 14deba68a0fe2..92f92e8ff63b5 100644 --- a/applicationset/generators/list_test.go +++ b/applicationset/generators/list_test.go @@ -5,6 +5,7 @@ import ( "github.com/stretchr/testify/assert" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" ) @@ -12,14 +13,14 @@ import ( func TestGenerateListParams(t *testing.T) { testCases := []struct { elements []apiextensionsv1.JSON - expected []map[string]string + expected []map[string]interface{} }{ { elements: []apiextensionsv1.JSON{{Raw: []byte(`{"cluster": "cluster","url": "url"}`)}}, - expected: []map[string]string{{"cluster": "cluster", "url": "url"}}, + expected: []map[string]interface{}{{"cluster": "cluster", "url": "url"}}, }, { elements: []apiextensionsv1.JSON{{Raw: []byte(`{"cluster": "cluster","url": "url","values":{"foo":"bar"}}`)}}, - expected: []map[string]string{{"cluster": "cluster", "url": "url", "values.foo": "bar"}}, + expected: []map[string]interface{}{{"cluster": "cluster", "url": "url", "values.foo": "bar"}}, }, } @@ -27,13 +28,57 @@ func TestGenerateListParams(t *testing.T) { var listGenerator = NewListGenerator() + applicationSetInfo := argoprojiov1alpha1.ApplicationSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "set", + }, + Spec: argoprojiov1alpha1.ApplicationSetSpec{}, + } + got, err := listGenerator.GenerateParams(&argoprojiov1alpha1.ApplicationSetGenerator{ List: &argoprojiov1alpha1.ListGenerator{ Elements: testCase.elements, - }}, nil) + }}, &applicationSetInfo) assert.NoError(t, err) assert.ElementsMatch(t, testCase.expected, got) } } + +func TestGenerateListParamsGoTemplate(t *testing.T) { + testCases := []struct { + elements []apiextensionsv1.JSON + expected []map[string]interface{} + }{ + { + elements: []apiextensionsv1.JSON{{Raw: []byte(`{"cluster": "cluster","url": "url"}`)}}, + expected: []map[string]interface{}{{"cluster": "cluster", "url": "url"}}, + }, { + elements: []apiextensionsv1.JSON{{Raw: []byte(`{"cluster": "cluster","url": "url","values":{"foo":"bar"}}`)}}, + expected: []map[string]interface{}{{"cluster": "cluster", "url": "url", "values": map[string]interface{}{"foo": "bar"}}}, + }, + } + + for _, testCase := range testCases { + + var listGenerator = NewListGenerator() + + applicationSetInfo := argoprojiov1alpha1.ApplicationSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "set", + }, + Spec: argoprojiov1alpha1.ApplicationSetSpec{ + GoTemplate: true, + }, + } + + got, err := listGenerator.GenerateParams(&argoprojiov1alpha1.ApplicationSetGenerator{ + List: &argoprojiov1alpha1.ListGenerator{ + Elements: testCase.elements, + }}, &applicationSetInfo) + + assert.NoError(t, err) + assert.ElementsMatch(t, testCase.expected, got) + } +} diff --git a/applicationset/generators/matrix.go b/applicationset/generators/matrix.go index 278d8c1cc6b2a..75a5de3aa2ab5 100644 --- a/applicationset/generators/matrix.go +++ b/applicationset/generators/matrix.go @@ -4,7 +4,10 @@ import ( "fmt" "time" + "github.com/imdario/mergo" + "github.com/argoproj/argo-cd/v2/applicationset/utils" + argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" ) @@ -28,7 +31,7 @@ func NewMatrixGenerator(supportedGenerators map[string]Generator) Generator { return m } -func (m *MatrixGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, appSet *argoprojiov1alpha1.ApplicationSet) ([]map[string]string, error) { +func (m *MatrixGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, appSet *argoprojiov1alpha1.ApplicationSet) ([]map[string]interface{}, error) { if appSetGenerator.Matrix == nil { return nil, EmptyAppSetGeneratorError @@ -42,7 +45,7 @@ func (m *MatrixGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.App return nil, ErrMoreThanTwoGenerators } - res := []map[string]string{} + res := []map[string]interface{}{} g0, err := m.getParams(appSetGenerator.Matrix.Generators[0], appSet, nil) if err != nil { @@ -51,21 +54,33 @@ func (m *MatrixGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.App for _, a := range g0 { g1, err := m.getParams(appSetGenerator.Matrix.Generators[1], appSet, a) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get params for second generator in the matrix generator: %w", err) } for _, b := range g1 { - val, err := utils.CombineStringMaps(a, b) - if err != nil { - return nil, err + + if appSet.Spec.GoTemplate { + tmp := map[string]interface{}{} + if err := mergo.Merge(&tmp, a); err != nil { + return nil, fmt.Errorf("failed to merge params from the first generator in the matrix generator with temp map: %w", err) + } + if err := mergo.Merge(&tmp, b); err != nil { + return nil, fmt.Errorf("failed to merge params from the first generator in the matrix generator with the second: %w", err) + } + res = append(res, tmp) + } else { + val, err := utils.CombineStringMaps(a, b) + if err != nil { + return nil, fmt.Errorf("failed to combine string maps with merging params for the matrix generator: %w", err) + } + res = append(res, utils.ConvertToMapStringInterface(val)) } - res = append(res, val) } } return res, nil } -func (m *MatrixGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.ApplicationSetNestedGenerator, appSet *argoprojiov1alpha1.ApplicationSet, params map[string]string) ([]map[string]string, error) { +func (m *MatrixGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.ApplicationSetNestedGenerator, appSet *argoprojiov1alpha1.ApplicationSet, params map[string]interface{}) ([]map[string]interface{}, error) { var matrix *argoprojiov1alpha1.MatrixGenerator if appSetBaseGenerator.Matrix != nil { // Since nested matrix generator is represented as a JSON object in the CRD, we unmarshall it back to a Go struct here. diff --git a/applicationset/generators/matrix_test.go b/applicationset/generators/matrix_test.go index cce467efcfa37..e476505eb976b 100644 --- a/applicationset/generators/matrix_test.go +++ b/applicationset/generators/matrix_test.go @@ -2,14 +2,15 @@ package generators import ( "context" + "testing" + "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" kubefake "k8s.io/client-go/kubernetes/fake" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -34,7 +35,7 @@ func TestMatrixGenerate(t *testing.T) { name string baseGenerators []argoprojiov1alpha1.ApplicationSetNestedGenerator expectedErr error - expected []map[string]string + expected []map[string]interface{} }{ { name: "happy flow - generate params", @@ -46,7 +47,7 @@ func TestMatrixGenerate(t *testing.T) { List: listGenerator, }, }, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1", "cluster": "Cluster", "url": "Url"}, {"path": "app2", "path.basename": "app2", "path.basenameNormalized": "app2", "cluster": "Cluster", "url": "Url"}, }, @@ -71,7 +72,7 @@ func TestMatrixGenerate(t *testing.T) { }, }, }, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"a": "1", "b": "1"}, {"a": "1", "b": "2"}, {"a": "2", "b": "1"}, @@ -135,7 +136,12 @@ func TestMatrixGenerate(t *testing.T) { t.Run(testCaseCopy.name, func(t *testing.T) { genMock := &generatorMock{} - appSet := &argoprojiov1alpha1.ApplicationSet{} + appSet := &argoprojiov1alpha1.ApplicationSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "set", + }, + Spec: argoprojiov1alpha1.ApplicationSetSpec{}, + } for _, g := range testCaseCopy.baseGenerators { @@ -143,7 +149,7 @@ func TestMatrixGenerate(t *testing.T) { Git: g.Git, List: g.List, } - genMock.On("GenerateParams", mock.AnythingOfType("*v1alpha1.ApplicationSetGenerator"), appSet).Return([]map[string]string{ + genMock.On("GenerateParams", mock.AnythingOfType("*v1alpha1.ApplicationSetGenerator"), appSet).Return([]map[string]interface{}{ { "path": "app1", "path.basename": "app1", @@ -175,7 +181,202 @@ func TestMatrixGenerate(t *testing.T) { }, appSet) if testCaseCopy.expectedErr != nil { - assert.EqualError(t, err, testCaseCopy.expectedErr.Error()) + assert.ErrorIs(t, err, testCaseCopy.expectedErr) + } else { + assert.NoError(t, err) + assert.Equal(t, testCaseCopy.expected, got) + } + + }) + + } +} + +func TestMatrixGenerateGoTemplate(t *testing.T) { + + gitGenerator := &argoprojiov1alpha1.GitGenerator{ + RepoURL: "RepoURL", + Revision: "Revision", + Directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, + } + + listGenerator := &argoprojiov1alpha1.ListGenerator{ + Elements: []apiextensionsv1.JSON{{Raw: []byte(`{"cluster": "Cluster","url": "Url"}`)}}, + } + + testCases := []struct { + name string + baseGenerators []argoprojiov1alpha1.ApplicationSetNestedGenerator + expectedErr error + expected []map[string]interface{} + }{ + { + name: "happy flow - generate params", + baseGenerators: []argoprojiov1alpha1.ApplicationSetNestedGenerator{ + { + Git: gitGenerator, + }, + { + List: listGenerator, + }, + }, + expected: []map[string]interface{}{ + { + "path": map[string]string{ + "path": "app1", + "basename": "app1", + "basenameNormalized": "app1", + }, + "cluster": "Cluster", + "url": "Url", + }, + { + "path": map[string]string{ + "path": "app2", + "basename": "app2", + "basenameNormalized": "app2", + }, + "cluster": "Cluster", + "url": "Url", + }, + }, + }, + { + name: "happy flow - generate params from two lists", + baseGenerators: []argoprojiov1alpha1.ApplicationSetNestedGenerator{ + { + List: &argoprojiov1alpha1.ListGenerator{ + Elements: []apiextensionsv1.JSON{ + {Raw: []byte(`{"a": "1"}`)}, + {Raw: []byte(`{"a": "2"}`)}, + }, + }, + }, + { + List: &argoprojiov1alpha1.ListGenerator{ + Elements: []apiextensionsv1.JSON{ + {Raw: []byte(`{"b": "1"}`)}, + {Raw: []byte(`{"b": "2"}`)}, + }, + }, + }, + }, + expected: []map[string]interface{}{ + {"a": "1", "b": "1"}, + {"a": "1", "b": "2"}, + {"a": "2", "b": "1"}, + {"a": "2", "b": "2"}, + }, + }, + { + name: "returns error if there is less than two base generators", + baseGenerators: []argoprojiov1alpha1.ApplicationSetNestedGenerator{ + { + Git: gitGenerator, + }, + }, + expectedErr: ErrLessThanTwoGenerators, + }, + { + name: "returns error if there is more than two base generators", + baseGenerators: []argoprojiov1alpha1.ApplicationSetNestedGenerator{ + { + List: listGenerator, + }, + { + List: listGenerator, + }, + { + List: listGenerator, + }, + }, + expectedErr: ErrMoreThanTwoGenerators, + }, + { + name: "returns error if there is more than one inner generator in the first base generator", + baseGenerators: []argoprojiov1alpha1.ApplicationSetNestedGenerator{ + { + Git: gitGenerator, + List: listGenerator, + }, + { + Git: gitGenerator, + }, + }, + expectedErr: ErrMoreThenOneInnerGenerators, + }, + { + name: "returns error if there is more than one inner generator in the second base generator", + baseGenerators: []argoprojiov1alpha1.ApplicationSetNestedGenerator{ + { + List: listGenerator, + }, + { + Git: gitGenerator, + List: listGenerator, + }, + }, + expectedErr: ErrMoreThenOneInnerGenerators, + }, + } + + for _, testCase := range testCases { + testCaseCopy := testCase // Since tests may run in parallel + + t.Run(testCaseCopy.name, func(t *testing.T) { + genMock := &generatorMock{} + appSet := &argoprojiov1alpha1.ApplicationSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "set", + }, + Spec: argoprojiov1alpha1.ApplicationSetSpec{ + GoTemplate: true, + }, + } + + for _, g := range testCaseCopy.baseGenerators { + + gitGeneratorSpec := argoprojiov1alpha1.ApplicationSetGenerator{ + Git: g.Git, + List: g.List, + } + genMock.On("GenerateParams", mock.AnythingOfType("*v1alpha1.ApplicationSetGenerator"), appSet).Return([]map[string]interface{}{ + { + "path": map[string]string{ + "path": "app1", + "basename": "app1", + "basenameNormalized": "app1", + }, + }, + { + "path": map[string]string{ + "path": "app2", + "basename": "app2", + "basenameNormalized": "app2", + }, + }, + }, nil) + + genMock.On("GetTemplate", &gitGeneratorSpec). + Return(&argoprojiov1alpha1.ApplicationSetTemplate{}) + } + + var matrixGenerator = NewMatrixGenerator( + map[string]Generator{ + "Git": genMock, + "List": &ListGenerator{}, + }, + ) + + got, err := matrixGenerator.GenerateParams(&argoprojiov1alpha1.ApplicationSetGenerator{ + Matrix: &argoprojiov1alpha1.MatrixGenerator{ + Generators: testCaseCopy.baseGenerators, + Template: argoprojiov1alpha1.ApplicationSetTemplate{}, + }, + }, appSet) + + if testCaseCopy.expectedErr != nil { + assert.ErrorIs(t, err, testCaseCopy.expectedErr) } else { assert.NoError(t, err) assert.Equal(t, testCaseCopy.expected, got) @@ -287,7 +488,7 @@ func TestInterpolatedMatrixGenerate(t *testing.T) { name string baseGenerators []argoprojiov1alpha1.ApplicationSetNestedGenerator expectedErr error - expected []map[string]string + expected []map[string]interface{} clientError bool }{ { @@ -300,7 +501,7 @@ func TestInterpolatedMatrixGenerate(t *testing.T) { Clusters: interpolatedClusterGenerator, }, }, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"path": "examples/git-generator-files-discovery/cluster-config/dev/config.json", "path.basename": "dev", "path.basenameNormalized": "dev", "name": "dev-01", "nameNormalized": "dev-01", "server": "https://dev-01.example.com", "metadata.labels.environment": "dev", "metadata.labels.argocd.argoproj.io/secret-type": "cluster"}, {"path": "examples/git-generator-files-discovery/cluster-config/prod/config.json", "path.basename": "prod", "path.basenameNormalized": "prod", "name": "prod-01", "nameNormalized": "prod-01", "server": "https://prod-01.example.com", "metadata.labels.environment": "prod", "metadata.labels.argocd.argoproj.io/secret-type": "cluster"}, }, @@ -376,7 +577,7 @@ func TestInterpolatedMatrixGenerate(t *testing.T) { Git: g.Git, Clusters: g.Clusters, } - genMock.On("GenerateParams", mock.AnythingOfType("*v1alpha1.ApplicationSetGenerator"), appSet).Return([]map[string]string{ + genMock.On("GenerateParams", mock.AnythingOfType("*v1alpha1.ApplicationSetGenerator"), appSet).Return([]map[string]interface{}{ { "path": "examples/git-generator-files-discovery/cluster-config/dev/config.json", "path.basename": "dev", @@ -406,7 +607,195 @@ func TestInterpolatedMatrixGenerate(t *testing.T) { }, appSet) if testCaseCopy.expectedErr != nil { - assert.EqualError(t, err, testCaseCopy.expectedErr.Error()) + assert.ErrorIs(t, err, testCaseCopy.expectedErr) + } else { + assert.NoError(t, err) + assert.Equal(t, testCaseCopy.expected, got) + } + + }) + } +} + +func TestInterpolatedMatrixGenerateGoTemplate(t *testing.T) { + interpolatedGitGenerator := &argoprojiov1alpha1.GitGenerator{ + RepoURL: "RepoURL", + Revision: "Revision", + Files: []argoprojiov1alpha1.GitFileGeneratorItem{ + {Path: "examples/git-generator-files-discovery/cluster-config/dev/config.json"}, + {Path: "examples/git-generator-files-discovery/cluster-config/prod/config.json"}, + }, + } + + interpolatedClusterGenerator := &argoprojiov1alpha1.ClusterGenerator{ + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{"environment": "{{.path.basename}}"}, + MatchExpressions: nil, + }, + } + testCases := []struct { + name string + baseGenerators []argoprojiov1alpha1.ApplicationSetNestedGenerator + expectedErr error + expected []map[string]interface{} + clientError bool + }{ + { + name: "happy flow - generate interpolated params", + baseGenerators: []argoprojiov1alpha1.ApplicationSetNestedGenerator{ + { + Git: interpolatedGitGenerator, + }, + { + Clusters: interpolatedClusterGenerator, + }, + }, + expected: []map[string]interface{}{ + { + "path": map[string]string{ + "path": "examples/git-generator-files-discovery/cluster-config/dev/config.json", + "basename": "dev", + "basenameNormalized": "dev", + }, + "name": "dev-01", + "nameNormalized": "dev-01", + "server": "https://dev-01.example.com", + "metadata": map[string]interface{}{ + "labels": map[string]string{ + "environment": "dev", + "argocd.argoproj.io/secret-type": "cluster", + }, + }, + }, + { + "path": map[string]string{ + "path": "examples/git-generator-files-discovery/cluster-config/prod/config.json", + "basename": "prod", + "basenameNormalized": "prod", + }, + "name": "prod-01", + "nameNormalized": "prod-01", + "server": "https://prod-01.example.com", + "metadata": map[string]interface{}{ + "labels": map[string]string{ + "environment": "prod", + "argocd.argoproj.io/secret-type": "cluster", + }, + }, + }, + }, + clientError: false, + }, + } + clusters := []client.Object{ + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "dev-01", + Namespace: "namespace", + Labels: map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "dev", + }, + }, + Data: map[string][]byte{ + "config": []byte("{}"), + "name": []byte("dev-01"), + "server": []byte("https://dev-01.example.com"), + }, + Type: corev1.SecretType("Opaque"), + }, + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "prod-01", + Namespace: "namespace", + Labels: map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "prod", + }, + }, + Data: map[string][]byte{ + "config": []byte("{}"), + "name": []byte("prod-01"), + "server": []byte("https://prod-01.example.com"), + }, + Type: corev1.SecretType("Opaque"), + }, + } + // convert []client.Object to []runtime.Object, for use by kubefake package + runtimeClusters := []runtime.Object{} + for _, clientCluster := range clusters { + runtimeClusters = append(runtimeClusters, clientCluster) + } + + for _, testCase := range testCases { + testCaseCopy := testCase // Since tests may run in parallel + + t.Run(testCaseCopy.name, func(t *testing.T) { + genMock := &generatorMock{} + appSet := &argoprojiov1alpha1.ApplicationSet{ + Spec: argoprojiov1alpha1.ApplicationSetSpec{ + GoTemplate: true, + }, + } + + appClientset := kubefake.NewSimpleClientset(runtimeClusters...) + fakeClient := fake.NewClientBuilder().WithObjects(clusters...).Build() + cl := &possiblyErroringFakeCtrlRuntimeClient{ + fakeClient, + testCase.clientError, + } + var clusterGenerator = NewClusterGenerator(cl, context.Background(), appClientset, "namespace") + + for _, g := range testCaseCopy.baseGenerators { + + gitGeneratorSpec := argoprojiov1alpha1.ApplicationSetGenerator{ + Git: g.Git, + Clusters: g.Clusters, + } + genMock.On("GenerateParams", mock.AnythingOfType("*v1alpha1.ApplicationSetGenerator"), appSet).Return([]map[string]interface{}{ + + { + "path": map[string]string{ + "path": "examples/git-generator-files-discovery/cluster-config/dev/config.json", + "basename": "dev", + "basenameNormalized": "dev", + }, + }, + { + "path": map[string]string{ + "path": "examples/git-generator-files-discovery/cluster-config/prod/config.json", + "basename": "prod", + "basenameNormalized": "prod", + }, + }, + }, nil) + genMock.On("GetTemplate", &gitGeneratorSpec). + Return(&argoprojiov1alpha1.ApplicationSetTemplate{}) + } + var matrixGenerator = NewMatrixGenerator( + map[string]Generator{ + "Git": genMock, + "Clusters": clusterGenerator, + }, + ) + + got, err := matrixGenerator.GenerateParams(&argoprojiov1alpha1.ApplicationSetGenerator{ + Matrix: &argoprojiov1alpha1.MatrixGenerator{ + Generators: testCaseCopy.baseGenerators, + Template: argoprojiov1alpha1.ApplicationSetTemplate{}, + }, + }, appSet) + + if testCaseCopy.expectedErr != nil { + assert.ErrorIs(t, err, testCaseCopy.expectedErr) } else { assert.NoError(t, err) assert.Equal(t, testCaseCopy.expected, got) @@ -427,10 +816,10 @@ func (g *generatorMock) GetTemplate(appSetGenerator *argoprojiov1alpha1.Applicat return args.Get(0).(*argoprojiov1alpha1.ApplicationSetTemplate) } -func (g *generatorMock) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, appSet *argoprojiov1alpha1.ApplicationSet) ([]map[string]string, error) { +func (g *generatorMock) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, appSet *argoprojiov1alpha1.ApplicationSet) ([]map[string]interface{}, error) { args := g.Called(appSetGenerator, appSet) - return args.Get(0).([]map[string]string), args.Error(1) + return args.Get(0).([]map[string]interface{}), args.Error(1) } func (g *generatorMock) GetRequeueAfter(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator) time.Duration { diff --git a/applicationset/generators/merge.go b/applicationset/generators/merge.go index 777d36b62e7c6..969603ac30eea 100644 --- a/applicationset/generators/merge.go +++ b/applicationset/generators/merge.go @@ -5,7 +5,10 @@ import ( "fmt" "time" + "github.com/imdario/mergo" + "github.com/argoproj/argo-cd/v2/applicationset/utils" + argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" ) @@ -32,8 +35,8 @@ func NewMergeGenerator(supportedGenerators map[string]Generator) Generator { // getParamSetsForAllGenerators generates params for each child generator in a MergeGenerator. Param sets are returned // in slices ordered according to the order of the given generators. -func (m *MergeGenerator) getParamSetsForAllGenerators(generators []argoprojiov1alpha1.ApplicationSetNestedGenerator, appSet *argoprojiov1alpha1.ApplicationSet) ([][]map[string]string, error) { - var paramSets [][]map[string]string +func (m *MergeGenerator) getParamSetsForAllGenerators(generators []argoprojiov1alpha1.ApplicationSetNestedGenerator, appSet *argoprojiov1alpha1.ApplicationSet) ([][]map[string]interface{}, error) { + var paramSets [][]map[string]interface{} for _, generator := range generators { generatorParamSets, err := m.getParams(generator, appSet) if err != nil { @@ -46,7 +49,7 @@ func (m *MergeGenerator) getParamSetsForAllGenerators(generators []argoprojiov1a } // GenerateParams gets the params produced by the MergeGenerator. -func (m *MergeGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, appSet *argoprojiov1alpha1.ApplicationSet) ([]map[string]string, error) { +func (m *MergeGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, appSet *argoprojiov1alpha1.ApplicationSet) ([]map[string]interface{}, error) { if appSetGenerator.Merge == nil { return nil, EmptyAppSetGeneratorError } @@ -73,16 +76,24 @@ func (m *MergeGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.Appl for mergeKeyValue, baseParamSet := range baseParamSetsByMergeKey { if overrideParamSet, exists := paramSetsByMergeKey[mergeKeyValue]; exists { - overriddenParamSet, err := utils.CombineStringMapsAllowDuplicates(baseParamSet, overrideParamSet) - if err != nil { - return nil, err + + if appSet.Spec.GoTemplate { + if err := mergo.Merge(&baseParamSet, overrideParamSet, mergo.WithOverride); err != nil { + return nil, fmt.Errorf("failed to merge base param set with override param set: %w", err) + } + baseParamSetsByMergeKey[mergeKeyValue] = baseParamSet + } else { + overriddenParamSet, err := utils.CombineStringMapsAllowDuplicates(baseParamSet, overrideParamSet) + if err != nil { + return nil, err + } + baseParamSetsByMergeKey[mergeKeyValue] = utils.ConvertToMapStringInterface(overriddenParamSet) } - baseParamSetsByMergeKey[mergeKeyValue] = overriddenParamSet } } } - mergedParamSets := make([]map[string]string, len(baseParamSetsByMergeKey)) + mergedParamSets := make([]map[string]interface{}, len(baseParamSetsByMergeKey)) var i = 0 for _, mergedParamSet := range baseParamSetsByMergeKey { mergedParamSets[i] = mergedParamSet @@ -95,7 +106,7 @@ func (m *MergeGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.Appl // getParamSetsByMergeKey converts the given list of parameter sets to a map of parameter sets where the key is the // unique key of the parameter set as determined by the given mergeKeys. If any two parameter sets share the same merge // key, getParamSetsByMergeKey will throw NonUniqueParamSets. -func getParamSetsByMergeKey(mergeKeys []string, paramSets []map[string]string) (map[string]map[string]string, error) { +func getParamSetsByMergeKey(mergeKeys []string, paramSets []map[string]interface{}) (map[string]map[string]interface{}, error) { if len(mergeKeys) < 1 { return nil, ErrNoMergeKeys } @@ -105,9 +116,9 @@ func getParamSetsByMergeKey(mergeKeys []string, paramSets []map[string]string) ( deDuplicatedMergeKeys[mergeKey] = false } - paramSetsByMergeKey := make(map[string]map[string]string, len(paramSets)) + paramSetsByMergeKey := make(map[string]map[string]interface{}, len(paramSets)) for _, paramSet := range paramSets { - paramSetKey := make(map[string]string) + paramSetKey := make(map[string]interface{}) for mergeKey := range deDuplicatedMergeKeys { paramSetKey[mergeKey] = paramSet[mergeKey] } @@ -126,7 +137,7 @@ func getParamSetsByMergeKey(mergeKeys []string, paramSets []map[string]string) ( } // getParams get the parameters generated by this generator. -func (m *MergeGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.ApplicationSetNestedGenerator, appSet *argoprojiov1alpha1.ApplicationSet) ([]map[string]string, error) { +func (m *MergeGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.ApplicationSetNestedGenerator, appSet *argoprojiov1alpha1.ApplicationSet) ([]map[string]interface{}, error) { var matrix *argoprojiov1alpha1.MatrixGenerator if appSetBaseGenerator.Matrix != nil { @@ -165,7 +176,7 @@ func (m *MergeGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.Applic m.supportedGenerators, argoprojiov1alpha1.ApplicationSetTemplate{}, appSet, - map[string]string{}) + map[string]interface{}{}) if err != nil { return nil, fmt.Errorf("child generator returned an error on parameter generation: %v", err) diff --git a/applicationset/generators/merge_test.go b/applicationset/generators/merge_test.go index e6c980e819c4d..3a91a51d58013 100644 --- a/applicationset/generators/merge_test.go +++ b/applicationset/generators/merge_test.go @@ -35,7 +35,7 @@ func getTerminalListGeneratorMultiple(jsons []string) argoprojiov1alpha1.Applica return generator } -func listOfMapsToSet(maps []map[string]string) (map[string]bool, error) { +func listOfMapsToSet(maps []map[string]interface{}) (map[string]bool, error) { set := make(map[string]bool, len(maps)) for _, paramMap := range maps { paramMapAsJson, err := json.Marshal(paramMap) @@ -55,7 +55,7 @@ func TestMergeGenerate(t *testing.T) { baseGenerators []argoprojiov1alpha1.ApplicationSetNestedGenerator mergeKeys []string expectedErr error - expected []map[string]string + expected []map[string]interface{} }{ { name: "no generators", @@ -79,7 +79,7 @@ func TestMergeGenerate(t *testing.T) { *getNestedListGenerator(`{"a": "3_1","b": "different","c": "3_3"}`), // gets ignored because its merge key value isn't in the base params set }, mergeKeys: []string{"b"}, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"a": "2_1", "b": "same", "c": "1_3"}, }, }, @@ -90,7 +90,7 @@ func TestMergeGenerate(t *testing.T) { *getNestedListGenerator(`{"a": "a"}`), }, mergeKeys: []string{"b"}, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"a": "a"}, }, }, @@ -101,7 +101,7 @@ func TestMergeGenerate(t *testing.T) { *getNestedListGenerator(`{"b": "b"}`), }, mergeKeys: []string{"b"}, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"a": "a"}, }, }, @@ -119,7 +119,7 @@ func TestMergeGenerate(t *testing.T) { *getNestedListGenerator(`{"a": "1", "b": "1", "c": "added"}`), }, mergeKeys: []string{"a", "b"}, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"a": "1", "b": "1", "c": "added"}, {"a": "1", "b": "2"}, {"a": "2", "b": "1"}, @@ -141,7 +141,7 @@ func TestMergeGenerate(t *testing.T) { *getNestedListGenerator(`{"a": "1", "b": "3", "d": "added"}`), }, mergeKeys: []string{"a", "b"}, - expected: []map[string]string{ + expected: []map[string]interface{}{ {"a": "1", "b": "3", "c": "added", "d": "added"}, {"a": "2", "b": "2"}, }, @@ -213,9 +213,9 @@ func TestParamSetsAreUniqueByMergeKeys(t *testing.T) { testCases := []struct { name string mergeKeys []string - paramSets []map[string]string + paramSets []map[string]interface{} expectedErr error - expected map[string]map[string]string + expected map[string]map[string]interface{} }{ { name: "no merge keys", @@ -225,28 +225,37 @@ func TestParamSetsAreUniqueByMergeKeys(t *testing.T) { { name: "no paramSets", mergeKeys: []string{"key"}, - expected: make(map[string]map[string]string), + expected: make(map[string]map[string]interface{}), }, { name: "simple key, unique paramSets", mergeKeys: []string{"key"}, - paramSets: []map[string]string{{"key": "a"}, {"key": "b"}}, - expected: map[string]map[string]string{ + paramSets: []map[string]interface{}{{"key": "a"}, {"key": "b"}}, + expected: map[string]map[string]interface{}{ `{"key":"a"}`: {"key": "a"}, `{"key":"b"}`: {"key": "b"}, }, }, + { + name: "simple key object, unique paramSets", + mergeKeys: []string{"key"}, + paramSets: []map[string]interface{}{{"key": map[string]interface{}{"hello": "world"}}, {"key": "b"}}, + expected: map[string]map[string]interface{}{ + `{"key":{"hello":"world"}}`: {"key": map[string]interface{}{"hello": "world"}}, + `{"key":"b"}`: {"key": "b"}, + }, + }, { name: "simple key, non-unique paramSets", mergeKeys: []string{"key"}, - paramSets: []map[string]string{{"key": "a"}, {"key": "b"}, {"key": "b"}}, + paramSets: []map[string]interface{}{{"key": "a"}, {"key": "b"}, {"key": "b"}}, expectedErr: fmt.Errorf("%w. Duplicate key was %s", ErrNonUniqueParamSets, `{"key":"b"}`), }, { name: "simple key, duplicated key name, unique paramSets", mergeKeys: []string{"key", "key"}, - paramSets: []map[string]string{{"key": "a"}, {"key": "b"}}, - expected: map[string]map[string]string{ + paramSets: []map[string]interface{}{{"key": "a"}, {"key": "b"}}, + expected: map[string]map[string]interface{}{ `{"key":"a"}`: {"key": "a"}, `{"key":"b"}`: {"key": "b"}, }, @@ -254,32 +263,46 @@ func TestParamSetsAreUniqueByMergeKeys(t *testing.T) { { name: "simple key, duplicated key name, non-unique paramSets", mergeKeys: []string{"key", "key"}, - paramSets: []map[string]string{{"key": "a"}, {"key": "b"}, {"key": "b"}}, + paramSets: []map[string]interface{}{{"key": "a"}, {"key": "b"}, {"key": "b"}}, expectedErr: fmt.Errorf("%w. Duplicate key was %s", ErrNonUniqueParamSets, `{"key":"b"}`), }, { name: "compound key, unique paramSets", mergeKeys: []string{"key1", "key2"}, - paramSets: []map[string]string{ + paramSets: []map[string]interface{}{ {"key1": "a", "key2": "a"}, {"key1": "a", "key2": "b"}, {"key1": "b", "key2": "a"}, }, - expected: map[string]map[string]string{ + expected: map[string]map[string]interface{}{ `{"key1":"a","key2":"a"}`: {"key1": "a", "key2": "a"}, `{"key1":"a","key2":"b"}`: {"key1": "a", "key2": "b"}, `{"key1":"b","key2":"a"}`: {"key1": "b", "key2": "a"}, }, }, + { + name: "compound key object, unique paramSets", + mergeKeys: []string{"key1", "key2"}, + paramSets: []map[string]interface{}{ + {"key1": "a", "key2": map[string]interface{}{"hello": "world"}}, + {"key1": "a", "key2": "b"}, + {"key1": "b", "key2": "a"}, + }, + expected: map[string]map[string]interface{}{ + `{"key1":"a","key2":{"hello":"world"}}`: {"key1": "a", "key2": map[string]interface{}{"hello": "world"}}, + `{"key1":"a","key2":"b"}`: {"key1": "a", "key2": "b"}, + `{"key1":"b","key2":"a"}`: {"key1": "b", "key2": "a"}, + }, + }, { name: "compound key, duplicate key names, unique paramSets", mergeKeys: []string{"key1", "key1", "key2"}, - paramSets: []map[string]string{ + paramSets: []map[string]interface{}{ {"key1": "a", "key2": "a"}, {"key1": "a", "key2": "b"}, {"key1": "b", "key2": "a"}, }, - expected: map[string]map[string]string{ + expected: map[string]map[string]interface{}{ `{"key1":"a","key2":"a"}`: {"key1": "a", "key2": "a"}, `{"key1":"a","key2":"b"}`: {"key1": "a", "key2": "b"}, `{"key1":"b","key2":"a"}`: {"key1": "b", "key2": "a"}, @@ -288,7 +311,7 @@ func TestParamSetsAreUniqueByMergeKeys(t *testing.T) { { name: "compound key, non-unique paramSets", mergeKeys: []string{"key1", "key2"}, - paramSets: []map[string]string{ + paramSets: []map[string]interface{}{ {"key1": "a", "key2": "a"}, {"key1": "a", "key2": "a"}, {"key1": "b", "key2": "a"}, @@ -298,7 +321,7 @@ func TestParamSetsAreUniqueByMergeKeys(t *testing.T) { { name: "compound key, duplicate key names, non-unique paramSets", mergeKeys: []string{"key1", "key1", "key2"}, - paramSets: []map[string]string{ + paramSets: []map[string]interface{}{ {"key1": "a", "key2": "a"}, {"key1": "a", "key2": "a"}, {"key1": "b", "key2": "a"}, diff --git a/applicationset/generators/pull_request.go b/applicationset/generators/pull_request.go index fd38c25b14d06..629e81cf7f17b 100644 --- a/applicationset/generators/pull_request.go +++ b/applicationset/generators/pull_request.go @@ -48,7 +48,7 @@ func (g *PullRequestGenerator) GetTemplate(appSetGenerator *argoprojiov1alpha1.A return &appSetGenerator.PullRequest.Template } -func (g *PullRequestGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, applicationSetInfo *argoprojiov1alpha1.ApplicationSet) ([]map[string]string, error) { +func (g *PullRequestGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, applicationSetInfo *argoprojiov1alpha1.ApplicationSet) ([]map[string]interface{}, error) { if appSetGenerator == nil { return nil, EmptyAppSetGeneratorError } @@ -67,7 +67,7 @@ func (g *PullRequestGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha if err != nil { return nil, fmt.Errorf("error listing repos: %v", err) } - params := make([]map[string]string, 0, len(pulls)) + params := make([]map[string]interface{}, 0, len(pulls)) // In order to follow the DNS label standard as defined in RFC 1123, // we need to limit the 'branch' to 50 to give room to append/suffix-ing it @@ -87,7 +87,7 @@ func (g *PullRequestGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha shortSHALength = len(pull.HeadSHA) } - params = append(params, map[string]string{ + params = append(params, map[string]interface{}{ "number": strconv.Itoa(pull.Number), "branch": pull.Branch, "branch_slug": slug.Make(pull.Branch), diff --git a/applicationset/generators/pull_request_test.go b/applicationset/generators/pull_request_test.go index 4cb18d64b3d1a..34b867dace0f9 100644 --- a/applicationset/generators/pull_request_test.go +++ b/applicationset/generators/pull_request_test.go @@ -18,7 +18,7 @@ func TestPullRequestGithubGenerateParams(t *testing.T) { ctx := context.Background() cases := []struct { selectFunc func(context.Context, *argoprojiov1alpha1.PullRequestGenerator, *argoprojiov1alpha1.ApplicationSet) (pullrequest.PullRequestService, error) - expected []map[string]string + expected []map[string]interface{} expectedErr error }{ { @@ -35,7 +35,7 @@ func TestPullRequestGithubGenerateParams(t *testing.T) { nil, ) }, - expected: []map[string]string{ + expected: []map[string]interface{}{ { "number": "1", "branch": "branch1", @@ -60,7 +60,7 @@ func TestPullRequestGithubGenerateParams(t *testing.T) { nil, ) }, - expected: []map[string]string{ + expected: []map[string]interface{}{ { "number": "2", "branch": "feat/areally+long_pull_request_name_to_test_argo_slugification_and_branch_name_shortening_feature", @@ -85,7 +85,7 @@ func TestPullRequestGithubGenerateParams(t *testing.T) { nil, ) }, - expected: []map[string]string{ + expected: []map[string]interface{}{ { "number": "1", "branch": "a-very-short-sha", diff --git a/applicationset/generators/scm_provider.go b/applicationset/generators/scm_provider.go index bd7b7a6adb3fc..4f3a624a87fc9 100644 --- a/applicationset/generators/scm_provider.go +++ b/applicationset/generators/scm_provider.go @@ -10,6 +10,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/argoproj/argo-cd/v2/applicationset/services/scm_provider" + "github.com/argoproj/argo-cd/v2/applicationset/utils" argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" ) @@ -43,7 +44,7 @@ func (g *SCMProviderGenerator) GetTemplate(appSetGenerator *argoprojiov1alpha1.A return &appSetGenerator.SCMProvider.Template } -func (g *SCMProviderGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, applicationSetInfo *argoprojiov1alpha1.ApplicationSet) ([]map[string]string, error) { +func (g *SCMProviderGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, applicationSetInfo *argoprojiov1alpha1.ApplicationSet) ([]map[string]interface{}, error) { if appSetGenerator == nil { return nil, EmptyAppSetGeneratorError } @@ -119,7 +120,7 @@ func (g *SCMProviderGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha if err != nil { return nil, fmt.Errorf("error listing repos: %v", err) } - params := make([]map[string]string, 0, len(repos)) + params := make([]map[string]interface{}, 0, len(repos)) var shortSHALength int for _, repo := range repos { shortSHALength = 8 @@ -127,7 +128,7 @@ func (g *SCMProviderGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha shortSHALength = len(repo.SHA) } - params = append(params, map[string]string{ + params = append(params, map[string]interface{}{ "organization": repo.Organization, "repository": repo.Repository, "url": repo.URL, @@ -135,7 +136,7 @@ func (g *SCMProviderGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha "sha": repo.SHA, "short_sha": repo.SHA[:shortSHALength], "labels": strings.Join(repo.Labels, ","), - "branchNormalized": sanitizeName(repo.Branch), + "branchNormalized": utils.SanitizeName(repo.Branch), }) } return params, nil diff --git a/applicationset/utils/map.go b/applicationset/utils/map.go index 709bdee7a2cd2..4e45e1c3fe2d2 100644 --- a/applicationset/utils/map.go +++ b/applicationset/utils/map.go @@ -4,7 +4,32 @@ import ( "fmt" ) -func CombineStringMaps(a map[string]string, b map[string]string) (map[string]string, error) { +func ConvertToMapStringString(mapStringInterface map[string]interface{}) map[string]string { + mapStringString := make(map[string]string, len(mapStringInterface)) + + for key, value := range mapStringInterface { + strKey := fmt.Sprintf("%v", key) + strValue := fmt.Sprintf("%v", value) + + mapStringString[strKey] = strValue + } + return mapStringString +} + +func ConvertToMapStringInterface(mapStringString map[string]string) map[string]interface{} { + mapStringInterface := make(map[string]interface{}, len(mapStringString)) + + for key, value := range mapStringString { + mapStringInterface[key] = value + } + return mapStringInterface +} + +func CombineStringMaps(aSI map[string]interface{}, bSI map[string]interface{}) (map[string]string, error) { + + a := ConvertToMapStringString(aSI) + b := ConvertToMapStringString(bSI) + res := map[string]string{} for k, v := range a { @@ -23,7 +48,11 @@ func CombineStringMaps(a map[string]string, b map[string]string) (map[string]str } // CombineStringMapsAllowDuplicates merges two maps. Where there are duplicates, take the latter map's value. -func CombineStringMapsAllowDuplicates(a map[string]string, b map[string]string) (map[string]string, error) { +func CombineStringMapsAllowDuplicates(aSI map[string]interface{}, bSI map[string]interface{}) (map[string]string, error) { + + a := ConvertToMapStringString(aSI) + b := ConvertToMapStringString(bSI) + res := map[string]string{} for k, v := range a { diff --git a/applicationset/utils/map_test.go b/applicationset/utils/map_test.go index adbba89098860..860bb046cc253 100644 --- a/applicationset/utils/map_test.go +++ b/applicationset/utils/map_test.go @@ -10,29 +10,29 @@ import ( func TestCombineStringMaps(t *testing.T) { testCases := []struct { name string - left map[string]string - right map[string]string + left map[string]interface{} + right map[string]interface{} expected map[string]string expectedErr error }{ { name: "combines the maps", - left: map[string]string{"foo": "bar"}, - right: map[string]string{"a": "b"}, + left: map[string]interface{}{"foo": "bar"}, + right: map[string]interface{}{"a": "b"}, expected: map[string]string{"a": "b", "foo": "bar"}, expectedErr: nil, }, { name: "fails if keys are the same but value isn't", - left: map[string]string{"foo": "bar", "a": "fail"}, - right: map[string]string{"a": "b", "c": "d"}, + left: map[string]interface{}{"foo": "bar", "a": "fail"}, + right: map[string]interface{}{"a": "b", "c": "d"}, expected: map[string]string{"a": "b", "foo": "bar"}, expectedErr: fmt.Errorf("found duplicate key a with different value, a: fail ,b: b"), }, { name: "pass if keys & values are the same", - left: map[string]string{"foo": "bar", "a": "b"}, - right: map[string]string{"a": "b", "c": "d"}, + left: map[string]interface{}{"foo": "bar", "a": "b"}, + right: map[string]interface{}{"a": "b", "c": "d"}, expected: map[string]string{"a": "b", "c": "d", "foo": "bar"}, expectedErr: nil, }, diff --git a/applicationset/utils/utils.go b/applicationset/utils/utils.go index b2b1445022776..b8eda72c38871 100644 --- a/applicationset/utils/utils.go +++ b/applicationset/utils/utils.go @@ -1,29 +1,169 @@ package utils import ( + "bytes" "encoding/json" "fmt" "io" "reflect" + "regexp" "sort" - "strconv" "strings" + "text/template" + "unsafe" - log "github.com/sirupsen/logrus" + "github.com/Masterminds/sprig" "github.com/valyala/fasttemplate" + log "github.com/sirupsen/logrus" + argoappsv1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" argoappsetv1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" ) +var sprigFuncMap = sprig.GenericFuncMap() // a singleton for better performance + +func init() { + // Avoid allowing the user to learn things about the environment. + delete(sprigFuncMap, "env") + delete(sprigFuncMap, "expandenv") + delete(sprigFuncMap, "getHostByName") + sprigFuncMap["normalize"] = SanitizeName +} + type Renderer interface { - RenderTemplateParams(tmpl *argoappsv1.Application, syncPolicy *argoappsetv1.ApplicationSetSyncPolicy, params map[string]string) (*argoappsv1.Application, error) + RenderTemplateParams(tmpl *argoappsv1.Application, syncPolicy *argoappsetv1.ApplicationSetSyncPolicy, params map[string]interface{}, useGoTemplate bool) (*argoappsv1.Application, error) } type Render struct { } -func (r *Render) RenderTemplateParams(tmpl *argoappsv1.Application, syncPolicy *argoappsetv1.ApplicationSetSyncPolicy, params map[string]string) (*argoappsv1.Application, error) { +func copyValueIntoUnexported(destination, value reflect.Value) { + reflect.NewAt(destination.Type(), unsafe.Pointer(destination.UnsafeAddr())). + Elem(). + Set(value) +} + +func copyUnexported(copy, original reflect.Value) { + var unexported = reflect.NewAt(original.Type(), unsafe.Pointer(original.UnsafeAddr())).Elem() + copyValueIntoUnexported(copy, unexported) +} + +// This function is in charge of searching all String fields of the object recursively and apply templating +// thanks to https://gist.github.com/randallmlough/1fd78ec8a1034916ca52281e3b886dc7 +func (r *Render) deeplyReplace(copy, original reflect.Value, replaceMap map[string]interface{}, useGoTemplate bool) error { + switch original.Kind() { + // The first cases handle nested structures and translate them recursively + // If it is a pointer we need to unwrap and call once again + case reflect.Ptr: + // To get the actual value of the original we have to call Elem() + // At the same time this unwraps the pointer so we don't end up in + // an infinite recursion + originalValue := original.Elem() + // Check if the pointer is nil + if !originalValue.IsValid() { + return nil + } + // Allocate a new object and set the pointer to it + if originalValue.CanSet() { + copy.Set(reflect.New(originalValue.Type())) + } else { + copyUnexported(copy, original) + } + // Unwrap the newly created pointer + if err := r.deeplyReplace(copy.Elem(), originalValue, replaceMap, useGoTemplate); err != nil { + return err + } + + // If it is an interface (which is very similar to a pointer), do basically the + // same as for the pointer. Though a pointer is not the same as an interface so + // note that we have to call Elem() after creating a new object because otherwise + // we would end up with an actual pointer + case reflect.Interface: + // Get rid of the wrapping interface + originalValue := original.Elem() + // Create a new object. Now new gives us a pointer, but we want the value it + // points to, so we have to call Elem() to unwrap it + copyValue := reflect.New(originalValue.Type()).Elem() + if err := r.deeplyReplace(copyValue, originalValue, replaceMap, useGoTemplate); err != nil { + return err + } + copy.Set(copyValue) + + // If it is a struct we translate each field + case reflect.Struct: + for i := 0; i < original.NumField(); i += 1 { + var currentType = fmt.Sprintf("%s.%s", original.Type().Field(i).Name, original.Type().PkgPath()) + // specific case time + if currentType == "time.Time" { + copy.Field(i).Set(original.Field(i)) + } else if err := r.deeplyReplace(copy.Field(i), original.Field(i), replaceMap, useGoTemplate); err != nil { + return err + } + } + + // If it is a slice we create a new slice and translate each element + case reflect.Slice: + if copy.CanSet() { + copy.Set(reflect.MakeSlice(original.Type(), original.Len(), original.Cap())) + } else { + copyValueIntoUnexported(copy, reflect.MakeSlice(original.Type(), original.Len(), original.Cap())) + } + + for i := 0; i < original.Len(); i += 1 { + if err := r.deeplyReplace(copy.Index(i), original.Index(i), replaceMap, useGoTemplate); err != nil { + return err + } + } + + // If it is a map we create a new map and translate each value + case reflect.Map: + if copy.CanSet() { + copy.Set(reflect.MakeMap(original.Type())) + } else { + copyValueIntoUnexported(copy, reflect.MakeMap(original.Type())) + } + for _, key := range original.MapKeys() { + originalValue := original.MapIndex(key) + if originalValue.Kind() != reflect.String && originalValue.IsNil() { + continue + } + // New gives us a pointer, but again we want the value + copyValue := reflect.New(originalValue.Type()).Elem() + + if err := r.deeplyReplace(copyValue, originalValue, replaceMap, useGoTemplate); err != nil { + return err + } + copy.SetMapIndex(key, copyValue) + } + + // Otherwise we cannot traverse anywhere so this finishes the the recursion + // If it is a string translate it (yay finally we're doing what we came for) + case reflect.String: + strToTemplate := original.String() + templated, err := r.Replace(strToTemplate, replaceMap, useGoTemplate) + if err != nil { + return err + } + if copy.CanSet() { + copy.SetString(templated) + } else { + copyValueIntoUnexported(copy, reflect.ValueOf(templated)) + } + return nil + + // And everything else will simply be taken from the original + default: + if copy.CanSet() { + copy.Set(original) + } else { + copyUnexported(copy, original) + } + } + return nil +} + +func (r *Render) RenderTemplateParams(tmpl *argoappsv1.Application, syncPolicy *argoappsetv1.ApplicationSetSyncPolicy, params map[string]interface{}, useGoTemplate bool) (*argoappsv1.Application, error) { if tmpl == nil { return nil, fmt.Errorf("application template is empty ") } @@ -32,22 +172,14 @@ func (r *Render) RenderTemplateParams(tmpl *argoappsv1.Application, syncPolicy * return tmpl, nil } - tmplBytes, err := json.Marshal(tmpl) - if err != nil { - return nil, err - } + original := reflect.ValueOf(tmpl) + copy := reflect.New(original.Type()).Elem() - fstTmpl := fasttemplate.New(string(tmplBytes), "{{", "}}") - replacedTmplStr, err := r.Replace(fstTmpl, params, true) - if err != nil { + if err := r.deeplyReplace(copy, original, params, useGoTemplate); err != nil { return nil, err } - var replacedTmpl argoappsv1.Application - err = json.Unmarshal([]byte(replacedTmplStr), &replacedTmpl) - if err != nil { - return nil, err - } + replacedTmpl := copy.Interface().(*argoappsv1.Application) // Add the 'resources-finalizer' finalizer if: // The template application doesn't have any finalizers, and: @@ -55,46 +187,50 @@ func (r *Render) RenderTemplateParams(tmpl *argoappsv1.Application, syncPolicy * // b) there IS a syncPolicy, but preserveResourcesOnDeletion is set to false // See TestRenderTemplateParamsFinalizers in util_test.go for test-based definition of behaviour if (syncPolicy == nil || !syncPolicy.PreserveResourcesOnDeletion) && - (replacedTmpl.ObjectMeta.Finalizers == nil || len(replacedTmpl.ObjectMeta.Finalizers) == 0) { + ((*replacedTmpl).ObjectMeta.Finalizers == nil || len((*replacedTmpl).ObjectMeta.Finalizers) == 0) { - replacedTmpl.ObjectMeta.Finalizers = []string{"resources-finalizer.argocd.argoproj.io"} + (*replacedTmpl).ObjectMeta.Finalizers = []string{"resources-finalizer.argocd.argoproj.io"} } - return &replacedTmpl, nil + return replacedTmpl, nil } +var isTemplatedRegex = regexp.MustCompile(".*{{.*}}.*") + // Replace executes basic string substitution of a template with replacement values. -// 'allowUnresolved' indicates whether it is acceptable to have unresolved variables // remaining in the substituted template. -func (r *Render) Replace(fstTmpl *fasttemplate.Template, replaceMap map[string]string, allowUnresolved bool) (string, error) { - var unresolvedErr error - replacedTmpl := fstTmpl.ExecuteFuncString(func(w io.Writer, tag string) (int, error) { +func (r *Render) Replace(tmpl string, replaceMap map[string]interface{}, useGoTemplate bool) (string, error) { + if useGoTemplate { + template, err := template.New("").Funcs(sprigFuncMap).Parse(tmpl) + if err != nil { + return "", fmt.Errorf("failed to parse template %s: %w", tmpl, err) + } - trimmedTag := strings.TrimSpace(tag) + var replacedTmplBuffer bytes.Buffer + if err = template.Execute(&replacedTmplBuffer, replaceMap); err != nil { + return "", fmt.Errorf("failed to execute go template %s: %w", tmpl, err) + } + + return replacedTmplBuffer.String(), nil + } + + if !isTemplatedRegex.MatchString(tmpl) { + return tmpl, nil + } - replacement, ok := replaceMap[trimmedTag] + fstTmpl := fasttemplate.New(tmpl, "{{", "}}") + replacedTmpl := fstTmpl.ExecuteFuncString(func(w io.Writer, tag string) (int, error) { + trimmedTag := strings.TrimSpace(tag) + replacement, ok := replaceMap[trimmedTag].(string) if len(trimmedTag) == 0 || !ok { - if allowUnresolved { - // just write the same string back - return w.Write([]byte(fmt.Sprintf("{{%s}}", tag))) - } - unresolvedErr = fmt.Errorf("failed to resolve {{%s}}", tag) - return 0, nil + return w.Write([]byte(fmt.Sprintf("{{%s}}", tag))) } - // The following escapes any special characters (e.g. newlines, tabs, etc...) - // in preparation for substitution - replacement = strconv.Quote(replacement) - replacement = replacement[1 : len(replacement)-1] return w.Write([]byte(replacement)) }) - if unresolvedErr != nil { - return "", unresolvedErr - } - return replacedTmpl, nil } -// Log a warning if there are unrecognized generators +// CheckInvalidGenerators logs a warning if there are unrecognized generators func CheckInvalidGenerators(applicationSetInfo *argoappsetv1.ApplicationSet) { hasInvalidGenerators, invalidGenerators := invalidGenerators(applicationSetInfo) if len(invalidGenerators) > 0 { @@ -187,3 +323,20 @@ func NormalizeBitbucketBasePath(basePath string) string { } return basePath } + +// SanitizeName sanitizes the name in accordance with the below rules +// 1. contain no more than 253 characters +// 2. contain only lowercase alphanumeric characters, '-' or '.' +// 3. start and end with an alphanumeric character +func SanitizeName(name string) string { + invalidDNSNameChars := regexp.MustCompile("[^-a-z0-9.]") + maxDNSNameLength := 253 + + name = strings.ToLower(name) + name = invalidDNSNameChars.ReplaceAllString(name, "-") + if len(name) > maxDNSNameLength { + name = name[:maxDNSNameLength] + } + + return strings.Trim(name, "-.") +} diff --git a/applicationset/utils/utils_test.go b/applicationset/utils/utils_test.go index 5dc9a477a7a6c..c0a057906acf6 100644 --- a/applicationset/utils/utils_test.go +++ b/applicationset/utils/utils_test.go @@ -2,12 +2,14 @@ package utils import ( "testing" + "time" "github.com/sirupsen/logrus" logtest "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" argoappsv1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" argoappsetv1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" @@ -29,6 +31,14 @@ func TestRenderTemplateParams(t *testing.T) { fieldMap["Project"] = func(app *argoappsv1.Application) *string { return &app.Spec.Project } emptyApplication := &argoappsv1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"annotation-key": "annotation-value", "annotation-key2": "annotation-value2"}, + Labels: map[string]string{"label-key": "label-value", "label-key2": "label-value2"}, + CreationTimestamp: metav1.NewTime(time.Now()), + UID: types.UID("d546da12-06b7-4f9a-8ea2-3adb16a20e2b"), + Name: "application-one", + Namespace: "default", + }, Spec: argoappsv1.ApplicationSpec{ Source: argoappsv1.ApplicationSource{ Path: "", @@ -48,14 +58,14 @@ func TestRenderTemplateParams(t *testing.T) { tests := []struct { name string fieldVal string - params map[string]string + params map[string]interface{} expectedVal string }{ { name: "simple substitution", fieldVal: "{{one}}", expectedVal: "two", - params: map[string]string{ + params: map[string]interface{}{ "one": "two", }, }, @@ -63,7 +73,7 @@ func TestRenderTemplateParams(t *testing.T) { name: "simple substitution with whitespace", fieldVal: "{{ one }}", expectedVal: "two", - params: map[string]string{ + params: map[string]interface{}{ "one": "two", }, }, @@ -72,7 +82,7 @@ func TestRenderTemplateParams(t *testing.T) { name: "template characters but not in a template", fieldVal: "}} {{", expectedVal: "}} {{", - params: map[string]string{ + params: map[string]interface{}{ "one": "two", }, }, @@ -81,7 +91,7 @@ func TestRenderTemplateParams(t *testing.T) { name: "nested template", fieldVal: "{{ }}", expectedVal: "{{ }}", - params: map[string]string{ + params: map[string]interface{}{ "one": "{{ }}", }, }, @@ -89,7 +99,7 @@ func TestRenderTemplateParams(t *testing.T) { name: "field with whitespace", fieldVal: "{{ }}", expectedVal: "{{ }}", - params: map[string]string{ + params: map[string]interface{}{ " ": "two", "": "three", }, @@ -99,7 +109,7 @@ func TestRenderTemplateParams(t *testing.T) { name: "template contains itself, containing itself", fieldVal: "{{one}}", expectedVal: "{{one}}", - params: map[string]string{ + params: map[string]interface{}{ "{{one}}": "{{one}}", }, }, @@ -108,7 +118,7 @@ func TestRenderTemplateParams(t *testing.T) { name: "template contains itself, containing something else", fieldVal: "{{one}}", expectedVal: "{{one}}", - params: map[string]string{ + params: map[string]interface{}{ "{{one}}": "{{two}}", }, }, @@ -117,7 +127,7 @@ func TestRenderTemplateParams(t *testing.T) { name: "templates are case sensitive", fieldVal: "{{ONE}}", expectedVal: "{{ONE}}", - params: map[string]string{ + params: map[string]interface{}{ "{{one}}": "two", }, }, @@ -125,7 +135,7 @@ func TestRenderTemplateParams(t *testing.T) { name: "multiple on a line", fieldVal: "{{one}}{{one}}", expectedVal: "twotwo", - params: map[string]string{ + params: map[string]interface{}{ "one": "two", }, }, @@ -133,11 +143,20 @@ func TestRenderTemplateParams(t *testing.T) { name: "multiple different on a line", fieldVal: "{{one}}{{three}}", expectedVal: "twofour", - params: map[string]string{ + params: map[string]interface{}{ "one": "two", "three": "four", }, }, + { + name: "multiple different on a line with quote", + fieldVal: "{{one}} {{three}}", + expectedVal: "\"hello\" world four", + params: map[string]interface{}{ + "one": "\"hello\" world", + "three": "four", + }, + }, } for _, test := range tests { @@ -154,14 +173,291 @@ func TestRenderTemplateParams(t *testing.T) { // Render the cloned application, into a new application render := Render{} - newApplication, err := render.RenderTemplateParams(application, nil, test.params) + newApplication, err := render.RenderTemplateParams(application, nil, test.params, false) // Retrieve the value of the target field from the newApplication, then verify that // the target field has been templated into the expected value actualValue := *getPtrFunc(newApplication) assert.Equal(t, test.expectedVal, actualValue, "Field '%s' had an unexpected value. expected: '%s' value: '%s'", fieldName, test.expectedVal, actualValue) + assert.Equal(t, newApplication.ObjectMeta.Annotations["annotation-key"], "annotation-value") + assert.Equal(t, newApplication.ObjectMeta.Annotations["annotation-key2"], "annotation-value2") + assert.Equal(t, newApplication.ObjectMeta.Labels["label-key"], "label-value") + assert.Equal(t, newApplication.ObjectMeta.Labels["label-key2"], "label-value2") + assert.Equal(t, newApplication.ObjectMeta.Name, "application-one") + assert.Equal(t, newApplication.ObjectMeta.Namespace, "default") + assert.Equal(t, newApplication.ObjectMeta.UID, types.UID("d546da12-06b7-4f9a-8ea2-3adb16a20e2b")) + assert.Equal(t, newApplication.ObjectMeta.CreationTimestamp, application.ObjectMeta.CreationTimestamp) assert.NoError(t, err) + } + }) + } + +} + +func TestRenderTemplateParamsGoTemplate(t *testing.T) { + + // Believe it or not, this is actually less complex than the equivalent solution using reflection + fieldMap := map[string]func(app *argoappsv1.Application) *string{} + fieldMap["Path"] = func(app *argoappsv1.Application) *string { return &app.Spec.Source.Path } + fieldMap["RepoURL"] = func(app *argoappsv1.Application) *string { return &app.Spec.Source.RepoURL } + fieldMap["TargetRevision"] = func(app *argoappsv1.Application) *string { return &app.Spec.Source.TargetRevision } + fieldMap["Chart"] = func(app *argoappsv1.Application) *string { return &app.Spec.Source.Chart } + + fieldMap["Server"] = func(app *argoappsv1.Application) *string { return &app.Spec.Destination.Server } + fieldMap["Namespace"] = func(app *argoappsv1.Application) *string { return &app.Spec.Destination.Namespace } + fieldMap["Name"] = func(app *argoappsv1.Application) *string { return &app.Spec.Destination.Name } + + fieldMap["Project"] = func(app *argoappsv1.Application) *string { return &app.Spec.Project } + + emptyApplication := &argoappsv1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"annotation-key": "annotation-value", "annotation-key2": "annotation-value2"}, + Labels: map[string]string{"label-key": "label-value", "label-key2": "label-value2"}, + CreationTimestamp: metav1.NewTime(time.Now()), + UID: types.UID("d546da12-06b7-4f9a-8ea2-3adb16a20e2b"), + Name: "application-one", + Namespace: "default", + }, + Spec: argoappsv1.ApplicationSpec{ + Source: argoappsv1.ApplicationSource{ + Path: "", + RepoURL: "", + TargetRevision: "", + Chart: "", + }, + Destination: argoappsv1.ApplicationDestination{ + Server: "", + Namespace: "", + Name: "", + }, + Project: "", + }, + } + + tests := []struct { + name string + fieldVal string + params map[string]interface{} + expectedVal string + errorMessage string + }{ + { + name: "simple substitution", + fieldVal: "{{ .one }}", + expectedVal: "two", + params: map[string]interface{}{ + "one": "two", + }, + }, + { + name: "simple substitution with whitespace", + fieldVal: "{{ .one }}", + expectedVal: "two", + params: map[string]interface{}{ + "one": "two", + }, + }, + { + name: "template contains itself, containing itself", + fieldVal: "{{ .one }}", + expectedVal: "{{one}}", + params: map[string]interface{}{ + "one": "{{one}}", + }, + }, + { + name: "template contains itself, containing something else", + fieldVal: "{{ .one }}", + expectedVal: "{{two}}", + params: map[string]interface{}{ + "one": "{{two}}", + }, + }, + { + name: "multiple on a line", + fieldVal: "{{.one}}{{.one}}", + expectedVal: "twotwo", + params: map[string]interface{}{ + "one": "two", + }, + }, + { + name: "multiple different on a line", + fieldVal: "{{.one}}{{.three}}", + expectedVal: "twofour", + params: map[string]interface{}{ + "one": "two", + "three": "four", + }, + }, + { + name: "multiple different on a line with quote", + fieldVal: "{{.one}} {{.three}}", + expectedVal: "\"hello\" world four", + params: map[string]interface{}{ + "one": "\"hello\" world", + "three": "four", + }, + }, + { + name: "depth", + fieldVal: "{{ .image.version }}", + expectedVal: "latest", + params: map[string]interface{}{ + "replicas": 3, + "image": map[string]interface{}{ + "name": "busybox", + "version": "latest", + }, + }, + }, + { + name: "multiple depth", + fieldVal: "{{ .image.name }}:{{ .image.version }}", + expectedVal: "busybox:latest", + params: map[string]interface{}{ + "replicas": 3, + "image": map[string]interface{}{ + "name": "busybox", + "version": "latest", + }, + }, + }, + { + name: "if ok", + fieldVal: "{{ if .hpa.enabled }}{{ .hpa.maxReplicas }}{{ else }}{{ .replicas }}{{ end }}", + expectedVal: "5", + params: map[string]interface{}{ + "replicas": 3, + "hpa": map[string]interface{}{ + "enabled": true, + "minReplicas": 1, + "maxReplicas": 5, + }, + }, + }, + { + name: "if not ok", + fieldVal: "{{ if .hpa.enabled }}{{ .hpa.maxReplicas }}{{ else }}{{ .replicas }}{{ end }}", + expectedVal: "3", + params: map[string]interface{}{ + "replicas": 3, + "hpa": map[string]interface{}{ + "enabled": false, + "minReplicas": 1, + "maxReplicas": 5, + }, + }, + }, + { + name: "loop", + fieldVal: "{{ range .volumes }}[{{ .name }}]{{ end }}", + expectedVal: "[volume-one][volume-two]", + params: map[string]interface{}{ + "replicas": 3, + "volumes": []map[string]interface{}{ + { + "name": "volume-one", + "emptyDir": map[string]interface{}{}, + }, + { + "name": "volume-two", + "emptyDir": map[string]interface{}{}, + }, + }, + }, + }, + { + name: "Index", + fieldVal: `{{ index .admin "admin-ca" }}, {{ index .admin "admin-jks" }}`, + expectedVal: "value admin ca, value admin jks", + params: map[string]interface{}{ + "admin": map[string]interface{}{ + "admin-ca": "value admin ca", + "admin-jks": "value admin jks", + }, + }, + }, + { + name: "Index", + fieldVal: `{{ index .admin "admin-ca" }}, \\ "Hello world", {{ index .admin "admin-jks" }}`, + expectedVal: `value "admin" ca with \, \\ "Hello world", value admin jks`, + params: map[string]interface{}{ + "admin": map[string]interface{}{ + "admin-ca": `value "admin" ca with \`, + "admin-jks": "value admin jks", + }, + }, + }, + { + name: "quote", + fieldVal: `{{.quote}}`, + expectedVal: `"`, + params: map[string]interface{}{ + "quote": `"`, + }, + }, + { + name: "Test No Data", + fieldVal: `{{.data}}`, + expectedVal: "{{.data}}", + params: map[string]interface{}{}, + }, + { + name: "Test Parse Error", + fieldVal: `{{functiondoesnotexist}}`, + expectedVal: "", + params: map[string]interface{}{ + "data": `a data string`, + }, + errorMessage: `failed to parse template {{functiondoesnotexist}}: template: :1: function "functiondoesnotexist" not defined`, + }, + { + name: "Test template error", + fieldVal: `{{.data.test}}`, + expectedVal: "", + params: map[string]interface{}{ + "data": `a data string`, + }, + errorMessage: `failed to execute go template {{.data.test}}: template: :1:7: executing "" at <.data.test>: can't evaluate field test in type interface {}`, + }, + } + + for _, test := range tests { + + t.Run(test.name, func(t *testing.T) { + + for fieldName, getPtrFunc := range fieldMap { + + // Clone the template application + application := emptyApplication.DeepCopy() + + // Set the value of the target field, to the test value + *getPtrFunc(application) = test.fieldVal + + // Render the cloned application, into a new application + render := Render{} + newApplication, err := render.RenderTemplateParams(application, nil, test.params, true) + + // Retrieve the value of the target field from the newApplication, then verify that + // the target field has been templated into the expected value + if test.errorMessage != "" { + assert.Error(t, err) + assert.Equal(t, test.errorMessage, err.Error()) + } else { + assert.NoError(t, err) + actualValue := *getPtrFunc(newApplication) + assert.Equal(t, test.expectedVal, actualValue, "Field '%s' had an unexpected value. expected: '%s' value: '%s'", fieldName, test.expectedVal, actualValue) + assert.Equal(t, newApplication.ObjectMeta.Annotations["annotation-key"], "annotation-value") + assert.Equal(t, newApplication.ObjectMeta.Annotations["annotation-key2"], "annotation-value2") + assert.Equal(t, newApplication.ObjectMeta.Labels["label-key"], "label-value") + assert.Equal(t, newApplication.ObjectMeta.Labels["label-key2"], "label-value2") + assert.Equal(t, newApplication.ObjectMeta.Name, "application-one") + assert.Equal(t, newApplication.ObjectMeta.Namespace, "default") + assert.Equal(t, newApplication.ObjectMeta.UID, types.UID("d546da12-06b7-4f9a-8ea2-3adb16a20e2b")) + assert.Equal(t, newApplication.ObjectMeta.CreationTimestamp, application.ObjectMeta.CreationTimestamp) + } } }) } @@ -255,14 +551,14 @@ func TestRenderTemplateParamsFinalizers(t *testing.T) { application := emptyApplication.DeepCopy() application.Finalizers = c.existingFinalizers - params := map[string]string{ + params := map[string]interface{}{ "one": "two", } // Render the cloned application, into a new application render := Render{} - res, err := render.RenderTemplateParams(application, c.syncPolicy, params) + res, err := render.RenderTemplateParams(application, c.syncPolicy, params, true) assert.Nil(t, err) assert.ElementsMatch(t, res.Finalizers, c.expectedFinalizers) diff --git a/docs/operator-manual/applicationset/Generators-Git.md b/docs/operator-manual/applicationset/Generators-Git.md index b235602e36492..4774454d330ff 100644 --- a/docs/operator-manual/applicationset/Generators-Git.md +++ b/docs/operator-manual/applicationset/Generators-Git.md @@ -45,13 +45,13 @@ spec: - path: applicationset/examples/git-generator-directory/cluster-addons/* template: metadata: - name: '{{path[0]}}' + name: '{{path.segments[0]}}' spec: project: "my-project" source: repoURL: https://github.com/argoproj/argo-cd.git targetRevision: HEAD - path: '{{path}}' + path: '{{path.path}}' destination: server: https://kubernetes.default.svc namespace: '{{path.basename}}' @@ -60,8 +60,8 @@ spec: The generator parameters are: -- `{{path}}`: The directory paths within the Git repository that match the `path` wildcard. -- `{{path[n]}}`: The directory paths within the Git repository that match the `path` wildcard, split into array elements (`n` - array index) +- `{{path.path}}`: The directory paths within the Git repository that match the `path` wildcard. +- `{{path.segments[n]}}`: The directory paths within the Git repository that match the `path` wildcard, split into array elements (`n` - array index) - `{{path.basename}}`: For any directory path within the Git repository that matches the `path` wildcard, the right-most path name is extracted (e.g. `/directory/directory2` would produce `directory2`). - `{{path.basenameNormalized}}`: This field is the same as `path.basename` with unsupported characters replaced with `-` (e.g. a `path` of `/directory/directory_2`, and `path.basename` of `directory_2` would produce `directory-2` here). @@ -100,7 +100,7 @@ spec: source: repoURL: https://github.com/argoproj/argo-cd.git targetRevision: HEAD - path: '{{path}}' + path: '{{path.path}}' destination: server: https://kubernetes.default.svc namespace: '{{path.basename}}' @@ -181,7 +181,7 @@ spec: source: repoURL: https://github.com/example/example-repo.git targetRevision: HEAD - path: '{{path}}' + path: '{{path.path}}' destination: server: https://kubernetes.default.svc namespace: '{{path.basename}}' diff --git a/docs/operator-manual/applicationset/Generators-Matrix.md b/docs/operator-manual/applicationset/Generators-Matrix.md index d844ff03b87e4..270643a616bbb 100644 --- a/docs/operator-manual/applicationset/Generators-Matrix.md +++ b/docs/operator-manual/applicationset/Generators-Matrix.md @@ -56,7 +56,7 @@ spec: source: repoURL: https://github.com/argoproj/argo-cd.git targetRevision: HEAD - path: '{{path}}' + path: '{{path.path}}' destination: server: '{{server}}' namespace: '{{path.basename}}' diff --git a/docs/operator-manual/applicationset/GoTemplate.md b/docs/operator-manual/applicationset/GoTemplate.md new file mode 100644 index 0000000000000..ace2794c559e8 --- /dev/null +++ b/docs/operator-manual/applicationset/GoTemplate.md @@ -0,0 +1,226 @@ +# Go Template + +> v2.5 + +## Introduction + +ApplicationSet is able to use [Go Text Template](https://pkg.go.dev/text/template). To activate this feature, add +`goTemplate: true` to your ApplicationSet manifest. + +The [Sprig function library](https://masterminds.github.io/sprig/) (except for `env`, `expandenv` and `getHostByName`) +is available in addition to the default Go Text Template functions. + +An additional `normalize` function makes any string parameter usable as a valid DNS name by replacing invalid characters +with hyphens and truncating at 253 characters. This is useful when making parameters safe for things like Application +names. + +## Motivation + +Go Template is the Go Standard for string templating. It is also more powerful than fasttemplate (the default templating +engine) as it allows doing complex templating logic. + +## Limitations + +Go templates are applied on a per-field basis, and only on string fields. Here are some examples of what is **not** +possible with Go text templates: + +- Templating a boolean field. + + ::yaml + apiVersion: argoproj.io/v1alpha1 + kind: ApplicationSet + spec: + goTemplate: true + template: + spec: + source: + helm: + useCredentials: "{{.useCredentials}}" # This field may NOT be templated, because it is a boolean field. + +- Templating an object field: + + ::yaml + apiVersion: argoproj.io/v1alpha1 + kind: ApplicationSet + spec: + goTemplate: true + template: + spec: + syncPolicy: "{{.syncPolicy}}" # This field may NOT be templated, because it is an object field. + +- Using control keywords across fields: + + ::yaml + apiVersion: argoproj.io/v1alpha1 + kind: ApplicationSet + spec: + goTemplate: true + template: + spec: + source: + helm: + parameters: + # Each of these fields is evaluated as an independent template, so the first one will fail with an error. + - name: "{{range .parameters}}" + - name: "{{.name}}" + value: "{{.value}}" + - name: throw-away + value: "{{end}}" + + +## Migration guide + +### Globals + +All your templates must replace parameters with GoTemplate Syntax: + +Example: `{{ some.value }}` becomes `{{ .some.value }}` + +### Git Generators + +By activating Go Templating, `{{ .path }}` becomes an object. Therefore, some changes must be made to the Git +generators' templating: + +- `{{ path }}` becomes `{{ .path.path }}` +- `{{ path[n] }}` becomes `{{path.segments[n]}}` + +Here is an example: + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: cluster-addons +spec: + generators: + - git: + repoURL: https://github.com/argoproj/argo-cd.git + revision: HEAD + directories: + - path: applicationset/examples/git-generator-directory/cluster-addons/* + template: + metadata: + name: '{{path.basename}}' + spec: + project: default + source: + repoURL: https://github.com/argoproj/argo-cd.git + targetRevision: HEAD + path: '{{path}}' + destination: + server: https://kubernetes.default.svc + namespace: '{{path.basename}}' +``` + +becomes + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: cluster-addons +spec: + goTemplate: true + generators: + - git: + repoURL: https://github.com/argoproj/argo-cd.git + revision: HEAD + directories: + - path: applicationset/examples/git-generator-directory/cluster-addons/* + template: + metadata: + name: '{{.path.basename}}' + spec: + project: default + source: + repoURL: https://github.com/argoproj/argo-cd.git + targetRevision: HEAD + path: '{{.path.path}}' + destination: + server: https://kubernetes.default.svc + namespace: '{{.path.basename}}' +``` + +It is also possible to use Sprig functions to construct the path variables manually: + +| with `goTemplate: false` | with `goTemplate: true` | with `goTemplate: true` + Sprig | +| ------------ | ----------- | --------------------- | +| `{{path}}` | `{{.path.path}}` | `{{.path.path}}` | +| `{{path.basename}}` | `{{.path.basename}}` | `{{base .path.path}}` | +| `{{path.filename}}` | `{{.path.filename}}` | `{{.path.filename}}` | +| `{{path.basenameNormalized}}` | `{{.path.basenameNormalized}}` | `{{normalize .path.path}}` | +| `{{path.filenameNormalized}}` | `{{.path.filenameNormalized}}` | `{{normalize .path.filename}}` | +| `{{path[N]}}` | `{{.path.segments[N]}}` | `{{index (splitList "/" .path.path) N}}` | + +## Examples + +### Basic Go template usage + +This example shows basic string parameter substitution. + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: guestbook +spec: + goTemplate: true + generators: + - list: + elements: + - cluster: engineering-dev + url: https://1.2.3.4 + - cluster: engineering-prod + url: https://2.4.6.8 + - cluster: finance-preprod + url: https://9.8.7.6 + template: + metadata: + name: '{{.cluster}}-guestbook' + spec: + project: my-project + source: + repoURL: https://github.com/infra-team/cluster-deployments.git + targetRevision: HEAD + path: guestbook/{{.cluster}} + destination: + server: '{{.url}}' + namespace: guestbook +``` + +### Fallbacks for unset parameters + +For some generators, a parameter of a certain name might not always be populated (for example, with the values generator +or the git files generator). In these cases, you can use a Go template to provide a fallback value. + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: guestbook +spec: + goTemplate: true + generators: + - list: + elements: + - cluster: engineering-dev + url: https://kubernetes.default.svc + - cluster: engineering-prod + url: https://kubernetes.default.svc + nameSuffix: -my-name-suffix + template: + metadata: + name: '{{.cluster}}{{default "" .nameSuffix}}' + spec: + project: default + source: + repoURL: https://github.com/argoproj/argo-cd.git + targetRevision: HEAD + path: applicationset/examples/list-generator/guestbook/{{.cluster}} + destination: + server: '{{.url}}' + namespace: guestbook +``` + +This ApplicationSet will produce an Application called `engineering-dev` and another called +`engineering-prod-my-name-suffix`. diff --git a/docs/operator-manual/applicationset/Template.md b/docs/operator-manual/applicationset/Template.md index 7892215525dec..f66a403586bbd 100644 --- a/docs/operator-manual/applicationset/Template.md +++ b/docs/operator-manual/applicationset/Template.md @@ -2,6 +2,8 @@ The template fields of the ApplicationSet `spec` are used to generate Argo CD `Application` resources. +ApplicationSet is using [fasttemplate](https://github.com/valyala/fasttemplate) but will be soon deprecated in favor of Go Template. + ## Template fields An Argo CD Application is created by combining the parameters from the generator with fields of the template (via `{{values}}`), and from that a concrete `Application` resource is produced and applied to the cluster. @@ -51,7 +53,7 @@ template as a Helm string literal. For example: ```yaml metadata: - name: '{{`{{cluster}}`}}-guestbook' + name: '{{`{{.cluster}}`}}-guestbook' ``` This _only_ applies if you use Helm to deploy your ApplicationSet resources. diff --git a/go.mod b/go.mod index 091983a581a11..5891e78cfab08 100644 --- a/go.mod +++ b/go.mod @@ -112,7 +112,7 @@ require ( github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd // indirect github.com/Masterminds/goutils v1.1.0 // indirect github.com/Masterminds/semver v1.5.0 // indirect - github.com/Masterminds/sprig v2.22.0+incompatible // indirect + github.com/Masterminds/sprig v2.22.0+incompatible github.com/Microsoft/go-winio v0.4.17 // indirect github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect diff --git a/manifests/core-install.yaml b/manifests/core-install.yaml index c8df856cf1e06..2eef55bd766b3 100644 --- a/manifests/core-install.yaml +++ b/manifests/core-install.yaml @@ -8599,6 +8599,8 @@ spec: type: object type: object type: array + goTemplate: + type: boolean syncPolicy: properties: preserveResourcesOnDeletion: diff --git a/manifests/crds/applicationset-crd.yaml b/manifests/crds/applicationset-crd.yaml index ecbbcb8dc0ede..3a73bbdccacee 100644 --- a/manifests/crds/applicationset-crd.yaml +++ b/manifests/crds/applicationset-crd.yaml @@ -6447,6 +6447,8 @@ spec: type: object type: object type: array + goTemplate: + type: boolean syncPolicy: properties: preserveResourcesOnDeletion: diff --git a/manifests/ha/install.yaml b/manifests/ha/install.yaml index fff2571ba8529..5412c96f2c93a 100644 --- a/manifests/ha/install.yaml +++ b/manifests/ha/install.yaml @@ -8599,6 +8599,8 @@ spec: type: object type: object type: array + goTemplate: + type: boolean syncPolicy: properties: preserveResourcesOnDeletion: diff --git a/manifests/install.yaml b/manifests/install.yaml index e2f85dd165854..2248720e9fd9d 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -8599,6 +8599,8 @@ spec: type: object type: object type: array + goTemplate: + type: boolean syncPolicy: properties: preserveResourcesOnDeletion: diff --git a/mkdocs.yml b/mkdocs.yml index ffa3657663629..70171a771e4be 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -91,7 +91,9 @@ nav: - operator-manual/applicationset/Generators-SCM-Provider.md - operator-manual/applicationset/Generators-Cluster-Decision-Resource.md - operator-manual/applicationset/Generators-Pull-Request.md - - Template fields: operator-manual/applicationset/Template.md + - Template fields: + - operator-manual/applicationset/Template.md + - operator-manual/applicationset/GoTemplate.md - Controlling Resource Modification: operator-manual/applicationset/Controlling-Resource-Modification.md - Application Pruning & Resource Deletion: operator-manual/applicationset/Application-Deletion.md - Server Configuration Parameters: diff --git a/pkg/apis/applicationset/v1alpha1/applicationset_types.go b/pkg/apis/applicationset/v1alpha1/applicationset_types.go index 0479f4aa0326c..93516a1df7cb3 100644 --- a/pkg/apis/applicationset/v1alpha1/applicationset_types.go +++ b/pkg/apis/applicationset/v1alpha1/applicationset_types.go @@ -49,6 +49,8 @@ type ApplicationSet struct { // ApplicationSetSpec represents a class of application set state. type ApplicationSetSpec struct { + // Define whereas to use GoTemplate or not and fallback to fasttemplate. Default to False. + GoTemplate bool `json:"goTemplate,omitempty"` Generators []ApplicationSetGenerator `json:"generators"` Template ApplicationSetTemplate `json:"template"` SyncPolicy *ApplicationSetSyncPolicy `json:"syncPolicy,omitempty"` diff --git a/test/e2e/applicationset_test.go b/test/e2e/applicationset_test.go index 2129273bd9eaa..576165129312e 100644 --- a/test/e2e/applicationset_test.go +++ b/test/e2e/applicationset_test.go @@ -132,6 +132,99 @@ func TestSimpleListGenerator(t *testing.T) { } +func TestSimpleListGeneratorGoTemplate(t *testing.T) { + + expectedApp := argov1alpha1.Application{ + TypeMeta: metav1.TypeMeta{ + Kind: "Application", + APIVersion: "argoproj.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster-guestbook", + Namespace: utils.ArgoCDNamespace, + Finalizers: []string{"resources-finalizer.argocd.argoproj.io"}, + }, + Spec: argov1alpha1.ApplicationSpec{ + Project: "default", + Source: argov1alpha1.ApplicationSource{ + RepoURL: "https://github.com/argoproj/argocd-example-apps.git", + TargetRevision: "HEAD", + Path: "guestbook", + }, + Destination: argov1alpha1.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: "guestbook", + }, + }, + } + var expectedAppNewNamespace *argov1alpha1.Application + var expectedAppNewMetadata *argov1alpha1.Application + + Given(t). + // Create a ListGenerator-based ApplicationSet + When().Create(v1alpha1.ApplicationSet{ObjectMeta: metav1.ObjectMeta{ + Name: "simple-list-generator", + }, + Spec: v1alpha1.ApplicationSetSpec{ + GoTemplate: true, + Template: v1alpha1.ApplicationSetTemplate{ + ApplicationSetTemplateMeta: v1alpha1.ApplicationSetTemplateMeta{Name: "{{.cluster}}-guestbook"}, + Spec: argov1alpha1.ApplicationSpec{ + Project: "default", + Source: argov1alpha1.ApplicationSource{ + RepoURL: "https://github.com/argoproj/argocd-example-apps.git", + TargetRevision: "HEAD", + Path: "guestbook", + }, + Destination: argov1alpha1.ApplicationDestination{ + Server: "{{.url}}", + Namespace: "guestbook", + }, + }, + }, + Generators: []v1alpha1.ApplicationSetGenerator{ + { + List: &v1alpha1.ListGenerator{ + Elements: []apiextensionsv1.JSON{{ + Raw: []byte(`{"cluster": "my-cluster","url": "https://kubernetes.default.svc"}`), + }}, + }, + }, + }, + }, + }).Then().Expect(ApplicationsExist([]argov1alpha1.Application{expectedApp})). + + // Update the ApplicationSet template namespace, and verify it updates the Applications + When(). + And(func() { + expectedAppNewNamespace = expectedApp.DeepCopy() + expectedAppNewNamespace.Spec.Destination.Namespace = "guestbook2" + }). + Update(func(appset *v1alpha1.ApplicationSet) { + appset.Spec.Template.Spec.Destination.Namespace = "guestbook2" + }).Then().Expect(ApplicationsExist([]argov1alpha1.Application{*expectedAppNewNamespace})). + + // Update the metadata fields in the appset template, and make sure it propagates to the apps + When(). + And(func() { + expectedAppNewMetadata = expectedAppNewNamespace.DeepCopy() + expectedAppNewMetadata.ObjectMeta.Annotations = map[string]string{"annotation-key": "annotation-value"} + expectedAppNewMetadata.ObjectMeta.Labels = map[string]string{"label-key": "label-value"} + }). + Update(func(appset *v1alpha1.ApplicationSet) { + appset.Spec.Template.Annotations = map[string]string{"annotation-key": "annotation-value"} + appset.Spec.Template.Labels = map[string]string{"label-key": "label-value"} + }).Then().Expect(ApplicationsExist([]argov1alpha1.Application{*expectedAppNewMetadata})). + + // verify the ApplicationSet status conditions were set correctly + Expect(ApplicationSetHasConditions("simple-list-generator", ExpectedConditions)). + + // Delete the ApplicationSet, and verify it deletes the Applications + When(). + Delete().Then().Expect(ApplicationsDoNotExist([]argov1alpha1.Application{*expectedAppNewMetadata})) + +} + func TestSimpleGitDirectoryGenerator(t *testing.T) { generateExpectedApp := func(name string) argov1alpha1.Application { return argov1alpha1.Application{ @@ -241,6 +334,116 @@ func TestSimpleGitDirectoryGenerator(t *testing.T) { Delete().Then().Expect(ApplicationsDoNotExist(expectedAppsNewNamespace)) } +func TestSimpleGitDirectoryGeneratorGoTemplate(t *testing.T) { + generateExpectedApp := func(name string) argov1alpha1.Application { + return argov1alpha1.Application{ + TypeMeta: metav1.TypeMeta{ + Kind: "Application", + APIVersion: "argoproj.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: utils.ArgoCDNamespace, + Finalizers: []string{"resources-finalizer.argocd.argoproj.io"}, + }, + Spec: argov1alpha1.ApplicationSpec{ + Project: "default", + Source: argov1alpha1.ApplicationSource{ + RepoURL: "https://github.com/argoproj/argocd-example-apps.git", + TargetRevision: "HEAD", + Path: name, + }, + Destination: argov1alpha1.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: name, + }, + }, + } + } + + expectedApps := []argov1alpha1.Application{ + generateExpectedApp("kustomize-guestbook"), + generateExpectedApp("helm-guestbook"), + generateExpectedApp("ksonnet-guestbook"), + } + + var expectedAppsNewNamespace []argov1alpha1.Application + var expectedAppsNewMetadata []argov1alpha1.Application + + Given(t). + When(). + // Create a GitGenerator-based ApplicationSet + Create(v1alpha1.ApplicationSet{ObjectMeta: metav1.ObjectMeta{ + Name: "simple-git-generator", + }, + Spec: v1alpha1.ApplicationSetSpec{ + GoTemplate: true, + Template: v1alpha1.ApplicationSetTemplate{ + ApplicationSetTemplateMeta: v1alpha1.ApplicationSetTemplateMeta{Name: "{{.path.basename}}"}, + Spec: argov1alpha1.ApplicationSpec{ + Project: "default", + Source: argov1alpha1.ApplicationSource{ + RepoURL: "https://github.com/argoproj/argocd-example-apps.git", + TargetRevision: "HEAD", + Path: "{{.path.path}}", + }, + Destination: argov1alpha1.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: "{{.path.basename}}", + }, + }, + }, + Generators: []v1alpha1.ApplicationSetGenerator{ + { + Git: &v1alpha1.GitGenerator{ + RepoURL: "https://github.com/argoproj/argocd-example-apps.git", + Directories: []v1alpha1.GitDirectoryGeneratorItem{ + { + Path: "*guestbook*", + }, + }, + }, + }, + }, + }, + }).Then().Expect(ApplicationsExist(expectedApps)). + + // Update the ApplicationSet template namespace, and verify it updates the Applications + When(). + And(func() { + for _, expectedApp := range expectedApps { + newExpectedApp := expectedApp.DeepCopy() + newExpectedApp.Spec.Destination.Namespace = "guestbook2" + expectedAppsNewNamespace = append(expectedAppsNewNamespace, *newExpectedApp) + } + }). + Update(func(appset *v1alpha1.ApplicationSet) { + appset.Spec.Template.Spec.Destination.Namespace = "guestbook2" + }).Then().Expect(ApplicationsExist(expectedAppsNewNamespace)). + + // Update the metadata fields in the appset template, and make sure it propagates to the apps + When(). + And(func() { + for _, expectedApp := range expectedAppsNewNamespace { + expectedAppNewMetadata := expectedApp.DeepCopy() + expectedAppNewMetadata.ObjectMeta.Annotations = map[string]string{"annotation-key": "annotation-value"} + expectedAppNewMetadata.ObjectMeta.Labels = map[string]string{"label-key": "label-value"} + expectedAppsNewMetadata = append(expectedAppsNewMetadata, *expectedAppNewMetadata) + } + }). + Update(func(appset *v1alpha1.ApplicationSet) { + appset.Spec.Template.Annotations = map[string]string{"annotation-key": "annotation-value"} + appset.Spec.Template.Labels = map[string]string{"label-key": "label-value"} + }).Then().Expect(ApplicationsExist(expectedAppsNewMetadata)). + + // verify the ApplicationSet status conditions were set correctly + Expect(ApplicationSetHasConditions("simple-git-generator", ExpectedConditions)). + + // Delete the ApplicationSet, and verify it deletes the Applications + When(). + Delete().Then().Expect(ApplicationsDoNotExist(expectedAppsNewNamespace)) +} + func TestSimpleGitFilesGenerator(t *testing.T) { generateExpectedApp := func(name string) argov1alpha1.Application { @@ -350,6 +553,116 @@ func TestSimpleGitFilesGenerator(t *testing.T) { Delete().Then().Expect(ApplicationsDoNotExist(expectedAppsNewNamespace)) } +func TestSimpleGitFilesGeneratorGoTemplate(t *testing.T) { + + generateExpectedApp := func(name string) argov1alpha1.Application { + return argov1alpha1.Application{ + TypeMeta: metav1.TypeMeta{ + Kind: "Application", + APIVersion: "argoproj.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: utils.ArgoCDNamespace, + Finalizers: []string{"resources-finalizer.argocd.argoproj.io"}, + }, + Spec: argov1alpha1.ApplicationSpec{ + Project: "default", + Source: argov1alpha1.ApplicationSource{ + RepoURL: "https://github.com/argoproj/argocd-example-apps.git", + TargetRevision: "HEAD", + Path: "guestbook", + }, + Destination: argov1alpha1.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: "guestbook", + }, + }, + } + } + + expectedApps := []argov1alpha1.Application{ + generateExpectedApp("engineering-dev-guestbook"), + generateExpectedApp("engineering-prod-guestbook"), + } + + var expectedAppsNewNamespace []argov1alpha1.Application + var expectedAppsNewMetadata []argov1alpha1.Application + + Given(t). + When(). + // Create a GitGenerator-based ApplicationSet + Create(v1alpha1.ApplicationSet{ObjectMeta: metav1.ObjectMeta{ + Name: "simple-git-generator", + }, + Spec: v1alpha1.ApplicationSetSpec{ + GoTemplate: true, + Template: v1alpha1.ApplicationSetTemplate{ + ApplicationSetTemplateMeta: v1alpha1.ApplicationSetTemplateMeta{Name: "{{.cluster.name}}-guestbook"}, + Spec: argov1alpha1.ApplicationSpec{ + Project: "default", + Source: argov1alpha1.ApplicationSource{ + RepoURL: "https://github.com/argoproj/argocd-example-apps.git", + TargetRevision: "HEAD", + Path: "guestbook", + }, + Destination: argov1alpha1.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: "guestbook", + }, + }, + }, + Generators: []v1alpha1.ApplicationSetGenerator{ + { + Git: &v1alpha1.GitGenerator{ + RepoURL: "https://github.com/argoproj/applicationset.git", + Files: []v1alpha1.GitFileGeneratorItem{ + { + Path: "examples/git-generator-files-discovery/cluster-config/**/config.json", + }, + }, + }, + }, + }, + }, + }).Then().Expect(ApplicationsExist(expectedApps)). + + // Update the ApplicationSet template namespace, and verify it updates the Applications + When(). + And(func() { + for _, expectedApp := range expectedApps { + newExpectedApp := expectedApp.DeepCopy() + newExpectedApp.Spec.Destination.Namespace = "guestbook2" + expectedAppsNewNamespace = append(expectedAppsNewNamespace, *newExpectedApp) + } + }). + Update(func(appset *v1alpha1.ApplicationSet) { + appset.Spec.Template.Spec.Destination.Namespace = "guestbook2" + }).Then().Expect(ApplicationsExist(expectedAppsNewNamespace)). + + // Update the metadata fields in the appset template, and make sure it propagates to the apps + When(). + And(func() { + for _, expectedApp := range expectedAppsNewNamespace { + expectedAppNewMetadata := expectedApp.DeepCopy() + expectedAppNewMetadata.ObjectMeta.Annotations = map[string]string{"annotation-key": "annotation-value"} + expectedAppNewMetadata.ObjectMeta.Labels = map[string]string{"label-key": "label-value"} + expectedAppsNewMetadata = append(expectedAppsNewMetadata, *expectedAppNewMetadata) + } + }). + Update(func(appset *v1alpha1.ApplicationSet) { + appset.Spec.Template.Annotations = map[string]string{"annotation-key": "annotation-value"} + appset.Spec.Template.Labels = map[string]string{"label-key": "label-value"} + }).Then().Expect(ApplicationsExist(expectedAppsNewMetadata)). + + // verify the ApplicationSet status conditions were set correctly + Expect(ApplicationSetHasConditions("simple-git-generator", ExpectedConditions)). + + // Delete the ApplicationSet, and verify it deletes the Applications + When(). + Delete().Then().Expect(ApplicationsDoNotExist(expectedAppsNewNamespace)) +} + func TestSimpleGitFilesPreserveResourcesOnDeletion(t *testing.T) { Given(t). @@ -380,37 +693,165 @@ func TestSimpleGitFilesPreserveResourcesOnDeletion(t *testing.T) { }, }, }, - SyncPolicy: &v1alpha1.ApplicationSetSyncPolicy{ - PreserveResourcesOnDeletion: true, - }, - Generators: []v1alpha1.ApplicationSetGenerator{ - { - Git: &v1alpha1.GitGenerator{ - RepoURL: "https://github.com/argoproj/applicationset.git", - Files: []v1alpha1.GitFileGeneratorItem{ - { - Path: "examples/git-generator-files-discovery/cluster-config/**/config.json", - }, + SyncPolicy: &v1alpha1.ApplicationSetSyncPolicy{ + PreserveResourcesOnDeletion: true, + }, + Generators: []v1alpha1.ApplicationSetGenerator{ + { + Git: &v1alpha1.GitGenerator{ + RepoURL: "https://github.com/argoproj/applicationset.git", + Files: []v1alpha1.GitFileGeneratorItem{ + { + Path: "examples/git-generator-files-discovery/cluster-config/**/config.json", + }, + }, + }, + }, + }, + }, + // We use an extra-long duration here, as we might need to wait for image pull. + }).Then().ExpectWithDuration(Pod(func(p corev1.Pod) bool { return strings.Contains(p.Name, "guestbook-ui") }), 6*time.Minute). + When(). + Delete(). + And(func() { + t.Log("Waiting 30 seconds to give the cluster a chance to delete the pods.") + // Wait 30 seconds to give the cluster a chance to deletes the pods, if it is going to do so. + // It should NOT delete the pods; to do so would be an ApplicationSet bug, and + // that is what we are testing here. + time.Sleep(30 * time.Second) + // The pod should continue to exist after 30 seconds. + }).Then().Expect(Pod(func(p corev1.Pod) bool { return strings.Contains(p.Name, "guestbook-ui") })) +} + +func TestSimpleGitFilesPreserveResourcesOnDeletionGoTemplate(t *testing.T) { + + Given(t). + When(). + CreateNamespace(). + // Create a GitGenerator-based ApplicationSet + Create(v1alpha1.ApplicationSet{ObjectMeta: metav1.ObjectMeta{ + Name: "simple-git-generator", + }, + Spec: v1alpha1.ApplicationSetSpec{ + GoTemplate: true, + Template: v1alpha1.ApplicationSetTemplate{ + ApplicationSetTemplateMeta: v1alpha1.ApplicationSetTemplateMeta{Name: "{{.cluster.name}}-guestbook"}, + Spec: argov1alpha1.ApplicationSpec{ + Project: "default", + Source: argov1alpha1.ApplicationSource{ + RepoURL: "https://github.com/argoproj/argocd-example-apps.git", + TargetRevision: "HEAD", + Path: "guestbook", + }, + Destination: argov1alpha1.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: utils.ApplicationSetNamespace, + }, + + // Automatically create resources + SyncPolicy: &argov1alpha1.SyncPolicy{ + Automated: &argov1alpha1.SyncPolicyAutomated{}, + }, + }, + }, + SyncPolicy: &v1alpha1.ApplicationSetSyncPolicy{ + PreserveResourcesOnDeletion: true, + }, + Generators: []v1alpha1.ApplicationSetGenerator{ + { + Git: &v1alpha1.GitGenerator{ + RepoURL: "https://github.com/argoproj/applicationset.git", + Files: []v1alpha1.GitFileGeneratorItem{ + { + Path: "examples/git-generator-files-discovery/cluster-config/**/config.json", + }, + }, + }, + }, + }, + }, + // We use an extra-long duration here, as we might need to wait for image pull. + }).Then().ExpectWithDuration(Pod(func(p corev1.Pod) bool { return strings.Contains(p.Name, "guestbook-ui") }), 6*time.Minute). + When(). + Delete(). + And(func() { + t.Log("Waiting 30 seconds to give the cluster a chance to delete the pods.") + // Wait 30 seconds to give the cluster a chance to deletes the pods, if it is going to do so. + // It should NOT delete the pods; to do so would be an ApplicationSet bug, and + // that is what we are testing here. + time.Sleep(30 * time.Second) + // The pod should continue to exist after 30 seconds. + }).Then().Expect(Pod(func(p corev1.Pod) bool { return strings.Contains(p.Name, "guestbook-ui") })) +} + +func TestSimpleSCMProviderGenerator(t *testing.T) { + expectedApp := argov1alpha1.Application{ + TypeMeta: metav1.TypeMeta{ + Kind: "Application", + APIVersion: "argoproj.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "argocd-example-apps-guestbook", + Namespace: utils.ArgoCDNamespace, + Finalizers: []string{"resources-finalizer.argocd.argoproj.io"}, + }, + Spec: argov1alpha1.ApplicationSpec{ + Project: "default", + Source: argov1alpha1.ApplicationSource{ + RepoURL: "git@github.com:argoproj/argocd-example-apps.git", + TargetRevision: "master", + Path: "guestbook", + }, + Destination: argov1alpha1.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: "guestbook", + }, + }, + } + + // Because you can't &"". + repoMatch := "example-apps" + + Given(t). + // Create an SCMProviderGenerator-based ApplicationSet + When().Create(v1alpha1.ApplicationSet{ObjectMeta: metav1.ObjectMeta{ + Name: "simple-scm-provider-generator", + }, + Spec: v1alpha1.ApplicationSetSpec{ + Template: v1alpha1.ApplicationSetTemplate{ + ApplicationSetTemplateMeta: v1alpha1.ApplicationSetTemplateMeta{Name: "{{ repository }}-guestbook"}, + Spec: argov1alpha1.ApplicationSpec{ + Project: "default", + Source: argov1alpha1.ApplicationSource{ + RepoURL: "{{ url }}", + TargetRevision: "{{ branch }}", + Path: "guestbook", + }, + Destination: argov1alpha1.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: "guestbook", + }, + }, + }, + Generators: []v1alpha1.ApplicationSetGenerator{ + { + SCMProvider: &v1alpha1.SCMProviderGenerator{ + Github: &v1alpha1.SCMProviderGeneratorGithub{ + Organization: "argoproj", + }, + Filters: []v1alpha1.SCMProviderGeneratorFilter{ + { + RepositoryMatch: &repoMatch, }, }, }, }, }, - // We use an extra-long duration here, as we might need to wait for image pull. - }).Then().ExpectWithDuration(Pod(func(p corev1.Pod) bool { return strings.Contains(p.Name, "guestbook-ui") }), 6*time.Minute). - When(). - Delete(). - And(func() { - t.Log("Waiting 30 seconds to give the cluster a chance to delete the pods.") - // Wait 30 seconds to give the cluster a chance to deletes the pods, if it is going to do so. - // It should NOT delete the pods; to do so would be an ApplicationSet bug, and - // that is what we are testing here. - time.Sleep(30 * time.Second) - // The pod should continue to exist after 30 seconds. - }).Then().Expect(Pod(func(p corev1.Pod) bool { return strings.Contains(p.Name, "guestbook-ui") })) + }, + }).Then().Expect(ApplicationsExist([]argov1alpha1.Application{expectedApp})) } -func TestSimpleSCMProviderGenerator(t *testing.T) { +func TestSimpleSCMProviderGeneratorGoTemplate(t *testing.T) { expectedApp := argov1alpha1.Application{ TypeMeta: metav1.TypeMeta{ Kind: "Application", @@ -444,13 +885,14 @@ func TestSimpleSCMProviderGenerator(t *testing.T) { Name: "simple-scm-provider-generator", }, Spec: v1alpha1.ApplicationSetSpec{ + GoTemplate: true, Template: v1alpha1.ApplicationSetTemplate{ - ApplicationSetTemplateMeta: v1alpha1.ApplicationSetTemplateMeta{Name: "{{ repository }}-guestbook"}, + ApplicationSetTemplateMeta: v1alpha1.ApplicationSetTemplateMeta{Name: "{{ .repository }}-guestbook"}, Spec: argov1alpha1.ApplicationSpec{ Project: "default", Source: argov1alpha1.ApplicationSource{ - RepoURL: "{{ url }}", - TargetRevision: "{{ branch }}", + RepoURL: "{{ .url }}", + TargetRevision: "{{ .branch }}", Path: "guestbook", }, Destination: argov1alpha1.ApplicationDestination{ @@ -543,6 +985,73 @@ func TestCustomApplicationFinalizers(t *testing.T) { Delete().Then().Expect(ApplicationsDoNotExist([]argov1alpha1.Application{expectedApp})) } +func TestCustomApplicationFinalizersGoTemplate(t *testing.T) { + expectedApp := argov1alpha1.Application{ + TypeMeta: metav1.TypeMeta{ + Kind: "Application", + APIVersion: "argoproj.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster-guestbook", + Namespace: utils.ArgoCDNamespace, + Finalizers: []string{"resources-finalizer.argocd.argoproj.io/background"}, + }, + Spec: argov1alpha1.ApplicationSpec{ + Project: "default", + Source: argov1alpha1.ApplicationSource{ + RepoURL: "https://github.com/argoproj/argocd-example-apps.git", + TargetRevision: "HEAD", + Path: "guestbook", + }, + Destination: argov1alpha1.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: "guestbook", + }, + }, + } + + Given(t). + // Create a ListGenerator-based ApplicationSet + When().Create(v1alpha1.ApplicationSet{ObjectMeta: metav1.ObjectMeta{ + Name: "simple-list-generator", + }, + Spec: v1alpha1.ApplicationSetSpec{ + GoTemplate: true, + Template: v1alpha1.ApplicationSetTemplate{ + ApplicationSetTemplateMeta: v1alpha1.ApplicationSetTemplateMeta{ + Name: "{{.cluster}}-guestbook", + Finalizers: []string{"resources-finalizer.argocd.argoproj.io/background"}, + }, + Spec: argov1alpha1.ApplicationSpec{ + Project: "default", + Source: argov1alpha1.ApplicationSource{ + RepoURL: "https://github.com/argoproj/argocd-example-apps.git", + TargetRevision: "HEAD", + Path: "guestbook", + }, + Destination: argov1alpha1.ApplicationDestination{ + Server: "{{.url}}", + Namespace: "guestbook", + }, + }, + }, + Generators: []v1alpha1.ApplicationSetGenerator{ + { + List: &v1alpha1.ListGenerator{ + Elements: []apiextensionsv1.JSON{{ + Raw: []byte(`{"cluster": "my-cluster","url": "https://kubernetes.default.svc"}`), + }}, + }, + }, + }, + }, + }).Then().Expect(ApplicationsExist([]argov1alpha1.Application{expectedApp})). + + // Delete the ApplicationSet, and verify it deletes the Applications + When(). + Delete().Then().Expect(ApplicationsDoNotExist([]argov1alpha1.Application{expectedApp})) +} + func TestSimplePullRequestGenerator(t *testing.T) { if utils.IsGitHubAPISkippedTest(t) { @@ -617,6 +1126,81 @@ func TestSimplePullRequestGenerator(t *testing.T) { }).Then().Expect(ApplicationsExist([]argov1alpha1.Application{expectedApp})) } +func TestSimplePullRequestGeneratorGoTemplate(t *testing.T) { + + if utils.IsGitHubAPISkippedTest(t) { + return + } + + expectedApp := argov1alpha1.Application{ + TypeMeta: metav1.TypeMeta{ + Kind: "Application", + APIVersion: "argoproj.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "guestbook-1", + Namespace: utils.ArgoCDNamespace, + Finalizers: []string{"resources-finalizer.argocd.argoproj.io"}, + }, + Spec: argov1alpha1.ApplicationSpec{ + Project: "default", + Source: argov1alpha1.ApplicationSource{ + RepoURL: "git@github.com:applicationset-test-org/argocd-example-apps.git", + TargetRevision: "824a5c987fdfb2b0629e9dbf5f31636c69ba4772", + Path: "kustomize-guestbook", + Kustomize: &argov1alpha1.ApplicationSourceKustomize{ + NamePrefix: "guestbook-1", + }, + }, + Destination: argov1alpha1.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: "guestbook-pull-request", + }, + }, + } + + Given(t). + // Create an PullRequestGenerator-based ApplicationSet + When().Create(v1alpha1.ApplicationSet{ObjectMeta: metav1.ObjectMeta{ + Name: "simple-pull-request-generator", + }, + Spec: v1alpha1.ApplicationSetSpec{ + GoTemplate: true, + Template: v1alpha1.ApplicationSetTemplate{ + ApplicationSetTemplateMeta: v1alpha1.ApplicationSetTemplateMeta{Name: "guestbook-{{ .number }}"}, + Spec: argov1alpha1.ApplicationSpec{ + Project: "default", + Source: argov1alpha1.ApplicationSource{ + RepoURL: "git@github.com:applicationset-test-org/argocd-example-apps.git", + TargetRevision: "{{ .head_sha }}", + Path: "kustomize-guestbook", + Kustomize: &argov1alpha1.ApplicationSourceKustomize{ + NamePrefix: "guestbook-{{ .number }}", + }, + }, + Destination: argov1alpha1.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: "guestbook-{{ .branch }}", + }, + }, + }, + Generators: []v1alpha1.ApplicationSetGenerator{ + { + PullRequest: &v1alpha1.PullRequestGenerator{ + Github: &v1alpha1.PullRequestGeneratorGithub{ + Owner: "applicationset-test-org", + Repo: "argocd-example-apps", + Labels: []string{ + "preview", + }, + }, + }, + }, + }, + }, + }).Then().Expect(ApplicationsExist([]argov1alpha1.Application{expectedApp})) +} + func TestGitGeneratorPrivateRepo(t *testing.T) { FailOnErr(fixture.RunCli("repo", "add", fixture.RepoURL(fixture.RepoURLTypeHTTPS), "--username", fixture.GitUsername, "--password", fixture.GitPassword, "--insecure-skip-server-verification")) generateExpectedApp := func(name string) argov1alpha1.Application { @@ -691,3 +1275,79 @@ func TestGitGeneratorPrivateRepo(t *testing.T) { When(). Delete().Then().Expect(ApplicationsDoNotExist(expectedAppsNewNamespace)) } + +func TestGitGeneratorPrivateRepoGoTemplate(t *testing.T) { + FailOnErr(fixture.RunCli("repo", "add", fixture.RepoURL(fixture.RepoURLTypeHTTPS), "--username", fixture.GitUsername, "--password", fixture.GitPassword, "--insecure-skip-server-verification")) + generateExpectedApp := func(name string) argov1alpha1.Application { + return argov1alpha1.Application{ + TypeMeta: metav1.TypeMeta{ + Kind: "Application", + APIVersion: "argoproj.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: utils.ArgoCDNamespace, + Finalizers: []string{"resources-finalizer.argocd.argoproj.io"}, + }, + Spec: argov1alpha1.ApplicationSpec{ + Project: "default", + Source: argov1alpha1.ApplicationSource{ + RepoURL: fixture.RepoURL(fixture.RepoURLTypeHTTPS), + TargetRevision: "HEAD", + Path: name, + }, + Destination: argov1alpha1.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: name, + }, + }, + } + } + + expectedApps := []argov1alpha1.Application{ + generateExpectedApp("https-kustomize-base"), + } + + var expectedAppsNewNamespace []argov1alpha1.Application + + Given(t). + When(). + // Create a GitGenerator-based ApplicationSet + Create(v1alpha1.ApplicationSet{ObjectMeta: metav1.ObjectMeta{ + Name: "simple-git-generator-private", + }, + Spec: v1alpha1.ApplicationSetSpec{ + GoTemplate: true, + Template: v1alpha1.ApplicationSetTemplate{ + ApplicationSetTemplateMeta: v1alpha1.ApplicationSetTemplateMeta{Name: "{{.path.basename}}"}, + Spec: argov1alpha1.ApplicationSpec{ + Project: "default", + Source: argov1alpha1.ApplicationSource{ + RepoURL: fixture.RepoURL(fixture.RepoURLTypeHTTPS), + TargetRevision: "HEAD", + Path: "{{.path.path}}", + }, + Destination: argov1alpha1.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: "{{.path.basename}}", + }, + }, + }, + Generators: []v1alpha1.ApplicationSetGenerator{ + { + Git: &v1alpha1.GitGenerator{ + RepoURL: fixture.RepoURL(fixture.RepoURLTypeHTTPS), + Directories: []v1alpha1.GitDirectoryGeneratorItem{ + { + Path: "*kustomize*", + }, + }, + }, + }, + }, + }, + }).Then().Expect(ApplicationsExist(expectedApps)). + // Delete the ApplicationSet, and verify it deletes the Applications + When(). + Delete().Then().Expect(ApplicationsDoNotExist(expectedAppsNewNamespace)) +}