feat: Matrix generator where a generator can reference items of another one

applicationset/controllers/applicationset_controller.go

@@ -430,7 +430,7 @@ func (r *ApplicationSetReconciler) generateApplications(applicationSetInfo argop
 	var applicationSetReason argoprojiov1alpha1.ApplicationSetReasonType
 
 	for _, requestedGenerator := range applicationSetInfo.Spec.Generators {
-		t, err := generators.Transform(requestedGenerator, r.Generators, applicationSetInfo.Spec.Template, &applicationSetInfo)
+		t, err := generators.Transform(requestedGenerator, r.Generators, applicationSetInfo.Spec.Template, &applicationSetInfo, map[string]string{})
 		if err != nil {
 			log.WithError(err).WithField("generator", requestedGenerator).
 				Error("error generating application from params")

applicationset/generators/generator_spec_processor.go

@@ -1,12 +1,14 @@
 package generators
 
 import (
+	"encoding/json"
+	"github.com/argoproj/argo-cd/v2/applicationset/utils"
+	"github.com/valyala/fasttemplate"
 	"reflect"
 
+	argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1"
 	"github.com/imdario/mergo"
 	log "github.com/sirupsen/logrus"
-
-	argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1"
 )
 
 type TransformResult struct {
@@ -15,9 +17,10 @@ type TransformResult struct {
 }
 
 //Transform a spec generator to list of paramSets and a template
-func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, allGenerators map[string]Generator, baseTemplate argoprojiov1alpha1.ApplicationSetTemplate, appSet *argoprojiov1alpha1.ApplicationSet) ([]TransformResult, error) {
+func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, allGenerators map[string]Generator, baseTemplate argoprojiov1alpha1.ApplicationSetTemplate, appSet *argoprojiov1alpha1.ApplicationSet, genParams map[string]string) ([]TransformResult, error) {
 	res := []TransformResult{}
 	var firstError error
+	interpolatedGenerator := requestedGenerator.DeepCopy()
 
 	generators := GetRelevantGenerators(&requestedGenerator, allGenerators)
 	for _, g := range generators {
@@ -31,8 +34,20 @@ func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, al
 			}
 			continue
 		}
-
-		params, err := g.GenerateParams(&requestedGenerator, appSet)
+		var params []map[string]string
+		if len(genParams) != 0 {
+			tempInterpolatedGenerator, err := interpolateGenerator(&requestedGenerator, genParams)
+			interpolatedGenerator = &tempInterpolatedGenerator
+			if err != nil {
+				log.WithError(err).WithField("genParams", genParams).
+					Error("error interpolating params for generator")
+				if firstError == nil {
+					firstError = err
+				}
+				continue
+			}
+		}
+		params, err = g.GenerateParams(interpolatedGenerator, appSet)
 		if err != nil {
 			log.WithError(err).WithField("generator", g).
 				Error("error generating params")
@@ -46,11 +61,9 @@ func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, al
 			Params:   params,
 			Template: mergedTemplate,
 		})
-
 	}
 
 	return res, firstError
-
 }
 
 func GetRelevantGenerators(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, generators map[string]Generator) []Generator {
@@ -81,3 +94,29 @@ func mergeGeneratorTemplate(g Generator, requestedGenerator *argoprojiov1alpha1.
 
 	return *dest, err
 }
+
+// Currently for Matrix Generator. Allows interpolating the matrix's 2nd child generator with values from the 1st child generator
+// "params" parameter is an array, where each index corresponds to a generator. Each index contains a map w/ that generator's parameters.
+func interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, params map[string]string) (argoprojiov1alpha1.ApplicationSetGenerator, error) {
+	interpolatedGenerator := requestedGenerator.DeepCopy()
+	tmplBytes, err := json.Marshal(interpolatedGenerator)
+	if err != nil {
+		log.WithError(err).WithField("requestedGenerator", interpolatedGenerator).Error("error marshalling requested generator for interpolation")
+		return *interpolatedGenerator, err
+	}
+
+	render := utils.Render{}
+	fstTmpl := fasttemplate.New(string(tmplBytes), "{{", "}}")
+	replacedTmplStr, err := render.Replace(fstTmpl, params, true)
+	if err != nil {
+		log.WithError(err).WithField("interpolatedGeneratorString", replacedTmplStr).Error("error interpolating generator with other generator's parameter")
+		return *interpolatedGenerator, err
+	}
+
+	err = json.Unmarshal([]byte(replacedTmplStr), interpolatedGenerator)
+	if err != nil {
+		log.WithError(err).WithField("requestedGenerator", interpolatedGenerator).Error("error unmarshalling requested generator for interpolation")
+		return *interpolatedGenerator, err
+	}
+	return *interpolatedGenerator, nil
+}

applicationset/generators/generator_spec_processor_test.go

@@ -0,0 +1,180 @@
+package generators
+
+import (
+	"context"
+	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	corev1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	kubefake "k8s.io/client-go/kubernetes/fake"
+	crtclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"testing"
+
+	argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1"
+)
+
+func getMockClusterGenerator() Generator {
+	clusters := []crtclient.Object{
+		&corev1.Secret{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "Secret",
+				APIVersion: "v1",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "staging-01",
+				Namespace: "namespace",
+				Labels: map[string]string{
+					"argocd.argoproj.io/secret-type": "cluster",
+					"environment":                    "staging",
+					"org":                            "foo",
+				},
+				Annotations: map[string]string{
+					"foo.argoproj.io": "staging",
+				},
+			},
+			Data: map[string][]byte{
+				"config": []byte("{}"),
+				"name":   []byte("staging-01"),
+				"server": []byte("https://staging-01.example.com"),
+			},
+			Type: corev1.SecretType("Opaque"),
+		},
+		&corev1.Secret{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "Secret",
+				APIVersion: "v1",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "production-01",
+				Namespace: "namespace",
+				Labels: map[string]string{
+					"argocd.argoproj.io/secret-type": "cluster",
+					"environment":                    "production",
+					"org":                            "bar",
+				},
+				Annotations: map[string]string{
+					"foo.argoproj.io": "production",
+				},
+			},
+			Data: map[string][]byte{
+				"config": []byte("{}"),
+				"name":   []byte("production_01/west"),
+				"server": []byte("https://production-01.example.com"),
+			},
+			Type: corev1.SecretType("Opaque"),
+		},
+	}
+	runtimeClusters := []runtime.Object{}
+	appClientset := kubefake.NewSimpleClientset(runtimeClusters...)
+
+	fakeClient := fake.NewClientBuilder().WithObjects(clusters...).Build()
+	return NewClusterGenerator(fakeClient, context.Background(), appClientset, "namespace")
+}
+
+func getMockGitGenerator() Generator {
+	argoCDServiceMock := argoCDServiceMock{mock: &mock.Mock{}}
+	argoCDServiceMock.mock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything).Return([]string{"app1", "app2", "app_3", "p1/app4"}, nil)
+	var gitGenerator = NewGitGenerator(argoCDServiceMock)
+	return gitGenerator
+}
+
+func TestGetRelevantGenerators(t *testing.T) {
+
+	testGenerators := map[string]Generator{
+		"Clusters": getMockClusterGenerator(),
+		"Git":      getMockGitGenerator(),
+	}
+
+	testGenerators["Matrix"] = NewMatrixGenerator(testGenerators)
+	testGenerators["Merge"] = NewMergeGenerator(testGenerators)
+	testGenerators["List"] = NewListGenerator()
+
+	requestedGenerator := &argoprojiov1alpha1.ApplicationSetGenerator{
+		List: &argoprojiov1alpha1.ListGenerator{
+			Elements: []apiextensionsv1.JSON{{Raw: []byte(`{"cluster": "cluster","url": "url","values":{"foo":"bar"}}`)}},
+		}}
+
+	relevantGenerators := GetRelevantGenerators(requestedGenerator, testGenerators)
+	assert.Len(t, relevantGenerators, 1)
+	assert.IsType(t, &ListGenerator{}, relevantGenerators[0])
+
+	requestedGenerator = &argoprojiov1alpha1.ApplicationSetGenerator{
+		Clusters: &argoprojiov1alpha1.ClusterGenerator{
+			Selector: metav1.LabelSelector{},
+			Template: argoprojiov1alpha1.ApplicationSetTemplate{},
+			Values:   nil,
+		},
+	}
+
+	relevantGenerators = GetRelevantGenerators(requestedGenerator, testGenerators)
+	assert.Len(t, relevantGenerators, 1)
+	assert.IsType(t, &ClusterGenerator{}, relevantGenerators[0])
+
+	requestedGenerator = &argoprojiov1alpha1.ApplicationSetGenerator{
+		Git: &argoprojiov1alpha1.GitGenerator{
+			RepoURL:             "",
+			Directories:         nil,
+			Files:               nil,
+			Revision:            "",
+			RequeueAfterSeconds: nil,
+			Template:            argoprojiov1alpha1.ApplicationSetTemplate{},
+		},
+	}
+
+	relevantGenerators = GetRelevantGenerators(requestedGenerator, testGenerators)
+	assert.Len(t, relevantGenerators, 1)
+	assert.IsType(t, &GitGenerator{}, relevantGenerators[0])
+}
+
+func TestInterpolateGenerator(t *testing.T) {
+	requestedGenerator := &argoprojiov1alpha1.ApplicationSetGenerator{
+		Clusters: &argoprojiov1alpha1.ClusterGenerator{
+			Selector: metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"argocd.argoproj.io/secret-type": "cluster",
+					"path-basename":                  "{{path.basename}}",
+					"path-zero":                      "{{path[0]}}",
+					"path-full":                      "{{path}}",
+				}},
+		},
+	}
+	gitGeneratorParams := map[string]string{
+		"path": "p1/p2/app3", "path.basename": "app3", "path[0]": "p1", "path[1]": "p2", "path.basenameNormalized": "app3",
+	}
+	interpolatedGenerator, err := interpolateGenerator(requestedGenerator, gitGeneratorParams)
+	if err != nil {
+		log.WithError(err).WithField("requestedGenerator", requestedGenerator).Error("error interpolating Generator")
+		return
+	}
+	assert.Equal(t, "app3", interpolatedGenerator.Clusters.Selector.MatchLabels["path-basename"])
+	assert.Equal(t, "p1", interpolatedGenerator.Clusters.Selector.MatchLabels["path-zero"])
+	assert.Equal(t, "p1/p2/app3", interpolatedGenerator.Clusters.Selector.MatchLabels["path-full"])
+
+	fileNamePath := argoprojiov1alpha1.GitFileGeneratorItem{
+		Path: "{{name}}",
+	}
+	fileServerPath := argoprojiov1alpha1.GitFileGeneratorItem{
+		Path: "{{server}}",
+	}
+
+	requestedGenerator = &argoprojiov1alpha1.ApplicationSetGenerator{
+		Git: &argoprojiov1alpha1.GitGenerator{
+			Files:    append([]argoprojiov1alpha1.GitFileGeneratorItem{}, fileNamePath, fileServerPath),
+			Template: argoprojiov1alpha1.ApplicationSetTemplate{},
+		},
+	}
+	clusterGeneratorParams := map[string]string{
+		"name": "production_01/west", "server": "https://production-01.example.com",
+	}
+	interpolatedGenerator, err = interpolateGenerator(requestedGenerator, clusterGeneratorParams)
+	if err != nil {
+		log.WithError(err).WithField("requestedGenerator", requestedGenerator).Error("error interpolating Generator")
+		return
+	}
+	assert.Equal(t, "production_01/west", interpolatedGenerator.Git.Files[0].Path)
+	assert.Equal(t, "https://production-01.example.com", interpolatedGenerator.Git.Files[1].Path)
+}

applicationset/generators/matrix.go

@@ -44,16 +44,15 @@ func (m *MatrixGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.App
 
 	res := []map[string]string{}
 
-	g0, err := m.getParams(appSetGenerator.Matrix.Generators[0], appSet)
+	g0, err := m.getParams(appSetGenerator.Matrix.Generators[0], appSet, nil)
 	if err != nil {
 		return nil, err
 	}
-	g1, err := m.getParams(appSetGenerator.Matrix.Generators[1], appSet)
-	if err != nil {
-		return nil, err
-	}
-
 	for _, a := range g0 {
+		g1, err := m.getParams(appSetGenerator.Matrix.Generators[1], appSet, a)
+		if err != nil {
+			return nil, err
+		}
 		for _, b := range g1 {
 			val, err := utils.CombineStringMaps(a, b)
 			if err != nil {
@@ -66,7 +65,7 @@ func (m *MatrixGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.App
 	return res, nil
 }
 
-func (m *MatrixGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.ApplicationSetNestedGenerator, appSet *argoprojiov1alpha1.ApplicationSet) ([]map[string]string, error) {
+func (m *MatrixGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.ApplicationSetNestedGenerator, appSet *argoprojiov1alpha1.ApplicationSet, params map[string]string) ([]map[string]string, error) {
 	var matrix *argoprojiov1alpha1.MatrixGenerator
 	if appSetBaseGenerator.Matrix != nil {
 		// Since nested matrix generator is represented as a JSON object in the CRD, we unmarshall it back to a Go struct here.
@@ -104,7 +103,8 @@ func (m *MatrixGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.Appli
 		},
 		m.supportedGenerators,
 		argoprojiov1alpha1.ApplicationSetTemplate{},
-		appSet)
+		appSet,
+		params)
 
 	if err != nil {
 		return nil, fmt.Errorf("child generator returned an error on parameter generation: %v", err)