Merge branch 'master' into master

.golangci.yaml

@@ -66,6 +66,8 @@ linters-settings:
         pkg: k8s.io/api/rbac/v1
       - alias: apierrors
         pkg: k8s.io/apimachinery/pkg/api/errors
+      - alias: apiextensionsv1
+        pkg: k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1
       - alias: metav1
         pkg: k8s.io/apimachinery/pkg/apis/meta/v1
       - alias: informersv1

USERS.md

@@ -30,7 +30,7 @@ Currently, the following organizations are **officially** using Argo CD:
 1. [Ant Group](https://www.antgroup.com/)
 1. [AppDirect](https://www.appdirect.com)
 1. [Arctiq Inc.](https://www.arctiq.ca)
-2. [Arturia](https://www.arturia.com)
+1. [Arturia](https://www.arturia.com)
 1. [ARZ Allgemeines Rechenzentrum GmbH](https://www.arz.at/)
 1. [Augury](https://www.augury.com/)
 1. [Autodesk](https://www.autodesk.com)
@@ -397,3 +397,4 @@ Currently, the following organizations are **officially** using Argo CD:
 1. [Zimpler](https://www.zimpler.com/)
 1. [ZipRecuiter](https://www.ziprecruiter.com/)
 1. [ZOZO](https://corp.zozo.com/)
+1. [hetao101](https://www.hetao101.com/)

applicationset/controllers/applicationset_controller.go

@@ -768,7 +768,7 @@ func (r *ApplicationSetReconciler) deleteInCluster(ctx context.Context, logCtx *
 }
 
 // removeFinalizerOnInvalidDestination removes the Argo CD resources finalizer if the application contains an invalid target (eg missing cluster)
-func (r *ApplicationSetReconciler) removeFinalizerOnInvalidDestination(ctx context.Context, applicationSet argov1alpha1.ApplicationSet, app *argov1alpha1.Application, clusterList *argov1alpha1.ClusterList, appLog *log.Entry) error {
+func (r *ApplicationSetReconciler) removeFinalizerOnInvalidDestination(ctx context.Context, applicationSet argov1alpha1.ApplicationSet, app *argov1alpha1.Application, clusterList []utils.ClusterSpecifier, appLog *log.Entry) error {
 	// Only check if the finalizers need to be removed IF there are finalizers to remove
 	if len(app.Finalizers) == 0 {
 		return nil
@@ -783,7 +783,7 @@ func (r *ApplicationSetReconciler) removeFinalizerOnInvalidDestination(ctx conte
 	} else {
 		// Detect if the destination's server field does not match an existing cluster
 		matchingCluster := false
-		for _, cluster := range clusterList.Items {
+		for _, cluster := range clusterList {
 			if destCluster.Server != cluster.Server {
 				continue
 			}

applicationset/generators/cluster.go

@@ -70,7 +70,7 @@ func (g *ClusterGenerator) GenerateParams(appSetGenerator *argoappsetv1alpha1.Ap
 	// - Since local clusters do not have secrets, they do not have labels to match against
 	ignoreLocalClusters := len(appSetGenerator.Clusters.Selector.MatchExpressions) > 0 || len(appSetGenerator.Clusters.Selector.MatchLabels) > 0
 
-	// ListCluster from Argo CD's util/db package will include the local cluster in the list of clusters
+	// ListCluster will include the local cluster in the list of clusters
 	clustersFromArgoCD, err := utils.ListClusters(g.ctx, g.clientset, g.namespace)
 	if err != nil {
 		return nil, fmt.Errorf("error listing clusters: %w", err)
@@ -93,7 +93,7 @@ func (g *ClusterGenerator) GenerateParams(appSetGenerator *argoappsetv1alpha1.Ap
 	logCtx.Debugf("Using flat mode = %t for cluster generator", isFlatMode)
 	clustersParams := make([]map[string]any, 0)
 
-	for _, cluster := range clustersFromArgoCD.Items {
+	for _, cluster := range clustersFromArgoCD {
 		// If there is a secret for this cluster, then it's a non-local cluster, so it will be
 		// handled by the next step.
 		if secretForCluster, exists := clusterSecrets[cluster.Name]; exists {

applicationset/generators/duck_type.go

@@ -165,9 +165,6 @@ func (g *DuckTypeGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.A
 	}
 	log.Infof("Number of decisions found: %v", len(clusterDecisions))
 
-	// Read this outside the loop to improve performance
-	argoClusters := clustersFromArgoCD.Items
-
 	if len(clusterDecisions) == 0 {
 		log.Warningf("clusterDecisionResource status.%s missing", statusListKey)
 		return nil, nil
@@ -188,7 +185,7 @@ func (g *DuckTypeGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.A
 
 		found := false
 
-		for _, argoCluster := range argoClusters {
+		for _, argoCluster := range clustersFromArgoCD {
 			if argoCluster.Name == strMatchValue {
 				log.WithField(matchKey, argoCluster.Name).Info("matched cluster in ArgoCD")
 				params["name"] = argoCluster.Name

applicationset/generators/generator_spec_processor.go

@@ -159,16 +159,3 @@ func InterpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetG
 
 	return *interpolatedGenerator, nil
 }
-
-// Fixes https://github.com/argoproj/argo-cd/issues/11982 while ensuring backwards compatibility.
-// This is only a short-term solution and should be removed in a future major version.
-func dropDisabledNestedSelectors(generators []argoprojiov1alpha1.ApplicationSetNestedGenerator) bool {
-	var foundSelector bool
-	for i := range generators {
-		if generators[i].Selector != nil {
-			foundSelector = true
-			generators[i].Selector = nil
-		}
-	}
-	return foundSelector
-}

applicationset/generators/git.go

@@ -83,13 +83,17 @@ func (g *GitGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.Applic
 		verifyCommit = len(appProject.Spec.SignatureKeys) > 0 && gpg.IsGPGEnabled()
 	}
 
+	// If the project field is templated, we cannot resolve the project name, so we pass an empty string to the repo-server.
+	// This means only "globally-scoped" repo credentials can be used for such appsets.
+	project := resolveProjectName(appSet.Spec.Template.Spec.Project)
+
 	var err error
 	var res []map[string]any
 	switch {
 	case len(appSetGenerator.Git.Directories) != 0:
-		res, err = g.generateParamsForGitDirectories(appSetGenerator, noRevisionCache, verifyCommit, appSet.Spec.GoTemplate, appSet.Spec.GoTemplateOptions)
+		res, err = g.generateParamsForGitDirectories(appSetGenerator, noRevisionCache, verifyCommit, appSet.Spec.GoTemplate, project, appSet.Spec.GoTemplateOptions)
 	case len(appSetGenerator.Git.Files) != 0:
-		res, err = g.generateParamsForGitFiles(appSetGenerator, noRevisionCache, verifyCommit, appSet.Spec.GoTemplate, appSet.Spec.GoTemplateOptions)
+		res, err = g.generateParamsForGitFiles(appSetGenerator, noRevisionCache, verifyCommit, appSet.Spec.GoTemplate, project, appSet.Spec.GoTemplateOptions)
 	default:
 		return nil, EmptyAppSetGeneratorError
 	}
@@ -100,9 +104,9 @@ func (g *GitGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.Applic
 	return res, nil
 }
 
-func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, noRevisionCache, verifyCommit bool, useGoTemplate bool, goTemplateOptions []string) ([]map[string]any, error) {
+func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, noRevisionCache, verifyCommit, useGoTemplate bool, project string, goTemplateOptions []string) ([]map[string]any, error) {
 	// Directories, not files
-	allPaths, err := g.repos.GetDirectories(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, noRevisionCache, verifyCommit)
+	allPaths, err := g.repos.GetDirectories(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, project, noRevisionCache, verifyCommit)
 	if err != nil {
 		return nil, fmt.Errorf("error getting directories from repo: %w", err)
 	}
@@ -125,11 +129,11 @@ func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoproj
 	return res, nil
 }
 
-func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, noRevisionCache, verifyCommit bool, useGoTemplate bool, goTemplateOptions []string) ([]map[string]any, error) {
+func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, noRevisionCache, verifyCommit, useGoTemplate bool, project string, goTemplateOptions []string) ([]map[string]any, error) {
 	// Get all files that match the requested path string, removing duplicates
 	allFiles := make(map[string][]byte)
 	for _, requestedPath := range appSetGenerator.Git.Files {
-		files, err := g.repos.GetFiles(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, requestedPath.Path, noRevisionCache, verifyCommit)
+		files, err := g.repos.GetFiles(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, project, requestedPath.Path, noRevisionCache, verifyCommit)
 		if err != nil {
 			return nil, err
 		}
@@ -335,3 +339,11 @@ func (g *GitGenerator) generateParamsFromApps(requestedApps []string, appSetGene
 
 	return res, nil
 }
+
+func resolveProjectName(project string) string {
+	if strings.Contains(project, "{{") {
+		return ""
+	}
+
+	return project
+}

applicationset/generators/git_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	"github.com/argoproj/argo-cd/v3/applicationset/services/mocks"
@@ -169,11 +170,12 @@ foo:
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			params, err := (*GitGenerator)(nil).generateParamsFromGitFile(tt.args.filePath, tt.args.fileContent, tt.args.values, tt.args.useGoTemplate, tt.args.goTemplateOptions, tt.args.pathParamPrefix)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("GitGenerator.generateParamsFromGitFile() error = %v, wantErr %v", err, tt.wantErr)
-				return
+			if tt.wantErr {
+				assert.Error(t, err, "GitGenerator.generateParamsFromGitFile()")
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, tt.want, params)
 			}
-			assert.Equal(t, tt.want, params)
 		})
 	}
 }
@@ -198,7 +200,6 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) {
 				"app_3",
 				"p1/app4",
 			},
-			repoError: nil,
 			expected: []map[string]any{
 				{"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1", "path[0]": "app1"},
 				{"path": "app2", "path.basename": "app2", "path.basenameNormalized": "app2", "path[0]": "app2"},
@@ -233,7 +234,6 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) {
 				"p1/p2/app3",
 				"p1/p2/p3/app4",
 			},
-			repoError: nil,
 			expected: []map[string]any{
 				{"path": "p1/app2", "path.basename": "app2", "path[0]": "p1", "path[1]": "app2", "path.basenameNormalized": "app2"},
 				{"path": "p1/p2/app3", "path.basename": "app3", "path[0]": "p1", "path[1]": "p2", "path[2]": "app3", "path.basenameNormalized": "app3"},
@@ -321,7 +321,7 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) {
 
 			argoCDServiceMock := mocks.Repos{}
 
-			argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(testCaseCopy.repoApps, testCaseCopy.repoError)
+			argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(testCaseCopy.repoApps, testCaseCopy.repoError)
 
 			gitGenerator := NewGitGenerator(&argoCDServiceMock, "")
 			applicationSetInfo := v1alpha1.ApplicationSet{
@@ -622,7 +622,7 @@ func TestGitGenerateParamsFromDirectoriesGoTemplate(t *testing.T) {
 
 			argoCDServiceMock := mocks.Repos{}
 
-			argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(testCaseCopy.repoApps, testCaseCopy.repoError)
+			argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(testCaseCopy.repoApps, testCaseCopy.repoError)
 
 			gitGenerator := NewGitGenerator(&argoCDServiceMock, "")
 			applicationSetInfo := v1alpha1.ApplicationSet{
@@ -1281,7 +1281,7 @@ cluster:
 			t.Parallel()
 
 			argoCDServiceMock := mocks.Repos{}
-			argoCDServiceMock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
+			argoCDServiceMock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
 				Return(testCaseCopy.repoFileContents, testCaseCopy.repoPathsError)
 
 			gitGenerator := NewGitGenerator(&argoCDServiceMock, "")
@@ -1637,7 +1637,7 @@ cluster:
 			t.Parallel()
 
 			argoCDServiceMock := mocks.Repos{}
-			argoCDServiceMock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
+			argoCDServiceMock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
 				Return(testCaseCopy.repoFileContents, testCaseCopy.repoPathsError)
 
 			gitGenerator := NewGitGenerator(&argoCDServiceMock, "")
@@ -1682,6 +1682,7 @@ cluster:
 func TestGitGenerator_GenerateParams(t *testing.T) {
 	cases := []struct {
 		name               string
+		appProject         v1alpha1.AppProject
 		directories        []v1alpha1.GitDirectoryGeneratorItem
 		pathParamPrefix    string
 		repoApps           []string
@@ -1690,6 +1691,7 @@ func TestGitGenerator_GenerateParams(t *testing.T) {
 		values             map[string]string
 		expected           []map[string]any
 		expectedError      error
+		expectedProject    *string
 		appset             v1alpha1.ApplicationSet
 		callGetDirectories bool
 	}{
@@ -1761,21 +1763,102 @@ func TestGitGenerator_GenerateParams(t *testing.T) {
 			expected:           []map[string]any{{"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1", "path[0]": "app1", "values.foo": "bar"}},
 			expectedError:      errors.New("error getting project project: appprojects.argoproj.io \"project\" not found"),
 		},
+		{
+			name: "Project field is not templated - verify that project is passed through to repo-server as-is",
+			repoApps: []string{
+				"app1",
+			},
+			callGetDirectories: true,
+			appProject: v1alpha1.AppProject{
+				TypeMeta: metav1.TypeMeta{},
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "project",
+					Namespace: "argocd",
+				},
+			},
+			appset: v1alpha1.ApplicationSet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "set",
+					Namespace: "namespace",
+				},
+				Spec: v1alpha1.ApplicationSetSpec{
+					Generators: []v1alpha1.ApplicationSetGenerator{{
+						Git: &v1alpha1.GitGenerator{
+							RepoURL:         "RepoURL",
+							Revision:        "Revision",
+							Directories:     []v1alpha1.GitDirectoryGeneratorItem{{Path: "*"}},
+							PathParamPrefix: "",
+							Values: map[string]string{
+								"foo": "bar",
+							},
+						},
+					}},
+					Template: v1alpha1.ApplicationSetTemplate{
+						Spec: v1alpha1.ApplicationSpec{
+							Project: "project",
+						},
+					},
+				},
+			},
+			expected:        []map[string]any{{"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1", "path[0]": "app1", "values.foo": "bar"}},
+			expectedProject: ptr.To("project"),
+			expectedError:   nil,
+		},
+		{
+			name: "Project field is templated - verify that project is passed through to repo-server as empty string",
+			repoApps: []string{
+				"app1",
+			},
+			callGetDirectories: true,
+			appset: v1alpha1.ApplicationSet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "set",
+					Namespace: "namespace",
+				},
+				Spec: v1alpha1.ApplicationSetSpec{
+					Generators: []v1alpha1.ApplicationSetGenerator{{
+						Git: &v1alpha1.GitGenerator{
+							RepoURL:         "RepoURL",
+							Revision:        "Revision",
+							Directories:     []v1alpha1.GitDirectoryGeneratorItem{{Path: "*"}},
+							PathParamPrefix: "",
+							Values: map[string]string{
+								"foo": "bar",
+							},
+						},
+					}},
+					Template: v1alpha1.ApplicationSetTemplate{
+						Spec: v1alpha1.ApplicationSpec{
+							Project: "{{.project}}",
+						},
+					},
+				},
+			},
+			expected:        []map[string]any{{"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1", "path[0]": "app1", "values.foo": "bar"}},
+			expectedProject: ptr.To(""),
+			expectedError:   nil,
+		},
 	}
 	for _, testCase := range cases {
 		argoCDServiceMock := mocks.Repos{}
 
 		if testCase.callGetDirectories {
-			argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(testCase.repoApps, testCase.repoPathsError)
+			var project any
+			if testCase.expectedProject != nil {
+				project = *testCase.expectedProject
+			} else {
+				project = mock.Anything
+			}
+
+			argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything, project, mock.Anything, mock.Anything).Return(testCase.repoApps, testCase.repoPathsError)
 		}
-		gitGenerator := NewGitGenerator(&argoCDServiceMock, "namespace")
+		gitGenerator := NewGitGenerator(&argoCDServiceMock, "argocd")
 
 		scheme := runtime.NewScheme()
 		err := v1alpha1.AddToScheme(scheme)
 		require.NoError(t, err)
-		appProject := v1alpha1.AppProject{}
 
-		client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&appProject).Build()
+		client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&testCase.appProject).Build()
 
 		got, err := gitGenerator.GenerateParams(&testCase.appset.Spec.Generators[0], &testCase.appset, client)
 

applicationset/generators/matrix.go

@@ -10,8 +10,6 @@ import (
 
 	"github.com/argoproj/argo-cd/v3/applicationset/utils"
 	argoprojiov1alpha1 "github.com/argoproj/argo-cd/v3/pkg/apis/application/v1alpha1"
-
-	log "github.com/sirupsen/logrus"
 )
 
 var _ Generator = (*MatrixGenerator)(nil)
@@ -86,22 +84,10 @@ func (m *MatrixGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.Appli
 	if err != nil {
 		return nil, err
 	}
-	if matrixGen != nil && !appSet.Spec.ApplyNestedSelectors {
-		foundSelector := dropDisabledNestedSelectors(matrixGen.Generators)
-		if foundSelector {
-			log.Warnf("AppSet '%v' defines selector on nested matrix generator's generator without enabling them via 'spec.applyNestedSelectors', ignoring nested selectors", appSet.Name)
-		}
-	}
 	mergeGen, err := getMergeGenerator(appSetBaseGenerator)
 	if err != nil {
 		return nil, fmt.Errorf("error retrieving merge generator: %w", err)
 	}
-	if mergeGen != nil && !appSet.Spec.ApplyNestedSelectors {
-		foundSelector := dropDisabledNestedSelectors(mergeGen.Generators)
-		if foundSelector {
-			log.Warnf("AppSet '%v' defines selector on nested merge generator's generator without enabling them via 'spec.applyNestedSelectors', ignoring nested selectors", appSet.Name)
-		}
-	}
 
 	t, err := Transform(
 		argoprojiov1alpha1.ApplicationSetGenerator{

applicationset/generators/matrix_test.go

@@ -1087,7 +1087,7 @@ func TestGitGenerator_GenerateParams_list_x_git_matrix_generator(t *testing.T) {
 	}
 
 	repoServiceMock := &mocks.Repos{}
-	repoServiceMock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(map[string][]byte{
+	repoServiceMock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(map[string][]byte{
 		"some/path.json": []byte("test: content"),
 	}, nil)
 	gitGenerator := NewGitGenerator(repoServiceMock, "")