[v16] Reduce resource consumption of ListUnifiedResources

api/go.mod

@@ -3,6 +3,7 @@ module github.com/gravitational/teleport/api
 go 1.23.0
 
 require (
+	github.com/charlievieth/strcase v0.0.5
 	github.com/coreos/go-semver v0.3.1
 	github.com/go-piv/piv-go v1.11.0
 	github.com/gobwas/ws v1.4.0

api/go.sum

@@ -3,6 +3,8 @@ github.com/beevik/etree v1.3.0 h1:hQTc+pylzIKDb23yYprodCWWTt+ojFfUZyzU09a/hmU=
 github.com/beevik/etree v1.3.0/go.mod h1:aiPf89g/1k3AShMVAzriilpcE4R/Vuor90y83zVZWFc=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/charlievieth/strcase v0.0.5 h1:gV4iXVyD6eI5KdfOV+/vIVCKXZwtCWOmDMcu7Uy00Rs=
+github.com/charlievieth/strcase v0.0.5/go.mod h1:FIOYY1aDBMSIOFqmVomHBpoK+bteGlESRsgsdWjrhx8=
 github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
 github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=

api/types/lock.go

@@ -298,5 +298,13 @@ func (t LockTarget) String() string {
 
 // Equals returns true when the two lock targets are equal.
 func (t LockTarget) Equals(t2 LockTarget) bool {
-	return proto.Equal(&t, &t2)
+	return t.User == t2.User &&
+		t.Role == t2.Role &&
+		t.Login == t2.Login &&
+		t.MFADevice == t2.MFADevice &&
+		t.WindowsDesktop == t2.WindowsDesktop &&
+		t.AccessRequest == t2.AccessRequest &&
+		t.Device == t2.Device &&
+		t.ServerID == t2.ServerID &&
+		t.Node == t2.Node
 }

api/types/lock_test.go

@@ -133,3 +133,39 @@ func TestLockTargetIsEmpty(t *testing.T) {
 		require.False(t, lt.IsEmpty(), "field name: %v", field.Name)
 	}
 }
+
+// TestLockTargetEquals checks that the implementation of [LockTarget.Equals]
+// is correct by filling one field at a time in for two LockTargets and expecting
+// Equals to return the appropriate value. Only the public fields that don't start with
+// `XXX_` are checked (as those are gogoproto-internal fields).
+func TestLockTargetEquals(t *testing.T) {
+	t.Run("equal", func(t *testing.T) {
+		require.True(t, (LockTarget{}).Equals(LockTarget{}), "empty targets equal")
+
+		for i, field := range reflect.VisibleFields(reflect.TypeOf(LockTarget{})) {
+			if strings.HasPrefix(field.Name, "XXX_") {
+				continue
+			}
+
+			var a, b LockTarget
+			// if we add non-string fields to LockTarget we need a type switch here
+			reflect.ValueOf(&a).Elem().Field(i).SetString("nonempty")
+			reflect.ValueOf(&b).Elem().Field(i).SetString("nonempty")
+			require.True(t, a.Equals(b), "field name: %v", field.Name)
+		}
+	})
+
+	t.Run("not equal", func(t *testing.T) {
+		for i, field := range reflect.VisibleFields(reflect.TypeOf(LockTarget{})) {
+			if strings.HasPrefix(field.Name, "XXX_") {
+				continue
+			}
+
+			var a, b LockTarget
+			// if we add non-string fields to LockTarget we need a type switch here
+			reflect.ValueOf(&a).Elem().Field(i).SetString("nonempty")
+			reflect.ValueOf(&b).Elem().Field(i).SetString("other")
+			require.False(t, a.Equals(b), "field name: %v", field.Name)
+		}
+	})
+}

api/types/resource.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/charlievieth/strcase"
 	"github.com/gravitational/trace"
 
 	"github.com/gravitational/teleport/api/defaults"
@@ -530,7 +531,7 @@ Outer:
 	for _, searchV := range searchVals {
 		// Iterate through field values to look for a match.
 		for _, fieldV := range fieldVals {
-			if containsFold(fieldV, searchV) {
+			if strcase.Contains(fieldV, searchV) {
 				continue Outer
 			}
 		}
@@ -546,23 +547,6 @@ Outer:
 	return true
 }
 
-// containsFold is a case-insensitive alternative to strings.Contains, used to help avoid excess allocations during searches.
-func containsFold(s, substr string) bool {
-	if len(s) < len(substr) {
-		return false
-	}
-
-	n := len(s) - len(substr)
-
-	for i := 0; i <= n; i++ {
-		if strings.EqualFold(s[i:i+len(substr)], substr) {
-			return true
-		}
-	}
-
-	return false
-}
-
 func stringCompare(a string, b string, isDesc bool) bool {
 	if isDesc {
 		return a > b

api/types/server.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/charlievieth/strcase"
 	"github.com/google/uuid"
 	"github.com/gravitational/trace"
 
@@ -575,25 +576,46 @@ func (s *ServerV2) MatchSearch(values []string) bool {
 	if s.GetKind() != KindNode {
 		return false
 	}
+Outer:
+	for _, searchV := range values {
+		for key, value := range s.Metadata.Labels {
+			if strcase.Contains(key, searchV) || strcase.Contains(value, searchV) {
+				continue Outer
+			}
+		}
+		for key, cmd := range s.Spec.CmdLabels {
+			if strcase.Contains(key, searchV) || strcase.Contains(cmd.Result, searchV) {
+				continue Outer
+			}
+		}
 
-	var custom func(val string) bool
-	if s.GetUseTunnel() {
-		custom = func(val string) bool {
-			return strings.EqualFold(val, "tunnel")
+		if strcase.Contains(s.Metadata.Name, searchV) {
+			continue
 		}
-	}
 
-	fieldVals := make([]string, 0, (len(s.Metadata.Labels)*2)+(len(s.Spec.CmdLabels)*2)+len(s.Spec.PublicAddrs)+3)
+		if strcase.Contains(s.Spec.Hostname, searchV) {
+			continue
+		}
 
-	labels := CombineLabels(s.Metadata.Labels, s.Spec.CmdLabels)
-	for key, value := range labels {
-		fieldVals = append(fieldVals, key, value)
-	}
+		if strcase.Contains(s.Spec.Addr, searchV) {
+			continue
+		}
+
+		for _, addr := range s.Spec.PublicAddrs {
+			if strcase.Contains(addr, searchV) {
+				continue Outer
+			}
+		}
 
-	fieldVals = append(fieldVals, s.Metadata.Name, s.Spec.Hostname, s.Spec.Addr)
-	fieldVals = append(fieldVals, s.Spec.PublicAddrs...)
+		if s.GetUseTunnel() && strings.EqualFold(searchV, "tunnel") {
+			continue
+		}
+
+		// When no fields matched a value, prematurely end if we can.
+		return false
+	}
 
-	return MatchSearch(fieldVals, values, custom)
+	return true
 }
 
 // DeepCopy creates a clone of this server value

go.mod

@@ -285,6 +285,7 @@ require (
 	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
+	github.com/charlievieth/strcase v0.0.5 // indirect
 	github.com/cloudflare/cfssl v1.6.4 // indirect
 	github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 // indirect
 	github.com/containerd/containerd v1.7.27 // indirect

go.sum

@@ -373,6 +373,8 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=
 github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
+github.com/charlievieth/strcase v0.0.5 h1:gV4iXVyD6eI5KdfOV+/vIVCKXZwtCWOmDMcu7Uy00Rs=
+github.com/charlievieth/strcase v0.0.5/go.mod h1:FIOYY1aDBMSIOFqmVomHBpoK+bteGlESRsgsdWjrhx8=
 github.com/charmbracelet/bubbles v0.18.0 h1:PYv1A036luoBGroX6VWjQIE9Syf2Wby2oOl/39KLfy0=
 github.com/charmbracelet/bubbles v0.18.0/go.mod h1:08qhZhtIwzgrtBjAcJnij1t1H0ZRjwHyGsy6AL11PSw=
 github.com/charmbracelet/bubbletea v0.26.1 h1:xujcQeF73rh4jwu3+zhfQsvV18x+7zIjlw7/CYbzGJ0=

integrations/event-handler/go.mod

@@ -97,6 +97,7 @@ require (
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
+	github.com/charlievieth/strcase v0.0.5 // indirect
 	github.com/chzyer/readline v1.5.1 // indirect
 	github.com/cloudflare/cfssl v1.6.4 // indirect
 	github.com/containerd/containerd v1.7.27 // indirect

integrations/event-handler/go.sum

@@ -200,6 +200,8 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=
 github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
+github.com/charlievieth/strcase v0.0.5 h1:gV4iXVyD6eI5KdfOV+/vIVCKXZwtCWOmDMcu7Uy00Rs=
+github.com/charlievieth/strcase v0.0.5/go.mod h1:FIOYY1aDBMSIOFqmVomHBpoK+bteGlESRsgsdWjrhx8=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM=
 github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ=

integrations/terraform/go.mod

@@ -111,6 +111,7 @@ require (
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
+	github.com/charlievieth/strcase v0.0.5 // indirect
 	github.com/cloudflare/cfssl v1.6.4 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b // indirect

integrations/terraform/go.sum

@@ -302,6 +302,8 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=
 github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
+github.com/charlievieth/strcase v0.0.5 h1:gV4iXVyD6eI5KdfOV+/vIVCKXZwtCWOmDMcu7Uy00Rs=
+github.com/charlievieth/strcase v0.0.5/go.mod h1:FIOYY1aDBMSIOFqmVomHBpoK+bteGlESRsgsdWjrhx8=
 github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=

lib/auth/auth_with_roles.go

@@ -22,6 +22,7 @@ import (
 	"cmp"
 	"context"
 	"fmt"
+	"maps"
 	"net/url"
 	"os"
 	"slices"
@@ -1347,6 +1348,21 @@ func (c *resourceAccess) checkAccess(resource types.ResourceWithLabels, filter s
 	return true, nil
 }
 
+var (
+	// supportedUnifiedResourceKinds is the set of kinds that
+	// may be requested via ListUnifiedResources.
+	supportedUnifiedResourceKinds = map[string]struct{}{
+		types.KindApp:                    {},
+		types.KindDatabase:               {},
+		types.KindKubernetesCluster:      {},
+		types.KindNode:                   {},
+		types.KindSAMLIdPServiceProvider: {},
+		types.KindWindowsDesktop:         {},
+	}
+
+	defaultUnifiedResourceKinds = slices.Collect(maps.Keys(supportedUnifiedResourceKinds))
+)
+
 // ListUnifiedResources returns a paginated list of unified resources filtered by user access.
 func (a *ServerWithRoles) ListUnifiedResources(ctx context.Context, req *proto.ListUnifiedResourcesRequest) (*proto.ListUnifiedResourcesResponse, error) {
 	filter := services.MatchResourceFilter{
@@ -1363,17 +1379,29 @@ func (a *ServerWithRoles) ListUnifiedResources(ctx context.Context, req *proto.L
 		filter.PredicateExpression = expression
 	}
 
+	// Validate the requested kinds and precheck that the user has read/list
+	// permissions for all requested resources before doing any listing of
+	// resources to conserve resources.
+	requested := req.Kinds
+	if len(req.Kinds) == 0 {
+		requested = defaultUnifiedResourceKinds
+	}
+
 	resourceAccess := &resourceAccess{
 		// Populate kindAccessMap with any access errors the user has for each possible
 		// resource kind. This allows the access check to occur a single time per resource
 		// kind instead of once per matching resource.
-		kindAccessMap: make(map[string]error, len(services.UnifiedResourceKinds)),
+		kindAccessMap: make(map[string]error, len(requested)),
 		// requestableMap is populated with resources that are being returned but can only
 		// be accessed to the user via an access request
 		requestableMap: make(map[string]struct{}),
 	}
 
-	for _, kind := range services.UnifiedResourceKinds {
+	for _, kind := range requested {
+		if _, ok := supportedUnifiedResourceKinds[kind]; !ok {
+			return nil, trace.BadParameter("Unsupported kind %q requested", kind)
+		}
+
 		actionVerbs := []string{types.VerbList, types.VerbRead}
 		if kind == types.KindNode {
 			// We are checking list only for Nodes to keep backwards compatibility.
@@ -1390,10 +1418,6 @@ func (a *ServerWithRoles) ListUnifiedResources(ctx context.Context, req *proto.L
 	// Before doing any listing, verify that the user is allowed to list
 	// at least one of the requested kinds. If no access is permitted, then
 	// return an access denied error.
-	requested := req.Kinds
-	if len(req.Kinds) == 0 {
-		requested = services.UnifiedResourceKinds
-	}
 	var rbacErrors int
 	for _, kind := range requested {
 		if err, ok := resourceAccess.kindAccessMap[kind]; ok && err != nil {

lib/auth/auth_with_roles_test.go

@@ -5595,7 +5595,7 @@ func TestListUnifiedResources_MixedAccess(t *testing.T) {
 	}
 
 	// Update the roles to prevent access to any resource kinds.
-	role.SetRules(types.Deny, []types.Rule{{Resources: services.UnifiedResourceKinds, Verbs: []string{types.VerbList, types.VerbRead}}})
+	role.SetRules(types.Deny, []types.Rule{{Resources: []string{types.Wildcard}, Verbs: []string{types.VerbList, types.VerbRead}}})
 	_, err = srv.Auth().UpsertRole(ctx, role)
 	require.NoError(t, err)
 
@@ -5618,7 +5618,7 @@ func TestListUnifiedResources_MixedAccess(t *testing.T) {
 	resp, err = clt.ListUnifiedResources(ctx, &proto.ListUnifiedResourcesRequest{
 		Limit:  20,
 		SortBy: types.SortBy{IsDesc: true, Field: types.ResourceMetadataName},
-		Kinds:  []string{types.KindNode, types.KindDatabaseServer},
+		Kinds:  []string{types.KindNode, types.KindDatabase},
 	})
 	require.True(t, trace.IsAccessDenied(err))
 	require.Nil(t, resp)
@@ -5729,7 +5729,9 @@ func BenchmarkListUnifiedResourcesFilter(b *testing.B) {
 		node, err := types.NewServerWithLabels(
 			name,
 			types.KindNode,
-			types.ServerSpecV2{},
+			types.ServerSpecV2{
+				Hostname: "node." + strconv.Itoa(i),
+			},
 			labels,
 		)
 		require.NoError(b, err)
@@ -5850,7 +5852,9 @@ func BenchmarkListUnifiedResources(b *testing.B) {
 		node, err := types.NewServerWithLabels(
 			name,
 			types.KindNode,
-			types.ServerSpecV2{},
+			types.ServerSpecV2{
+				Hostname: "node." + strconv.Itoa(i),
+			},
 			map[string]string{
 				"key":   id,
 				"group": "users",
@@ -5927,6 +5931,7 @@ func benchmarkListUnifiedResources(
 	for n := 0; n < b.N; n++ {
 		var resources []*proto.PaginatedResource
 		req := &proto.ListUnifiedResourcesRequest{
+			Kinds:  []string{types.KindNode},
 			SortBy: types.SortBy{IsDesc: false, Field: types.ResourceMetadataName},
 			Limit:  1_000,
 		}

lib/services/unified_resource.go

@@ -39,16 +39,6 @@ import (
 	"github.com/gravitational/teleport/lib/utils"
 )
 
-// UnifiedResourceKinds is a list of all kinds that are stored in the unified resource cache.
-var UnifiedResourceKinds []string = []string{
-	types.KindNode,
-	types.KindKubeServer,
-	types.KindDatabaseServer,
-	types.KindAppServer,
-	types.KindWindowsDesktop,
-	types.KindSAMLIdPServiceProvider,
-}
-
 // UnifiedResourceCacheConfig is used to configure a UnifiedResourceCache
 type UnifiedResourceCacheConfig struct {
 	// BTreeDegree is a degree of B-Tree, 2 for example, will create a
@@ -677,12 +667,14 @@ func (c *UnifiedResourceCache) processEventsAndUpdateCurrent(ctx context.Context
 
 // resourceKinds returns a list of resources to be watched.
 func (c *UnifiedResourceCache) resourceKinds() []types.WatchKind {
-	watchKinds := make([]types.WatchKind, 0, len(UnifiedResourceKinds))
-	for _, kind := range UnifiedResourceKinds {
-		watchKinds = append(watchKinds, types.WatchKind{Kind: kind})
+	return []types.WatchKind{
+		{Kind: types.KindNode},
+		{Kind: types.KindKubeServer},
+		{Kind: types.KindDatabaseServer},
+		{Kind: types.KindAppServer},
+		{Kind: types.KindWindowsDesktop},
+		{Kind: types.KindSAMLIdPServiceProvider},
 	}
-
-	return watchKinds
 }
 
 func (c *UnifiedResourceCache) defineCollectorAsInitialized() {