[v17] feat: Bot instances at scale UI (RFD0222)

api/constants/constants.go

@@ -544,3 +544,7 @@ const MaxPIVPINCacheTTL = time.Hour
 // routine running in every auth server. Any report older than this period should
 // be considered stale.
 const AutoUpdateAgentReportPeriod = time.Minute
+
+// AutoUpdateBotInstanceReportPeriod is the period of the autoupdate bot instance
+// reporting routine.
+const AutoUpdateBotInstanceReportPeriod = time.Minute

api/proto/teleport/machineid/v1/bot_instance_service.proto

@@ -50,6 +50,39 @@ message ListBotInstancesRequest {
   types.SortBy sort = 5;
 }
 
+// Request for ListBotInstancesV2.
+//
+// Follows the pagination semantics of
+// https://cloud.google.com/apis/design/standard_methods#list
+message ListBotInstancesV2Request {
+  // The maximum number of items to return.
+  // The server may impose a different page size at its discretion.
+  int32 page_size = 1;
+  // The page_token value returned from a previous ListBotInstancesV2 request,
+  // if any.
+  string page_token = 2;
+  // The sort field to use for the results. If empty, the default sort field is
+  // used.
+  string sort_field = 3;
+  // The sort order to use for the results. If empty, the default sort order is
+  // used.
+  bool sort_desc = 4;
+  // Fields used to filter the results
+  Filters filter = 5;
+
+  // Filters contains fields to be used to filter the results.
+  message Filters {
+    // The name of the Bot to list BotInstances for. If non-empty, only
+    // BotInstances for that bot will be listed.
+    string bot_name = 1;
+    // A search term used to filter the results. If non-empty, it's used to
+    // match against supported fields.
+    string search_term = 2;
+    // A Teleport predicate language query used to filter the results.
+    string query = 3;
+  }
+}
+
 // Response for ListBotInstances.
 message ListBotInstancesResponse {
   // BotInstance that matched the search.
@@ -86,7 +119,12 @@ service BotInstanceService {
   // GetBotInstance returns the specified BotInstance resource.
   rpc GetBotInstance(GetBotInstanceRequest) returns (BotInstance);
   // ListBotInstances returns a page of BotInstance resources.
-  rpc ListBotInstances(ListBotInstancesRequest) returns (ListBotInstancesResponse);
+  // Deprecated: Use ListBotInstancesV2 instead
+  rpc ListBotInstances(ListBotInstancesRequest) returns (ListBotInstancesResponse) {
+    option deprecated = true;
+  }
+  // ListBotInstancesV2 returns a page of BotInstance resources.
+  rpc ListBotInstancesV2(ListBotInstancesV2Request) returns (ListBotInstancesResponse);
   // DeleteBotInstance hard deletes the specified BotInstance resource.
   rpc DeleteBotInstance(DeleteBotInstanceRequest) returns (google.protobuf.Empty);
   // SubmitHeartbeat submits a heartbeat for a BotInstance.

docs/pages/reference/cli/tctl.mdx

@@ -581,6 +581,16 @@ $ tctl bots instances list [<name>]
 - `[<name>]` - an optional bot name. If provided, filters the result to show
   only instances for the named bot. Otherwise, shows all instances for all bots.
 
+### Flags
+
+| Name | Default Value(s) | Allowed Value(s) | Description |
+| - | - | - | - |
+| `--search` | none | A search term | Optional. Filters the returned bot instances using a fuzzy search based on the term provided. |
+| `--query` | none | Teleport predicate language query | Optional. Filters the returned bot instances based on the Teleport predicate language query provided. |
+| `--sort-index` | `bot_name` | `bot_name`, `active_at_latest`, `version_latest`, `host_name_latest` | Optional. Sorts the returned bot instances using the given field. |
+| `--sort-order` | `ascending` | `ascending`, `descending` | Optional. Sorts the returned bot instances in the given order. |
+| `--format` | `text` | `text`, `json` | If set to `json`, returns results as a machine-readable JSON string. |
+
 ### Examples
 
 This shows all known instances for the bot named "example":
@@ -589,6 +599,25 @@ This shows all known instances for the bot named "example":
 $ tctl bots instance list example
 ```
 
+This shows all known instances which contain the term "github";
+
+```code
+$ tctl bots instance list --search github
+```
+
+Searchable fields include; bot name, instance id, hostname, join method, version
+
+This shows all known instances with a version older than "18.0.0";
+
+```code
+$ tctl bots instance list --query `older_than(status.latest_heartbeat.version, "18.0.0")`
+```
+Version-specific functions include;
+
+- `newer_than(version, comparison)`
+- `older_than(version, comparison)`
+- `between(version, lower (inclusive), upper (exclusive))`
+
 ### Global flags
 
 These flags are available for all commands: `--debug, --config`. Run

lib/auth/auth.go

@@ -1571,7 +1571,7 @@ func (a *Server) runPeriodicOperations() {
 		})
 		ticker.Push(interval.SubInterval[periodicIntervalKey]{
 			Key:           autoUpdateBotInstanceReportKey,
-			Duration:      constants.AutoUpdateAgentReportPeriod,
+			Duration:      constants.AutoUpdateBotInstanceReportPeriod,
 			FirstDuration: retryutils.HalfJitter(10 * time.Second),
 			Jitter:        retryutils.SeventhJitter,
 		})

lib/auth/authclient/api.go

@@ -1354,7 +1354,7 @@ type Cache interface {
 	GetBotInstance(ctx context.Context, botName, instanceID string) (*machineidv1.BotInstance, error)
 
 	// ListBotInstances returns a page of BotInstance resources.
-	ListBotInstances(ctx context.Context, botName string, pageSize int, lastToken string, search string, sort *types.SortBy) ([]*machineidv1.BotInstance, string, error)
+	ListBotInstances(ctx context.Context, pageSize int, lastToken string, options *services.ListBotInstancesRequestOptions) ([]*machineidv1.BotInstance, string, error)
 
 	// ListProvisionTokens returns a paginated list of provision tokens.
 	ListProvisionTokens(ctx context.Context, pageSize int, pageToken string, anyRoles types.SystemRoles, botName string) ([]types.ProvisionToken, string, error)

lib/auth/autoupdate/autoupdatev1/service.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"log/slog"
 	"maps"
+	"slices"
 
 	"github.com/gravitational/trace"
 	"github.com/jonboulle/clockwork"
@@ -1052,8 +1053,15 @@ func (s *Service) GetAutoUpdateBotInstanceReport(ctx context.Context, _ *autoupd
 		return nil, trace.Wrap(err)
 	}
 
-	if err := authCtx.CheckAccessToKind(types.KindAutoUpdateBotInstanceReport, types.VerbRead); err != nil {
-		return nil, trace.Wrap(err)
+	// Because this report also powers the bot instance dashboard in the Web UI
+	// we allow users with `bot_instance:list` as well as `autoupdate_bot_instance_report:read`
+	// and return the first error if both checks fail.
+	authErrors := []error{
+		authCtx.CheckAccessToKind(types.KindAutoUpdateBotInstanceReport, types.VerbRead),
+		authCtx.CheckAccessToKind(types.KindBotInstance, types.VerbList),
+	}
+	if !slices.Contains(authErrors, nil) {
+		return nil, trace.NewAggregate(authErrors...)
 	}
 
 	report, err := s.backend.GetAutoUpdateBotInstanceReport(ctx)

lib/auth/autoupdate/autoupdatev1/service_test.go

@@ -362,6 +362,17 @@ func TestServiceAccess(t *testing.T) {
 			kind:         types.KindAutoUpdateBotInstanceReport,
 			allowedVerbs: []string{types.VerbRead},
 		},
+		{
+			name: "GetAutoUpdateBotInstanceReport",
+			allowedStates: []authz.AdminActionAuthState{
+				authz.AdminActionAuthUnauthorized,
+				authz.AdminActionAuthNotRequired,
+				authz.AdminActionAuthMFAVerified,
+				authz.AdminActionAuthMFAVerifiedWithReuse,
+			},
+			kind:         types.KindBotInstance,
+			allowedVerbs: []string{types.VerbList},
+		},
 		{
 			name: "DeleteAutoUpdateBotInstanceReport",
 			allowedStates: []authz.AdminActionAuthState{

lib/auth/machineid/machineidv1/auto_update_version_reporter.go

@@ -242,10 +242,8 @@ func (r *AutoUpdateVersionReporter) generateReport(ctx context.Context) error {
 		)
 		instances, nextToken, err = r.cache.ListBotInstances(
 			ctx,
-			"",
 			defaults.DefaultChunkSize,
 			nextToken,
-			"",
 			nil,
 		)
 		if err != nil {

lib/auth/machineid/machineidv1/bot_instance_service.go

@@ -62,7 +62,7 @@ type BotInstancesCache interface {
 	GetBotInstance(ctx context.Context, botName, instanceID string) (*pb.BotInstance, error)
 
 	// ListBotInstances returns a page of BotInstance resources.
-	ListBotInstances(ctx context.Context, botName string, pageSize int, lastToken string, search string, sort *types.SortBy) ([]*pb.BotInstance, string, error)
+	ListBotInstances(ctx context.Context, pageSize int, lastToken string, options *services.ListBotInstancesRequestOptions) ([]*pb.BotInstance, string, error)
 }
 
 // BotInstanceServiceConfig holds configuration options for the BotInstance gRPC
@@ -156,6 +156,26 @@ func (b *BotInstanceService) GetBotInstance(ctx context.Context, req *pb.GetBotI
 
 // ListBotInstances returns a list of bot instances matching the criteria in the request
 func (b *BotInstanceService) ListBotInstances(ctx context.Context, req *pb.ListBotInstancesRequest) (*pb.ListBotInstancesResponse, error) {
+	var sortField string
+	var sortDesc bool
+	if req.GetSort() != nil {
+		sortField = req.GetSort().Field
+		sortDesc = req.GetSort().IsDesc
+	}
+	return b.ListBotInstancesV2(ctx, &pb.ListBotInstancesV2Request{
+		PageSize:  req.GetPageSize(),
+		PageToken: req.GetPageToken(),
+		SortField: sortField,
+		SortDesc:  sortDesc,
+		Filter: &pb.ListBotInstancesV2Request_Filters{
+			BotName:    req.GetFilterBotName(),
+			SearchTerm: req.GetFilterSearchTerm(),
+		},
+	})
+}
+
+// ListBotInstancesV2 returns a list of bot instances matching the criteria in the request
+func (b *BotInstanceService) ListBotInstancesV2(ctx context.Context, req *pb.ListBotInstancesV2Request) (*pb.ListBotInstancesResponse, error) {
 	authCtx, err := b.authorizer.Authorize(ctx)
 	if err != nil {
 		return nil, trace.Wrap(err)
@@ -165,7 +185,13 @@ func (b *BotInstanceService) ListBotInstances(ctx context.Context, req *pb.ListB
 		return nil, trace.Wrap(err)
 	}
 
-	res, nextToken, err := b.cache.ListBotInstances(ctx, req.FilterBotName, int(req.PageSize), req.PageToken, req.FilterSearchTerm, req.Sort)
+	res, nextToken, err := b.cache.ListBotInstances(ctx, int(req.PageSize), req.PageToken, &services.ListBotInstancesRequestOptions{
+		SortField:        req.GetSortField(),
+		SortDesc:         req.GetSortDesc(),
+		FilterBotName:    req.GetFilter().GetBotName(),
+		FilterSearchTerm: req.GetFilter().GetSearchTerm(),
+		FilterQuery:      req.GetFilter().GetQuery(),
+	})
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}

lib/auth/machineid/machineidv1/bot_instance_service_test.go

@@ -70,6 +70,14 @@ func TestBotInstanceServiceAccess(t *testing.T) {
 			},
 			allowedVerbs: []string{types.VerbRead, types.VerbList},
 		},
+		{
+			name: "ListBotInstancesV2",
+			allowedStates: []authz.AdminActionAuthState{
+				authz.AdminActionAuthUnauthorized, authz.AdminActionAuthNotRequired,
+				authz.AdminActionAuthMFAVerified, authz.AdminActionAuthMFAVerifiedWithReuse,
+			},
+			allowedVerbs: []string{types.VerbRead, types.VerbList},
+		},
 		{
 			name: "DeleteBotInstance",
 			allowedStates: []authz.AdminActionAuthState{