[v18] feat: Bot instance upgrade status dashboard

api/constants/constants.go

@@ -579,3 +579,7 @@ const MaxPIVPINCacheTTL = time.Hour
 // routine running in every auth server. Any report older than this period should
 // be considered stale.
 const AutoUpdateAgentReportPeriod = time.Minute
+
+// AutoUpdateBotInstanceReportPeriod is the period of the autoupdate bot instance
+// reporting routine.
+const AutoUpdateBotInstanceReportPeriod = time.Minute

gen/preset-roles.json

@@ -1051,6 +1051,15 @@
               "read"
             ]
           },
+          {
+            "resources": [
+              "autoupdate_bot_instance_report"
+            ],
+            "verbs": [
+              "list",
+              "read"
+            ]
+          },
           {
             "resources": [
               "git_server"

lib/auth/auth.go

@@ -1714,7 +1714,7 @@ func (a *Server) runPeriodicOperations() {
 		})
 		ticker.Push(interval.SubInterval[periodicIntervalKey]{
 			Key:           autoUpdateBotInstanceReportKey,
-			Duration:      constants.AutoUpdateAgentReportPeriod,
+			Duration:      constants.AutoUpdateBotInstanceReportPeriod,
 			FirstDuration: retryutils.HalfJitter(10 * time.Second),
 			Jitter:        retryutils.SeventhJitter,
 		})

lib/auth/autoupdate/autoupdatev1/service.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"log/slog"
 	"maps"
+	"slices"
 
 	"github.com/gravitational/trace"
 	"github.com/jonboulle/clockwork"
@@ -1055,8 +1056,15 @@ func (s *Service) GetAutoUpdateBotInstanceReport(ctx context.Context, _ *autoupd
 		return nil, trace.Wrap(err)
 	}
 
-	if err := authCtx.CheckAccessToKind(types.KindAutoUpdateBotInstanceReport, types.VerbRead); err != nil {
-		return nil, trace.Wrap(err)
+	// Because this report also powers the bot instance dashboard in the Web UI
+	// we allow users with `bot_instance:list` as well as `autoupdate_bot_instance_report:read`
+	// and return the first error if both checks fail.
+	authErrors := []error{
+		authCtx.CheckAccessToKind(types.KindAutoUpdateBotInstanceReport, types.VerbRead),
+		authCtx.CheckAccessToKind(types.KindBotInstance, types.VerbList),
+	}
+	if !slices.Contains(authErrors, nil) {
+		return nil, trace.NewAggregate(authErrors...)
 	}
 
 	report, err := s.backend.GetAutoUpdateBotInstanceReport(ctx)

lib/auth/autoupdate/autoupdatev1/service_test.go

@@ -362,6 +362,17 @@ func TestServiceAccess(t *testing.T) {
 			kind:         types.KindAutoUpdateBotInstanceReport,
 			allowedVerbs: []string{types.VerbRead},
 		},
+		{
+			name: "GetAutoUpdateBotInstanceReport",
+			allowedStates: []authz.AdminActionAuthState{
+				authz.AdminActionAuthUnauthorized,
+				authz.AdminActionAuthNotRequired,
+				authz.AdminActionAuthMFAVerified,
+				authz.AdminActionAuthMFAVerifiedWithReuse,
+			},
+			kind:         types.KindBotInstance,
+			allowedVerbs: []string{types.VerbList},
+		},
 		{
 			name: "DeleteAutoUpdateBotInstanceReport",
 			allowedStates: []authz.AdminActionAuthState{

lib/auth/machineid/machineidv1/expression/expression.go

@@ -52,11 +52,8 @@ func NewBotInstanceExpressionParser() (*typical.Parser[*Environment, bool], erro
 		"status.latest_heartbeat.one_shot": typical.DynamicVariable(func(env *Environment) (bool, error) {
 			return env.GetLatestHeartbeat().GetOneShot(), nil
 		}),
-		"status.latest_heartbeat.version": typical.DynamicVariable(func(env *Environment) (*semver.Version, error) {
-			if env.GetLatestHeartbeat().GetVersion() == "" {
-				return nil, nil
-			}
-			return semver.NewVersion(env.LatestHeartbeat.Version)
+		"status.latest_heartbeat.version": typical.DynamicVariable(func(env *Environment) (string, error) {
+			return env.GetLatestHeartbeat().GetVersion(), nil
 		}),
 		"status.latest_authentication.join_method": typical.DynamicVariable(func(env *Environment) (string, error) {
 			return env.GetLatestAuthentication().GetJoinMethod(), nil
@@ -69,8 +66,6 @@ func NewBotInstanceExpressionParser() (*typical.Parser[*Environment, bool], erro
 	spec.Functions["older_than"] = typical.BinaryFunction[*Environment](semverLt)
 	// e.g. `between(status.latest_heartbeat.version, "19.0.0", "19.0.2")`
 	spec.Functions["between"] = typical.TernaryFunction[*Environment](semverBetween)
-	// e.g. `exact_version(status.latest_heartbeat.version, "19.1.0")`
-	spec.Functions["exact_version"] = typical.BinaryFunction[*Environment](semverEq)
 
 	return typical.NewParser[*Environment, bool](spec)
 }

lib/auth/machineid/machineidv1/expression/expression_test.go

@@ -108,8 +108,8 @@ func TestBotInstanceExpressionParser(t *testing.T) {
 		},
 		{
 			name:     "exact version",
-			expTrue:  `exact_version(status.latest_heartbeat.version, "19.0.1")`,
-			expFalse: `exact_version(status.latest_heartbeat.version, "19.0.2-rc.1+56001")`,
+			expTrue:  `status.latest_heartbeat.version == "19.0.1"`,
+			expFalse: `status.latest_heartbeat.version == "19.0.2-rc.1+56001"`,
 		},
 		{
 			name:     "between versions - lower",

lib/services/presets.go

@@ -212,6 +212,7 @@ func NewPresetEditorRole() types.Role {
 					types.NewRule(types.KindAutoUpdateVersion, RW()),
 					types.NewRule(types.KindAutoUpdateConfig, RW()),
 					types.NewRule(types.KindAutoUpdateAgentRollout, RO()),
+					types.NewRule(types.KindAutoUpdateBotInstanceReport, RO()),
 					types.NewRule(types.KindGitServer, RW()),
 					types.NewRule(types.KindWorkloadIdentityX509Revocation, RW()),
 					types.NewRule(types.KindHealthCheckConfig, RW()),

lib/web/apiserver.go

@@ -1175,6 +1175,8 @@ func (h *Handler) bindDefaultEndpoints() {
 	h.GET("/webapi/sites/:site/machine-id/bot-instance", h.WithClusterAuth(h.listBotInstances))
 	// GET Machine ID bot instances (paged)
 	h.GET("/v2/webapi/sites/:site/machine-id/bot-instance", h.WithClusterAuth(h.listBotInstancesV2))
+	// GET Machine ID bot instance metrics.
+	h.GET("/webapi/sites/:site/machine-id/bot-instance/metrics", h.WithClusterAuth(h.botInstanceMetrics))
 
 	// List workload identities
 	h.GET("/webapi/sites/:site/workload-identity", h.WithClusterAuth(h.listWorkloadIdentities))

lib/web/machineid.go

@@ -19,17 +19,20 @@ package web
 import (
 	"cmp"
 	"context"
+	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/coreos/go-semver/semver"
 	yaml "github.com/ghodss/yaml"
 	"github.com/gravitational/trace"
 	"github.com/julienschmidt/httprouter"
 	"google.golang.org/protobuf/types/known/durationpb"
 	"google.golang.org/protobuf/types/known/fieldmaskpb"
 
+	"github.com/gravitational/teleport/api/constants"
 	headerv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/header/v1"
 	machineidv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/machineid/v1"
 	"github.com/gravitational/teleport/api/types"
@@ -539,3 +542,161 @@ type BotInstance struct {
 	ActiveAtLatest   string `json:"active_at_latest,omitempty"`
 	OSLatest         string `json:"os_latest,omitempty"`
 }
+
+func (h *Handler) botInstanceMetrics(_ http.ResponseWriter, r *http.Request, _ httprouter.Params, sctx *SessionContext, cluster reversetunnelclient.Cluster) (any, error) {
+	ctx := r.Context()
+
+	clt, err := sctx.GetUserClient(ctx, cluster)
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	rsp := BotInstanceMetricsResponse{
+		RefreshAfterSeconds: int(constants.AutoUpdateBotInstanceReportPeriod.Seconds()),
+	}
+
+	// If no report is available yet, `UpgradeStatuses` will be nil.
+	report, err := clt.GetAutoUpdateBotInstanceReport(ctx)
+	switch {
+	case trace.IsNotFound(err):
+		return rsp, nil
+	case err != nil:
+		return nil, trace.Wrap(err)
+	}
+
+	// Our target version is the operator's selected auto-update tools version,
+	// or if there isn't one configured: the proxy version.
+	autoUpdateVersion, err := h.cfg.AccessPoint.GetAutoUpdateVersion(ctx)
+	if err != nil && !trace.IsNotFound(err) {
+		return nil, trace.Wrap(err)
+	}
+	targetVersion, err := getToolsVersion(autoUpdateVersion)
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	// Returns the earliest possible version in a major release. It's based on:
+	// lib/utils.VersionBeforeAlpha.
+	lowerBound := func(major int64) semver.Version {
+		return semver.Version{Major: major, PreRelease: "aa"}
+	}
+
+	const versionField = "status.latest_heartbeat.version"
+
+	rsp.UpgradeStatuses = &BotInstanceUpgradeStatuses{
+		UpdatedAt: report.GetSpec().GetTimestamp().AsTime(),
+		UpToDate: BotInstanceUpgradeStatus{
+			Filter: fmt.Sprintf("%[1]s == %[2]q", versionField, targetVersion),
+		},
+		Unsupported: BotInstanceUpgradeStatus{
+			Filter: fmt.Sprintf(
+				"older_than(%[1]s, %[2]q) || %[1]s == %[3]q || newer_than(%[1]s, %[3]q)",
+				versionField,
+				lowerBound(targetVersion.Major-1),
+				lowerBound(targetVersion.Major+1),
+			),
+		},
+		PatchAvailable: BotInstanceUpgradeStatus{
+			Filter: fmt.Sprintf(
+				"between(%[1]s, %[2]q, %[3]q)",
+				versionField,
+				lowerBound(targetVersion.Major),
+				targetVersion,
+			),
+		},
+		RequiresUpgrade: BotInstanceUpgradeStatus{
+			Filter: fmt.Sprintf(
+				"between(%[1]s, %[2]q, %[3]q)",
+				versionField,
+				lowerBound(targetVersion.Major-1),
+				lowerBound(targetVersion.Major),
+			),
+		},
+	}
+
+	for _, groupMetrics := range report.GetSpec().GetGroups() {
+		for versionString, versionMetrics := range groupMetrics.GetVersions() {
+			version, err := semver.NewVersion(versionString)
+			if err != nil {
+				h.logger.ErrorContext(ctx,
+					"Failed to parse bot instance version string",
+					"version_string", versionString,
+					"error", err,
+				)
+				continue
+			}
+
+			switch {
+			case targetVersion.Equal(*version):
+				// Bot is up to date.
+				rsp.UpgradeStatuses.UpToDate.Count += int(versionMetrics.Count)
+
+			case targetVersion.LessThan(*version):
+				// Bot is running a newer version, we don't support this.
+				rsp.UpgradeStatuses.Unsupported.Count += int(versionMetrics.Count)
+
+			case targetVersion.Major == version.Major:
+				// Bot is running the right major version, but there's a minor
+				// or patch update available
+				rsp.UpgradeStatuses.PatchAvailable.Count += int(versionMetrics.Count)
+
+			case version.Major == targetVersion.Major-1:
+				// Bot is running the previous major version and should upgrade.
+				rsp.UpgradeStatuses.RequiresUpgrade.Count += int(versionMetrics.Count)
+
+			case version.Major < targetVersion.Major-1:
+				// Bot is running a version that is too old. In this case, the
+				// connection would be terminated so we shouldn't really see it.
+				rsp.UpgradeStatuses.Unsupported.Count += int(versionMetrics.Count)
+
+			default:
+				// The branches of this switch should be exhaustive, but just in case!
+				h.logger.DebugContext(ctx,
+					"Bot instance version comparison is missing a branch",
+					"bot_instance_version", version,
+					"target_version", targetVersion,
+				)
+			}
+		}
+	}
+	return rsp, nil
+}
+
+type BotInstanceMetricsResponse struct {
+	// RefreshAfterSeconds is the amount of time (in seconds) after receiving
+	// this response the client should poll for new metrics.
+	RefreshAfterSeconds int `json:"refresh_after_seconds"`
+
+	// UpgradeStatuses contains instance counts by "upgrade status".
+	UpgradeStatuses *BotInstanceUpgradeStatuses `json:"upgrade_statuses"`
+}
+
+type BotInstanceUpgradeStatuses struct {
+	// UpdatedAt is when these metrics were last updated.
+	UpdatedAt time.Time `json:"updated_at"`
+
+	// UpToDate means the instance matches the desired version.
+	UpToDate BotInstanceUpgradeStatus `json:"up_to_date"`
+
+	// Unsupported means the instance is running a release that is too old or
+	// too new for us to support.
+	Unsupported BotInstanceUpgradeStatus `json:"unsupported"`
+
+	// RequiresUpgrade means the instance is running a release from the previous
+	// major series. We can support it for now, but the next major upgrade will
+	// break compatibility.
+	RequiresUpgrade BotInstanceUpgradeStatus `json:"requires_upgrade"`
+
+	// PatchAvailable means the instance is running a release from the desired
+	// major series, but they're behind on a minor or patch release.
+	PatchAvailable BotInstanceUpgradeStatus `json:"patch_available"`
+}
+
+type BotInstanceUpgradeStatus struct {
+	// Count is the number of bot instances.
+	Count int `json:"count"`
+
+	// Filter is a predicate language filter that can be applied to the bot
+	// instance list to find matching instances.
+	Filter string `json:"filter"`
+}