[v17] MWI: Heartbeat bot service health

.github/workflows/lint.yaml

@@ -4,6 +4,7 @@ on:
   merge_group:
 env:
   GOFLAGS: '-buildvcs=false'
+  GOEXPERIMENT: 'synctest'
 
 jobs:
   changes:

Makefile

@@ -941,7 +941,7 @@ test-go-prepare: ensure-webassets bpf-bytecode $(TEST_LOG_DIR) ensure-gotestsum
 test-go-unit: FLAGS ?= -race -shuffle on
 test-go-unit: SUBJECT ?= $(shell go list ./... | grep -vE 'teleport/(e2e|integration|tool/tsh|integrations/operator|integrations/access|integrations/lib)')
 test-go-unit:
-	$(CGOFLAG) go test -cover -json -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG) $(LIBFIDO2_TEST_TAG) $(TOUCHID_TAG) $(PIV_TEST_TAG) $(VNETDAEMON_TAG)" $(PACKAGES) $(SUBJECT) $(FLAGS) $(ADDFLAGS) \
+	$(CGOFLAG) GOEXPERIMENT=synctest go test -cover -json -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG) $(LIBFIDO2_TEST_TAG) $(TOUCHID_TAG) $(PIV_TEST_TAG) $(VNETDAEMON_TAG)" $(PACKAGES) $(SUBJECT) $(FLAGS) $(ADDFLAGS) \
 		| tee $(TEST_LOG_DIR)/unit.json \
 		| gotestsum --raw-command -- cat
 

api/proto/teleport/machineid/v1/bot_instance.proto

@@ -167,4 +167,47 @@ message BotInstanceStatus {
   BotInstanceStatusHeartbeat initial_heartbeat = 3;
   // The N most recent heartbeats for this bot instance.
   repeated BotInstanceStatusHeartbeat latest_heartbeats = 4;
+  // The health of the services/output `tbot` is running.
+  repeated BotInstanceServiceHealth service_health = 5;
+}
+
+// BotInstanceServiceHealth is a snapshot of a `tbot` service's health.
+message BotInstanceServiceHealth {
+  // Service identifies the service.
+  BotInstanceServiceIdentifier service = 1;
+
+  // Status describes the service's healthiness.
+  BotInstanceHealthStatus status = 2;
+
+  // Reason is a human-readable explanation for the service's status. It might
+  // include an error message.
+  optional string reason = 3;
+
+  // UpdatedAt is the time at which the service's health last changed.
+  google.protobuf.Timestamp updated_at = 4;
+}
+
+// BotInstanceServiceIdentifier uniquely identifies a `tbot` service.
+message BotInstanceServiceIdentifier {
+  // Type of service (e.g. database-tunnel, ssh-multiplexer).
+  string type = 1;
+
+  // Name of the service, either given by the user or auto-generated.
+  string name = 2;
+}
+
+// BotInstanceHealthStatus describes the healthiness of a `tbot` service.
+enum BotInstanceHealthStatus {
+  // The enum zero-value, it means no status was included.
+  BOT_INSTANCE_HEALTH_STATUS_UNSPECIFIED = 0;
+
+  // Means the service is still "starting up" and hasn't reported its status.
+  BOT_INSTANCE_HEALTH_STATUS_INITIALIZING = 1;
+
+  // Means the service is healthy and ready to serve traffic, or it has
+  // recently succeeded in generating an output.
+  BOT_INSTANCE_HEALTH_STATUS_HEALTHY = 2;
+
+  // Means the service is failing to serve traffic or generate output.
+  BOT_INSTANCE_HEALTH_STATUS_UNHEALTHY = 3;
 }

api/proto/teleport/machineid/v1/bot_instance_service.proto

@@ -71,6 +71,9 @@ message DeleteBotInstanceRequest {
 message SubmitHeartbeatRequest {
   // The heartbeat data to submit.
   BotInstanceStatusHeartbeat heartbeat = 1;
+
+  // The health of the services/output `tbot` is running.
+  repeated BotInstanceServiceHealth service_health = 2;
 }
 
 // The response for SubmitHeartbeat.

docs/pages/reference/machine-id/diagnostics-service.mdx

@@ -93,8 +93,8 @@ Content-Type: application/json
 }
 ```
 
-By default, `tbot` generates service names based on their configuration such as
-the output destination. You can override this by providing your own name in the
+By default, `tbot` generates service names based on their type (e.g.
+`application-output-1`). You can override this by providing your own name in the
 `tbot` configuration file.
 
 ```yaml

lib/auth/machineid/machineidv1/bot_instance_service.go

@@ -21,6 +21,8 @@ package machineidv1
 import (
 	"context"
 	"log/slog"
+	"os"
+	"strconv"
 	"time"
 
 	"github.com/gravitational/trace"
@@ -46,6 +48,12 @@ const (
 	// ensure the instance remains accessible until shortly after the last
 	// issued certificate expires.
 	ExpiryMargin = time.Minute * 5
+
+	// serviceNameLimit is the maximum length in bytes of a bot service name.
+	serviceNameLimit = 64
+
+	// statusReasonLimit is the maximum length in bytes of a service status reason.
+	statusReasonLimit = 256
 )
 
 // BotInstancesCache is the subset of the cached resources that the Service queries.
@@ -178,6 +186,17 @@ func (b *BotInstanceService) SubmitHeartbeat(ctx context.Context, req *pb.Submit
 		return nil, trace.BadParameter("heartbeat: must be non-nil")
 	}
 
+	for _, svcHealth := range req.GetServiceHealth() {
+		name := svcHealth.GetService().GetName()
+		if len(name) > serviceNameLimit {
+			return nil, trace.BadParameter("service name %q is longer than %d bytes", name, serviceNameLimit)
+		}
+		reason := svcHealth.GetReason()
+		if len(reason) > statusReasonLimit {
+			return nil, trace.BadParameter("service %q has a status reason longer than %d bytes", name, statusReasonLimit)
+		}
+	}
+
 	// Enforce that the connecting client is a bot and has a bot instance ID.
 	botName := authCtx.Identity.GetIdentity().BotName
 	botInstanceID := authCtx.Identity.GetIdentity().BotInstanceID
@@ -212,6 +231,11 @@ func (b *BotInstanceService) SubmitHeartbeat(ctx context.Context, req *pb.Submit
 		// Append the new heartbeat to the end.
 		instance.Status.LatestHeartbeats = append(instance.Status.LatestHeartbeats, req.Heartbeat)
 
+		if storeHeartbeatExtras() {
+			// Overwrite the service health.
+			instance.Status.ServiceHealth = req.ServiceHealth
+		}
+
 		return instance, nil
 	})
 	if err != nil {
@@ -220,3 +244,15 @@ func (b *BotInstanceService) SubmitHeartbeat(ctx context.Context, req *pb.Submit
 
 	return &pb.SubmitHeartbeatResponse{}, nil
 }
+
+// storeHeartbeatExtras returns whether we should store "extra" data submitted
+// with tbot heartbeats, such as the service health. Defaults to true unless the
+// TELEPORT_DISABLE_TBOT_HEARTBEAT_EXTRAS environment variable is set to true on
+// the auth server.
+func storeHeartbeatExtras() bool {
+	disabled, err := strconv.ParseBool(os.Getenv("TELEPORT_DISABLE_TBOT_HEARTBEAT_EXTRAS"))
+	if err != nil {
+		return true
+	}
+	return !disabled
+}

lib/auth/machineid/machineidv1/bot_instance_service_test.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"slices"
 	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -257,6 +258,7 @@ func TestBotInstanceServiceSubmitHeartbeat(t *testing.T) {
 		createBotInstance bool
 		assertErr         assert.ErrorAssertionFunc
 		wantHeartbeat     bool
+		wantServiceHealth []*machineidv1.BotInstanceServiceHealth
 	}{
 		{
 			name:              "success",
@@ -265,10 +267,30 @@ func TestBotInstanceServiceSubmitHeartbeat(t *testing.T) {
 				Heartbeat: &machineidv1.BotInstanceStatusHeartbeat{
 					Hostname: "llama",
 				},
+				ServiceHealth: []*machineidv1.BotInstanceServiceHealth{
+					{
+						Service: &machineidv1.BotInstanceServiceIdentifier{
+							Type: "application-tunnel",
+							Name: "my-application-tunnel",
+						},
+						Status: machineidv1.BotInstanceHealthStatus_BOT_INSTANCE_HEALTH_STATUS_UNHEALTHY,
+						Reason: ptr("application is broken"),
+					},
+				},
 			},
 			identity:      goodIdentity,
 			assertErr:     assert.NoError,
 			wantHeartbeat: true,
+			wantServiceHealth: []*machineidv1.BotInstanceServiceHealth{
+				{
+					Service: &machineidv1.BotInstanceServiceIdentifier{
+						Type: "application-tunnel",
+						Name: "my-application-tunnel",
+					},
+					Status: machineidv1.BotInstanceHealthStatus_BOT_INSTANCE_HEALTH_STATUS_UNHEALTHY,
+					Reason: ptr("application is broken"),
+				},
+			},
 		},
 		{
 			name:              "missing bot name",
@@ -327,6 +349,54 @@ func TestBotInstanceServiceSubmitHeartbeat(t *testing.T) {
 			},
 			wantHeartbeat: false,
 		},
+		{
+			name:              "service name too long",
+			createBotInstance: true,
+			req: &machineidv1.SubmitHeartbeatRequest{
+				Heartbeat: &machineidv1.BotInstanceStatusHeartbeat{
+					Hostname: "llama",
+				},
+				ServiceHealth: []*machineidv1.BotInstanceServiceHealth{
+					{
+						Service: &machineidv1.BotInstanceServiceIdentifier{
+							Type: "application-tunnel",
+							Name: strings.Repeat("a", 100),
+						},
+						Status: machineidv1.BotInstanceHealthStatus_BOT_INSTANCE_HEALTH_STATUS_UNHEALTHY,
+						Reason: ptr("application is broken"),
+					},
+				},
+			},
+			identity: goodIdentity,
+			assertErr: func(t assert.TestingT, err error, i ...any) bool {
+				return assert.True(t, trace.IsBadParameter(err)) && assert.Contains(t, err.Error(), "is longer than 64 bytes")
+			},
+			wantHeartbeat: false,
+		},
+		{
+			name:              "status reason too long",
+			createBotInstance: true,
+			req: &machineidv1.SubmitHeartbeatRequest{
+				Heartbeat: &machineidv1.BotInstanceStatusHeartbeat{
+					Hostname: "llama",
+				},
+				ServiceHealth: []*machineidv1.BotInstanceServiceHealth{
+					{
+						Service: &machineidv1.BotInstanceServiceIdentifier{
+							Type: "application-tunnel",
+							Name: "my-application-tunnel",
+						},
+						Status: machineidv1.BotInstanceHealthStatus_BOT_INSTANCE_HEALTH_STATUS_UNHEALTHY,
+						Reason: ptr(strings.Repeat("a", 300)),
+					},
+				},
+			},
+			identity: goodIdentity,
+			assertErr: func(t assert.TestingT, err error, i ...any) bool {
+				return assert.True(t, trace.IsBadParameter(err)) && assert.Contains(t, err.Error(), "status reason longer than 256 bytes")
+			},
+			wantHeartbeat: false,
+		},
 	}
 
 	for _, tt := range tests {
@@ -377,6 +447,13 @@ func TestBotInstanceServiceSubmitHeartbeat(t *testing.T) {
 					assert.Nil(t, bi.Status.InitialHeartbeat)
 					assert.Empty(t, bi.Status.LatestHeartbeats)
 				}
+				assert.Empty(t,
+					cmp.Diff(
+						bi.Status.ServiceHealth,
+						tt.wantServiceHealth,
+						protocmp.Transform(),
+					),
+				)
 			}
 		})
 	}
@@ -591,3 +668,5 @@ func newBotInstanceService(
 
 	return service
 }
+
+func ptr[T any](v T) *T { return &v }