[v17] MWI: Add a useful /readyz endpoint to tbot

docs/pages/includes/machine-id/common-output-config.yaml

@@ -25,3 +25,8 @@ roles:
 # on the next invocation, but don't want long-lived workload certificates on-disk.
 credential_ttl: 30m
 renewal_interval: 15m
+
+# name optionally overrides the name of the service used in logs and the `/readyz`
+# endpoint. It must only contain letters, numbers, hyphens, underscores, and plus
+# symbols.
+name: my-service-name

docs/pages/reference/machine-id/configuration.mdx

@@ -366,6 +366,11 @@ renewal_interval: 15m
 # plugin is used to automatically refresh the credentials within a single
 # invocation of `kubectl`. Defaults to `false`.
 disable_exec_plugin: false
+
+# name optionally overrides the name of the service used in logs and the `/readyz`
+# endpoint. It must only contain letters, numbers, hyphens, underscores, and plus
+# symbols.
+name: my-service-name
 ```
 
 Each Kubernetes cluster matching a selector will result in a new context in the
@@ -456,6 +461,10 @@ audiences:
  - foo.example.com
 (!docs/pages/includes/machine-id/workload-identity-selector-config.yaml!)
 (!docs/pages/includes/machine-id/common-output-config.yaml!)
+# name optionally overrides the name of the service used in logs and the `/readyz`
+# endpoint. It must only contain letters, numbers, hyphens, underscores, and plus
+# symbols.
+name: my-service-name
 ```
 
 ### `workload-identity-aws-roles-anywhere`
@@ -523,6 +532,10 @@ artifact_name: my-credentials-file
 # defaults to `false`.
 overwrite_credential_file: false
 (!docs/pages/includes/machine-id/workload-identity-selector-config.yaml!)
+# name optionally overrides the name of the service used in logs and the `/readyz`
+# endpoint. It must only contain letters, numbers, hyphens, underscores, and plus
+# symbols.
+name: my-service-name
 ```
 
 ### `spiffe-svid`
@@ -780,6 +793,10 @@ svids:
           #
           # If unspecified, the GID is not checked.
           gid: 50
+# name optionally overrides the name of the service used in logs and the `/readyz`
+# endpoint. It must only contain letters, numbers, hyphens, underscores, and plus
+# symbols.
+name: my-service-name
 ```
 
 #### Envoy SDS
@@ -947,6 +964,10 @@ database: postgres
 # username is the name of the user on the specified database server to open a
 # tunnel for.
 username: postgres
+# name optionally overrides the name of the service used in logs and the `/readyz`
+# endpoint. It must only contain letters, numbers, hyphens, underscores, and plus
+# symbols.
+name: my-service-name
 ```
 
 The `database-tunnel` service will not start if `tbot` has been configured
@@ -978,6 +999,10 @@ listen: tcp://127.0.0.1:8084
 # app_name is the name of the application, as configured in Teleport, that
 # the service should open a tunnel to.
 app_name: my-application
+# name optionally overrides the name of the service used in logs and the `/readyz`
+# endpoint. It must only contain letters, numbers, hyphens, underscores, and plus
+# symbols.
+name: my-service-name
 ```
 
 The `application-tunnel` service will not start if `tbot` has been configured
@@ -1036,6 +1061,10 @@ proxy_command:
 #
 # If unspecified, proxy templates will not be used.
 proxy_templates_path: /etc/my-proxy-templates.yaml
+# name optionally overrides the name of the service used in logs and the `/readyz`
+# endpoint. It must only contain letters, numbers, hyphens, underscores, and plus
+# symbols.
+name: my-service-name
 ```
 
 Once configured, `tbot` will create the following artifacts in the specified

docs/pages/reference/machine-id/diagnostics-service.mdx

@@ -40,13 +40,65 @@ to determine if the `tbot` process is running and has not crashed or hung.
 If deploying to Kubernetes, we recommend this endpoint is used for your
 Liveness Probe.
 
-### `/readyz`
+### `/readyz` and `/readyz/{service}`
 
-The `/readyz` endpoint currently returns the same information as `/livez`.
+The `/readyz` endpoint returns the overall health of `tbot`, including all of
+its internal and user-defined services. If all services are healthy, it will
+respond with a 200 status code. If any service is unhealthy, it will respond
+with a 503 status code.
 
-In the future, this endpoint will be expanded to indicate whether the internal
-components of `tbot` have been able to generate certificates and are ready
-to serve requests.
+```code
+$ curl -v http://127.0.0.1:3001/readyz
+
+HTTP/1.1 503 Service Unavailable
+Content-Type: application/json
+
+{
+  "status": "unhealthy",
+  "services": {
+    "ca-rotation": {
+      "status": "healthy"
+    },
+    "heartbeat": {
+      "status": "healthy"
+    },
+    "identity": {
+      "status": "healthy"
+    },
+    "aws-roles-anywhere": {
+      "status": "unhealthy",
+      "reason": "access denied to perform action \"read\" on \"workload_identity\""
+    }
+  }
+}
+```
+
+If deploying to Kubernetes, we recommend this endpoint is used for your
+Readiness Probe.
+
+You can also use the `/readyz/{service}` endpoint to query the health of a
+specific service.
+
+```code
+$ curl -v http://127.0.0.1:3001/readyz/aws-roles-anywhere
+
+HTTP/1.1 200 OK
+Content-Type: application/json
+
+{
+  "status": "healthy"
+}
+```
+
+By default, `tbot` generates service names based on their configuration such as
+the output destination. You can override this by providing your own name in the
+`tbot` configuration file.
+
+```yaml
+services:
+  - type: identity
+    name: my-service-123
+```
 
 ### `/metrics`
 

lib/tbot/config/config.go

@@ -26,6 +26,7 @@ import (
 	"fmt"
 	"io"
 	"net/url"
+	"regexp"
 	"slices"
 	"strings"
 	"time"
@@ -65,6 +66,38 @@ var SupportedJoinMethods = []string{
 	string(types.JoinMethodTerraformCloud),
 }
 
+// ReservedServiceNames are the service names reserved for internal use.
+var ReservedServiceNames = []string{
+	"ca-rotation",
+	"crl-cache",
+	"heartbeat",
+	"identity",
+	"spiffe-trust-bundle-cache",
+}
+
+var reservedServiceNamesMap = func() map[string]struct{} {
+	m := make(map[string]struct{}, len(ReservedServiceNames))
+	for _, k := range ReservedServiceNames {
+		m[k] = struct{}{}
+	}
+	return m
+}()
+
+var serviceNameRegex = regexp.MustCompile(`\A[a-z\d_\-+]+\z`)
+
+func validateServiceName(name string) error {
+	if name == "" {
+		return nil
+	}
+	if _, ok := reservedServiceNamesMap[name]; ok {
+		return trace.BadParameter("service name %q is reserved for internal use", name)
+	}
+	if !serviceNameRegex.MatchString(name) {
+		return trace.BadParameter("invalid service name: %q, may only contain lowercase letters, numbers, hyphens, underscores, or plus symbols", name)
+	}
+	return nil
+}
+
 var log = logutils.NewPackageLogger(teleport.ComponentKey, teleport.ComponentTBot)
 
 // AzureOnboardingConfig holds configuration relevant to the "azure" join method.
@@ -288,13 +321,23 @@ func (conf *BotConfig) CheckAndSetDefaults() error {
 
 	// We've migrated Outputs to Services, so copy all Outputs to Services.
 	conf.Services = append(conf.Services, conf.Outputs...)
+	uniqueNames := make(map[string]struct{}, len(conf.Services))
 	for i, service := range conf.Services {
 		if err := service.CheckAndSetDefaults(); err != nil {
 			return trace.Wrap(err, "validating service[%d]", i)
 		}
 		if err := service.GetCredentialLifetime().Validate(conf.Oneshot); err != nil {
 			return trace.Wrap(err, "validating service[%d]", i)
 		}
+		if name := service.GetName(); name != "" {
+			if err := validateServiceName(name); err != nil {
+				return trace.Wrap(err, "validating service[%d]", i)
+			}
+			if _, seen := uniqueNames[name]; seen {
+				return trace.BadParameter("validating service[%d]: duplicate name: %q", i, name)
+			}
+			uniqueNames[name] = struct{}{}
+		}
 	}
 
 	destinationPaths := map[string]int{}
@@ -386,6 +429,10 @@ type ServiceConfig interface {
 	// RenewalInterval. It's used for validation purposes; services that do not
 	// support these options should return the zero value.
 	GetCredentialLifetime() CredentialLifetime
+
+	// GetName returns the user-given name of the service, used for validation
+	// purposes.
+	GetName() string
 }
 
 // ServiceConfigs assists polymorphic unmarshaling of a slice of ServiceConfigs.

lib/tbot/config/config_test.go

@@ -578,3 +578,59 @@ func TestBotConfig_Base64(t *testing.T) {
 		})
 	}
 }
+
+func TestBotConfig_NameValidation(t *testing.T) {
+	t.Parallel()
+
+	testCases := map[string]struct {
+		cfg *BotConfig
+		err string
+	}{
+		"duplicate names": {
+			cfg: &BotConfig{
+				Version: V2,
+				Services: ServiceConfigs{
+					&IdentityOutput{
+						Name:        "foo",
+						Destination: &DestinationMemory{},
+					},
+					&IdentityOutput{
+						Name:        "foo",
+						Destination: &DestinationMemory{},
+					},
+				},
+			},
+			err: `duplicate name: "foo`,
+		},
+		"reserved name": {
+			cfg: &BotConfig{
+				Version: V2,
+				Services: ServiceConfigs{
+					&IdentityOutput{
+						Name:        "identity",
+						Destination: &DestinationMemory{},
+					},
+				},
+			},
+			err: `service name "identity" is reserved for internal use`,
+		},
+		"invalid name": {
+			cfg: &BotConfig{
+				Version: V2,
+				Services: ServiceConfigs{
+					&IdentityOutput{
+						Name:        "hello, world!",
+						Destination: &DestinationMemory{},
+					},
+				},
+			},
+			err: `may only contain lowercase letters`,
+		},
+	}
+	for desc, tc := range testCases {
+		t.Run(desc, func(t *testing.T) {
+			t.Parallel()
+			require.ErrorContains(t, tc.cfg.CheckAndSetDefaults(), tc.err)
+		})
+	}
+}

lib/tbot/config/service_application.go

@@ -35,6 +35,8 @@ var (
 const ApplicationOutputType = "application"
 
 type ApplicationOutput struct {
+	// Name of the service for logs and the /readyz endpoint.
+	Name string `yaml:"name,omitempty"`
 	// Destination is where the credentials should be written to.
 	Destination bot.Destination `yaml:"destination"`
 	// Roles is the list of roles to request for the generated credentials.
@@ -68,6 +70,11 @@ func (o *ApplicationOutput) CheckAndSetDefaults() error {
 	return nil
 }
 
+// GetName returns the user-given name of the service, used for validation purposes.
+func (o *ApplicationOutput) GetName() string {
+	return o.Name
+}
+
 func (o *ApplicationOutput) GetDestination() bot.Destination {
 	return o.Destination
 }

lib/tbot/config/service_application_tunnel.go

@@ -35,6 +35,8 @@ const ApplicationTunnelServiceType = "application-tunnel"
 // ApplicationTunnelService opens an authenticated tunnel for Application
 // Access.
 type ApplicationTunnelService struct {
+	// Name of the service for logs and the /readyz endpoint.
+	Name string `yaml:"name,omitempty"`
 	// Listen is the address on which database tunnel should listen. Example:
 	// - "tcp://127.0.0.1:3306"
 	// - "tcp://0.0.0.0:3306
@@ -59,7 +61,12 @@ func (s *ApplicationTunnelService) Type() string {
 	return ApplicationTunnelServiceType
 }
 
-func (s *ApplicationTunnelService) MarshalYAML() (interface{}, error) {
+// GetName returns the user-given name of the service, used for validation purposes.
+func (o *ApplicationTunnelService) GetName() string {
+	return o.Name
+}
+
+func (s *ApplicationTunnelService) MarshalYAML() (any, error) {
 	type raw ApplicationTunnelService
 	return withTypeHeader((*raw)(s), ApplicationTunnelServiceType)
 }

lib/tbot/config/service_client_credential.go

@@ -47,11 +47,19 @@ var (
 // be modified. This output is currently part of an experiment and could be
 // removed in a future release.
 type UnstableClientCredentialOutput struct {
+	// Name of the service for logs and the /readyz endpoint.
+	Name string `yaml:"name,omitempty"`
+
 	mu     sync.Mutex
 	facade *identity.Facade
 	ready  chan struct{}
 }
 
+// GetName returns the user-given name of the service, used for validation purposes.
+func (o *UnstableClientCredentialOutput) GetName() string {
+	return o.Name
+}
+
 // Ready returns a channel which closes when the Output is ready to be used
 // as a client credential. Using this as a credential before Ready closes is
 // unsupported.

lib/tbot/config/service_database.go

@@ -76,6 +76,8 @@ var (
 // DatabaseOutput produces credentials which can be used to connect to a
 // database through teleport.
 type DatabaseOutput struct {
+	// Name of the service for logs and the /readyz endpoint.
+	Name string `yaml:"name,omitempty"`
 	// Destination is where the credentials should be written to.
 	Destination bot.Destination `yaml:"destination"`
 	// Roles is the list of roles to request for the generated credentials.
@@ -101,6 +103,11 @@ type DatabaseOutput struct {
 	CredentialLifetime CredentialLifetime `yaml:",inline"`
 }
 
+// GetName returns the user-given name of the service, used for validation purposes.
+func (o *DatabaseOutput) GetName() string {
+	return o.Name
+}
+
 func (o *DatabaseOutput) Init(ctx context.Context) error {
 	subDirs := []string{}
 	if o.Format == CockroachDatabaseFormat {