diff --git a/docs/pages/reference/cli/tctl.mdx b/docs/pages/reference/cli/tctl.mdx index dd9002a0ae9ae..f8588ab351a81 100644 --- a/docs/pages/reference/cli/tctl.mdx +++ b/docs/pages/reference/cli/tctl.mdx @@ -1356,7 +1356,7 @@ $ tctl rm saml/okta # Delete a local user called "admin": $ tctl rm users/admin -# Delete a lock +# Delete a lock $ tctl rm lock/ed7038cb-a3cc-4f59-8063-09553665b773 ``` @@ -1850,12 +1850,18 @@ $ tctl tokens rm [] Reports diagnostic information. -The diagnostic metrics endpoint must be enabled with `teleport start --diag-addr=` for `tctl top` to work. +`tctl top` can consume metrics from a HTTP diagnostic endpoint. ```code $ tctl top [] [] ``` +When a specific endpoint is provided, `tctl top` will always attempt to connect to it. +The endpoint should be a valid HTTP URL, corresponding to a matching +diagnostic metrics service configuration such as `teleport start --diag-addr=`. + +When no endpoint is specified, `tctl top` will attempt to connect via the debug UNIX socket endpoint, falling back to localhost. + ### Argument - `[]` Diagnostic HTTP URL (HTTPS not supported) @@ -1867,6 +1873,8 @@ $ tctl top [] [] $ sudo teleport start --diag-addr=127.0.0.1:3000 # View stats with a refresh period of 5 seconds $ tctl top http://127.0.0.1:3000 5s +# Use configured defaults +$ tctl top ``` ## tctl users add @@ -2159,7 +2167,7 @@ the two user filter flags imply that if a user - was provisioned into Teleport by Okta (from `--user-origin okta`), OR - has the label values `role=aws-admin` AND `dept=engineering` (from `--user-label "role=aws-admin,dept=engineering"`) -then they will be provisioned into AWS Identity Center by Teleport. +then they will be provisioned into AWS Identity Center by Teleport. ## tctl plugins install okta @@ -2176,7 +2184,7 @@ Install the Okta integration. | `--api-token` | none | string | Optional. Okta API token for the plugin to use. | | `--[no-]scim` | `--no-scim` | boolean | Optional. Enable SCIM Okta integration. | | `--[no-]users-sync` | `--users-sync` | none | Optional. Enable user synchronization. | -| `-o`, `--owner` | none | string | Optional. Add a default owner for synced Access Lists. | +| `-o`, `--owner` | none | string | Optional. Add a default owner for synced Access Lists. | | `--[no-]accesslist-sync` | `--accesslist-sync` | none | Optional. Enable or disable group to Access List synchronization. | | `--[no-]appgroup-sync` | `--appgroup-sync` | none | Optional. Enable or disable Okta Applications and Groups sync. | | `-g`, `--group-filter` | none | string | Optional. Add a group filter. Supports globbing by default. Enclose in `^pattern$` for full regex support. | diff --git a/lib/autoupdate/agent/updater.go b/lib/autoupdate/agent/updater.go index 3f4e967e6d6f2..cb609fa7136d0 100644 --- a/lib/autoupdate/agent/updater.go +++ b/lib/autoupdate/agent/updater.go @@ -66,9 +66,7 @@ const ( // reservedFreeDisk is the minimum required free space left on disk during downloads. // TODO(sclevine): This value is arbitrary and could be replaced by, e.g., min(1%, 200mb) in the future // to account for a range of disk sizes. - reservedFreeDisk = 10_000_000 - // debugSocketFileName is the name of Teleport's debug socket in the data dir. - debugSocketFileName = "debug.sock" // 10 MB + reservedFreeDisk = 10_000_000 // 10 MB // requiredUmask must be set before this package can be used. // Use syscall.Umask to set when no other goroutines are running. requiredUmask = 0o022 @@ -125,7 +123,7 @@ func NewLocalUpdater(cfg LocalUpdaterConfig, ns *Namespace) (*Updater, error) { cfg.SystemDir = packageSystemDir } validator := Validator{Log: cfg.Log} - debugClient := debug.NewClient(filepath.Join(ns.dataDir, debugSocketFileName)) + debugClient := debug.NewClient(ns.dataDir) return &Updater{ Log: cfg.Log, Pool: certPool, diff --git a/lib/client/debug/debug.go b/lib/client/debug/debug.go index 368a4eadc3ab1..a8b51a5fb2ef8 100644 --- a/lib/client/debug/debug.go +++ b/lib/client/debug/debug.go @@ -24,10 +24,14 @@ import ( "net" "net/http" "net/url" + "path/filepath" "strconv" "github.com/gravitational/trace" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + "github.com/gravitational/teleport" apidefaults "github.com/gravitational/teleport/api/defaults" ) @@ -47,11 +51,13 @@ var SupportedProfiles = map[string]struct{}{ // Client represents the debug service client. type Client struct { - clt *http.Client + clt *http.Client + socketPath string } // NewClient generates a new debug service client. -func NewClient(socketPath string) *Client { +func NewClient(dataDir string) *Client { + socketPath := filepath.Join(dataDir, teleport.DebugServiceSocketName) return &Client{ clt: &http.Client{ Timeout: apidefaults.DefaultIOTimeout, @@ -66,9 +72,15 @@ func NewClient(socketPath string) *Client { return trace.Errorf("redirect via socket not allowed") }, }, + socketPath: socketPath, } } +// SocketPath returns the absolute path to the UNIX socket that the debug service is exposed on. +func (c *Client) SocketPath() string { + return c.socketPath +} + // SetLogLevel changes the application's log level and a change status message. func (c *Client) SetLogLevel(ctx context.Context, level string) (string, error) { resp, err := c.do(ctx, http.MethodPut, url.URL{Path: "/log-level"}, []byte(level)) @@ -173,6 +185,22 @@ func (c *Client) GetReadiness(ctx context.Context) (Readiness, error) { return ready, nil } +// GetMetrics returns prometheus metrics as a map keyed by metric name. +func (c *Client) GetMetrics(ctx context.Context) (map[string]*dto.MetricFamily, error) { + resp, err := c.do(ctx, http.MethodGet, url.URL{Path: "/metrics"}, nil) + if err != nil { + return nil, trace.Wrap(err) + } + defer resp.Body.Close() + var parser expfmt.TextParser + metrics, err := parser.TextToMetricFamilies(resp.Body) + if err != nil { + return nil, trace.Wrap(err) + } + + return metrics, nil +} + func (c *Client) do(ctx context.Context, method string, u url.URL, body []byte) (*http.Response, error) { u.Scheme = "http" u.Host = "debug" diff --git a/lib/client/debug/debug_test.go b/lib/client/debug/debug_test.go index 52ced041ca0d6..c82b945a57fe9 100644 --- a/lib/client/debug/debug_test.go +++ b/lib/client/debug/debug_test.go @@ -206,7 +206,7 @@ func newSocketMockService(t *testing.T, status int, contents []byte) (string, fu }() t.Cleanup(func() { srv.Shutdown(context.Background()) }) - return socketPath, func() []string { + return socketDir, func() []string { srv.Shutdown(context.Background()) return requests } diff --git a/tool/tctl/common/top/client/diag/client.go b/tool/tctl/common/top/client/diag/client.go new file mode 100644 index 0000000000000..feea1e50fffc4 --- /dev/null +++ b/tool/tctl/common/top/client/diag/client.go @@ -0,0 +1,102 @@ +// Teleport +// Copyright (C) 2025 Gravitational, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package diag + +import ( + "context" + "net" + "net/http" + "net/url" + + "github.com/gravitational/trace" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + + "github.com/gravitational/teleport/lib/defaults" +) + +// Client is a wrapper around [*http.Client] that provides +// helpers for fetching metrics from the diagnostic endpoint of Teleport. +type Client struct { + endpoint string + clt *http.Client +} + +// parseAddress takes a string address and attempts to parse it into a valid URL. +// The input can either be a valid string URL or a : pair. +func parseAddress(addr string) (*url.URL, error) { + u, err := url.Parse(addr) + + if err != nil || u.Scheme == "" || u.Host == "" { + // Attempt to parse the input as a host:port tuple instead. + _, _, err = net.SplitHostPort(addr) + if err != nil { + return nil, trace.Errorf("address %s is neither a valid URL nor :", addr) + } + + u = &url.URL{ + Scheme: "http", + Host: addr, + } + } + + return u, nil +} + +// NewClient creates a new Client for a given address. +func NewClient(addr string) (*Client, error) { + clt, err := defaults.HTTPClient() + if err != nil { + return nil, trace.Wrap(err) + } + + u, err := parseAddress(addr) + if err != nil { + return nil, trace.Wrap(err) + } + + if u.Scheme != "http" { + return nil, trace.Errorf("unsupported scheme: %s, please provide a http address", u.Scheme) + } + + return &Client{ + endpoint: u.JoinPath("metrics").String(), + clt: clt, + }, nil +} + +// GetMetrics returns prometheus metrics as a map keyed by metric name. +func (c *Client) GetMetrics(ctx context.Context) (map[string]*dto.MetricFamily, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.endpoint, http.NoBody) + if err != nil { + return nil, trace.Wrap(err) + } + + resp, err := c.clt.Do(req) + if err != nil { + return nil, trace.Wrap(err) + } + defer resp.Body.Close() + + var parser expfmt.TextParser + metrics, err := parser.TextToMetricFamilies(resp.Body) + if err != nil { + return nil, trace.Wrap(err) + } + + return metrics, nil +} diff --git a/tool/tctl/common/top/client/diag/client_test.go b/tool/tctl/common/top/client/diag/client_test.go new file mode 100644 index 0000000000000..047e53b1c9f2e --- /dev/null +++ b/tool/tctl/common/top/client/diag/client_test.go @@ -0,0 +1,87 @@ +/* + * Teleport + * Copyright (C) 2025 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package diag + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestClientAddressParsing(t *testing.T) { + t.Parallel() + + testCases := []struct { + addr string + url *url.URL + }{ + { + addr: "http://127.0.0.1:3000", + url: &url.URL{ + Scheme: "http", + Host: "127.0.0.1:3000", + }, + }, + { + addr: "http://localhost:3000", + url: &url.URL{ + Scheme: "http", + Host: "localhost:3000", + }, + }, + { + addr: "localhost:3000", + url: &url.URL{ + Scheme: "http", + Host: "localhost:3000", + }, + }, + { + addr: "127.0.0.1:3000", + url: &url.URL{ + Scheme: "http", + Host: "127.0.0.1:3000", + }, + }, + { + addr: "badurl:300:9:1", + }, + { + addr: "http//badurl", + }, + { + addr: "/var/lib/file.sock", + }, + } + + for _, tc := range testCases { + t.Run(tc.addr, func(t *testing.T) { + u, err := parseAddress(tc.addr) + if tc.url == nil { + require.Error(t, err) + require.Nil(t, u) + } else { + require.NoError(t, err) + require.Equal(t, tc.url.Host, u.Host) + require.Equal(t, tc.url.Scheme, u.Scheme) + } + }) + } +} diff --git a/tool/tctl/common/top/command.go b/tool/tctl/common/top/command.go index b917ff5fb9fcb..14aeaf20597ea 100644 --- a/tool/tctl/common/top/command.go +++ b/tool/tctl/common/top/command.go @@ -22,12 +22,14 @@ import ( "github.com/alecthomas/kingpin/v2" tea "github.com/charmbracelet/bubbletea" - "github.com/gravitational/roundtrip" "github.com/gravitational/trace" + dto "github.com/prometheus/client_model/go" + "github.com/gravitational/teleport/lib/client/debug" "github.com/gravitational/teleport/lib/service/servicecfg" commonclient "github.com/gravitational/teleport/tool/tctl/common/client" tctlcfg "github.com/gravitational/teleport/tool/tctl/common/config" + "github.com/gravitational/teleport/tool/tctl/common/top/client/diag" ) // Command is a debug command that consumes the @@ -40,27 +42,70 @@ type Command struct { refreshPeriod time.Duration } +const defaultDiagAddr = "http://127.0.0.1:3000" + // Initialize sets up the "tctl top" command. func (c *Command) Initialize(app *kingpin.Application, _ *tctlcfg.GlobalCLIFlags, config *servicecfg.Config) { c.config = config c.top = app.Command("top", "Report diagnostic information.") - c.top.Arg("diag-addr", "Diagnostic HTTP URL").Default("http://127.0.0.1:3000").StringVar(&c.diagURL) + c.top.Arg("diag-addr", "Diagnostic HTTP URL").StringVar(&c.diagURL) c.top.Arg("refresh", "Refresh period").Default("5s").DurationVar(&c.refreshPeriod) } +type MetricsClient interface { + GetMetrics(ctx context.Context) (map[string]*dto.MetricFamily, error) +} + +func (c *Command) newMetricsClient(ctx context.Context) (string, MetricsClient, error) { + if c.diagURL != "" { + clt, err := diag.NewClient(c.diagURL) + return c.diagURL, clt, trace.Wrap(err) + } + + // Try the local UNIX debug service client first. + debugClient := debug.NewClient(c.config.DataDir) + _, debugErr := debugClient.GetMetrics(ctx) + if debugErr == nil { + return debugClient.SocketPath(), debugClient, nil + } + debugErr = trace.Wrap(debugErr, "retrieving metrics from debug service") + + // Try default diagnostic address + diagClient, defErr := diag.NewClient(defaultDiagAddr) + if defErr != nil { + return "", nil, trace.Wrap( + trace.NewAggregate( + trace.Wrap(defErr, "creating diagnostics client for default address %q", defaultDiagAddr), + debugErr), + "unable to connect to Teleport metrics server") + } + + _, defErr = diagClient.GetMetrics(ctx) + if defErr == nil { + return defaultDiagAddr, diagClient, nil + } + + return "", nil, trace.Wrap( + trace.NewAggregate( + trace.Wrap(defErr, "getting metrics from diagnostics client at default address %q", defaultDiagAddr), + debugErr, + ), + "connecting to Teleport metrics server") +} + // TryRun attempts to run subcommands. func (c *Command) TryRun(ctx context.Context, cmd string, _ commonclient.InitFunc) (match bool, err error) { if cmd != c.top.FullCommand() { return false, nil } - diagClient, err := roundtrip.NewClient(c.diagURL, "") + addr, metricsClient, err := c.newMetricsClient(ctx) if err != nil { return true, trace.Wrap(err) } p := tea.NewProgram( - newTopModel(c.refreshPeriod, diagClient), + newTopModel(c.refreshPeriod, metricsClient, addr), tea.WithAltScreen(), tea.WithContext(ctx), ) diff --git a/tool/tctl/common/top/model.go b/tool/tctl/common/top/model.go index 4a2d06f5162b9..f6a443e94e5a0 100644 --- a/tool/tctl/common/top/model.go +++ b/tool/tctl/common/top/model.go @@ -30,7 +30,6 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" "github.com/dustin/go-humanize" - "github.com/gravitational/roundtrip" "github.com/gravitational/trace" "github.com/guptarohit/asciigraph" @@ -46,16 +45,18 @@ type topModel struct { selected int help help.Model refreshInterval time.Duration - clt *roundtrip.Client + clt MetricsClient report *Report reportError error + addr string } -func newTopModel(refreshInterval time.Duration, clt *roundtrip.Client) *topModel { +func newTopModel(refreshInterval time.Duration, clt MetricsClient, addr string) *topModel { return &topModel{ help: help.New(), clt: clt, refreshInterval: refreshInterval, + addr: addr, } } @@ -171,15 +172,16 @@ func (m *topModel) footerView() string { if m.reportError != nil { if trace.IsConnectionProblem(m.reportError) { - leftContent = fmt.Sprintf("Could not connect to metrics service: %v", m.clt.Endpoint()) + leftContent = fmt.Sprintf("Could not connect to metrics service: %v", m.addr) } else { leftContent = fmt.Sprintf("Failed to generate report: %v", m.reportError) } } if leftContent == "" && m.report != nil { - leftContent = fmt.Sprintf("Report generated at %s for host %s", + leftContent = fmt.Sprintf("Report generated at %s for host %s (%s)", m.report.Timestamp.Format(constants.HumanDateFormatSeconds), m.report.Hostname, + m.addr, ) } left := lipgloss.NewStyle(). diff --git a/tool/tctl/common/top/report.go b/tool/tctl/common/top/report.go index ce7ef1f7850c4..f9af7a604de32 100644 --- a/tool/tctl/common/top/report.go +++ b/tool/tctl/common/top/report.go @@ -22,17 +22,14 @@ import ( "fmt" "iter" "math" - "net/url" "os" "slices" "strings" "time" - "github.com/gravitational/roundtrip" "github.com/gravitational/trace" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" - "github.com/prometheus/common/expfmt" "github.com/gravitational/teleport" "github.com/gravitational/teleport/api/types" @@ -377,14 +374,8 @@ type Bucket struct { UpperBound float64 } -func fetchAndGenerateReport(ctx context.Context, client *roundtrip.Client, prev *Report, period time.Duration) (*Report, error) { - re, err := client.Get(ctx, client.Endpoint("metrics"), url.Values{}) - if err != nil { - return nil, trace.Wrap(trace.ConvertSystemError(err)) - } - - var parser expfmt.TextParser - metrics, err := parser.TextToMetricFamilies(re.Reader()) +func fetchAndGenerateReport(ctx context.Context, client MetricsClient, prev *Report, period time.Duration) (*Report, error) { + metrics, err := client.GetMetrics(ctx) if err != nil { return nil, trace.Wrap(err) } diff --git a/tool/teleport/common/debug.go b/tool/teleport/common/debug.go index c993b78742f6a..2d1d8d59a11e9 100644 --- a/tool/teleport/common/debug.go +++ b/tool/teleport/common/debug.go @@ -24,14 +24,12 @@ import ( "errors" "fmt" "io" - "path/filepath" "slices" "strings" "time" "github.com/gravitational/trace" - "github.com/gravitational/teleport" debugclient "github.com/gravitational/teleport/lib/client/debug" "github.com/gravitational/teleport/lib/config" "github.com/gravitational/teleport/lib/defaults" @@ -46,18 +44,19 @@ type DebugClient interface { GetLogLevel(context.Context) (string, error) // CollectProfile collects a pprof profile. CollectProfile(context.Context, string, int) ([]byte, error) + SocketPath() string } func onSetLogLevel(configPath string, level string) error { ctx := context.Background() - clt, dataDir, socketPath, err := newDebugClient(configPath) + clt, dataDir, err := newDebugClient(configPath) if err != nil { return trace.Wrap(err) } setMessage, err := setLogLevel(ctx, clt, level) if err != nil { - return convertToReadableErr(err, dataDir, socketPath) + return convertToReadableErr(err, dataDir, clt.SocketPath()) } fmt.Println(setMessage) @@ -74,14 +73,14 @@ func setLogLevel(ctx context.Context, clt DebugClient, level string) (string, er func onGetLogLevel(configPath string) error { ctx := context.Background() - clt, dataDir, socketPath, err := newDebugClient(configPath) + clt, dataDir, err := newDebugClient(configPath) if err != nil { return trace.Wrap(err) } currentLogLevel, err := getLogLevel(ctx, clt) if err != nil { - return convertToReadableErr(err, dataDir, socketPath) + return convertToReadableErr(err, dataDir, clt.SocketPath()) } fmt.Printf("Current log level %q\n", currentLogLevel) @@ -111,14 +110,14 @@ func onCollectProfiles(configPath string, rawProfiles string, seconds int) error ctx, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() - clt, dataDir, socketPath, err := newDebugClient(configPath) + clt, dataDir, err := newDebugClient(configPath) if err != nil { return trace.Wrap(err) } var output bytes.Buffer if err := collectProfiles(ctx, clt, &output, rawProfiles, seconds); err != nil { - return convertToReadableErr(err, dataDir, socketPath) + return convertToReadableErr(err, dataDir, clt.SocketPath()) } fmt.Print(output.String()) @@ -169,10 +168,10 @@ func collectProfiles(ctx context.Context, clt DebugClient, buf io.Writer, rawPro // newDebugClient initializes the debug client based on the Teleport // configuration. It also returns the data dir and socket path used. -func newDebugClient(configPath string) (DebugClient, string, string, error) { +func newDebugClient(configPath string) (DebugClient, string, error) { cfg, err := config.ReadConfigFile(configPath) if err != nil { - return nil, "", "", trace.Wrap(err) + return nil, "", trace.Wrap(err) } // ReadConfigFile returns nil configuration if the file doesn't exists. @@ -183,8 +182,7 @@ func newDebugClient(configPath string) (DebugClient, string, string, error) { dataDir = cfg.DataDir } - socketPath := filepath.Join(dataDir, teleport.DebugServiceSocketName) - return debugclient.NewClient(socketPath), dataDir, socketPath, nil + return debugclient.NewClient(dataDir), dataDir, nil } // convertToReadableErr converts debug service client error into a more friendly