diff --git a/go.mod b/go.mod index bdc18c7e34788..77a2c5c4ba1bb 100644 --- a/go.mod +++ b/go.mod @@ -138,7 +138,7 @@ require ( github.com/google/uuid v1.6.0 github.com/googleapis/gax-go/v2 v2.14.1 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 - github.com/grafana/pyroscope-go v1.2.2 + github.com/grafana/pyroscope-go v1.2.3-0.20250624101038-0410c20e9693 github.com/gravitational/license v0.0.0-20250329001817-070456fa8ec1 github.com/gravitational/roundtrip v1.0.2 github.com/gravitational/teleport/api v0.0.0 diff --git a/go.sum b/go.sum index 07fc2fdc8f2db..0a73f003bbee4 100644 --- a/go.sum +++ b/go.sum @@ -1534,8 +1534,8 @@ github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5T github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= -github.com/grafana/pyroscope-go v1.2.2 h1:uvKCyZMD724RkaCEMrSTC38Yn7AnFe8S2wiAIYdDPCE= -github.com/grafana/pyroscope-go v1.2.2/go.mod h1:zzT9QXQAp2Iz2ZdS216UiV8y9uXJYQiGE1q8v1FyhqU= +github.com/grafana/pyroscope-go v1.2.3-0.20250624101038-0410c20e9693 h1:oAnuX7YdawNRJN9jAJqs4uWfv11R+9sgBBMalTp2Xz0= +github.com/grafana/pyroscope-go v1.2.3-0.20250624101038-0410c20e9693/go.mod h1:zzT9QXQAp2Iz2ZdS216UiV8y9uXJYQiGE1q8v1FyhqU= github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/gravitational/go-cassandra-native-protocol v0.0.0-teleport.1 h1:zGsdDzqXSuXI+1t+2TRRzdYiv+B3M4IgOPA8W/raFOA= diff --git a/integrations/terraform-mwi/go.mod b/integrations/terraform-mwi/go.mod index e4e6350e7b151..38f2c4fa23661 100644 --- a/integrations/terraform-mwi/go.mod +++ b/integrations/terraform-mwi/go.mod @@ -259,7 +259,7 @@ require ( github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/gosuri/uitable v0.0.4 // indirect - github.com/grafana/pyroscope-go v1.2.2 // indirect + github.com/grafana/pyroscope-go v1.2.3-0.20250624101038-0410c20e9693 // indirect github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect github.com/gravitational/license v0.0.0-20250329001817-070456fa8ec1 // indirect github.com/gravitational/roundtrip v1.0.2 // indirect diff --git a/integrations/terraform-mwi/go.sum b/integrations/terraform-mwi/go.sum index 08604ae674738..7891c2c2046c9 100644 --- a/integrations/terraform-mwi/go.sum +++ b/integrations/terraform-mwi/go.sum @@ -687,8 +687,8 @@ github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5T github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= -github.com/grafana/pyroscope-go v1.2.2 h1:uvKCyZMD724RkaCEMrSTC38Yn7AnFe8S2wiAIYdDPCE= -github.com/grafana/pyroscope-go v1.2.2/go.mod h1:zzT9QXQAp2Iz2ZdS216UiV8y9uXJYQiGE1q8v1FyhqU= +github.com/grafana/pyroscope-go v1.2.3-0.20250624101038-0410c20e9693 h1:oAnuX7YdawNRJN9jAJqs4uWfv11R+9sgBBMalTp2Xz0= +github.com/grafana/pyroscope-go v1.2.3-0.20250624101038-0410c20e9693/go.mod h1:zzT9QXQAp2Iz2ZdS216UiV8y9uXJYQiGE1q8v1FyhqU= github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/gravitational/go-cassandra-native-protocol v0.0.0-teleport.1 h1:zGsdDzqXSuXI+1t+2TRRzdYiv+B3M4IgOPA8W/raFOA= diff --git a/integrations/terraform/go.mod b/integrations/terraform/go.mod index dc8ceaa8e3897..1e04609713b13 100644 --- a/integrations/terraform/go.mod +++ b/integrations/terraform/go.mod @@ -260,7 +260,7 @@ require ( github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/gosuri/uitable v0.0.4 // indirect - github.com/grafana/pyroscope-go v1.2.2 // indirect + github.com/grafana/pyroscope-go v1.2.3-0.20250624101038-0410c20e9693 // indirect github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect github.com/gravitational/license v0.0.0-20250329001817-070456fa8ec1 // indirect github.com/gravitational/roundtrip v1.0.2 // indirect diff --git a/integrations/terraform/go.sum b/integrations/terraform/go.sum index c5c4565dbb14c..c181852f7ddff 100644 --- a/integrations/terraform/go.sum +++ b/integrations/terraform/go.sum @@ -785,8 +785,8 @@ github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5T github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= -github.com/grafana/pyroscope-go v1.2.2 h1:uvKCyZMD724RkaCEMrSTC38Yn7AnFe8S2wiAIYdDPCE= -github.com/grafana/pyroscope-go v1.2.2/go.mod h1:zzT9QXQAp2Iz2ZdS216UiV8y9uXJYQiGE1q8v1FyhqU= +github.com/grafana/pyroscope-go v1.2.3-0.20250624101038-0410c20e9693 h1:oAnuX7YdawNRJN9jAJqs4uWfv11R+9sgBBMalTp2Xz0= +github.com/grafana/pyroscope-go v1.2.3-0.20250624101038-0410c20e9693/go.mod h1:zzT9QXQAp2Iz2ZdS216UiV8y9uXJYQiGE1q8v1FyhqU= github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/gravitational/go-cassandra-native-protocol v0.0.0-teleport.1 h1:zGsdDzqXSuXI+1t+2TRRzdYiv+B3M4IgOPA8W/raFOA= diff --git a/lib/service/pyroscope.go b/lib/service/pyroscope.go index e3393b4b428e6..c5ec7b8a90ef8 100644 --- a/lib/service/pyroscope.go +++ b/lib/service/pyroscope.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "log/slog" + "net/http" "os" "time" @@ -27,6 +28,7 @@ import ( "github.com/gravitational/trace" "github.com/gravitational/teleport" + logutils "github.com/gravitational/teleport/lib/utils/log" ) // TODO: Replace logger when pyroscope uses slog @@ -34,7 +36,25 @@ type pyroscopeLogger struct { l *slog.Logger } -func (l pyroscopeLogger) Infof(format string, args ...interface{}) { +type roundTripper struct { + tripper http.RoundTripper + timeout time.Duration + logger *slog.Logger +} + +// CloseIdleConnections ensures idle connections of the wrapped +// [http.RoundTripper] are closed. +func (rt roundTripper) CloseIdleConnections() { + type closeIdler interface { + CloseIdleConnections() + } + + if tr, ok := rt.tripper.(closeIdler); ok { + tr.CloseIdleConnections() + } +} + +func (l pyroscopeLogger) Infof(format string, args ...any) { if !l.l.Handler().Enabled(context.Background(), slog.LevelInfo) { return } @@ -42,7 +62,7 @@ func (l pyroscopeLogger) Infof(format string, args ...interface{}) { l.l.Info(fmt.Sprintf(format, args...)) } -func (l pyroscopeLogger) Debugf(format string, args ...interface{}) { +func (l pyroscopeLogger) Debugf(format string, args ...any) { if !l.l.Handler().Enabled(context.Background(), slog.LevelDebug) { return } @@ -51,7 +71,7 @@ func (l pyroscopeLogger) Debugf(format string, args ...interface{}) { l.l.Debug(fmt.Sprintf(format, args...)) } -func (l pyroscopeLogger) Errorf(format string, args ...interface{}) { +func (l pyroscopeLogger) Errorf(format string, args ...any) { if !l.l.Handler().Enabled(context.Background(), slog.LevelError) { return } @@ -60,6 +80,19 @@ func (l pyroscopeLogger) Errorf(format string, args ...interface{}) { l.l.Error(fmt.Sprintf(format, args...)) } +func (rt roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + start := time.Now() + resp, err := rt.tripper.RoundTrip(req) + duration := time.Since(start) + + threshold := rt.timeout * 90 / 100 + if duration > threshold { + rt.logger.DebugContext(req.Context(), "Pyroscope upload exceeded threshold", "upload_duration", duration, "upload_threshold", threshold, "upload_url", logutils.StringerAttr(req.URL)) + } + + return resp, err +} + // createPyroscopeConfig generates the Pyroscope configuration for the Teleport process. func createPyroscopeConfig(ctx context.Context, logger *slog.Logger, address string) (pyroscope.Config, error) { if address == "" { @@ -71,6 +104,17 @@ func createPyroscopeConfig(ctx context.Context, logger *slog.Logger, address str hostname = "unknown" } + const httpTimeout = 60 * time.Second + + httpClient := &http.Client{ + Timeout: httpTimeout, + Transport: roundTripper{ + tripper: http.DefaultTransport, + timeout: httpTimeout, + logger: logger, + }, + } + config := pyroscope.Config{ ApplicationName: teleport.ComponentTeleport, ServerAddress: address, @@ -80,6 +124,8 @@ func createPyroscopeConfig(ctx context.Context, logger *slog.Logger, address str "version": teleport.Version, "git_ref": teleport.Gitref, }, + HTTPClient: httpClient, + UploadRate: 60 * time.Second, } // Evaluate if profile configuration is customized @@ -90,21 +136,14 @@ func createPyroscopeConfig(ctx context.Context, logger *slog.Logger, address str logger.InfoContext(ctx, "Pyroscope will configure profiles from env") } - var uploadRate *time.Duration if rate := os.Getenv("TELEPORT_PYROSCOPE_UPLOAD_RATE"); rate != "" { parsedRate, err := time.ParseDuration(rate) if err != nil { logger.InfoContext(ctx, "invalid TELEPORT_PYROSCOPE_UPLOAD_RATE, ignoring value", "provided_value", rate, "error", err) } else { - uploadRate = &parsedRate + logger.InfoContext(ctx, "TELEPORT_PYROSCOPE_UPLOAD_RATE configured", "rate", parsedRate) + config.UploadRate = parsedRate } - } else { - logger.InfoContext(ctx, "TELEPORT_PYROSCOPE_UPLOAD_RATE not specified, using default") - } - - // Set UploadRate or fall back to defaults - if uploadRate != nil { - config.UploadRate = *uploadRate } if value, isSet := os.LookupEnv("TELEPORT_PYROSCOPE_KUBE_COMPONENT"); isSet { @@ -131,13 +170,13 @@ func (process *TeleportProcess) initPyroscope(address string) { if err != nil { logger.ErrorContext(process.ExitContext(), "error starting pyroscope profiler", "address", address, "error", err) } else { + logger.InfoContext(process.ExitContext(), "Pyroscope has successfully started") process.OnExit("pyroscope.profiler", func(payload any) { // Observed rare and inconsistent panics, short term solution is to not wait for flush profiler.Flush(false) _ = profiler.Stop() }) } - logger.InfoContext(process.ExitContext(), "Pyroscope has successfully started") } // getPyroscopeProfileTypesFromEnv sets the profile types based on environment variables.