Skip to content

[v18] Pyroscope client timeout and default upload rate #56730

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
evanfreed merged 2 commits into from
Jul 14, 2025
Merged

[v18] Pyroscope client timeout and default upload rate #56730

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d58d546
Pyroscope client timeout and default upload rate (#56584)
evanfreed Jul 11, 2025
258b6e7
Merge branch 'branch/v18' into evan/backport-56584-branch/v18
evanfreed Jul 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ require (
github.com/google/uuid v1.6.0
github.com/googleapis/gax-go/v2 v2.14.1
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674
github.com/grafana/pyroscope-go v1.2.2
github.com/grafana/pyroscope-go v1.2.3-0.20250624101038-0410c20e9693
github.com/gravitational/license v0.0.0-20250329001817-070456fa8ec1
github.com/gravitational/roundtrip v1.0.2
github.com/gravitational/teleport/api v0.0.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1534,8 +1534,8 @@ github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5T
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY=
github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo=
github.com/grafana/pyroscope-go v1.2.2 h1:uvKCyZMD724RkaCEMrSTC38Yn7AnFe8S2wiAIYdDPCE=
github.com/grafana/pyroscope-go v1.2.2/go.mod h1:zzT9QXQAp2Iz2ZdS216UiV8y9uXJYQiGE1q8v1FyhqU=
github.com/grafana/pyroscope-go v1.2.3-0.20250624101038-0410c20e9693 h1:oAnuX7YdawNRJN9jAJqs4uWfv11R+9sgBBMalTp2Xz0=
github.com/grafana/pyroscope-go v1.2.3-0.20250624101038-0410c20e9693/go.mod h1:zzT9QXQAp2Iz2ZdS216UiV8y9uXJYQiGE1q8v1FyhqU=
github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg=
github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU=
github.com/gravitational/go-cassandra-native-protocol v0.0.0-teleport.1 h1:zGsdDzqXSuXI+1t+2TRRzdYiv+B3M4IgOPA8W/raFOA=
Expand Down
2 changes: 1 addition & 1 deletion integrations/terraform-mwi/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,7 @@ require (
github.com/gorilla/securecookie v1.1.2 // indirect
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
github.com/gosuri/uitable v0.0.4 // indirect
github.com/grafana/pyroscope-go v1.2.2 // indirect
github.com/grafana/pyroscope-go v1.2.3-0.20250624101038-0410c20e9693 // indirect
github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect
github.com/gravitational/license v0.0.0-20250329001817-070456fa8ec1 // indirect
github.com/gravitational/roundtrip v1.0.2 // indirect
Expand Down
4 changes: 2 additions & 2 deletions integrations/terraform-mwi/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -687,8 +687,8 @@ github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5T
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY=
github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo=
github.com/grafana/pyroscope-go v1.2.2 h1:uvKCyZMD724RkaCEMrSTC38Yn7AnFe8S2wiAIYdDPCE=
github.com/grafana/pyroscope-go v1.2.2/go.mod h1:zzT9QXQAp2Iz2ZdS216UiV8y9uXJYQiGE1q8v1FyhqU=
github.com/grafana/pyroscope-go v1.2.3-0.20250624101038-0410c20e9693 h1:oAnuX7YdawNRJN9jAJqs4uWfv11R+9sgBBMalTp2Xz0=
github.com/grafana/pyroscope-go v1.2.3-0.20250624101038-0410c20e9693/go.mod h1:zzT9QXQAp2Iz2ZdS216UiV8y9uXJYQiGE1q8v1FyhqU=
github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg=
github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU=
github.com/gravitational/go-cassandra-native-protocol v0.0.0-teleport.1 h1:zGsdDzqXSuXI+1t+2TRRzdYiv+B3M4IgOPA8W/raFOA=
Expand Down
2 changes: 1 addition & 1 deletion integrations/terraform/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ require (
github.com/gorilla/securecookie v1.1.2 // indirect
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
github.com/gosuri/uitable v0.0.4 // indirect
github.com/grafana/pyroscope-go v1.2.2 // indirect
github.com/grafana/pyroscope-go v1.2.3-0.20250624101038-0410c20e9693 // indirect
github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect
github.com/gravitational/license v0.0.0-20250329001817-070456fa8ec1 // indirect
github.com/gravitational/roundtrip v1.0.2 // indirect
Expand Down
4 changes: 2 additions & 2 deletions integrations/terraform/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -785,8 +785,8 @@ github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5T
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY=
github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo=
github.com/grafana/pyroscope-go v1.2.2 h1:uvKCyZMD724RkaCEMrSTC38Yn7AnFe8S2wiAIYdDPCE=
github.com/grafana/pyroscope-go v1.2.2/go.mod h1:zzT9QXQAp2Iz2ZdS216UiV8y9uXJYQiGE1q8v1FyhqU=
github.com/grafana/pyroscope-go v1.2.3-0.20250624101038-0410c20e9693 h1:oAnuX7YdawNRJN9jAJqs4uWfv11R+9sgBBMalTp2Xz0=
github.com/grafana/pyroscope-go v1.2.3-0.20250624101038-0410c20e9693/go.mod h1:zzT9QXQAp2Iz2ZdS216UiV8y9uXJYQiGE1q8v1FyhqU=
github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg=
github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU=
github.com/gravitational/go-cassandra-native-protocol v0.0.0-teleport.1 h1:zGsdDzqXSuXI+1t+2TRRzdYiv+B3M4IgOPA8W/raFOA=
Expand Down
65 changes: 52 additions & 13 deletions lib/service/pyroscope.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,29 +20,49 @@ import (
"context"
"fmt"
"log/slog"
"net/http"
"os"
"time"

"github.com/grafana/pyroscope-go"
"github.com/gravitational/trace"

"github.com/gravitational/teleport"
logutils "github.com/gravitational/teleport/lib/utils/log"
)

// TODO: Replace logger when pyroscope uses slog
type pyroscopeLogger struct {
l *slog.Logger
}

func (l pyroscopeLogger) Infof(format string, args ...interface{}) {
type roundTripper struct {
tripper http.RoundTripper
timeout time.Duration
logger *slog.Logger
}

// CloseIdleConnections ensures idle connections of the wrapped
// [http.RoundTripper] are closed.
func (rt roundTripper) CloseIdleConnections() {
type closeIdler interface {
CloseIdleConnections()
}

if tr, ok := rt.tripper.(closeIdler); ok {
tr.CloseIdleConnections()
}
}

func (l pyroscopeLogger) Infof(format string, args ...any) {
if !l.l.Handler().Enabled(context.Background(), slog.LevelInfo) {
return
}
//nolint:sloglint // msg cannot be constant
l.l.Info(fmt.Sprintf(format, args...))
}

func (l pyroscopeLogger) Debugf(format string, args ...interface{}) {
func (l pyroscopeLogger) Debugf(format string, args ...any) {
if !l.l.Handler().Enabled(context.Background(), slog.LevelDebug) {
return
}
Expand All @@ -51,7 +71,7 @@ func (l pyroscopeLogger) Debugf(format string, args ...interface{}) {
l.l.Debug(fmt.Sprintf(format, args...))
}

func (l pyroscopeLogger) Errorf(format string, args ...interface{}) {
func (l pyroscopeLogger) Errorf(format string, args ...any) {
if !l.l.Handler().Enabled(context.Background(), slog.LevelError) {
return
}
Expand All @@ -60,6 +80,19 @@ func (l pyroscopeLogger) Errorf(format string, args ...interface{}) {
l.l.Error(fmt.Sprintf(format, args...))
}

func (rt roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
start := time.Now()
resp, err := rt.tripper.RoundTrip(req)
duration := time.Since(start)

threshold := rt.timeout * 90 / 100
if duration > threshold {
rt.logger.DebugContext(req.Context(), "Pyroscope upload exceeded threshold", "upload_duration", duration, "upload_threshold", threshold, "upload_url", logutils.StringerAttr(req.URL))
}

return resp, err
}

// createPyroscopeConfig generates the Pyroscope configuration for the Teleport process.
func createPyroscopeConfig(ctx context.Context, logger *slog.Logger, address string) (pyroscope.Config, error) {
if address == "" {
Expand All @@ -71,6 +104,17 @@ func createPyroscopeConfig(ctx context.Context, logger *slog.Logger, address str
hostname = "unknown"
}

const httpTimeout = 60 * time.Second

httpClient := &http.Client{
Timeout: httpTimeout,
Transport: roundTripper{
tripper: http.DefaultTransport,
timeout: httpTimeout,
logger: logger,
},
}

config := pyroscope.Config{
ApplicationName: teleport.ComponentTeleport,
ServerAddress: address,
Expand All @@ -80,6 +124,8 @@ func createPyroscopeConfig(ctx context.Context, logger *slog.Logger, address str
"version": teleport.Version,
"git_ref": teleport.Gitref,
},
HTTPClient: httpClient,
UploadRate: 60 * time.Second,
}

// Evaluate if profile configuration is customized
Expand All @@ -90,21 +136,14 @@ func createPyroscopeConfig(ctx context.Context, logger *slog.Logger, address str
logger.InfoContext(ctx, "Pyroscope will configure profiles from env")
}

var uploadRate *time.Duration
if rate := os.Getenv("TELEPORT_PYROSCOPE_UPLOAD_RATE"); rate != "" {
parsedRate, err := time.ParseDuration(rate)
if err != nil {
logger.InfoContext(ctx, "invalid TELEPORT_PYROSCOPE_UPLOAD_RATE, ignoring value", "provided_value", rate, "error", err)
} else {
uploadRate = &parsedRate
logger.InfoContext(ctx, "TELEPORT_PYROSCOPE_UPLOAD_RATE configured", "rate", parsedRate)
config.UploadRate = parsedRate
}
} else {
logger.InfoContext(ctx, "TELEPORT_PYROSCOPE_UPLOAD_RATE not specified, using default")
}

// Set UploadRate or fall back to defaults
if uploadRate != nil {
config.UploadRate = *uploadRate
}

if value, isSet := os.LookupEnv("TELEPORT_PYROSCOPE_KUBE_COMPONENT"); isSet {
Expand All @@ -131,13 +170,13 @@ func (process *TeleportProcess) initPyroscope(address string) {
if err != nil {
logger.ErrorContext(process.ExitContext(), "error starting pyroscope profiler", "address", address, "error", err)
} else {
logger.InfoContext(process.ExitContext(), "Pyroscope has successfully started")
process.OnExit("pyroscope.profiler", func(payload any) {
// Observed rare and inconsistent panics, short term solution is to not wait for flush
profiler.Flush(false)
_ = profiler.Stop()
})
}
logger.InfoContext(process.ExitContext(), "Pyroscope has successfully started")
}

// getPyroscopeProfileTypesFromEnv sets the profile types based on environment variables.
Expand Down
Loading