From 2fbfb7bf3594cc0240c459401ff35a3b36ce40a3 Mon Sep 17 00:00:00 2001 From: Evan Freed Date: Tue, 21 Jan 2025 09:59:26 -0600 Subject: [PATCH 1/8] feat: add pyroscope sdk support Signed-off-by: Evan Freed --- go.mod | 2 + go.sum | 4 ++ integrations/terraform/go.sum | 4 ++ lib/service/pyroscope.go | 130 ++++++++++++++++++++++++++++++++++ lib/service/service.go | 5 ++ 5 files changed, 145 insertions(+) create mode 100644 lib/service/pyroscope.go diff --git a/go.mod b/go.mod index c4f647a1b9af8..375f4811fc5f6 100644 --- a/go.mod +++ b/go.mod @@ -131,6 +131,7 @@ require ( github.com/google/uuid v1.6.0 github.com/googleapis/gax-go/v2 v2.14.1 github.com/gorilla/websocket v1.5.3 + github.com/grafana/pyroscope-go v1.2.0 github.com/gravitational/license v0.0.0-20240313232707-8312e719d624 github.com/gravitational/roundtrip v1.0.2 github.com/gravitational/teleport/api v0.0.0 @@ -387,6 +388,7 @@ require ( github.com/gorilla/handlers v1.5.2 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/gosuri/uitable v0.0.4 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect diff --git a/go.sum b/go.sum index a0b782ccc5e73..9946a4c88a0c5 100644 --- a/go.sum +++ b/go.sum @@ -1549,6 +1549,10 @@ github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aN github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= +github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8= +github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/gravitational/go-cassandra-native-protocol v0.0.0-teleport.1 h1:zGsdDzqXSuXI+1t+2TRRzdYiv+B3M4IgOPA8W/raFOA= github.com/gravitational/go-cassandra-native-protocol v0.0.0-teleport.1/go.mod h1:6FzirJfdffakAVqmHjwVfFkpru/gNbIazUOK5rIhndc= github.com/gravitational/go-libfido2 v1.5.3-teleport.1 h1:nPfxiTH2Sr3J6zan280fbHOkWE7gRF/lMqvhcXKh2ek= diff --git a/integrations/terraform/go.sum b/integrations/terraform/go.sum index 9dcdeb6f28d8f..02be37417e39b 100644 --- a/integrations/terraform/go.sum +++ b/integrations/terraform/go.sum @@ -670,6 +670,10 @@ github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aN github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= +github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8= +github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/gravitational/go-cassandra-native-protocol v0.0.0-teleport.1 h1:zGsdDzqXSuXI+1t+2TRRzdYiv+B3M4IgOPA8W/raFOA= github.com/gravitational/go-cassandra-native-protocol v0.0.0-teleport.1/go.mod h1:6FzirJfdffakAVqmHjwVfFkpru/gNbIazUOK5rIhndc= github.com/gravitational/go-libfido2 v1.5.3-teleport.1 h1:nPfxiTH2Sr3J6zan280fbHOkWE7gRF/lMqvhcXKh2ek= diff --git a/lib/service/pyroscope.go b/lib/service/pyroscope.go new file mode 100644 index 0000000000000..15de9d7000b4c --- /dev/null +++ b/lib/service/pyroscope.go @@ -0,0 +1,130 @@ +/* + * Teleport + * Copyright (C) 2023 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package service + +import ( + "fmt" + "log/slog" + "os" + "time" + + "github.com/grafana/pyroscope-go" + + "github.com/gravitational/teleport" +) + +// TODO: Replace logger when pyroscope uses slog +type logger struct { + l *slog.Logger +} + +func (l logger) Infof(format string, args ...interface{}) { + //nolint:sloglint // msg cannot be constant + l.l.Info(fmt.Sprintf(format, args...)) +} + +func (l logger) Debugf(format string, args ...interface{}) { + //nolint:sloglint // msg cannot be constant + l.l.Debug(fmt.Sprintf(format, args...)) +} + +func (l logger) Errorf(format string, args ...interface{}) { + //nolint:sloglint // msg cannot be constant + l.l.Error(fmt.Sprintf(format, args...)) +} + +// initPyroscope instruments Teleport to run with continuous profiling for Pyroscope +func (process *TeleportProcess) initPyroscope() { + address := os.Getenv("TELEPORT_PYROSCOPE_SERVER_ADDRESS") + hostname, err := os.Hostname() + if err != nil { + hostname = "unknown" + } + + // Build pyroscope config + config := pyroscope.Config{ + ApplicationName: "teleport", + ServerAddress: address, + Logger: pyroscope.Logger(logger{l: slog.Default()}), + Tags: map[string]string{ + "host": hostname, + "version": teleport.Version, + "git_ref": teleport.Gitref, + }, + } + + // Evaluate if profile configuration is customized + if p := getPyroscopeProfileTypesFromEnv(); len(p) == 0 { + slog.InfoContext(process.ExitContext(), "No profile types enabled, using default") + } else { + config.ProfileTypes = p + } + + var uploadRate *time.Duration + if rate := os.Getenv("TELEPORT_PYROSCOPE_UPLOAD_RATE"); rate != "" { + parsedRate, err := time.ParseDuration(rate) + if err != nil { + slog.InfoContext(process.ExitContext(), "invalid TELEPORT_PYROSCOPE_UPLOAD_RATE, ignoring value", "provided_value", rate, "error", err) + } else { + uploadRate = &parsedRate + } + } else { + slog.InfoContext(process.ExitContext(), "TELEPORT_PYROSCOPE_UPLOAD_RATE not specified, using default") + } + + // Set UploadRate or fall back to defaults + if uploadRate != nil { + config.UploadRate = *uploadRate + } + + profiler, err := pyroscope.Start(config) + if err != nil { + slog.ErrorContext(process.ExitContext(), "error starting pyroscope profiler", "error", err) + } else { + process.OnExit("pyroscope.profiler", func(payload any) { + profiler.Flush(payload == nil) + _ = profiler.Stop() + }) + } + slog.InfoContext(process.ExitContext(), "Pyroscope has successfully started") +} + +// getPyroscopeProfileTypesFromEnv sets the profile types based on environment variables. +func getPyroscopeProfileTypesFromEnv() []pyroscope.ProfileType { + var profileTypes []pyroscope.ProfileType + + if os.Getenv("TELEPORT_PYROSCOPE_PROFILE_MEMORY_ENABLED") == "true" { + profileTypes = append(profileTypes, + pyroscope.ProfileAllocObjects, + pyroscope.ProfileAllocSpace, + pyroscope.ProfileInuseObjects, + pyroscope.ProfileInuseSpace, + ) + } + + if os.Getenv("TELEPORT_PYROSCOPE_PROFILE_CPU_ENABLED") == "true" { + profileTypes = append(profileTypes, pyroscope.ProfileCPU) + } + + if os.Getenv("TELEPORT_PYROSCOPE_PROFILE_GOROUTINES_ENABLED") == "true" { + profileTypes = append(profileTypes, pyroscope.ProfileGoroutines) + } + + return profileTypes +} diff --git a/lib/service/service.go b/lib/service/service.go index 8faebdc01a9fd..0d72ca8a6d920 100644 --- a/lib/service/service.go +++ b/lib/service/service.go @@ -1350,6 +1350,11 @@ func NewTeleport(cfg *servicecfg.Config) (*TeleportProcess, error) { } } + if address := os.Getenv("TELEPORT_PYROSCOPE_SERVER_ADDRESS"); address != "" { + process.initPyroscope() + serviceStarted = true + } + if cfg.DebugService.Enabled { if err := process.initDebugService(); err != nil { return nil, trace.Wrap(err) From 2ad96089fe46442c1e641baba63fd8fd3591dd9d Mon Sep 17 00:00:00 2001 From: Evan Freed <2314084+evanfreed@users.noreply.github.com> Date: Wed, 22 Jan 2025 12:47:06 -0600 Subject: [PATCH 2/8] Update lib/service/pyroscope.go Co-authored-by: rosstimothy <39066650+rosstimothy@users.noreply.github.com> --- lib/service/pyroscope.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/service/pyroscope.go b/lib/service/pyroscope.go index 15de9d7000b4c..398283b9ed143 100644 --- a/lib/service/pyroscope.go +++ b/lib/service/pyroscope.go @@ -59,7 +59,7 @@ func (process *TeleportProcess) initPyroscope() { // Build pyroscope config config := pyroscope.Config{ - ApplicationName: "teleport", + ApplicationName: teleport.ComponentTeleport, ServerAddress: address, Logger: pyroscope.Logger(logger{l: slog.Default()}), Tags: map[string]string{ From bde8c38f7403cbdd7d98da89f11232eed5417d9a Mon Sep 17 00:00:00 2001 From: Evan Freed <2314084+evanfreed@users.noreply.github.com> Date: Wed, 22 Jan 2025 12:47:20 -0600 Subject: [PATCH 3/8] Update lib/service/pyroscope.go Co-authored-by: rosstimothy <39066650+rosstimothy@users.noreply.github.com> --- lib/service/pyroscope.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/service/pyroscope.go b/lib/service/pyroscope.go index 398283b9ed143..2f726a64f9558 100644 --- a/lib/service/pyroscope.go +++ b/lib/service/pyroscope.go @@ -30,7 +30,7 @@ import ( ) // TODO: Replace logger when pyroscope uses slog -type logger struct { +type pyroscopeLogger struct { l *slog.Logger } From 20cded469636454bd457d13c493521a325422c10 Mon Sep 17 00:00:00 2001 From: Evan Freed <2314084+evanfreed@users.noreply.github.com> Date: Wed, 22 Jan 2025 12:47:29 -0600 Subject: [PATCH 4/8] Update lib/service/service.go Co-authored-by: rosstimothy <39066650+rosstimothy@users.noreply.github.com> --- lib/service/service.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/service/service.go b/lib/service/service.go index 0d72ca8a6d920..c0af648c9bed5 100644 --- a/lib/service/service.go +++ b/lib/service/service.go @@ -1351,8 +1351,7 @@ func NewTeleport(cfg *servicecfg.Config) (*TeleportProcess, error) { } if address := os.Getenv("TELEPORT_PYROSCOPE_SERVER_ADDRESS"); address != "" { - process.initPyroscope() - serviceStarted = true + process.initPyroscope(address) } if cfg.DebugService.Enabled { From 4711fae86ec675eeda07bb6c5400bdf58438c9d7 Mon Sep 17 00:00:00 2001 From: Evan Freed Date: Wed, 22 Jan 2025 12:50:00 -0600 Subject: [PATCH 5/8] review comments Signed-off-by: Evan Freed --- lib/service/pyroscope.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/service/pyroscope.go b/lib/service/pyroscope.go index 2f726a64f9558..e36f22406e74d 100644 --- a/lib/service/pyroscope.go +++ b/lib/service/pyroscope.go @@ -34,24 +34,23 @@ type pyroscopeLogger struct { l *slog.Logger } -func (l logger) Infof(format string, args ...interface{}) { +func (l pyroscopeLogger) Infof(format string, args ...interface{}) { //nolint:sloglint // msg cannot be constant l.l.Info(fmt.Sprintf(format, args...)) } -func (l logger) Debugf(format string, args ...interface{}) { +func (l pyroscopeLogger) Debugf(format string, args ...interface{}) { //nolint:sloglint // msg cannot be constant l.l.Debug(fmt.Sprintf(format, args...)) } -func (l logger) Errorf(format string, args ...interface{}) { +func (l pyroscopeLogger) Errorf(format string, args ...interface{}) { //nolint:sloglint // msg cannot be constant l.l.Error(fmt.Sprintf(format, args...)) } // initPyroscope instruments Teleport to run with continuous profiling for Pyroscope -func (process *TeleportProcess) initPyroscope() { - address := os.Getenv("TELEPORT_PYROSCOPE_SERVER_ADDRESS") +func (process *TeleportProcess) initPyroscope(address string) { hostname, err := os.Hostname() if err != nil { hostname = "unknown" @@ -61,7 +60,7 @@ func (process *TeleportProcess) initPyroscope() { config := pyroscope.Config{ ApplicationName: teleport.ComponentTeleport, ServerAddress: address, - Logger: pyroscope.Logger(logger{l: slog.Default()}), + Logger: pyroscope.Logger(pyroscopeLogger{l: slog.Default()}), Tags: map[string]string{ "host": hostname, "version": teleport.Version, From 25fa67fcd16b3ceeff35188d4b95a6db2598749e Mon Sep 17 00:00:00 2001 From: Evan Freed Date: Wed, 22 Jan 2025 12:52:25 -0600 Subject: [PATCH 6/8] license fix Signed-off-by: Evan Freed --- lib/service/pyroscope.go | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/lib/service/pyroscope.go b/lib/service/pyroscope.go index e36f22406e74d..ee4529a8569eb 100644 --- a/lib/service/pyroscope.go +++ b/lib/service/pyroscope.go @@ -1,20 +1,18 @@ -/* - * Teleport - * Copyright (C) 2023 Gravitational, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +// Teleport +// Copyright (C) 2025 Gravitational, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . package service From b2ee247c5f4567e92fcd282bee463af87c97eaef Mon Sep 17 00:00:00 2001 From: Evan Freed <2314084+evanfreed@users.noreply.github.com> Date: Thu, 23 Jan 2025 11:33:10 -0600 Subject: [PATCH 7/8] Update lib/service/pyroscope.go Co-authored-by: rosstimothy <39066650+rosstimothy@users.noreply.github.com> --- lib/service/pyroscope.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/service/pyroscope.go b/lib/service/pyroscope.go index ee4529a8569eb..b6a362d5a3182 100644 --- a/lib/service/pyroscope.go +++ b/lib/service/pyroscope.go @@ -49,6 +49,10 @@ func (l pyroscopeLogger) Errorf(format string, args ...interface{}) { // initPyroscope instruments Teleport to run with continuous profiling for Pyroscope func (process *TeleportProcess) initPyroscope(address string) { + if address == "" { + return + } + hostname, err := os.Hostname() if err != nil { hostname = "unknown" From 8926faf3d13a85f67e0bb55e6e2ccc26534e56b6 Mon Sep 17 00:00:00 2001 From: Evan Freed Date: Thu, 23 Jan 2025 12:51:42 -0600 Subject: [PATCH 8/8] fmt Signed-off-by: Evan Freed --- lib/service/pyroscope.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/service/pyroscope.go b/lib/service/pyroscope.go index b6a362d5a3182..34b289004a081 100644 --- a/lib/service/pyroscope.go +++ b/lib/service/pyroscope.go @@ -52,7 +52,7 @@ func (process *TeleportProcess) initPyroscope(address string) { if address == "" { return } - + hostname, err := os.Hostname() if err != nil { hostname = "unknown"