From 1b5f94be8ac551690710721c2686231c361ee784 Mon Sep 17 00:00:00 2001 From: Evan Freed <2314084+evanfreed@users.noreply.github.com> Date: Thu, 23 Jan 2025 13:37:23 -0600 Subject: [PATCH 1/2] Add pyroscope support (#51294) * feat: add pyroscope sdk support Signed-off-by: Evan Freed * Update lib/service/pyroscope.go Co-authored-by: rosstimothy <39066650+rosstimothy@users.noreply.github.com> * Update lib/service/pyroscope.go Co-authored-by: rosstimothy <39066650+rosstimothy@users.noreply.github.com> * Update lib/service/service.go Co-authored-by: rosstimothy <39066650+rosstimothy@users.noreply.github.com> * review comments Signed-off-by: Evan Freed * license fix Signed-off-by: Evan Freed * Update lib/service/pyroscope.go Co-authored-by: rosstimothy <39066650+rosstimothy@users.noreply.github.com> * fmt Signed-off-by: Evan Freed --------- Signed-off-by: Evan Freed Co-authored-by: rosstimothy <39066650+rosstimothy@users.noreply.github.com> --- go.mod | 2 + go.sum | 4 ++ integrations/terraform/go.sum | 4 ++ lib/service/pyroscope.go | 131 ++++++++++++++++++++++++++++++++++ lib/service/service.go | 4 ++ 5 files changed, 145 insertions(+) create mode 100644 lib/service/pyroscope.go diff --git a/go.mod b/go.mod index f1dc5feeb3f44..9ddd068ac1eb6 100644 --- a/go.mod +++ b/go.mod @@ -114,6 +114,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/gravitational/license v0.0.0-20231228155916-928ed9ac0335 github.com/gravitational/oxy v0.0.0-20221029012416-9fbf4c444680 + github.com/grafana/pyroscope-go v1.2.0 github.com/gravitational/roundtrip v1.0.2 github.com/gravitational/teleport/api v0.0.0 github.com/gravitational/trace v1.3.1 @@ -366,6 +367,7 @@ require ( github.com/gorilla/handlers v1.5.2 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/gosuri/uitable v0.0.4 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect diff --git a/go.sum b/go.sum index 6f7bb4c11a9bb..689fab230c3ad 100644 --- a/go.sum +++ b/go.sum @@ -831,6 +831,10 @@ github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= github.com/gravitational/go-cassandra-native-protocol v0.0.0-20221005103706-b9e66c056e90 h1:fPNJE2kaWC0Oy2YKxk1tbnqhKl3aTeXVAfjXzphJmI8= github.com/gravitational/go-cassandra-native-protocol v0.0.0-20221005103706-b9e66c056e90/go.mod h1:6FzirJfdffakAVqmHjwVfFkpru/gNbIazUOK5rIhndc= +github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8= +github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/gravitational/go-libfido2 v1.5.3-teleport.1 h1:nPfxiTH2Sr3J6zan280fbHOkWE7gRF/lMqvhcXKh2ek= github.com/gravitational/go-libfido2 v1.5.3-teleport.1/go.mod h1:92J9LtSBl0UyUWljElJpTbMMNhC6VeY8dshsu40qjjo= github.com/gravitational/go-mssqldb v0.11.1-0.20230331180905-0f76f1751cd3 h1:3JGTacvAeV5tIC4/9XsdLC2K5K7FWaXxIwpW4t+dGH0= diff --git a/integrations/terraform/go.sum b/integrations/terraform/go.sum index 9551cffd38418..140f46eada92e 100644 --- a/integrations/terraform/go.sum +++ b/integrations/terraform/go.sum @@ -643,6 +643,10 @@ github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= github.com/gravitational/go-cassandra-native-protocol v0.0.0-20221005103706-b9e66c056e90 h1:fPNJE2kaWC0Oy2YKxk1tbnqhKl3aTeXVAfjXzphJmI8= github.com/gravitational/go-cassandra-native-protocol v0.0.0-20221005103706-b9e66c056e90/go.mod h1:6FzirJfdffakAVqmHjwVfFkpru/gNbIazUOK5rIhndc= +github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8= +github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/gravitational/go-libfido2 v1.5.3-teleport.1 h1:nPfxiTH2Sr3J6zan280fbHOkWE7gRF/lMqvhcXKh2ek= github.com/gravitational/go-libfido2 v1.5.3-teleport.1/go.mod h1:92J9LtSBl0UyUWljElJpTbMMNhC6VeY8dshsu40qjjo= github.com/gravitational/go-mssqldb v0.11.1-0.20230331180905-0f76f1751cd3 h1:3JGTacvAeV5tIC4/9XsdLC2K5K7FWaXxIwpW4t+dGH0= diff --git a/lib/service/pyroscope.go b/lib/service/pyroscope.go new file mode 100644 index 0000000000000..34b289004a081 --- /dev/null +++ b/lib/service/pyroscope.go @@ -0,0 +1,131 @@ +// Teleport +// Copyright (C) 2025 Gravitational, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package service + +import ( + "fmt" + "log/slog" + "os" + "time" + + "github.com/grafana/pyroscope-go" + + "github.com/gravitational/teleport" +) + +// TODO: Replace logger when pyroscope uses slog +type pyroscopeLogger struct { + l *slog.Logger +} + +func (l pyroscopeLogger) Infof(format string, args ...interface{}) { + //nolint:sloglint // msg cannot be constant + l.l.Info(fmt.Sprintf(format, args...)) +} + +func (l pyroscopeLogger) Debugf(format string, args ...interface{}) { + //nolint:sloglint // msg cannot be constant + l.l.Debug(fmt.Sprintf(format, args...)) +} + +func (l pyroscopeLogger) Errorf(format string, args ...interface{}) { + //nolint:sloglint // msg cannot be constant + l.l.Error(fmt.Sprintf(format, args...)) +} + +// initPyroscope instruments Teleport to run with continuous profiling for Pyroscope +func (process *TeleportProcess) initPyroscope(address string) { + if address == "" { + return + } + + hostname, err := os.Hostname() + if err != nil { + hostname = "unknown" + } + + // Build pyroscope config + config := pyroscope.Config{ + ApplicationName: teleport.ComponentTeleport, + ServerAddress: address, + Logger: pyroscope.Logger(pyroscopeLogger{l: slog.Default()}), + Tags: map[string]string{ + "host": hostname, + "version": teleport.Version, + "git_ref": teleport.Gitref, + }, + } + + // Evaluate if profile configuration is customized + if p := getPyroscopeProfileTypesFromEnv(); len(p) == 0 { + slog.InfoContext(process.ExitContext(), "No profile types enabled, using default") + } else { + config.ProfileTypes = p + } + + var uploadRate *time.Duration + if rate := os.Getenv("TELEPORT_PYROSCOPE_UPLOAD_RATE"); rate != "" { + parsedRate, err := time.ParseDuration(rate) + if err != nil { + slog.InfoContext(process.ExitContext(), "invalid TELEPORT_PYROSCOPE_UPLOAD_RATE, ignoring value", "provided_value", rate, "error", err) + } else { + uploadRate = &parsedRate + } + } else { + slog.InfoContext(process.ExitContext(), "TELEPORT_PYROSCOPE_UPLOAD_RATE not specified, using default") + } + + // Set UploadRate or fall back to defaults + if uploadRate != nil { + config.UploadRate = *uploadRate + } + + profiler, err := pyroscope.Start(config) + if err != nil { + slog.ErrorContext(process.ExitContext(), "error starting pyroscope profiler", "error", err) + } else { + process.OnExit("pyroscope.profiler", func(payload any) { + profiler.Flush(payload == nil) + _ = profiler.Stop() + }) + } + slog.InfoContext(process.ExitContext(), "Pyroscope has successfully started") +} + +// getPyroscopeProfileTypesFromEnv sets the profile types based on environment variables. +func getPyroscopeProfileTypesFromEnv() []pyroscope.ProfileType { + var profileTypes []pyroscope.ProfileType + + if os.Getenv("TELEPORT_PYROSCOPE_PROFILE_MEMORY_ENABLED") == "true" { + profileTypes = append(profileTypes, + pyroscope.ProfileAllocObjects, + pyroscope.ProfileAllocSpace, + pyroscope.ProfileInuseObjects, + pyroscope.ProfileInuseSpace, + ) + } + + if os.Getenv("TELEPORT_PYROSCOPE_PROFILE_CPU_ENABLED") == "true" { + profileTypes = append(profileTypes, pyroscope.ProfileCPU) + } + + if os.Getenv("TELEPORT_PYROSCOPE_PROFILE_GOROUTINES_ENABLED") == "true" { + profileTypes = append(profileTypes, pyroscope.ProfileGoroutines) + } + + return profileTypes +} diff --git a/lib/service/service.go b/lib/service/service.go index fd50a20e38a74..a8d21e023bcf4 100644 --- a/lib/service/service.go +++ b/lib/service/service.go @@ -1257,6 +1257,10 @@ func NewTeleport(cfg *servicecfg.Config) (*TeleportProcess, error) { } } + if address := os.Getenv("TELEPORT_PYROSCOPE_SERVER_ADDRESS"); address != "" { + process.initPyroscope(address) + } + if cfg.DebugService.Enabled { if err := process.initDebugService(); err != nil { return nil, trace.Wrap(err) From 4607df70c3d647b1822b85eee7cbb7310ece2845 Mon Sep 17 00:00:00 2001 From: Evan Freed Date: Mon, 27 Jan 2025 09:31:36 -0600 Subject: [PATCH 2/2] fix lint Signed-off-by: Evan Freed --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 9ddd068ac1eb6..32186f65c24ba 100644 --- a/go.mod +++ b/go.mod @@ -112,9 +112,9 @@ require ( github.com/google/uuid v1.6.0 github.com/googleapis/gax-go/v2 v2.12.3 github.com/gorilla/websocket v1.5.1 + github.com/grafana/pyroscope-go v1.2.0 github.com/gravitational/license v0.0.0-20231228155916-928ed9ac0335 github.com/gravitational/oxy v0.0.0-20221029012416-9fbf4c444680 - github.com/grafana/pyroscope-go v1.2.0 github.com/gravitational/roundtrip v1.0.2 github.com/gravitational/teleport/api v0.0.0 github.com/gravitational/trace v1.3.1