[v15] Add pyroscope support

go.mod

@@ -114,6 +114,7 @@ require (
 	github.com/gorilla/websocket v1.5.1
 	github.com/gravitational/license v0.0.0-20231228155916-928ed9ac0335
 	github.com/gravitational/oxy v0.0.0-20221029012416-9fbf4c444680
+	github.com/grafana/pyroscope-go v1.2.0
 	github.com/gravitational/roundtrip v1.0.2
 	github.com/gravitational/teleport/api v0.0.0
 	github.com/gravitational/trace v1.3.1
@@ -366,6 +367,7 @@ require (
 	github.com/gorilla/handlers v1.5.2 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/gosuri/uitable v0.0.4 // indirect
+	github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect
 	github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect

go.sum

@@ -831,6 +831,10 @@ github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY=
 github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo=
 github.com/gravitational/go-cassandra-native-protocol v0.0.0-20221005103706-b9e66c056e90 h1:fPNJE2kaWC0Oy2YKxk1tbnqhKl3aTeXVAfjXzphJmI8=
 github.com/gravitational/go-cassandra-native-protocol v0.0.0-20221005103706-b9e66c056e90/go.mod h1:6FzirJfdffakAVqmHjwVfFkpru/gNbIazUOK5rIhndc=
+github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8=
+github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk=
+github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg=
+github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU=
 github.com/gravitational/go-libfido2 v1.5.3-teleport.1 h1:nPfxiTH2Sr3J6zan280fbHOkWE7gRF/lMqvhcXKh2ek=
 github.com/gravitational/go-libfido2 v1.5.3-teleport.1/go.mod h1:92J9LtSBl0UyUWljElJpTbMMNhC6VeY8dshsu40qjjo=
 github.com/gravitational/go-mssqldb v0.11.1-0.20230331180905-0f76f1751cd3 h1:3JGTacvAeV5tIC4/9XsdLC2K5K7FWaXxIwpW4t+dGH0=

integrations/terraform/go.sum

@@ -643,6 +643,10 @@ github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY=
 github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo=
 github.com/gravitational/go-cassandra-native-protocol v0.0.0-20221005103706-b9e66c056e90 h1:fPNJE2kaWC0Oy2YKxk1tbnqhKl3aTeXVAfjXzphJmI8=
 github.com/gravitational/go-cassandra-native-protocol v0.0.0-20221005103706-b9e66c056e90/go.mod h1:6FzirJfdffakAVqmHjwVfFkpru/gNbIazUOK5rIhndc=
+github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8=
+github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk=
+github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg=
+github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU=
 github.com/gravitational/go-libfido2 v1.5.3-teleport.1 h1:nPfxiTH2Sr3J6zan280fbHOkWE7gRF/lMqvhcXKh2ek=
 github.com/gravitational/go-libfido2 v1.5.3-teleport.1/go.mod h1:92J9LtSBl0UyUWljElJpTbMMNhC6VeY8dshsu40qjjo=
 github.com/gravitational/go-mssqldb v0.11.1-0.20230331180905-0f76f1751cd3 h1:3JGTacvAeV5tIC4/9XsdLC2K5K7FWaXxIwpW4t+dGH0=

lib/service/pyroscope.go

@@ -0,0 +1,131 @@
+// Teleport
+// Copyright (C) 2025 Gravitational, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package service
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"time"
+
+	"github.com/grafana/pyroscope-go"
+
+	"github.com/gravitational/teleport"
+)
+
+// TODO: Replace logger when pyroscope uses slog
+type pyroscopeLogger struct {
+	l *slog.Logger
+}
+
+func (l pyroscopeLogger) Infof(format string, args ...interface{}) {
+	//nolint:sloglint // msg cannot be constant
+	l.l.Info(fmt.Sprintf(format, args...))
+}
+
+func (l pyroscopeLogger) Debugf(format string, args ...interface{}) {
+	//nolint:sloglint // msg cannot be constant
+	l.l.Debug(fmt.Sprintf(format, args...))
+}
+
+func (l pyroscopeLogger) Errorf(format string, args ...interface{}) {
+	//nolint:sloglint // msg cannot be constant
+	l.l.Error(fmt.Sprintf(format, args...))
+}
+
+// initPyroscope instruments Teleport to run with continuous profiling for Pyroscope
+func (process *TeleportProcess) initPyroscope(address string) {
+	if address == "" {
+		return
+	}
+
+	hostname, err := os.Hostname()
+	if err != nil {
+		hostname = "unknown"
+	}
+
+	// Build pyroscope config
+	config := pyroscope.Config{
+		ApplicationName: teleport.ComponentTeleport,
+		ServerAddress:   address,
+		Logger:          pyroscope.Logger(pyroscopeLogger{l: slog.Default()}),
+		Tags: map[string]string{
+			"host":    hostname,
+			"version": teleport.Version,
+			"git_ref": teleport.Gitref,
+		},
+	}
+
+	// Evaluate if profile configuration is customized
+	if p := getPyroscopeProfileTypesFromEnv(); len(p) == 0 {
+		slog.InfoContext(process.ExitContext(), "No profile types enabled, using default")
+	} else {
+		config.ProfileTypes = p
+	}
+
+	var uploadRate *time.Duration
+	if rate := os.Getenv("TELEPORT_PYROSCOPE_UPLOAD_RATE"); rate != "" {
+		parsedRate, err := time.ParseDuration(rate)
+		if err != nil {
+			slog.InfoContext(process.ExitContext(), "invalid TELEPORT_PYROSCOPE_UPLOAD_RATE, ignoring value", "provided_value", rate, "error", err)
+		} else {
+			uploadRate = &parsedRate
+		}
+	} else {
+		slog.InfoContext(process.ExitContext(), "TELEPORT_PYROSCOPE_UPLOAD_RATE not specified, using default")
+	}
+
+	// Set UploadRate or fall back to defaults
+	if uploadRate != nil {
+		config.UploadRate = *uploadRate
+	}
+
+	profiler, err := pyroscope.Start(config)
+	if err != nil {
+		slog.ErrorContext(process.ExitContext(), "error starting pyroscope profiler", "error", err)
+	} else {
+		process.OnExit("pyroscope.profiler", func(payload any) {
+			profiler.Flush(payload == nil)
+			_ = profiler.Stop()
+		})
+	}
+	slog.InfoContext(process.ExitContext(), "Pyroscope has successfully started")
+}
+
+// getPyroscopeProfileTypesFromEnv sets the profile types based on environment variables.
+func getPyroscopeProfileTypesFromEnv() []pyroscope.ProfileType {
+	var profileTypes []pyroscope.ProfileType
+
+	if os.Getenv("TELEPORT_PYROSCOPE_PROFILE_MEMORY_ENABLED") == "true" {
+		profileTypes = append(profileTypes,
+			pyroscope.ProfileAllocObjects,
+			pyroscope.ProfileAllocSpace,
+			pyroscope.ProfileInuseObjects,
+			pyroscope.ProfileInuseSpace,
+		)
+	}
+
+	if os.Getenv("TELEPORT_PYROSCOPE_PROFILE_CPU_ENABLED") == "true" {
+		profileTypes = append(profileTypes, pyroscope.ProfileCPU)
+	}
+
+	if os.Getenv("TELEPORT_PYROSCOPE_PROFILE_GOROUTINES_ENABLED") == "true" {
+		profileTypes = append(profileTypes, pyroscope.ProfileGoroutines)
+	}
+
+	return profileTypes
+}

lib/service/service.go

@@ -1257,6 +1257,10 @@ func NewTeleport(cfg *servicecfg.Config) (*TeleportProcess, error) {
 		}
 	}
 
+	if address := os.Getenv("TELEPORT_PYROSCOPE_SERVER_ADDRESS"); address != "" {
+		process.initPyroscope(address)
+	}
+
 	if cfg.DebugService.Enabled {
 		if err := process.initDebugService(); err != nil {
 			return nil, trace.Wrap(err)