Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: initial pyroscope integration #1006

Merged
merged 12 commits into from
Aug 3, 2023
Merged
4 changes: 2 additions & 2 deletions abci/client/grpc_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"context"

"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"

"github.com/tendermint/tendermint/abci/types"
cmtnet "github.com/tendermint/tendermint/libs/net"
Expand Down Expand Up @@ -88,8 +89,7 @@ func (cli *grpcClient) OnStart() error {

RETRY_LOOP:
for {
//nolint:staticcheck // SA1019 Existing use of deprecated but supported dial option.
conn, err := grpc.Dial(cli.addr, grpc.WithInsecure(), grpc.WithContextDialer(dialerFunc))
conn, err := grpc.Dial(cli.addr, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(dialerFunc))
Comment on lines -91 to +92
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I do not know why the linter started complaining for this here, but this seemed like the best solution

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

seems resolved by 310cd2a

if err != nil {
if cli.mustConnect {
return err
Expand Down
4 changes: 2 additions & 2 deletions abci/example/example_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/require"

"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"

"golang.org/x/net/context"

Expand Down Expand Up @@ -148,8 +149,7 @@ func testGRPCSync(t *testing.T, app types.ABCIApplicationServer) {
})

// Connect to the socket
//nolint:staticcheck // SA1019 Existing use of deprecated but supported dial option.
conn, err := grpc.Dial(socket, grpc.WithInsecure(), grpc.WithContextDialer(dialerFunc))
conn, err := grpc.Dial(socket, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(dialerFunc))
if err != nil {
t.Fatalf("Error dialing GRPC server: %v", err.Error())
}
Expand Down
11 changes: 11 additions & 0 deletions cmd/cometbft/commands/run_node.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,17 @@ func AddNodeFlags(cmd *cobra.Command) {
trace.FlagInfluxDBTokenDescription,
)

cmd.PersistentFlags().String(
trace.FlagPyroscopeURL,
config.Instrumentation.PyroscopeURL,
trace.FlagPyroscopeURLDescription,
)

cmd.PersistentFlags().Bool(
trace.FlagPyroscopeTrace,
config.Instrumentation.PyroscopeTrace,
trace.FlagPyroscopeTraceDescription,
)
}

// NewRunNodeCmd returns the command that allows the CLI to start a node.
Expand Down
12 changes: 12 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -1190,6 +1190,13 @@ type InstrumentationConfig struct {

// InfluxBatchSize is the number of points to write in a single batch.
InfluxBatchSize int `mapstructure:"influx_batch_size"`

// PyroscopeURL is the pyroscope url used to establish a connection with a
// pyroscope continuous profiling server.
PyroscopeURL string `mapstructure:"pyroscope_url"`

// PyroscopeProfile is a flag that enables tracing with pyroscope.
PyroscopeTrace bool `mapstructure:"pyroscope_trace"`
}

// DefaultInstrumentationConfig returns a default configuration for metrics
Expand All @@ -1204,6 +1211,8 @@ func DefaultInstrumentationConfig() *InstrumentationConfig {
InfluxOrg: "celestia",
InfluxBucket: "e2e",
InfluxBatchSize: 20,
PyroscopeURL: "",
PyroscopeTrace: false,
}
}

Expand All @@ -1219,6 +1228,9 @@ func (cfg *InstrumentationConfig) ValidateBasic() error {
if cfg.MaxOpenConnections < 0 {
return errors.New("max_open_connections can't be negative")
}
if cfg.PyroscopeTrace && cfg.PyroscopeURL == "" {
return errors.New("pyroscope_trace can't be enabled if profiling is disabled")
}
// if there is not InfluxURL configured, then we do not need to validate the rest
// of the config because we are not connecting.
if cfg.InfluxURL == "" {
Expand Down
9 changes: 9 additions & 0 deletions config/toml.go
Original file line number Diff line number Diff line change
Expand Up @@ -556,6 +556,15 @@ influx_org = "{{ .Instrumentation.InfluxOrg }}"

# The size of the batches that are sent to the database.
influx_batch_size = {{ .Instrumentation.InfluxBatchSize }}

# The URL of the pyroscope instance to use for continuous profiling.
# If empty, continuous profiling is disabled.
pyroscope_url = "{{ .Instrumentation.PyroscopeURL }}"

# When true, tracing data is added to the continuous profiling
# performed by pyroscope.
pyroscope_trace = {{ .Instrumentation.PyroscopeTrace }}

`

/****** these are for test settings ***********/
Expand Down
19 changes: 13 additions & 6 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ require (
github.com/snikch/goodman v0.0.0-20171125024755-10e37e294daa
github.com/spf13/cobra v1.6.1
github.com/spf13/viper v1.13.0
github.com/stretchr/testify v1.8.1
github.com/stretchr/testify v1.8.2
)

require (
Expand Down Expand Up @@ -62,7 +62,14 @@ require (
google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8
)

require github.com/influxdata/influxdb-client-go/v2 v2.12.2
require (
github.com/influxdata/influxdb-client-go/v2 v2.12.2
github.com/pyroscope-io/client v0.7.0
github.com/pyroscope-io/otel-profiling-go v0.4.0
go.opentelemetry.io/otel v1.15.1
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1
go.opentelemetry.io/otel/sdk v1.15.1
)

require (
4d63.com/gochecknoglobals v0.1.0 // indirect
Expand Down Expand Up @@ -131,7 +138,7 @@ require (
github.com/go-critic/go-critic v0.6.5 // indirect
github.com/go-git/gcfg v1.5.0 // indirect
github.com/go-git/go-billy/v5 v5.3.1 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-toolsmith/astcast v1.0.0 // indirect
github.com/go-toolsmith/astcopy v1.0.2 // indirect
Expand Down Expand Up @@ -229,6 +236,7 @@ require (
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/pyroscope-io/godeltaprof v0.1.0 // indirect
github.com/quasilyte/go-ruleguard v0.3.18 // indirect
github.com/quasilyte/gogrep v0.0.0-20220828223005-86e4605de09f // indirect
github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 // indirect
Expand Down Expand Up @@ -276,17 +284,16 @@ require (
go.etcd.io/bbolt v1.3.6 // indirect
go.opencensus.io v0.23.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.3 // indirect
go.opentelemetry.io/otel v1.11.0 // indirect
go.opentelemetry.io/otel/metric v0.32.3 // indirect
go.opentelemetry.io/otel/trace v1.11.0 // indirect
go.opentelemetry.io/otel/trace v1.15.1 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.23.0 // indirect
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
golang.org/x/exp/typeparams v0.0.0-20220827204233-334a2380cb91 // indirect
golang.org/x/mod v0.7.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.5.0 // indirect
golang.org/x/sys v0.7.0 // indirect
golang.org/x/term v0.5.0 // indirect
golang.org/x/text v0.7.0 // indirect
golang.org/x/tools v0.4.0 // indirect
Expand Down
33 changes: 23 additions & 10 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -311,8 +311,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG
github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
Expand Down Expand Up @@ -423,6 +423,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
Expand Down Expand Up @@ -744,6 +745,12 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
github.com/pyroscope-io/client v0.7.0 h1:LWuuqPQ1oa6x7BnmUOuo/aGwdX85QGhWZUBYWWW3zdk=
github.com/pyroscope-io/client v0.7.0/go.mod h1:4h21iOU4pUOq0prKyDlvYRL+SCKsBc5wKiEtV+rJGqU=
github.com/pyroscope-io/godeltaprof v0.1.0 h1:UBqtjt0yZi4jTxqZmLAs34XG6ycS3vUTlhEUSq4NHLE=
github.com/pyroscope-io/godeltaprof v0.1.0/go.mod h1:psMITXp90+8pFenXkKIpNhrfmI9saQnPbba27VIaiQE=
github.com/pyroscope-io/otel-profiling-go v0.4.0 h1:Hk/rbUqOWoByoWy1tt4r5BX5xoKAvs5drr0511Ki8ic=
github.com/pyroscope-io/otel-profiling-go v0.4.0/go.mod h1:MXaofiWU7PgLP7eISUZJYVO4Z8WYMqpkYgeP4XrPLyg=
github.com/quasilyte/go-ruleguard v0.3.1-0.20210203134552-1b5a410e1cc8/go.mod h1:KsAh3x0e7Fkpgs+Q9pNLS5XpFSvYCEVl5gP9Pp1xp30=
github.com/quasilyte/go-ruleguard v0.3.18 h1:sd+abO1PEI9fkYennwzHn9kl3nqP6M5vE7FiOzZ+5CE=
github.com/quasilyte/go-ruleguard v0.3.18/go.mod h1:lOIzcYlgxrQ2sGJ735EHXmf/e9MJ516j16K/Ifcttvs=
Expand Down Expand Up @@ -855,8 +862,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs=
github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
Expand Down Expand Up @@ -931,12 +938,18 @@ go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.3 h1:syAz40OyelLZo42+3U68Phisvrx4qh+4wpdZw7eUUdY=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.3/go.mod h1:Dts42MGkzZne2yCru741+bFiTMWkIj/LLRizad7b9tw=
go.opentelemetry.io/otel v1.11.0 h1:kfToEGMDq6TrVrJ9Vht84Y8y9enykSZzDDZglV0kIEk=
go.opentelemetry.io/otel v1.11.0/go.mod h1:H2KtuEphyMvlhZ+F7tg9GRhAOe60moNx61Ex+WmiKkk=
go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4=
go.opentelemetry.io/otel v1.15.1 h1:3Iwq3lfRByPaws0f6bU3naAqOR1n5IeDWd9390kWHa8=
go.opentelemetry.io/otel v1.15.1/go.mod h1:mHHGEHVDLal6YrKMmk9LqC4a3sF5g+fHfrttQIB1NTc=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 h1:2PunuO5SbkN5MhCbuHCd3tC6qrcaj+uDAkX/qBU5BAs=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1/go.mod h1:q8+Tha+5LThjeSU8BW93uUC5w5/+DnYHMKBMpRCsui0=
go.opentelemetry.io/otel/metric v0.32.3 h1:dMpnJYk2KULXr0j8ph6N7+IcuiIQXlPXD4kix9t7L9c=
go.opentelemetry.io/otel/metric v0.32.3/go.mod h1:pgiGmKohxHyTPHGOff+vrtIH39/R9fiO/WoenUQ3kcc=
go.opentelemetry.io/otel/trace v1.11.0 h1:20U/Vj42SX+mASlXLmSGBg6jpI1jQtv682lZtTAOVFI=
go.opentelemetry.io/otel/trace v1.11.0/go.mod h1:nyYjis9jy0gytE9LXGU+/m1sHTKbRY0fX0hulNNDP1U=
go.opentelemetry.io/otel/sdk v1.15.1 h1:5FKR+skgpzvhPQHIEfcwMYjCBr14LWzs3uSqKiQzETI=
go.opentelemetry.io/otel/sdk v1.15.1/go.mod h1:8rVtxQfrbmbHKfqzpQkT5EzZMcbMBwTzNAggbEAM0KA=
go.opentelemetry.io/otel/trace v1.4.1/go.mod h1:iYEVbroFCNut9QkwEczV9vMRPHNKSSwYZjulEtsmhFc=
go.opentelemetry.io/otel/trace v1.15.1 h1:uXLo6iHJEzDfrNC0L0mNjItIp06SyaBQxu5t3xMlngY=
go.opentelemetry.io/otel/trace v1.15.1/go.mod h1:IWdQG/5N1x7f6YUlmdLeJvH9yxtuJAfc4VW5Agv9r/8=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
Expand Down Expand Up @@ -1176,8 +1189,8 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
Expand Down
30 changes: 30 additions & 0 deletions node/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ import (
dbm "github.com/cometbft/cometbft-db"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/pyroscope-io/client/pyroscope"
"github.com/rs/cors"
sdktrace "go.opentelemetry.io/otel/sdk/trace"

abci "github.com/tendermint/tendermint/abci/types"
bcv0 "github.com/tendermint/tendermint/blockchain/v0"
Expand Down Expand Up @@ -233,6 +235,8 @@ type Node struct {
indexerService *txindex.IndexerService
prometheusSrv *http.Server
influxDBClient *trace.Client
pyroscopeProfiler *pyroscope.Profiler
pyroscopeTracer *sdktrace.TracerProvider
}

func initDBs(config *cfg.Config, dbProvider DBProvider) (blockStore *store.BlockStore, stateDB dbm.DB, err error) {
Expand Down Expand Up @@ -1024,6 +1028,19 @@ func (n *Node) OnStart() error {
n.prometheusSrv = n.startPrometheusServer(n.config.Instrumentation.PrometheusListenAddr)
}

if n.config.Instrumentation.PyroscopeURL != "" {
profiler, tracer, err := n.setupPyroscope(
n.config.Instrumentation.PyroscopeURL,
string(n.nodeKey.ID()),
n.config.Instrumentation.PyroscopeTrace,
)
if err != nil {
return err
}
n.pyroscopeProfiler = profiler
n.pyroscopeTracer = tracer
}

// Start the transport.
addr, err := p2p.NewNetAddressString(p2p.IDAddressString(n.nodeKey.ID(), n.config.P2P.ListenAddress))
if err != nil {
Expand Down Expand Up @@ -1124,6 +1141,19 @@ func (n *Node) OnStop() {
if n.influxDBClient != nil {
n.influxDBClient.Stop()
}

if n.pyroscopeProfiler != nil {
if err := n.pyroscopeProfiler.Stop(); err != nil {
n.Logger.Error("Pyroscope profiler Stop", "err", err)
}
}

if n.pyroscopeTracer != nil {
if err := n.pyroscopeTracer.Shutdown(context.Background()); err != nil {
n.Logger.Error("Pyroscope tracer Shutdown", "err", err)
}
}

}

// ConfigureRPC makes sure RPC has all the objects it needs to operate.
Expand Down
75 changes: 75 additions & 0 deletions node/tracing.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
package node

import (
"github.com/pyroscope-io/client/pyroscope"

otelpyroscope "github.com/pyroscope-io/otel-profiling-go"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
"go.opentelemetry.io/otel/propagation"

sdktrace "go.opentelemetry.io/otel/sdk/trace"
)

// setupPyroscope sets up pyroscope profiler and optionally tracing.
func (n *Node) setupPyroscope(addr, nodeID string, tracing bool) (*pyroscope.Profiler, *sdktrace.TracerProvider, error) {
staheri14 marked this conversation as resolved.
Show resolved Hide resolved
tp, err := tracerProviderDebug()
if err != nil {
return nil, nil, err
}

labels := map[string]string{"node_id": nodeID}

if tracing {
if _, err = setupTracing(addr, labels); err != nil {
return nil, nil, err
}
} else {
tp = nil
}

pflr, err := pyroscope.Start(pyroscope.Config{
ApplicationName: "celestia",
ServerAddress: addr,
Logger: pyroscope.StandardLogger,
Tags: labels,
})

return pflr, tp, err
}

func setupTracing(addr string, labels map[string]string) (tp *sdktrace.TracerProvider, err error) {
tp, err = tracerProviderDebug()
if err != nil {
return nil, err
}

// Set the Tracer Provider and the W3C Trace Context propagator as globals.
// We wrap the tracer provider to also annotate goroutines with Span ID so
// that pprof would add corresponding labels to profiling samples.
otel.SetTracerProvider(otelpyroscope.NewTracerProvider(tp,
otelpyroscope.WithAppName("celestia"),
otelpyroscope.WithRootSpanOnly(true),
otelpyroscope.WithAddSpanName(true),
otelpyroscope.WithPyroscopeURL(addr),
otelpyroscope.WithProfileBaselineLabels(labels),
otelpyroscope.WithProfileBaselineURL(true),
otelpyroscope.WithProfileURL(true),
))

// Register the trace context and baggage propagators so data is propagated across services/processes.
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
propagation.TraceContext{},
propagation.Baggage{},
))

return tp, err
}

func tracerProviderDebug() (*sdktrace.TracerProvider, error) {
exp, err := stdouttrace.New(stdouttrace.WithPrettyPrint())
if err != nil {
return nil, err
}
return sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sdktrace.NewBatchSpanProcessor(exp))), nil
}
5 changes: 5 additions & 0 deletions pkg/trace/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,9 @@ const (
FlagInfluxDBToken = "influxdb-token"
FlagInfluxDBURLDescription = "URL of the InfluxDB instance to use for arbitrary data collection. If not specified, data will not be collected"
FlagInfluxDBTokenDescription = "Token to use when writing to the InfluxDB instance. Must be specified if 'influxdb-url' is specified" //nolint:gosec

FlagPyroscopeURL = "pyroscope-url"
FlagPyroscopeURLDescription = "URL of the Pyroscope instance to use for continuous profiling. If not specified, profiling will not be enabled"
FlagPyroscopeTrace = "pyroscope-trace"
FlagPyroscopeTraceDescription = "enable adding trace data to pyroscope profiling"
)
Loading
Loading