diff --git a/abci/client/grpc_client.go b/abci/client/grpc_client.go index 78d680accb..c2f5adaea8 100644 --- a/abci/client/grpc_client.go +++ b/abci/client/grpc_client.go @@ -9,6 +9,7 @@ import ( "context" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "github.com/tendermint/tendermint/abci/types" cmtnet "github.com/tendermint/tendermint/libs/net" @@ -88,8 +89,7 @@ func (cli *grpcClient) OnStart() error { RETRY_LOOP: for { - //nolint:staticcheck // SA1019 Existing use of deprecated but supported dial option. - conn, err := grpc.Dial(cli.addr, grpc.WithInsecure(), grpc.WithContextDialer(dialerFunc)) + conn, err := grpc.Dial(cli.addr, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(dialerFunc)) if err != nil { if cli.mustConnect { return err diff --git a/abci/example/example_test.go b/abci/example/example_test.go index 0bd172eb25..c53f5d9230 100644 --- a/abci/example/example_test.go +++ b/abci/example/example_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "golang.org/x/net/context" @@ -148,8 +149,7 @@ func testGRPCSync(t *testing.T, app types.ABCIApplicationServer) { }) // Connect to the socket - //nolint:staticcheck // SA1019 Existing use of deprecated but supported dial option. - conn, err := grpc.Dial(socket, grpc.WithInsecure(), grpc.WithContextDialer(dialerFunc)) + conn, err := grpc.Dial(socket, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(dialerFunc)) if err != nil { t.Fatalf("Error dialing GRPC server: %v", err.Error()) } diff --git a/cmd/cometbft/commands/run_node.go b/cmd/cometbft/commands/run_node.go index 1d79e21b57..58b40b2bed 100644 --- a/cmd/cometbft/commands/run_node.go +++ b/cmd/cometbft/commands/run_node.go @@ -107,6 +107,17 @@ func AddNodeFlags(cmd *cobra.Command) { trace.FlagInfluxDBTokenDescription, ) + cmd.PersistentFlags().String( + trace.FlagPyroscopeURL, + config.Instrumentation.PyroscopeURL, + trace.FlagPyroscopeURLDescription, + ) + + cmd.PersistentFlags().Bool( + trace.FlagPyroscopeTrace, + config.Instrumentation.PyroscopeTrace, + trace.FlagPyroscopeTraceDescription, + ) } // NewRunNodeCmd returns the command that allows the CLI to start a node. diff --git a/config/config.go b/config/config.go index 649ae176f0..1e67251e9a 100644 --- a/config/config.go +++ b/config/config.go @@ -1190,6 +1190,19 @@ type InstrumentationConfig struct { // InfluxBatchSize is the number of points to write in a single batch. InfluxBatchSize int `mapstructure:"influx_batch_size"` + + // PyroscopeURL is the pyroscope url used to establish a connection with a + // pyroscope continuous profiling server. + PyroscopeURL string `mapstructure:"pyroscope_url"` + + // PyroscopeProfile is a flag that enables tracing with pyroscope. + PyroscopeTrace bool `mapstructure:"pyroscope_trace"` + + // PyroscopeProfileTypes is a list of profile types to be traced with + // pyroscope. Available profile types are: cpu, alloc_objects, alloc_space, + // inuse_objects, inuse_space, goroutines, mutex_count, mutex_duration, + // block_count, block_duration. + PyroscopeProfileTypes []string `mapstructure:"pyroscope_profile_types"` } // DefaultInstrumentationConfig returns a default configuration for metrics @@ -1204,6 +1217,18 @@ func DefaultInstrumentationConfig() *InstrumentationConfig { InfluxOrg: "celestia", InfluxBucket: "e2e", InfluxBatchSize: 20, + PyroscopeURL: "", + PyroscopeTrace: false, + PyroscopeProfileTypes: []string{ + "cpu", + "alloc_objects", + "inuse_objects", + "goroutines", + "mutex_count", + "mutex_duration", + "block_count", + "block_duration", + }, } } @@ -1219,6 +1244,9 @@ func (cfg *InstrumentationConfig) ValidateBasic() error { if cfg.MaxOpenConnections < 0 { return errors.New("max_open_connections can't be negative") } + if cfg.PyroscopeTrace && cfg.PyroscopeURL == "" { + return errors.New("pyroscope_trace can't be enabled if profiling is disabled") + } // if there is not InfluxURL configured, then we do not need to validate the rest // of the config because we are not connecting. if cfg.InfluxURL == "" { diff --git a/config/toml.go b/config/toml.go index 9f45f0b636..417e93cfbf 100644 --- a/config/toml.go +++ b/config/toml.go @@ -556,6 +556,21 @@ influx_org = "{{ .Instrumentation.InfluxOrg }}" # The size of the batches that are sent to the database. influx_batch_size = {{ .Instrumentation.InfluxBatchSize }} + +# The URL of the pyroscope instance to use for continuous profiling. +# If empty, continuous profiling is disabled. +pyroscope_url = "{{ .Instrumentation.PyroscopeURL }}" + +# When true, tracing data is added to the continuous profiling +# performed by pyroscope. +pyroscope_trace = {{ .Instrumentation.PyroscopeTrace }} + +# pyroscope_profile_types is a list of profile types to be traced with +# pyroscope. Available profile types are: cpu, alloc_objects, alloc_space, +# inuse_objects, inuse_space, goroutines, mutex_count, mutex_duration, +# block_count, block_duration. +pyroscope_profile_types = [{{ range .Instrumentation.PyroscopeProfileTypes }}{{ printf "%q, " . }}{{end}}] + ` /****** these are for test settings ***********/ diff --git a/go.mod b/go.mod index 75c6b1acce..0a68985eab 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/snikch/goodman v0.0.0-20171125024755-10e37e294daa github.com/spf13/cobra v1.6.1 github.com/spf13/viper v1.13.0 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.2 ) require ( @@ -62,7 +62,14 @@ require ( google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8 ) -require github.com/influxdata/influxdb-client-go/v2 v2.12.2 +require ( + github.com/influxdata/influxdb-client-go/v2 v2.12.2 + github.com/pyroscope-io/client v0.7.0 + github.com/pyroscope-io/otel-profiling-go v0.4.0 + go.opentelemetry.io/otel v1.15.1 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 + go.opentelemetry.io/otel/sdk v1.15.1 +) require ( 4d63.com/gochecknoglobals v0.1.0 // indirect @@ -131,7 +138,7 @@ require ( github.com/go-critic/go-critic v0.6.5 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.3.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-toolsmith/astcast v1.0.0 // indirect github.com/go-toolsmith/astcopy v1.0.2 // indirect @@ -229,6 +236,7 @@ require ( github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect + github.com/pyroscope-io/godeltaprof v0.1.0 // indirect github.com/quasilyte/go-ruleguard v0.3.18 // indirect github.com/quasilyte/gogrep v0.0.0-20220828223005-86e4605de09f // indirect github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 // indirect @@ -276,9 +284,8 @@ require ( go.etcd.io/bbolt v1.3.6 // indirect go.opencensus.io v0.23.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.3 // indirect - go.opentelemetry.io/otel v1.11.0 // indirect go.opentelemetry.io/otel/metric v0.32.3 // indirect - go.opentelemetry.io/otel/trace v1.11.0 // indirect + go.opentelemetry.io/otel/trace v1.15.1 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.23.0 // indirect @@ -286,7 +293,7 @@ require ( golang.org/x/exp/typeparams v0.0.0-20220827204233-334a2380cb91 // indirect golang.org/x/mod v0.7.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.5.0 // indirect + golang.org/x/sys v0.7.0 // indirect golang.org/x/term v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect golang.org/x/tools v0.4.0 // indirect diff --git a/go.sum b/go.sum index a8f0c7cae4..5b1927006e 100644 --- a/go.sum +++ b/go.sum @@ -311,8 +311,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -423,6 +423,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -744,6 +745,12 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/pyroscope-io/client v0.7.0 h1:LWuuqPQ1oa6x7BnmUOuo/aGwdX85QGhWZUBYWWW3zdk= +github.com/pyroscope-io/client v0.7.0/go.mod h1:4h21iOU4pUOq0prKyDlvYRL+SCKsBc5wKiEtV+rJGqU= +github.com/pyroscope-io/godeltaprof v0.1.0 h1:UBqtjt0yZi4jTxqZmLAs34XG6ycS3vUTlhEUSq4NHLE= +github.com/pyroscope-io/godeltaprof v0.1.0/go.mod h1:psMITXp90+8pFenXkKIpNhrfmI9saQnPbba27VIaiQE= +github.com/pyroscope-io/otel-profiling-go v0.4.0 h1:Hk/rbUqOWoByoWy1tt4r5BX5xoKAvs5drr0511Ki8ic= +github.com/pyroscope-io/otel-profiling-go v0.4.0/go.mod h1:MXaofiWU7PgLP7eISUZJYVO4Z8WYMqpkYgeP4XrPLyg= github.com/quasilyte/go-ruleguard v0.3.1-0.20210203134552-1b5a410e1cc8/go.mod h1:KsAh3x0e7Fkpgs+Q9pNLS5XpFSvYCEVl5gP9Pp1xp30= github.com/quasilyte/go-ruleguard v0.3.18 h1:sd+abO1PEI9fkYennwzHn9kl3nqP6M5vE7FiOzZ+5CE= github.com/quasilyte/go-ruleguard v0.3.18/go.mod h1:lOIzcYlgxrQ2sGJ735EHXmf/e9MJ516j16K/Ifcttvs= @@ -855,8 +862,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -931,12 +938,18 @@ go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.3 h1:syAz40OyelLZo42+3U68Phisvrx4qh+4wpdZw7eUUdY= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.3/go.mod h1:Dts42MGkzZne2yCru741+bFiTMWkIj/LLRizad7b9tw= -go.opentelemetry.io/otel v1.11.0 h1:kfToEGMDq6TrVrJ9Vht84Y8y9enykSZzDDZglV0kIEk= -go.opentelemetry.io/otel v1.11.0/go.mod h1:H2KtuEphyMvlhZ+F7tg9GRhAOe60moNx61Ex+WmiKkk= +go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4= +go.opentelemetry.io/otel v1.15.1 h1:3Iwq3lfRByPaws0f6bU3naAqOR1n5IeDWd9390kWHa8= +go.opentelemetry.io/otel v1.15.1/go.mod h1:mHHGEHVDLal6YrKMmk9LqC4a3sF5g+fHfrttQIB1NTc= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 h1:2PunuO5SbkN5MhCbuHCd3tC6qrcaj+uDAkX/qBU5BAs= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1/go.mod h1:q8+Tha+5LThjeSU8BW93uUC5w5/+DnYHMKBMpRCsui0= go.opentelemetry.io/otel/metric v0.32.3 h1:dMpnJYk2KULXr0j8ph6N7+IcuiIQXlPXD4kix9t7L9c= go.opentelemetry.io/otel/metric v0.32.3/go.mod h1:pgiGmKohxHyTPHGOff+vrtIH39/R9fiO/WoenUQ3kcc= -go.opentelemetry.io/otel/trace v1.11.0 h1:20U/Vj42SX+mASlXLmSGBg6jpI1jQtv682lZtTAOVFI= -go.opentelemetry.io/otel/trace v1.11.0/go.mod h1:nyYjis9jy0gytE9LXGU+/m1sHTKbRY0fX0hulNNDP1U= +go.opentelemetry.io/otel/sdk v1.15.1 h1:5FKR+skgpzvhPQHIEfcwMYjCBr14LWzs3uSqKiQzETI= +go.opentelemetry.io/otel/sdk v1.15.1/go.mod h1:8rVtxQfrbmbHKfqzpQkT5EzZMcbMBwTzNAggbEAM0KA= +go.opentelemetry.io/otel/trace v1.4.1/go.mod h1:iYEVbroFCNut9QkwEczV9vMRPHNKSSwYZjulEtsmhFc= +go.opentelemetry.io/otel/trace v1.15.1 h1:uXLo6iHJEzDfrNC0L0mNjItIp06SyaBQxu5t3xMlngY= +go.opentelemetry.io/otel/trace v1.15.1/go.mod h1:IWdQG/5N1x7f6YUlmdLeJvH9yxtuJAfc4VW5Agv9r/8= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= @@ -1176,8 +1189,8 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/node/node.go b/node/node.go index 542a8f0dec..489c24ee79 100644 --- a/node/node.go +++ b/node/node.go @@ -13,7 +13,9 @@ import ( dbm "github.com/cometbft/cometbft-db" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/pyroscope-io/client/pyroscope" "github.com/rs/cors" + sdktrace "go.opentelemetry.io/otel/sdk/trace" abci "github.com/tendermint/tendermint/abci/types" bcv0 "github.com/tendermint/tendermint/blockchain/v0" @@ -233,6 +235,8 @@ type Node struct { indexerService *txindex.IndexerService prometheusSrv *http.Server influxDBClient *trace.Client + pyroscopeProfiler *pyroscope.Profiler + pyroscopeTracer *sdktrace.TracerProvider } func initDBs(config *cfg.Config, dbProvider DBProvider) (blockStore *store.BlockStore, stateDB dbm.DB, err error) { @@ -1024,6 +1028,18 @@ func (n *Node) OnStart() error { n.prometheusSrv = n.startPrometheusServer(n.config.Instrumentation.PrometheusListenAddr) } + if n.config.Instrumentation.PyroscopeURL != "" { + profiler, tracer, err := setupPyroscope( + n.config.Instrumentation, + string(n.nodeKey.ID()), + ) + if err != nil { + return err + } + n.pyroscopeProfiler = profiler + n.pyroscopeTracer = tracer + } + // Start the transport. addr, err := p2p.NewNetAddressString(p2p.IDAddressString(n.nodeKey.ID(), n.config.P2P.ListenAddress)) if err != nil { @@ -1124,6 +1140,19 @@ func (n *Node) OnStop() { if n.influxDBClient != nil { n.influxDBClient.Stop() } + + if n.pyroscopeProfiler != nil { + if err := n.pyroscopeProfiler.Stop(); err != nil { + n.Logger.Error("Pyroscope profiler Stop", "err", err) + } + } + + if n.pyroscopeTracer != nil { + if err := n.pyroscopeTracer.Shutdown(context.Background()); err != nil { + n.Logger.Error("Pyroscope tracer Shutdown", "err", err) + } + } + } // ConfigureRPC makes sure RPC has all the objects it needs to operate. diff --git a/node/tracing.go b/node/tracing.go new file mode 100644 index 0000000000..4e2e00f76e --- /dev/null +++ b/node/tracing.go @@ -0,0 +1,85 @@ +package node + +import ( + "github.com/pyroscope-io/client/pyroscope" + "github.com/tendermint/tendermint/config" + + otelpyroscope "github.com/pyroscope-io/otel-profiling-go" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/propagation" + + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +// setupPyroscope sets up pyroscope profiler and optionally tracing. +func setupPyroscope(instCfg *config.InstrumentationConfig, nodeID string) (*pyroscope.Profiler, *sdktrace.TracerProvider, error) { + tp, err := tracerProviderDebug() + if err != nil { + return nil, nil, err + } + + labels := map[string]string{"node_id": nodeID} + + if instCfg.PyroscopeTrace { + if _, err = setupTracing(instCfg.PyroscopeURL, labels); err != nil { + return nil, nil, err + } + } else { + tp = nil + } + + pflr, err := pyroscope.Start(pyroscope.Config{ + ApplicationName: "celestia", + ServerAddress: instCfg.PyroscopeURL, + Logger: nil, // use the noop logger by passing nil + Tags: labels, + ProfileTypes: toPyroscopeProfiles(instCfg.PyroscopeProfileTypes), + }) + + return pflr, tp, err +} + +func setupTracing(addr string, labels map[string]string) (tp *sdktrace.TracerProvider, err error) { + tp, err = tracerProviderDebug() + if err != nil { + return nil, err + } + + // Set the Tracer Provider and the W3C Trace Context propagator as globals. + // We wrap the tracer provider to also annotate goroutines with Span ID so + // that pprof would add corresponding labels to profiling samples. + otel.SetTracerProvider(otelpyroscope.NewTracerProvider(tp, + otelpyroscope.WithAppName("celestia"), + otelpyroscope.WithRootSpanOnly(true), + otelpyroscope.WithAddSpanName(true), + otelpyroscope.WithPyroscopeURL(addr), + otelpyroscope.WithProfileBaselineLabels(labels), + otelpyroscope.WithProfileBaselineURL(true), + otelpyroscope.WithProfileURL(true), + )) + + // Register the trace context and baggage propagators so data is propagated across services/processes. + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + )) + + return tp, err +} + +func tracerProviderDebug() (*sdktrace.TracerProvider, error) { + exp, err := stdouttrace.New(stdouttrace.WithPrettyPrint()) + if err != nil { + return nil, err + } + return sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sdktrace.NewBatchSpanProcessor(exp))), nil +} + +func toPyroscopeProfiles(profiles []string) []pyroscope.ProfileType { + pts := make([]pyroscope.ProfileType, 0, len(profiles)) + for _, p := range profiles { + pts = append(pts, pyroscope.ProfileType(p)) + } + return pts +} diff --git a/pkg/trace/flags.go b/pkg/trace/flags.go index a703b427ad..5d8b2a44ad 100644 --- a/pkg/trace/flags.go +++ b/pkg/trace/flags.go @@ -5,4 +5,9 @@ const ( FlagInfluxDBToken = "influxdb-token" FlagInfluxDBURLDescription = "URL of the InfluxDB instance to use for arbitrary data collection. If not specified, data will not be collected" FlagInfluxDBTokenDescription = "Token to use when writing to the InfluxDB instance. Must be specified if 'influxdb-url' is specified" //nolint:gosec + + FlagPyroscopeURL = "pyroscope-url" + FlagPyroscopeURLDescription = "URL of the Pyroscope instance to use for continuous profiling. If not specified, profiling will not be enabled" + FlagPyroscopeTrace = "pyroscope-trace" + FlagPyroscopeTraceDescription = "enable adding trace data to pyroscope profiling" ) diff --git a/rpc/grpc/client_server.go b/rpc/grpc/client_server.go index 39c6859c13..ed00f61dc3 100644 --- a/rpc/grpc/client_server.go +++ b/rpc/grpc/client_server.go @@ -5,6 +5,7 @@ import ( "golang.org/x/net/context" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" cmtnet "github.com/tendermint/tendermint/libs/net" ) @@ -26,8 +27,7 @@ func StartGRPCServer(ln net.Listener) error { // StartGRPCClient dials the gRPC server using protoAddr and returns a new // BroadcastAPIClient. func StartGRPCClient(protoAddr string) BroadcastAPIClient { - //nolint:staticcheck // SA1019 Existing use of deprecated but supported dial option. - conn, err := grpc.Dial(protoAddr, grpc.WithInsecure(), grpc.WithContextDialer(dialerFunc)) + conn, err := grpc.Dial(protoAddr, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(dialerFunc)) if err != nil { panic(err) } diff --git a/test/e2e/pkg/infrastructure.go b/test/e2e/pkg/infrastructure.go index a9e1abfa5d..a1bec6fbf2 100644 --- a/test/e2e/pkg/infrastructure.go +++ b/test/e2e/pkg/infrastructure.go @@ -40,6 +40,16 @@ type InfrastructureData struct { // InfluxDBToken is the token to use when writing to the InfluxDB instance. // Must be specified if 'influxdb-url' is specified. InfluxDBToken string `json:"influxdb_token,omitempty"` + + // PyroscopeURL is the URL of the pyroscope instance to use for continuous + // profiling. If not specified, data will not be collected. + PyroscopeURL string `json:"pyroscope_url,omitempty"` + + // PyroscopeTrace enables adding trace data to pyroscope profiling. + PyroscopeTrace bool `json:"pyroscope_trace,omitempty"` + + // PyroscopeProfileTypes is the list of profile types to collect. + PyroscopeProfileTypes []string `json:"pyroscope_profile_types,omitempty"` } // InstanceData contains the relevant information for a machine instance backing diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go index de98494deb..2cb9a58f4e 100644 --- a/test/e2e/pkg/testnet.go +++ b/test/e2e/pkg/testnet.go @@ -81,33 +81,36 @@ type Testnet struct { // Node represents a CometBFT node in a testnet. type Node struct { - Name string - Version string - Testnet *Testnet - Mode Mode - PrivvalKey crypto.PrivKey - NodeKey crypto.PrivKey - IP net.IP - ProxyPort uint32 - StartAt int64 - FastSync string - StateSync bool - Mempool string - Database string - ABCIProtocol Protocol - PrivvalProtocol Protocol - PersistInterval uint64 - SnapshotInterval uint64 - RetainBlocks uint64 - Seeds []*Node - PersistentPeers []*Node - Perturbations []Perturbation - Misbehaviors map[int64]string - SendNoLoad bool - Prometheus bool - PrometheusProxyPort uint32 - InfluxDBURL string - InfluxDBToken string + Name string + Version string + Testnet *Testnet + Mode Mode + PrivvalKey crypto.PrivKey + NodeKey crypto.PrivKey + IP net.IP + ProxyPort uint32 + StartAt int64 + FastSync string + StateSync bool + Mempool string + Database string + ABCIProtocol Protocol + PrivvalProtocol Protocol + PersistInterval uint64 + SnapshotInterval uint64 + RetainBlocks uint64 + Seeds []*Node + PersistentPeers []*Node + Perturbations []Perturbation + Misbehaviors map[int64]string + SendNoLoad bool + Prometheus bool + PrometheusProxyPort uint32 + InfluxDBURL string + InfluxDBToken string + PyroscopeURL string + PyroscopeTrace bool + PyroscopeProfileTypes []string } // LoadTestnet loads a testnet from a manifest file, using the filename to @@ -185,30 +188,33 @@ func LoadTestnet(manifest Manifest, fname string, ifd InfrastructureData) (*Test } node := &Node{ - Name: name, - Version: v, - Testnet: testnet, - PrivvalKey: keyGen.Generate(manifest.KeyType), - NodeKey: keyGen.Generate("ed25519"), - IP: ind.IPAddress, - ProxyPort: proxyPortGen.Next(), - Mode: ModeValidator, - Database: "goleveldb", - ABCIProtocol: Protocol(testnet.ABCIProtocol), - PrivvalProtocol: ProtocolFile, - StartAt: nodeManifest.StartAt, - FastSync: nodeManifest.FastSync, - Mempool: nodeManifest.Mempool, - StateSync: nodeManifest.StateSync, - PersistInterval: 1, - SnapshotInterval: nodeManifest.SnapshotInterval, - RetainBlocks: nodeManifest.RetainBlocks, - Perturbations: []Perturbation{}, - Misbehaviors: make(map[int64]string), - SendNoLoad: nodeManifest.SendNoLoad, - InfluxDBURL: ifd.InfluxDBURL, - InfluxDBToken: ifd.InfluxDBToken, - Prometheus: testnet.Prometheus, + Name: name, + Version: v, + Testnet: testnet, + PrivvalKey: keyGen.Generate(manifest.KeyType), + NodeKey: keyGen.Generate("ed25519"), + IP: ind.IPAddress, + ProxyPort: proxyPortGen.Next(), + Mode: ModeValidator, + Database: "goleveldb", + ABCIProtocol: Protocol(testnet.ABCIProtocol), + PrivvalProtocol: ProtocolFile, + StartAt: nodeManifest.StartAt, + FastSync: nodeManifest.FastSync, + Mempool: nodeManifest.Mempool, + StateSync: nodeManifest.StateSync, + PersistInterval: 1, + SnapshotInterval: nodeManifest.SnapshotInterval, + RetainBlocks: nodeManifest.RetainBlocks, + Perturbations: []Perturbation{}, + Misbehaviors: make(map[int64]string), + SendNoLoad: nodeManifest.SendNoLoad, + InfluxDBURL: ifd.InfluxDBURL, + InfluxDBToken: ifd.InfluxDBToken, + PyroscopeURL: ifd.PyroscopeURL, + PyroscopeTrace: ifd.PyroscopeTrace, + PyroscopeProfileTypes: ifd.PyroscopeProfileTypes, + Prometheus: testnet.Prometheus, } if node.StartAt == testnet.InitialHeight { node.StartAt = 0 // normalize to 0 for initial nodes, since code expects this diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go index 6e2f922577..425c2fa4a0 100644 --- a/test/e2e/runner/main.go +++ b/test/e2e/runner/main.go @@ -91,6 +91,19 @@ func NewCLI() *CLI { ifd.InfluxDBToken = itoken } + purl, err := cmd.Flags().GetString(trace.FlagPyroscopeURL) + if err != nil { + return err + } + pTrace, err := cmd.Flags().GetBool(trace.FlagPyroscopeTrace) + if err != nil { + return err + } + if ifd.PyroscopeURL == "" { + ifd.PyroscopeURL = purl + ifd.PyroscopeTrace = pTrace + } + testnet, err := e2e.LoadTestnet(m, file, ifd) if err != nil { return fmt.Errorf("loading testnet: %s", err) @@ -177,6 +190,10 @@ func NewCLI() *CLI { cli.root.PersistentFlags().String(trace.FlagInfluxDBToken, "", trace.FlagInfluxDBTokenDescription) + cli.root.PersistentFlags().String(trace.FlagPyroscopeURL, "", trace.FlagPyroscopeURLDescription) + + cli.root.PersistentFlags().Bool(trace.FlagPyroscopeTrace, false, trace.FlagPyroscopeTraceDescription) + cli.root.Flags().BoolVarP(&cli.preserve, "preserve", "p", false, "Preserves the running of the test net after tests are completed") diff --git a/test/e2e/runner/setup.go b/test/e2e/runner/setup.go index 5a49d3b1d2..8fdecf6a77 100644 --- a/test/e2e/runner/setup.go +++ b/test/e2e/runner/setup.go @@ -170,6 +170,9 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) { cfg.Instrumentation.InfluxBucket = "e2e" cfg.Instrumentation.InfluxURL = node.InfluxDBURL cfg.Instrumentation.InfluxToken = node.InfluxDBToken + cfg.Instrumentation.PyroscopeTrace = node.PyroscopeTrace + cfg.Instrumentation.PyroscopeURL = node.PyroscopeURL + cfg.Instrumentation.PyroscopeProfileTypes = node.PyroscopeProfileTypes switch node.ABCIProtocol { case e2e.ProtocolUNIX: