Skip to content

Commit

Permalink
feat: Add solver metrics (#435)
Browse files Browse the repository at this point in the history 
* refactor: Initialize tracer provider with noop provider

* refactor: Lift resource instantiation to SetupOTelSDK

We will also want to use this resource for meter and log providers.

* feat: Add solver metrics options

* feat: Add metrics config and provider

* chore: Configure services with metrics (or not)

We only configure metrics on the solver for now.

* feat: Add meter to solver controller

* feat: Add solver matcher metrics

* feat: Add solver system metrics

* feat: Add GetDealsAll store method

* feat: Add solver deal state metrics

* fix: Acquire lock in GetDealsAll

* refactor: Simplify metrics options check

* chore: Add report deal metrics trace errors
  • Loading branch information
@bgins
bgins authored Nov 21, 2024
1 parent c077ba8 commit bff3def
Show file tree
Hide file tree
Showing 18 changed files with 872 additions and 43 deletions.
2 changes: 1 addition & 1 deletion cmd/lilypad/jobcreator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func runJobCreator(cmd *cobra.Command, options jobcreator.JobCreatorOptions, net
commandCtx := system.NewCommandContext(cmd)
defer commandCtx.Cleanup()

telemetry, err := configureTelemetry(commandCtx.Ctx, system.JobCreatorService, network, options.Telemetry, options.Web3)
telemetry, err := configureTelemetry(commandCtx.Ctx, system.JobCreatorService, network, options.Telemetry, nil, options.Web3)
if err != nil {
log.Warn().Msgf("failed to setup opentelemetry: %s", err)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/lilypad/resource-provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func runResourceProvider(cmd *cobra.Command, options resourceprovider.ResourcePr
commandCtx := system.NewCommandContext(cmd)
defer commandCtx.Cleanup()

telemetry, err := configureTelemetry(commandCtx.Ctx, system.ResourceProviderService, network, options.Telemetry, options.Web3)
telemetry, err := configureTelemetry(commandCtx.Ctx, system.ResourceProviderService, network, options.Telemetry, nil, options.Web3)
if err != nil {
log.Warn().Msgf("failed to setup opentelemetry: %s", err)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/lilypad/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ func runJob(cmd *cobra.Command, options jobcreator.JobCreatorOptions, network st
commandCtx := system.NewCommandContext(cmd)
defer commandCtx.Cleanup()

telemetry, err := configureTelemetry(commandCtx.Ctx, system.JobCreatorService, network, options.Telemetry, options.Web3)
telemetry, err := configureTelemetry(commandCtx.Ctx, system.JobCreatorService, network, options.Telemetry, nil, options.Web3)
if err != nil {
log.Warn().Msgf("failed to setup opentelemetry: %s", err)
}
Expand Down
11 changes: 9 additions & 2 deletions cmd/lilypad/solver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,19 @@ func runSolver(cmd *cobra.Command, options solver.SolverOptions, network string)
commandCtx := system.NewCommandContext(cmd)
defer commandCtx.Cleanup()

telemetry, err := configureTelemetry(commandCtx.Ctx, system.SolverService, network, options.Telemetry, options.Web3)
telemetry, err := configureTelemetry(commandCtx.Ctx, system.SolverService, network, options.Telemetry, &options.Metrics, options.Web3)
if err != nil {
log.Warn().Msgf("failed to setup opentelemetry: %s", err)
}
commandCtx.Cm.RegisterCallbackWithContext(telemetry.Shutdown)
tracer := telemetry.TracerProvider.Tracer(system.GetOTelServiceName(system.SolverService))
meter := telemetry.MeterProvider.Meter(system.GetOTelServiceName(system.SolverService))

unregisterMetrics, err := system.NewMetrics(meter)
if err != nil {
log.Warn().Msgf("failed to start system metrics: %s", err)
}
commandCtx.Cm.RegisterCallback(unregisterMetrics)

web3SDK, err := web3.NewContractSDK(commandCtx.Ctx, options.Web3, tracer)
if err != nil {
Expand All @@ -55,7 +62,7 @@ func runSolver(cmd *cobra.Command, options solver.SolverOptions, network string)
return err
}

solverService, err := solver.NewSolver(options, solverStore, web3SDK, tracer)
solverService, err := solver.NewSolver(options, solverStore, web3SDK, tracer, meter)
if err != nil {
return err
}
Expand Down
19 changes: 18 additions & 1 deletion cmd/lilypad/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ func configureTelemetry(ctx context.Context,
service system.Service,
network string,
options system.TelemetryOptions,
metricsOptions *system.MetricsOptions,
web3Options web3.Web3Options,
) (*system.Telemetry, error) {
privateKey, err := web3.ParsePrivateKey(web3Options.PrivateKey)
Expand All @@ -61,7 +62,23 @@ func configureTelemetry(ctx context.Context,
Address: address.String(),
GPU: system.GetGPUInfo(),
}
telemetry, err := system.SetupOTelSDK(ctx, tc)

var mc system.MetricsConfig
if metricsOptions != nil {
mc = system.MetricsConfig{
MetricsURL: metricsOptions.URL,
MetricsToken: metricsOptions.Token,
Enabled: metricsOptions.Enable,
}
} else {
mc = system.MetricsConfig{
MetricsURL: "",
MetricsToken: "",
Enabled: false,
}
}

telemetry, err := system.SetupOTelSDK(ctx, tc, mc)

return &telemetry, err
}
Expand Down
34 changes: 20 additions & 14 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ require (
github.com/stretchr/testify v1.9.0
github.com/theckman/yacspin v0.13.12
go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.55.0
go.opentelemetry.io/otel v1.30.0
go.opentelemetry.io/otel v1.32.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0
go.opentelemetry.io/otel/sdk v1.28.0
go.opentelemetry.io/otel/trace v1.30.0
golang.org/x/crypto v0.25.0
go.opentelemetry.io/otel/sdk v1.32.0
go.opentelemetry.io/otel/trace v1.32.0
golang.org/x/crypto v0.28.0
gorgonia.org/cu v0.9.7-0.20240623234718-3cd40db700e9
k8s.io/apimachinery v0.29.0
)
Expand Down Expand Up @@ -62,6 +62,7 @@ require (
github.com/deckarep/golang-set/v2 v2.1.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/ebitengine/purego v0.8.1 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/ethereum/c-kzg-4844 v0.3.1 // indirect
github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
Expand All @@ -78,7 +79,7 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
Expand Down Expand Up @@ -160,6 +161,7 @@ require (
github.com/samber/lo v1.46.0 // indirect
github.com/sergi/go-diff v1.3.1 // indirect
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
github.com/shirou/gopsutil/v4 v4.24.10 // indirect
github.com/skeema/knownhosts v1.2.0 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/afero v1.10.0 // indirect
Expand All @@ -176,26 +178,30 @@ require (
github.com/whyrusleeping/cbor-gen v0.1.2 // indirect
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel/metric v1.30.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.32.0 // indirect
go.opentelemetry.io/otel/metric v1.32.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.32.0 // indirect
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/mod v0.19.0 // indirect
golang.org/x/net v0.27.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.24.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/net v0.30.0 // indirect
golang.org/x/sync v0.9.0 // indirect
golang.org/x/sys v0.27.0 // indirect
golang.org/x/text v0.20.0 // indirect
golang.org/x/tools v0.23.0 // indirect
golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
gonum.org/v1/gonum v0.15.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240730163845-b1a4ccb954bf // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect
google.golang.org/grpc v1.64.1 // indirect
google.golang.org/protobuf v1.34.2 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect
google.golang.org/grpc v1.67.1 // indirect
google.golang.org/protobuf v1.35.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
Expand Down
Loading

0 comments on commit bff3def

Please sign in to comment.