Add builder demotion metrics

metrics/metrics.go

@@ -22,6 +22,8 @@ var (
 	GetPayloadLatencyHistogram   otelapi.Float64Histogram
 	PublishBlockLatencyHistogram otelapi.Float64Histogram
 
+	BuilderDemotionCount otelapi.Int64Counter
+
 	latencyBoundaries = otelapi.WithExplicitBucketBoundaries(func() []float64 {
 		base := math.Exp(math.Log(12.0) / 15.0)
 		res := make([]float64, 0, 31)
@@ -37,6 +39,7 @@ func Setup(ctx context.Context) error {
 		setupMeter, // must come first
 		setupGetPayloadLatency,
 		setupPublishBlockLatency,
+		setupBuilderDemotionCount,
 	} {
 		if err := setup(ctx); err != nil {
 			return err
@@ -98,3 +101,15 @@ func setupPublishBlockLatency(ctx context.Context) error {
 	}
 	return nil
 }
+
+func setupBuilderDemotionCount(ctx context.Context) error {
+	counter, err := meter.Int64Counter(
+		"builder_demotion_count",
+		otelapi.WithDescription("number of times a builder has been demoted"),
+	)
+	BuilderDemotionCount = counter
+	if err != nil {
+		return err
+	}
+	return nil
+}

services/api/service.go

@@ -41,6 +41,8 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/attribute"
+	otelapi "go.opentelemetry.io/otel/metric"
 	uberatomic "go.uber.org/atomic"
 	"golang.org/x/exp/slices"
 )
@@ -600,6 +602,15 @@ func (api *RelayAPI) simulateBlock(ctx context.Context, opts blockSimOptions) (r
 }
 
 func (api *RelayAPI) demoteBuilder(pubkey string, req *common.VersionedSubmitBlockRequest, simError error) {
+	metrics.BuilderDemotionCount.Add(
+		context.Background(),
+		1,
+		otelapi.WithAttributes(
+			attribute.String("builderPubkey", pubkey),
+			attribute.String("simError", simError.Error()),
+		),
+	)
+
 	builderEntry, ok := api.blockBuildersCache[pubkey]
 	if !ok {
 		api.log.Warnf("builder %v not in the builder cache", pubkey)