cleanup validator not found

ethpandaops · Jan 23, 2025 · 69d59c2 · 69d59c2
1 parent 1d357f0
commit 69d59c2
Show file tree

Hide file tree

Showing 2 changed files with 57 additions and 2 deletions.
diff --git a/pkg/monitor/beaconchain/request.go b/pkg/monitor/beaconchain/request.go
@@ -70,9 +70,18 @@ func (c *client) getValidators(ctx context.Context, pubkeys []string) (*Response
 		return nil, err
 	}
 
+	// response can be a single or list of validators
+	// still returns OK if validators are not found
 	resp := new(Response[[]Validator])
 	if err := json.Unmarshal(data, resp); err != nil {
-		return nil, err
+		// Try single validator response
+		singleResp := new(Response[Validator])
+		if err2 := json.Unmarshal(data, singleResp); err2 != nil {
+			return nil, err
+		}
+
+		resp.Status = singleResp.Status
+		resp.Data = []Validator{singleResp.Data}
 	}
 
 	return resp, nil

diff --git a/pkg/monitor/service/validator/group/group.go b/pkg/monitor/service/validator/group/group.go
@@ -145,11 +145,36 @@ func (g *Group) checkBeaconAPI(ctx context.Context, state *State) {
 	for _, node := range g.ethereumPool.GetHealthyBeaconNodes() {
 		validators, err := node.Node().FetchValidators(ctx, "head", nil, g.pubkeys)
 		if err != nil {
-			g.log.WithError(err).WithField("node", node.Name()).Error("Error fetching validators")
+			g.log.WithError(err).WithField("source", node.Name()).Error("Error fetching validators")
+
+			for _, pubkey := range g.pubkeys {
+				g.metrics.UpdateStatus(MetricsStatusUnknown, []string{g.name, pubkey.String(), node.Name()})
+			}
+
+			continue
 		}
 
+		foundPubkeys := make(map[string]bool)
+
 		for _, validator := range validators {
 			g.updateValidatorBeaconAPI(validator, node.Name(), state)
+
+			pubkey, err := validator.PubKey(ctx)
+			if err != nil {
+				g.log.WithError(err).WithField("source", node.Name()).WithField("validator_index", validator.Index).Error("Error getting pubkey")
+
+				continue
+			}
+
+			foundPubkeys[pubkey.String()] = true
+		}
+
+		for _, pubkey := range g.pubkeys {
+			if !foundPubkeys[pubkey.String()] {
+				g.log.WithField("pubkey", pubkey.String()).WithField("source", node.Name()).Warn("Validator not found")
+
+				g.metrics.UpdateStatus(MetricsStatusUnknown, []string{g.name, pubkey.String(), node.Name()})
+			}
 		}
 	}
 }
@@ -205,19 +230,40 @@ func (g *Group) getValidatorsBeaconchain(ctx context.Context, validators []strin
 	if len(validators) == 1 {
 		response, err := g.beaconchain.GetValidator(ctx, validators[0])
 		if err != nil {
+			g.log.WithError(err).WithField("source", "beaconcha.in").WithField("pubkey", validators[0]).Error("Error getting validator")
+			g.metrics.UpdateStatus(MetricsStatusUnknown, []string{g.name, validators[0], "beaconcha.in"})
+
 			return err
 		}
 
 		g.updateValidatorBeaconchain(response, state)
 	} else {
 		response, err := g.beaconchain.GetValidators(ctx, validators)
 		if err != nil {
+			for _, pubkey := range validators {
+				g.metrics.UpdateStatus(MetricsStatusUnknown, []string{g.name, pubkey, "beaconcha.in"})
+			}
+
+			g.log.WithError(err).WithField("source", "beaconcha.in").WithField("pubkeys", validators).Error("Error getting validators")
+
 			return err
 		}
 
+		foundPubkeys := make(map[string]bool)
+
 		for _, validator := range response {
 			if validator != nil {
 				g.updateValidatorBeaconchain(validator, state)
+
+				foundPubkeys[validator.Pubkey] = true
+			}
+		}
+
+		for _, pubkey := range g.pubkeys {
+			if !foundPubkeys[pubkey.String()] {
+				g.log.WithField("pubkey", pubkey.String()).WithField("source", "beaconcha.in").Warn("Validator not found")
+
+				g.metrics.UpdateStatus(MetricsStatusUnknown, []string{g.name, pubkey.String(), "beaconcha.in"})
 			}
 		}
 	}