fix: Improve logging and fix function

op-monitorism/faultproof_withdrawals/README.md

@@ -38,7 +38,7 @@ DESCRIPTION:
 
 OPTIONS:
    --l1.geth.url value             L1 execution layer node URL [$FAULTPROOF_WITHDRAWAL_MON_L1_GETH_URL]
-   --l2.node.url value             L2 rollup node consensus layer (op-node) URL [$FAULTPROOF_WITHDRAWAL_MON_L2_OP_NODE_URL]
+   --l2.node.url value             [DEPRECATED] L2 rollup node consensus layer (op-node) URL [$FAULTPROOF_WITHDRAWAL_MON_L2_OP_NODE_URL]
    --l2.geth.url value             L2 OP Stack execution layer client(op-geth) URL [$FAULTPROOF_WITHDRAWAL_MON_L2_OP_GETH_URL]
    --event.block.range value       Max block range when scanning for events (default: 1000) [$FAULTPROOF_WITHDRAWAL_MON_EVENT_BLOCK_RANGE]
    --start.block.height value      Starting height to scan for events. This will take precedence if set. (default: 0) [$FAULTPROOF_WITHDRAWAL_MON_START_BLOCK_HEIGHT]
@@ -59,7 +59,7 @@ OPTIONS:
 
 ```bash
 L1_GETH_URL="https://..."
-L2_OP_NODE_URL="https://..."
+L2_OP_NODE_URL="https://..."  # [DEPRECATED] This URL is no longer required
 L2_OP_GETH_URL="https://..."
 
 export MONITORISM_LOOP_INTERVAL_MSEC=100

op-monitorism/faultproof_withdrawals/cli.go

@@ -11,9 +11,10 @@ import (
 )
 
 const (
-	L1GethURLFlagName = "l1.geth.url"
-	L2NodeURLFlagName = "l2.node.url"
-	L2GethURLFlagName = "l2.geth.url"
+	L1GethURLFlagName        = "l1.geth.url"
+	L2NodeURLFlagName        = "l2.node.url"
+	L2GethURLFlagName        = "l2.geth.url"
+	L2GethBackupURLsFlagName = "l2.geth.backup.urls"
 
 	EventBlockRangeFlagName           = "event.block.range"
 	StartingL1BlockHeightFlagName     = "start.block.height"
@@ -23,9 +24,10 @@ const (
 )
 
 type CLIConfig struct {
-	L1GethURL   string
-	L2OpGethURL string
-	L2OpNodeURL string
+	L1GethURL        string
+	L2OpGethURL      string
+	L2OpNodeURL      string
+	L2GethBackupURLs []string
 
 	EventBlockRange           uint64
 	StartingL1BlockHeight     int64
@@ -38,7 +40,8 @@ func ReadCLIFlags(ctx *cli.Context) (CLIConfig, error) {
 	cfg := CLIConfig{
 		L1GethURL:                 ctx.String(L1GethURLFlagName),
 		L2OpGethURL:               ctx.String(L2GethURLFlagName),
-		L2OpNodeURL:               ctx.String(L2NodeURLFlagName),
+		L2GethBackupURLs:          ctx.StringSlice(L2GethBackupURLsFlagName),
+		L2OpNodeURL:               "", // Ignored since deprecated
 		EventBlockRange:           ctx.Uint64(EventBlockRangeFlagName),
 		StartingL1BlockHeight:     ctx.Int64(StartingL1BlockHeightFlagName),
 		HoursInThePastToStartFrom: ctx.Uint64(HoursInThePastToStartFromFlagName),
@@ -62,14 +65,20 @@ func CLIFlags(envVar string) []cli.Flag {
 		},
 		&cli.StringFlag{
 			Name:    L2NodeURLFlagName,
-			Usage:   "L2 rollup node consensus layer (op-node) URL",
+			Usage:   "[DEPRECATED] L2 rollup node consensus layer (op-node) URL - this flag is ignored",
 			EnvVars: opservice.PrefixEnvVar(envVar, "L2_OP_NODE_URL"),
+			Hidden:  true,
 		},
 		&cli.StringFlag{
 			Name:    L2GethURLFlagName,
 			Usage:   "L2 OP Stack execution layer client(op-geth) URL",
 			EnvVars: opservice.PrefixEnvVar(envVar, "L2_OP_GETH_URL"),
 		},
+		&cli.StringSliceFlag{
+			Name:    L2GethBackupURLsFlagName,
+			Usage:   "Backup L2 OP Stack execution layer client URLs (format: name=url,name2=url2)",
+			EnvVars: opservice.PrefixEnvVar(envVar, "L2_OP_GETH_BACKUP_URLS"),
+		},
 		&cli.Uint64Flag{
 			Name:    EventBlockRangeFlagName,
 			Usage:   "Max block range when scanning for events",

op-monitorism/faultproof_withdrawals/monitor.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"math/big"
+	"strings"
 	"time"
 
 	"github.com/ethereum-optimism/monitorism/op-monitorism/faultproof_withdrawals/validator"
@@ -26,12 +27,12 @@ type Monitor struct {
 	ctx context.Context
 
 	// user arguments
-	l1GethClient   *ethclient.Client
-	l2OpGethClient *ethclient.Client
-	l2OpNodeClient *ethclient.Client
-	l1ChainID      *big.Int
-	l2ChainID      *big.Int
-	maxBlockRange  uint64
+	l1GethClient    *ethclient.Client
+	l2OpGethClient  *ethclient.Client
+	l2BackupClients map[string]*ethclient.Client
+	l1ChainID       *big.Int
+	l2ChainID       *big.Int
+	maxBlockRange   uint64
 
 	// helpers
 	withdrawalValidator validator.ProvenWithdrawalValidator
@@ -51,16 +52,32 @@ func NewMonitor(ctx context.Context, log log.Logger, m metrics.Factory, cfg CLIC
 	if err != nil {
 		return nil, fmt.Errorf("failed to dial l1: %w", err)
 	}
+
+	l1ChainID, err := l1GethClient.ChainID(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get l1 chain id: %w", err)
+	}
+
 	l2OpGethClient, err := ethclient.Dial(cfg.L2OpGethURL)
 	if err != nil {
 		return nil, fmt.Errorf("failed to dial l2: %w", err)
 	}
-	l2OpNodeClient, err := ethclient.Dial(cfg.L2OpNodeURL)
+	l2ChainID, err := l2OpGethClient.ChainID(ctx)
 	if err != nil {
-		return nil, fmt.Errorf("failed to dial l2: %w", err)
+		return nil, fmt.Errorf("failed to get l2 chain id: %w", err)
 	}
 
-	withdrawalValidator, err := validator.NewWithdrawalValidator(ctx, l1GethClient, l2OpGethClient, l2OpNodeClient, cfg.OptimismPortalAddress)
+	// if backup urls are provided, create a backup client for each
+	var l2OpGethBackupClients map[string]*ethclient.Client
+	if len(cfg.L2GethBackupURLs) > 0 {
+		l2OpGethBackupClients, err = GethBackupClientsDictionary(ctx, cfg.L2GethBackupURLs, l2ChainID)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create backup clients: %w", err)
+		}
+
+	}
+
+	withdrawalValidator, err := validator.NewWithdrawalValidator(ctx, log, l1GethClient, l2OpGethClient, l2OpGethBackupClients, cfg.OptimismPortalAddress)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create withdrawal validator: %w", err)
 	}
@@ -70,24 +87,15 @@ func NewMonitor(ctx context.Context, log log.Logger, m metrics.Factory, cfg CLIC
 		return nil, fmt.Errorf("failed to query latest block number: %w", err)
 	}
 
-	l1ChainID, err := l1GethClient.ChainID(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get l1 chain id: %w", err)
-	}
-	l2ChainID, err := l2OpGethClient.ChainID(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get l2 chain id: %w", err)
-	}
-
 	metrics := NewMetrics(m)
 
 	ret := &Monitor{
 		log: log,
 
-		ctx:            ctx,
-		l1GethClient:   l1GethClient,
-		l2OpGethClient: l2OpGethClient,
-		l2OpNodeClient: l2OpNodeClient,
+		ctx:             ctx,
+		l1GethClient:    l1GethClient,
+		l2OpGethClient:  l2OpGethClient,
+		l2BackupClients: l2OpGethBackupClients,
 
 		l1ChainID: l1ChainID,
 		l2ChainID: l2ChainID,
@@ -124,7 +132,11 @@ func NewMonitor(ctx context.Context, log log.Logger, m metrics.Factory, cfg CLIC
 		startingL1BlockHeight = uint64(cfg.StartingL1BlockHeight)
 	}
 
-	state, err := NewState(log, startingL1BlockHeight, latestL1Height, ret.withdrawalValidator.GetLatestL2Height())
+	latestL2Height, err := ret.withdrawalValidator.L2NodeHelper.BlockNumber()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get latest L2 height: %w", err)
+	}
+	state, err := NewState(log, startingL1BlockHeight, latestL1Height, latestL2Height)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create state: %w", err)
 	}
@@ -137,6 +149,30 @@ func NewMonitor(ctx context.Context, log log.Logger, m metrics.Factory, cfg CLIC
 	return ret, nil
 }
 
+func GethBackupClientsDictionary(ctx context.Context, L2GethBackupURLs []string, l2ChainID *big.Int) (map[string]*ethclient.Client, error) {
+	dictionary := make(map[string]*ethclient.Client)
+	for _, rawURL := range L2GethBackupURLs {
+		parts := strings.Split(rawURL, "=")
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("invalid backup URL format, expected name=url, got %s", rawURL)
+		}
+		name, url := parts[0], parts[1]
+		backupClient, err := ethclient.Dial(url)
+		if err != nil {
+			return nil, fmt.Errorf("failed to dial l2 backup, error: %w", err)
+		}
+		backupChainID, err := backupClient.ChainID(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get backup L2 chain ID, error: %w", err)
+		}
+		if backupChainID.Cmp(l2ChainID) != 0 {
+			return nil, fmt.Errorf("backup L2 client chain ID mismatch, expected: %d, got: %d", l2ChainID, backupChainID)
+		}
+		dictionary[name] = backupClient
+	}
+	return dictionary, nil
+}
+
 // getBlockAtApproximateTimeBinarySearch finds the block number corresponding to the timestamp from two weeks ago using a binary search approach.
 func (m *Monitor) getBlockAtApproximateTimeBinarySearch(ctx context.Context, client *ethclient.Client, latestBlockNumber *big.Int, hoursInThePast *big.Int) (*big.Int, error) {
 
@@ -236,6 +272,7 @@ func (m *Monitor) Run(ctx context.Context) {
 	start := m.state.nextL1Height
 
 	stop, err := m.GetMaxBlock()
+	m.state.nodeConnections++
 	if err != nil {
 		m.state.nodeConnectionFailures++
 		m.log.Error("failed to get max block", "error", err)
@@ -244,6 +281,7 @@ func (m *Monitor) Run(ctx context.Context) {
 
 	// review previous invalidProposalWithdrawalsEvents
 	err = m.ConsumeEvents(m.state.potentialAttackOnInProgressGames)
+	m.state.nodeConnections++
 	if err != nil {
 		m.state.nodeConnectionFailures++
 		m.log.Error("failed to consume events", "error", err)
@@ -253,6 +291,7 @@ func (m *Monitor) Run(ctx context.Context) {
 	// get new events
 	m.log.Info("getting enriched withdrawal events", "start", fmt.Sprintf("%d", start), "stop", fmt.Sprintf("%d", stop))
 	newEvents, err := m.withdrawalValidator.GetEnrichedWithdrawalsEventsMap(start, &stop)
+	m.state.nodeConnections++
 	if err != nil {
 		if start >= stop {
 			m.log.Info("no new events to process", "start", start, "stop", stop)
@@ -267,6 +306,7 @@ func (m *Monitor) Run(ctx context.Context) {
 	}
 
 	err = m.ConsumeEvents(newEvents)
+	m.state.nodeConnections++
 	if err != nil {
 		m.state.nodeConnectionFailures++
 		m.log.Error("failed to consume events", "error", err)
@@ -288,16 +328,20 @@ func (m *Monitor) ConsumeEvents(enrichedWithdrawalEvents map[common.Hash]*valida
 		}
 		m.log.Info("processing withdrawal event", "event", enrichedWithdrawalEvent)
 		err := m.withdrawalValidator.UpdateEnrichedWithdrawalEvent(enrichedWithdrawalEvent)
+		if err != nil {
+			m.log.Error("failed to update enriched withdrawal event", "error", err)
+			return err
+		}
 		//upgrade state to the latest L2 height	after the event is processed
-		m.state.latestL2Height = m.withdrawalValidator.GetLatestL2Height()
+		m.state.latestL2Height, err = m.withdrawalValidator.L2NodeHelper.BlockNumber()
 		if err != nil {
 			m.log.Error("failed to update enriched withdrawal event", "error", err)
 			return err
 		}
 
 		err = m.ConsumeEvent(enrichedWithdrawalEvent)
 		if err != nil {
-			m.log.Error("failed to consume event", "error", err)
+			m.log.Error("failed to consume event", "error", err, "enrichedWithdrawalEvent", enrichedWithdrawalEvent)
 			return err
 		}
 	}
@@ -313,7 +357,7 @@ func (m *Monitor) ConsumeEvent(enrichedWithdrawalEvent *validator.EnrichedProven
 	}
 	valid, err := m.withdrawalValidator.IsWithdrawalEventValid(enrichedWithdrawalEvent)
 	if err != nil {
-		m.log.Error("failed to check if forgery detected", "error", err)
+		m.log.Error("failed to check if forgery detected", "error", err, "enrichedWithdrawalEvent", enrichedWithdrawalEvent)
 		return err
 	}
 

op-monitorism/faultproof_withdrawals/state.go

@@ -29,6 +29,7 @@ type State struct {
 	withdrawalsProcessed uint64 // This counts the withdrawals that have being completed and processed and we are not tracking anymore. eventProcessed >= withdrawalsProcessed. withdrawalsProcessed does not includes potential attacks with games in progress.
 
 	nodeConnectionFailures uint64
+	nodeConnections        uint64
 
 	// possible attacks detected
 
@@ -74,6 +75,7 @@ func NewState(logger log.Logger, nextL1Height uint64, latestL1Height uint64, lat
 
 		withdrawalsProcessed:   0,
 		nodeConnectionFailures: 0,
+		nodeConnections:        0,
 
 		nextL1Height:    nextL1Height,
 		latestL1Height:  latestL1Height,
@@ -100,7 +102,7 @@ func (s *State) LogState() {
 
 		"eventsProcessed", fmt.Sprintf("%d", s.eventsProcessed),
 		"nodeConnectionFailures", fmt.Sprintf("%d", s.nodeConnectionFailures),
-
+		"nodeConnections", fmt.Sprintf("%d", s.nodeConnections),
 		"potentialAttackOnDefenderWinsGames", fmt.Sprintf("%d", s.numberOfPotentialAttacksOnDefenderWinsGames),
 		"potentialAttackOnInProgressGames", fmt.Sprintf("%d", s.numberOfPotentialAttackOnInProgressGames),
 		"suspiciousEventsOnChallengerWinsGames", fmt.Sprintf("%d", s.numberOfSuspiciousEventsOnChallengerWinsGames),
@@ -185,8 +187,8 @@ type Metrics struct {
 	EventsProcessedCounter      prometheus.Counter
 	WithdrawalsProcessedCounter prometheus.Counter
 
-	NodeConnectionFailuresCounter prometheus.Counter
-
+	NodeConnectionFailuresCounter              prometheus.Counter
+	NodeConnectionsCounter                     prometheus.Counter
 	PotentialAttackOnDefenderWinsGamesGauge    prometheus.Gauge
 	PotentialAttackOnInProgressGamesGauge      prometheus.Gauge
 	SuspiciousEventsOnChallengerWinsGamesGauge prometheus.Gauge
@@ -199,6 +201,7 @@ type Metrics struct {
 	previousEventsProcessed        uint64
 	previousWithdrawalsProcessed   uint64
 	previousNodeConnectionFailures uint64
+	previousNodeConnections        uint64
 }
 
 func (m *Metrics) String() string {
@@ -212,6 +215,7 @@ func (m *Metrics) String() string {
 	eventsProcessedCounterValue, _ := GetCounterValue(m.EventsProcessedCounter)
 
 	nodeConnectionFailuresCounterValue, _ := GetCounterValue(m.NodeConnectionFailuresCounter)
+	nodeConnectionsCounterValue, _ := GetCounterValue(m.NodeConnectionsCounter)
 
 	potentialAttackOnDefenderWinsGamesGaugeValue, _ := GetGaugeValue(m.PotentialAttackOnDefenderWinsGamesGauge)
 	potentialAttackOnInProgressGamesGaugeValue, _ := GetGaugeValue(m.PotentialAttackOnInProgressGamesGauge)
@@ -220,7 +224,7 @@ func (m *Metrics) String() string {
 	invalidProposalWithdrawalsEventsGaugeVecValue, _ := GetGaugeVecValue(m.PotentialAttackOnInProgressGamesGaugeVec, prometheus.Labels{})
 
 	return fmt.Sprintf(
-		"Up: %d\nInitialL1HeightGauge: %d\nNextL1HeightGauge: %d\nLatestL1HeightGauge: %d\n latestL2HeightGaugeValue: %d\n eventsProcessedCounterValue: %d\nwithdrawalsProcessedCounterValue: %d\nnodeConnectionFailuresCounterValue: %d\n potentialAttackOnDefenderWinsGamesGaugeValue: %d\n potentialAttackOnInProgressGamesGaugeValue: %d\n  forgeriesWithdrawalsEventsGaugeVecValue: %d\n invalidProposalWithdrawalsEventsGaugeVecValue: %d\n previousEventsProcessed: %d\n previousWithdrawalsProcessed: %d\n previousNodeConnectionFailures: %d\n",
+		"Up: %d\nInitialL1HeightGauge: %d\nNextL1HeightGauge: %d\nLatestL1HeightGauge: %d\n latestL2HeightGaugeValue: %d\n eventsProcessedCounterValue: %d\nwithdrawalsProcessedCounterValue: %d\nnodeConnectionFailuresCounterValue: %d\nnodeConnectionsCounterValue: %d\n potentialAttackOnDefenderWinsGamesGaugeValue: %d\n potentialAttackOnInProgressGamesGaugeValue: %d\n  forgeriesWithdrawalsEventsGaugeVecValue: %d\n invalidProposalWithdrawalsEventsGaugeVecValue: %d\n previousEventsProcessed: %d\n previousWithdrawalsProcessed: %d\n previousNodeConnectionFailures: %d\n previousNodeConnections: %d\n",
 		uint64(upGaugeValue),
 		uint64(initialL1HeightGaugeValue),
 		uint64(nextL1HeightGaugeValue),
@@ -229,13 +233,15 @@ func (m *Metrics) String() string {
 		uint64(eventsProcessedCounterValue),
 		uint64(withdrawalsProcessedCounterValue),
 		uint64(nodeConnectionFailuresCounterValue),
+		uint64(nodeConnectionsCounterValue),
 		uint64(potentialAttackOnDefenderWinsGamesGaugeValue),
 		uint64(potentialAttackOnInProgressGamesGaugeValue),
 		uint64(forgeriesWithdrawalsEventsGaugeVecValue),
 		uint64(invalidProposalWithdrawalsEventsGaugeVecValue),
 		m.previousEventsProcessed,
 		m.previousWithdrawalsProcessed,
 		m.previousNodeConnectionFailures,
+		m.previousNodeConnections,
 	)
 }
 
@@ -316,6 +322,11 @@ func NewMetrics(m metrics.Factory) *Metrics {
 			Name:      "node_connection_failures_total",
 			Help:      "Total number of node connection failures",
 		}),
+		NodeConnectionsCounter: m.NewCounter(prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Name:      "node_connections_total",
+			Help:      "Total number of node connections",
+		}),
 		PotentialAttackOnDefenderWinsGamesGauge: m.NewGauge(prometheus.GaugeOpts{
 			Namespace: MetricsNamespace,
 			Name:      "potential_attack_on_defender_wins_games_count",