Downloader: don't fail when see unusual file, skip it (backward/forwa…

…rd compatibility) (#8170)

cl/phase1/forkchoice/on_block.go

@@ -2,11 +2,8 @@ package forkchoice
 
 import (
 	"fmt"
-	"runtime"
 	"time"
 
-	libcommon "github.com/ledgerwatch/erigon-lib/common"
-	"github.com/ledgerwatch/erigon-lib/common/dbg"
 	"github.com/ledgerwatch/log/v3"
 
 	"github.com/ledgerwatch/erigon/cl/cltypes"
@@ -100,9 +97,6 @@ func (f *ForkChoiceStore) OnBlock(block *cltypes.SignedBeaconBlock, newPayload,
 	if blockEpoch < currentEpoch {
 		f.updateCheckpoints(lastProcessedState.CurrentJustifiedCheckpoint().Copy(), lastProcessedState.FinalizedCheckpoint().Copy())
 	}
-	var m runtime.MemStats
-	dbg.ReadMemStats(&m)
-	log.Debug("OnBlock", "elapsed", time.Since(start), "alloc", libcommon.ByteCount(m.Alloc),
-		"sys", libcommon.ByteCount(m.Sys))
+	log.Debug("OnBlock", "elapsed", time.Since(start))
 	return nil
 }

cl/phase1/network/gossip_manager.go

@@ -2,15 +2,12 @@ package network
 
 import (
 	"context"
-	"runtime"
 	"sync"
 
 	"github.com/ledgerwatch/erigon/cl/freezer"
 	"github.com/ledgerwatch/erigon/cl/phase1/forkchoice"
 	"github.com/ledgerwatch/erigon/cl/sentinel/peers"
 
-	libcommon "github.com/ledgerwatch/erigon-lib/common"
-	"github.com/ledgerwatch/erigon-lib/common/dbg"
 	"github.com/ledgerwatch/erigon-lib/gointerfaces/sentinel"
 	"github.com/ledgerwatch/erigon-lib/types/ssz"
 	"github.com/ledgerwatch/erigon/cl/clparams"
@@ -102,14 +99,10 @@ func (g *GossipManager) onRecv(ctx context.Context, data *sentinel.GossipData, l
 			l["at"] = "sentinel peer count"
 			return err
 		}
-		var m runtime.MemStats
-		dbg.ReadMemStats(&m)
 
 		log.Debug("Received block via gossip",
 			"peers", count.Amount,
 			"slot", block.Block.Slot,
-			"alloc/sys", libcommon.ByteCount(m.Alloc)+"/"+libcommon.ByteCount(m.Sys),
-			"numGC", m.NumGC,
 		)
 
 		if err := freezer.PutObjectSSZIntoFreezer("signedBeaconBlock", "caplin_core", block.Block.Slot, block, g.recorder); err != nil {

go.mod

@@ -4,8 +4,8 @@ go 1.19
 
 require (
 	github.com/erigontech/mdbx-go v0.27.14
-	github.com/ledgerwatch/erigon-lib v0.0.0-20230906031217-cb36bf043c5c
-	github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230909101632-42a1d412f95f
+	github.com/ledgerwatch/erigon-lib v0.0.0-20230910084438-ac62ce517774
+	github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230908120722-853b40162b46
 	github.com/ledgerwatch/log/v3 v3.9.0
 	github.com/ledgerwatch/secp256k1 v1.0.0
 )

go.sum

@@ -497,10 +497,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0=
 github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
-github.com/ledgerwatch/erigon-lib v0.0.0-20230906031217-cb36bf043c5c h1:ZUgAQR5/2D72hoq+8nz9f8l6e5KQxQW4CqCo2lm4CfU=
-github.com/ledgerwatch/erigon-lib v0.0.0-20230906031217-cb36bf043c5c/go.mod h1:VLDj7l088dUzwBBSORGy5GGdUPM8XhLNMURmm1F1QMA=
-github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230909101632-42a1d412f95f h1:DVjZZpZBRTUXdJn6iZlwJkJ8zF0hwdBucIZFheNfF6w=
-github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230909101632-42a1d412f95f/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
+github.com/ledgerwatch/erigon-lib v0.0.0-20230910084438-ac62ce517774 h1:MQEecIsUqCkqvZ6N8v8665dVcJgQWdTEi2A0F98hOso=
+github.com/ledgerwatch/erigon-lib v0.0.0-20230910084438-ac62ce517774/go.mod h1:VLDj7l088dUzwBBSORGy5GGdUPM8XhLNMURmm1F1QMA=
+github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230908120722-853b40162b46 h1:YmP/prsc1YkrOzHhM4+ZGSPtpE+JTP0aOt+mxl20gGg=
+github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230908120722-853b40162b46/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
 github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk=
 github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE=
 github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=

turbo/snapshotsync/freezeblocks/block_snapshots.go

@@ -548,9 +548,8 @@ func (s *RoSnapshots) ReopenList(fileNames []string, optimistic bool) error {
 	var segmentsMaxSet bool
 Loop:
 	for _, fName := range fileNames {
-		f, err := snaptype.ParseFileName(s.dir, fName)
-		if err != nil {
-			s.logger.Warn("invalid segment name", "err", err, "name", fName)
+		f, ok := snaptype.ParseFileName(s.dir, fName)
+		if !ok {
 			continue
 		}
 		var processed bool = true
@@ -2243,7 +2242,10 @@ func (m *Merger) Merge(ctx context.Context, snapshots *RoSnapshots, mergeRanges
 
 		for _, t := range snaptype.AllSnapshotTypes {
 			segName := snaptype.SegmentFileName(r.from, r.to, t)
-			f, _ := snaptype.ParseFileName(snapDir, segName)
+			f, ok := snaptype.ParseFileName(snapDir, segName)
+			if !ok {
+				continue
+			}
 			if err := m.merge(ctx, toMerge[t], f.Path, logEvery); err != nil {
 				return fmt.Errorf("mergeByAppendSegments: %w", err)
 			}

turbo/snapshotsync/freezeblocks/block_snapshots_test.go

@@ -212,23 +212,23 @@ func TestParseCompressedFileName(t *testing.T) {
 		require.NoError(err)
 		return s.Name()
 	}
-	_, err := snaptype.ParseFileName("", stat("a"))
-	require.Error(err)
-	_, err = snaptype.ParseFileName("", stat("1-a"))
-	require.Error(err)
-	_, err = snaptype.ParseFileName("", stat("1-2-a"))
-	require.Error(err)
-	_, err = snaptype.ParseFileName("", stat("1-2-bodies.info"))
-	require.Error(err)
-	_, err = snaptype.ParseFileName("", stat("1-2-bodies.seg"))
-	require.Error(err)
-	_, err = snaptype.ParseFileName("", stat("v2-1-2-bodies.seg"))
-	require.Error(err)
-	_, err = snaptype.ParseFileName("", stat("v0-1-2-bodies.seg"))
-	require.Error(err)
-
-	f, err := snaptype.ParseFileName("", stat("v1-1-2-bodies.seg"))
-	require.NoError(err)
+	_, ok := snaptype.ParseFileName("", stat("a"))
+	require.False(ok)
+	_, ok = snaptype.ParseFileName("", stat("1-a"))
+	require.False(ok)
+	_, ok = snaptype.ParseFileName("", stat("1-2-a"))
+	require.False(ok)
+	_, ok = snaptype.ParseFileName("", stat("1-2-bodies.info"))
+	require.False(ok)
+	_, ok = snaptype.ParseFileName("", stat("1-2-bodies.seg"))
+	require.False(ok)
+	_, ok = snaptype.ParseFileName("", stat("v2-1-2-bodies.seg"))
+	require.True(ok)
+	_, ok = snaptype.ParseFileName("", stat("v0-1-2-bodies.seg"))
+	require.True(ok)
+
+	f, ok := snaptype.ParseFileName("", stat("v1-1-2-bodies.seg"))
+	require.True(ok)
 	require.Equal(f.T, snaptype.Bodies)
 	require.Equal(1_000, int(f.From))
 	require.Equal(2_000, int(f.To))

turbo/snapshotsync/freezeblocks/bor_snapshots.go

@@ -797,9 +797,9 @@ func (s *BorRoSnapshots) ReopenList(fileNames []string, optimistic bool) error {
 	var segmentsMaxSet bool
 Loop:
 	for _, fName := range fileNames {
-		f, err := snaptype.ParseFileName(s.dir, fName)
-		if err != nil {
-			s.logger.Warn("invalid segment name", "err", err, "name", fName)
+		f, ok := snaptype.ParseFileName(s.dir, fName)
+		if !ok {
+			s.logger.Trace("BorRoSnapshots.ReopenList: skip", "file", fName)
 			continue
 		}
 
@@ -1136,7 +1136,10 @@ func (m *BorMerger) Merge(ctx context.Context, snapshots *BorRoSnapshots, mergeR
 
 		for _, t := range []snaptype.Type{snaptype.BorEvents, snaptype.BorSpans} {
 			segName := snaptype.SegmentFileName(r.from, r.to, t)
-			f, _ := snaptype.ParseFileName(snapDir, segName)
+			f, ok := snaptype.ParseFileName(snapDir, segName)
+			if !ok {
+				continue
+			}
 			if err := m.merge(ctx, toMerge[t], f.Path, logEvery); err != nil {
 				return fmt.Errorf("mergeByAppendSegments: %w", err)
 			}