From 2318138c6cf8cbdc12aa00903bffcd432e2dd51c Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Sun, 10 Sep 2023 15:46:30 +0700 Subject: [PATCH] Downloader: don't fail when see unusual file, skip it (backward/forward compatibility) (#8170) --- cl/phase1/forkchoice/on_block.go | 8 +---- cl/phase1/network/gossip_manager.go | 7 ---- go.mod | 4 +-- go.sum | 8 ++--- .../freezeblocks/block_snapshots.go | 10 +++--- .../freezeblocks/block_snapshots_test.go | 34 +++++++++---------- .../freezeblocks/bor_snapshots.go | 11 +++--- 7 files changed, 37 insertions(+), 45 deletions(-) diff --git a/cl/phase1/forkchoice/on_block.go b/cl/phase1/forkchoice/on_block.go index 5030d56882a..1db03f35e21 100644 --- a/cl/phase1/forkchoice/on_block.go +++ b/cl/phase1/forkchoice/on_block.go @@ -2,11 +2,8 @@ package forkchoice import ( "fmt" - "runtime" "time" - libcommon "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon-lib/common/dbg" "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/erigon/cl/cltypes" @@ -100,9 +97,6 @@ func (f *ForkChoiceStore) OnBlock(block *cltypes.SignedBeaconBlock, newPayload, if blockEpoch < currentEpoch { f.updateCheckpoints(lastProcessedState.CurrentJustifiedCheckpoint().Copy(), lastProcessedState.FinalizedCheckpoint().Copy()) } - var m runtime.MemStats - dbg.ReadMemStats(&m) - log.Debug("OnBlock", "elapsed", time.Since(start), "alloc", libcommon.ByteCount(m.Alloc), - "sys", libcommon.ByteCount(m.Sys)) + log.Debug("OnBlock", "elapsed", time.Since(start)) return nil } diff --git a/cl/phase1/network/gossip_manager.go b/cl/phase1/network/gossip_manager.go index 3e803e31ad6..2d15095c4d0 100644 --- a/cl/phase1/network/gossip_manager.go +++ b/cl/phase1/network/gossip_manager.go @@ -2,15 +2,12 @@ package network import ( "context" - "runtime" "sync" "github.com/ledgerwatch/erigon/cl/freezer" "github.com/ledgerwatch/erigon/cl/phase1/forkchoice" "github.com/ledgerwatch/erigon/cl/sentinel/peers" - libcommon "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon-lib/common/dbg" "github.com/ledgerwatch/erigon-lib/gointerfaces/sentinel" "github.com/ledgerwatch/erigon-lib/types/ssz" "github.com/ledgerwatch/erigon/cl/clparams" @@ -102,14 +99,10 @@ func (g *GossipManager) onRecv(ctx context.Context, data *sentinel.GossipData, l l["at"] = "sentinel peer count" return err } - var m runtime.MemStats - dbg.ReadMemStats(&m) log.Debug("Received block via gossip", "peers", count.Amount, "slot", block.Block.Slot, - "alloc/sys", libcommon.ByteCount(m.Alloc)+"/"+libcommon.ByteCount(m.Sys), - "numGC", m.NumGC, ) if err := freezer.PutObjectSSZIntoFreezer("signedBeaconBlock", "caplin_core", block.Block.Slot, block, g.recorder); err != nil { diff --git a/go.mod b/go.mod index 3d470d2c48c..ff210362e73 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.19 require ( github.com/erigontech/mdbx-go v0.27.14 - github.com/ledgerwatch/erigon-lib v0.0.0-20230906031217-cb36bf043c5c - github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230909101632-42a1d412f95f + github.com/ledgerwatch/erigon-lib v0.0.0-20230910084438-ac62ce517774 + github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230908120722-853b40162b46 github.com/ledgerwatch/log/v3 v3.9.0 github.com/ledgerwatch/secp256k1 v1.0.0 ) diff --git a/go.sum b/go.sum index b14c98f0954..5ac1a5857d2 100644 --- a/go.sum +++ b/go.sum @@ -497,10 +497,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230906031217-cb36bf043c5c h1:ZUgAQR5/2D72hoq+8nz9f8l6e5KQxQW4CqCo2lm4CfU= -github.com/ledgerwatch/erigon-lib v0.0.0-20230906031217-cb36bf043c5c/go.mod h1:VLDj7l088dUzwBBSORGy5GGdUPM8XhLNMURmm1F1QMA= -github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230909101632-42a1d412f95f h1:DVjZZpZBRTUXdJn6iZlwJkJ8zF0hwdBucIZFheNfF6w= -github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230909101632-42a1d412f95f/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= +github.com/ledgerwatch/erigon-lib v0.0.0-20230910084438-ac62ce517774 h1:MQEecIsUqCkqvZ6N8v8665dVcJgQWdTEi2A0F98hOso= +github.com/ledgerwatch/erigon-lib v0.0.0-20230910084438-ac62ce517774/go.mod h1:VLDj7l088dUzwBBSORGy5GGdUPM8XhLNMURmm1F1QMA= +github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230908120722-853b40162b46 h1:YmP/prsc1YkrOzHhM4+ZGSPtpE+JTP0aOt+mxl20gGg= +github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230908120722-853b40162b46/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk= github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE= github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ= diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots.go b/turbo/snapshotsync/freezeblocks/block_snapshots.go index da84cf4a2f6..ff7be95da4d 100644 --- a/turbo/snapshotsync/freezeblocks/block_snapshots.go +++ b/turbo/snapshotsync/freezeblocks/block_snapshots.go @@ -548,9 +548,8 @@ func (s *RoSnapshots) ReopenList(fileNames []string, optimistic bool) error { var segmentsMaxSet bool Loop: for _, fName := range fileNames { - f, err := snaptype.ParseFileName(s.dir, fName) - if err != nil { - s.logger.Warn("invalid segment name", "err", err, "name", fName) + f, ok := snaptype.ParseFileName(s.dir, fName) + if !ok { continue } var processed bool = true @@ -2243,7 +2242,10 @@ func (m *Merger) Merge(ctx context.Context, snapshots *RoSnapshots, mergeRanges for _, t := range snaptype.AllSnapshotTypes { segName := snaptype.SegmentFileName(r.from, r.to, t) - f, _ := snaptype.ParseFileName(snapDir, segName) + f, ok := snaptype.ParseFileName(snapDir, segName) + if !ok { + continue + } if err := m.merge(ctx, toMerge[t], f.Path, logEvery); err != nil { return fmt.Errorf("mergeByAppendSegments: %w", err) } diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots_test.go b/turbo/snapshotsync/freezeblocks/block_snapshots_test.go index 7e8294cfdbe..68a3508d412 100644 --- a/turbo/snapshotsync/freezeblocks/block_snapshots_test.go +++ b/turbo/snapshotsync/freezeblocks/block_snapshots_test.go @@ -212,23 +212,23 @@ func TestParseCompressedFileName(t *testing.T) { require.NoError(err) return s.Name() } - _, err := snaptype.ParseFileName("", stat("a")) - require.Error(err) - _, err = snaptype.ParseFileName("", stat("1-a")) - require.Error(err) - _, err = snaptype.ParseFileName("", stat("1-2-a")) - require.Error(err) - _, err = snaptype.ParseFileName("", stat("1-2-bodies.info")) - require.Error(err) - _, err = snaptype.ParseFileName("", stat("1-2-bodies.seg")) - require.Error(err) - _, err = snaptype.ParseFileName("", stat("v2-1-2-bodies.seg")) - require.Error(err) - _, err = snaptype.ParseFileName("", stat("v0-1-2-bodies.seg")) - require.Error(err) - - f, err := snaptype.ParseFileName("", stat("v1-1-2-bodies.seg")) - require.NoError(err) + _, ok := snaptype.ParseFileName("", stat("a")) + require.False(ok) + _, ok = snaptype.ParseFileName("", stat("1-a")) + require.False(ok) + _, ok = snaptype.ParseFileName("", stat("1-2-a")) + require.False(ok) + _, ok = snaptype.ParseFileName("", stat("1-2-bodies.info")) + require.False(ok) + _, ok = snaptype.ParseFileName("", stat("1-2-bodies.seg")) + require.False(ok) + _, ok = snaptype.ParseFileName("", stat("v2-1-2-bodies.seg")) + require.True(ok) + _, ok = snaptype.ParseFileName("", stat("v0-1-2-bodies.seg")) + require.True(ok) + + f, ok := snaptype.ParseFileName("", stat("v1-1-2-bodies.seg")) + require.True(ok) require.Equal(f.T, snaptype.Bodies) require.Equal(1_000, int(f.From)) require.Equal(2_000, int(f.To)) diff --git a/turbo/snapshotsync/freezeblocks/bor_snapshots.go b/turbo/snapshotsync/freezeblocks/bor_snapshots.go index 70d17078008..d8ea5b39668 100644 --- a/turbo/snapshotsync/freezeblocks/bor_snapshots.go +++ b/turbo/snapshotsync/freezeblocks/bor_snapshots.go @@ -797,9 +797,9 @@ func (s *BorRoSnapshots) ReopenList(fileNames []string, optimistic bool) error { var segmentsMaxSet bool Loop: for _, fName := range fileNames { - f, err := snaptype.ParseFileName(s.dir, fName) - if err != nil { - s.logger.Warn("invalid segment name", "err", err, "name", fName) + f, ok := snaptype.ParseFileName(s.dir, fName) + if !ok { + s.logger.Trace("BorRoSnapshots.ReopenList: skip", "file", fName) continue } @@ -1136,7 +1136,10 @@ func (m *BorMerger) Merge(ctx context.Context, snapshots *BorRoSnapshots, mergeR for _, t := range []snaptype.Type{snaptype.BorEvents, snaptype.BorSpans} { segName := snaptype.SegmentFileName(r.from, r.to, t) - f, _ := snaptype.ParseFileName(snapDir, segName) + f, ok := snaptype.ParseFileName(snapDir, segName) + if !ok { + continue + } if err := m.merge(ctx, toMerge[t], f.Path, logEvery); err != nil { return fmt.Errorf("mergeByAppendSegments: %w", err) }