From 05fb0239a10e7dbb0dc859b783593addb4daf75e Mon Sep 17 00:00:00 2001 From: milen <94537774+taratorio@users.noreply.github.com> Date: Wed, 9 Oct 2024 18:03:59 +0100 Subject: [PATCH] freezeblocks: fix max visible segment when there is a gap in .idx files (#12263) We have a gap in our bor mainnet snapshot toml between 'v1-051900-052000-headers.idx' = 'd97b3c598f520888eb67244ba28d994998584e65' 'v1-054000-054100-headers.idx' = 'a20efd5aca03ebb528215b24ce6aa0c20cc90c55' When syncing bor mainnet from scratch the node fails to sync due to that. We will fix the gap in our toml however the node should auto-recover by regenerating the .idx files After digging further it seems like we have a regression in https://github.com/erigontech/erigon/pull/11557 idxAvailability used to stop when finding a seg which is not indexed after that PR it does not, but instead it keeps on going so idxMax gets set to 62 mil although there is a gap (it should have been set to 52 mil where the gap is) Tested on my node and it recreated the gap of indices ``` [INFO] [10-09|16:32:27.574] [snapshots:missed-idx] Stat blocks=62.44M indices=52.00M alloc=26.8GB sys=27.4GB [DBUG] [10-09|16:32:27.957] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-052500-052600-headers.idx.tmp [DBUG] [10-09|16:32:27.957] [index] calculating file=v1-052500-052600-headers.idx [DBUG] [10-09|16:32:27.977] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-053200-053300-headers.idx.tmp [DBUG] [10-09|16:32:27.977] [index] calculating file=v1-053200-053300-headers.idx [DBUG] [10-09|16:32:27.978] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-052800-052900-headers.idx.tmp [DBUG] [10-09|16:32:27.978] [index] calculating file=v1-052800-052900-headers.idx [DBUG] [10-09|16:32:27.978] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-053000-053100-headers.idx.tmp [DBUG] [10-09|16:32:27.978] [index] calculating file=v1-053000-053100-headers.idx [DBUG] [10-09|16:32:27.980] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-052300-052400-headers.idx.tmp [DBUG] [10-09|16:32:27.980] [index] calculating file=v1-052300-052400-headers.idx [DBUG] [10-09|16:32:27.983] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-052700-052800-headers.idx.tmp [DBUG] [10-09|16:32:27.983] [index] calculating file=v1-052700-052800-headers.idx [DBUG] [10-09|16:32:27.990] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-052400-052500-headers.idx.tmp [DBUG] [10-09|16:32:27.990] [index] calculating file=v1-052400-052500-headers.idx [DBUG] [10-09|16:32:27.994] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-053300-053400-headers.idx.tmp [DBUG] [10-09|16:32:27.994] [index] calculating file=v1-053300-053400-headers.idx [DBUG] [10-09|16:32:27.996] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-052200-052300-headers.idx.tmp [DBUG] [10-09|16:32:27.996] [index] calculating file=v1-052200-052300-headers.idx [DBUG] [10-09|16:32:27.997] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-052600-052700-headers.idx.tmp [DBUG] [10-09|16:32:27.997] [index] calculating file=v1-052600-052700-headers.idx [DBUG] [10-09|16:32:28.001] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-052100-052200-headers.idx.tmp [DBUG] [10-09|16:32:28.001] [index] calculating file=v1-052100-052200-headers.idx [DBUG] [10-09|16:32:28.002] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-052000-052100-headers.idx.tmp [DBUG] [10-09|16:32:28.002] [index] calculating file=v1-052000-052100-headers.idx [DBUG] [10-09|16:32:28.016] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-053100-053200-headers.idx.tmp [DBUG] [10-09|16:32:28.016] [index] calculating file=v1-053100-053200-headers.idx [DBUG] [10-09|16:32:28.020] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-052900-053000-headers.idx.tmp [DBUG] [10-09|16:32:28.020] [index] calculating file=v1-052900-053000-headers.idx [DBUG] [10-09|16:32:28.023] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-053400-053500-headers.idx.tmp [DBUG] [10-09|16:32:28.023] [index] calculating file=v1-053400-053500-headers.idx [DBUG] [10-09|16:32:28.235] [index] write file=v1-053200-053300-headers.idx [DBUG] [10-09|16:32:28.237] [index] write file=v1-052800-052900-headers.idx [DBUG] [10-09|16:32:28.237] [index] write file=v1-052700-052800-headers.idx [DBUG] [10-09|16:32:28.244] [index] write file=v1-052400-052500-headers.idx [DBUG] [10-09|16:32:28.244] [index] write file=v1-052100-052200-headers.idx [DBUG] [10-09|16:32:28.265] [index] write file=v1-053000-053100-headers.idx [DBUG] [10-09|16:32:28.268] [index] write file=v1-052300-052400-headers.idx [DBUG] [10-09|16:32:28.270] [index] write file=v1-052500-052600-headers.idx [DBUG] [10-09|16:32:28.286] [index] write file=v1-052900-053000-headers.idx [DBUG] [10-09|16:32:28.288] [index] write file=v1-052600-052700-headers.idx [DBUG] [10-09|16:32:28.292] [index] write file=v1-052000-052100-headers.idx [DBUG] [10-09|16:32:28.296] [index] write file=v1-053100-053200-headers.idx [DBUG] [10-09|16:32:28.319] [index] write file=v1-052200-052300-headers.idx [DBUG] [10-09|16:32:28.322] [index] write file=v1-053300-053400-headers.idx [DBUG] [10-09|16:32:28.339] [index] write file=v1-053400-053500-headers.idx [DBUG] [10-09|16:32:28.508] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-053800-053900-headers.idx.tmp [DBUG] [10-09|16:32:28.508] [index] calculating file=v1-053800-053900-headers.idx [DBUG] [10-09|16:32:28.520] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-053900-054000-headers.idx.tmp [DBUG] [10-09|16:32:28.520] [index] calculating file=v1-053900-054000-headers.idx [DBUG] [10-09|16:32:28.521] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-053600-053700-headers.idx.tmp [DBUG] [10-09|16:32:28.521] [index] calculating file=v1-053600-053700-headers.idx [DBUG] [10-09|16:32:28.526] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-053700-053800-headers.idx.tmp [DBUG] [10-09|16:32:28.526] [index] calculating file=v1-053700-053800-headers.idx [DBUG] [10-09|16:32:28.529] [index] created file=/home/admin/erigon-data/e3-bor-heimdall-stage-mainnet/snapshots/v1-053500-053600-headers.idx.tmp [DBUG] [10-09|16:32:28.529] [index] calculating file=v1-053500-053600-headers.idx [DBUG] [10-09|16:32:28.756] [index] write file=v1-053800-053900-headers.idx [DBUG] [10-09|16:32:28.780] [index] write file=v1-053900-054000-headers.idx [DBUG] [10-09|16:32:28.782] [index] write file=v1-053600-053700-headers.idx [DBUG] [10-09|16:32:28.783] [index] write file=v1-053700-053800-headers.idx [DBUG] [10-09|16:32:28.805] [index] write file=v1-053500-053600-headers.idx [INFO] [10-09|16:32:30.944] [snapshots:missed-idx:reopen] Stat blocks=62.44M indices=62.44M alloc=21.7GB sys=32.5GB ``` --- .../freezeblocks/block_snapshots.go | 2 +- .../freezeblocks/block_snapshots_test.go | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots.go b/turbo/snapshotsync/freezeblocks/block_snapshots.go index 9971fde51b8..f518564f505 100644 --- a/turbo/snapshotsync/freezeblocks/block_snapshots.go +++ b/turbo/snapshotsync/freezeblocks/block_snapshots.go @@ -609,7 +609,7 @@ func (s *RoSnapshots) recalcVisibleFiles() { continue } if seg.indexes == nil { - continue + break } for len(newVisibleSegments) > 0 && newVisibleSegments[len(newVisibleSegments)-1].src.isSubSetOf(seg) { newVisibleSegments[len(newVisibleSegments)-1].src = nil diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots_test.go b/turbo/snapshotsync/freezeblocks/block_snapshots_test.go index f18c6ba694c..5350ed8032e 100644 --- a/turbo/snapshotsync/freezeblocks/block_snapshots_test.go +++ b/turbo/snapshotsync/freezeblocks/block_snapshots_test.go @@ -18,6 +18,7 @@ package freezeblocks import ( "context" + "os" "path/filepath" "testing" "testing/fstest" @@ -26,6 +27,7 @@ import ( "golang.org/x/exp/slices" "github.com/erigontech/erigon-lib/log/v3" + "github.com/erigontech/erigon/turbo/testlog" "github.com/erigontech/erigon-lib/chain/networkname" "github.com/erigontech/erigon-lib/chain/snapcfg" @@ -599,3 +601,32 @@ func TestCalculateVisibleSegments(t *testing.T) { require.Equal(5, getSeg(s, coresnaptype.Enums.Transactions).DirtySegments.Len()) } } + +func TestCalculateVisibleSegmentsWhenGapsInIdx(t *testing.T) { + logger := testlog.Logger(t, log.LvlCrit) + dir, require := t.TempDir(), require.New(t) + createFile := func(from, to uint64, name snaptype.Type) { + createTestSegmentFile(t, from, to, name.Enum(), dir, 1, logger) + } + + for i := uint64(0); i < 3; i++ { + createFile(i*500_000, (i+1)*500_000, coresnaptype.Headers) + createFile(i*500_000, (i+1)*500_000, coresnaptype.Bodies) + createFile(i*500_000, (i+1)*500_000, coresnaptype.Transactions) + } + + missingIdxFile := filepath.Join(dir, snaptype.IdxFileName(1, 500_000, 1_000_000, coresnaptype.Headers.Name())) + err := os.Remove(missingIdxFile) + require.NoError(err) + + cfg := ethconfig.BlocksFreezing{ChainName: networkname.MainnetChainName} + s := NewRoSnapshots(cfg, dir, 0, logger) + defer s.Close() + + require.NoError(s.ReopenFolder()) + idx := s.idxAvailability() + require.Equal(500_000-1, int(idx)) + + require.Equal(1, len(getSeg(s, coresnaptype.Enums.Headers).VisibleSegments)) + require.Equal(3, getSeg(s, coresnaptype.Enums.Headers).DirtySegments.Len()) +}