diff --git a/core/state/dump.go b/core/state/dump.go index 6a52cd49f2f..c684b6870ac 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -136,6 +136,10 @@ func NewDumper(db kv.Tx, txNumsReader rawdbv3.TxNumsReader, blockNumber uint64) } } +var TooMuchIterations = fmt.Errorf("[rpc] Dumper: too much iterations protection triggered") + +const DumperIterationsHardLimit = 10_000_000 + func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bool, startAddress libcommon.Address, maxResults int) ([]byte, error) { var emptyCodeHash = crypto.Keccak256Hash(nil) var emptyHash = libcommon.Hash{} @@ -160,6 +164,8 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo return nil, err } + var hardLimit = DumperIterationsHardLimit + t := time.Now() var nextKey []byte it, err := ttx.DomainRange(kv.AccountsDomain, startAddress[:], nil, txNum, order.Asc, maxResults) @@ -168,7 +174,6 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo } fmt.Printf("[dbg] after DomainRange: %s\n", time.Since(t)) defer it.Close() - i := 0 for it.HasNext() { k, v, err := it.Next() if err != nil { @@ -181,8 +186,6 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo if len(v) == 0 { continue } - i++ - fmt.Printf("[dbg] iter: %d\n", i) if e := accounts.DeserialiseV3(&acc, v); e != nil { return nil, fmt.Errorf("decoding %x for %x: %w", v, k, e) @@ -211,12 +214,15 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo addrList = append(addrList, libcommon.BytesToAddress(k)) numberOfResults++ + + if hardLimit--; hardLimit < 0 { + return nil, TooMuchIterations + } } it.Close() fmt.Printf("[dbg] first loop done: %d\n", len(addrList)) - j := 0 for i, addr := range addrList { account := accountList[i] @@ -229,8 +235,6 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo } defer r.Close() for r.HasNext() { - j++ - fmt.Printf("[dbg] iter j: %d\n", j) k, vs, err := r.Next() if err != nil { return nil, fmt.Errorf("walking over storage for %x: %w", addr, err) @@ -242,6 +246,10 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo account.Storage[libcommon.BytesToHash(loc).String()] = common.Bytes2Hex(vs) h, _ := libcommon.HashData(loc) t.Update(h.Bytes(), libcommon.Copy(vs)) + + if hardLimit--; hardLimit < 0 { + return nil, TooMuchIterations + } } r.Close() diff --git a/erigon-lib/seg/decompress.go b/erigon-lib/seg/decompress.go index 01c4bfec0d6..5c55c1d381a 100644 --- a/erigon-lib/seg/decompress.go +++ b/erigon-lib/seg/decompress.go @@ -1066,6 +1066,7 @@ func (g *Getter) FastNext(buf []byte) ([]byte, uint64) { return buf[:wordLen], postLoopPos } +// BinarySearch - doesn't know if file is sorted func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (offset uint64)) (foundOffset uint64) { //TODO: unit tests with asserts foundItem := sort.Search(count, func(i int) bool { @@ -1073,18 +1074,27 @@ func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (of g.Reset(offset) if g.HasNext() { key, _ := g.Next(nil) - return bytes.Compare(fromPrefix, key) >= 0 + fmt.Printf("ee: %s, %s, %d, %d\n", key, fromPrefix, i, bytes.Compare(key, fromPrefix)) + return bytes.Compare(key, fromPrefix) >= 0 + } else { + panic(1) } return false }) + fmt.Printf("found: %d, %d\n", foundItem, count) if foundItem < count { foundOffset = f(uint64(foundItem)) } - + fmt.Printf("found offset: %d\n", foundOffset) + g.Reset(foundOffset) + if g.HasNext() { + key, _ := g.Next(nil) + fmt.Printf("found key: %s\n", key) + } if dbg.AssertEnabled && foundItem > 2 { g.Reset(f(uint64(foundItem - 2))) // prev key prevKey, _ := g.Next(nil) - if bytes.Compare(fromPrefix, prevKey) < 0 { + if bytes.Compare(prevKey, fromPrefix) >= 0 { panic(fmt.Errorf("see smaller key: fromPrefix=%x, prevKey=%x", fromPrefix, prevKey)) } } diff --git a/erigon-lib/seg/decompress_test.go b/erigon-lib/seg/decompress_test.go index f43f0e08048..3ed0ecb2bdc 100644 --- a/erigon-lib/seg/decompress_test.go +++ b/erigon-lib/seg/decompress_test.go @@ -24,6 +24,7 @@ import ( "math/rand" "os" "path/filepath" + "slices" "strings" "testing" "time" @@ -261,6 +262,7 @@ func prepareLoremDictUncompressed(t *testing.T) *Decompressor { t.Fatal(err) } defer c.Close() + slices.Sort(loremStrings) for k, w := range loremStrings { if err = c.AddUncompressedWord([]byte(fmt.Sprintf("%s %d", w, k))); err != nil { t.Fatal(err) @@ -281,16 +283,25 @@ func TestUncompressed(t *testing.T) { defer d.Close() g := d.MakeGetter() i := 0 + var offsets []uint64 for g.HasNext() { w := loremStrings[i] expected := []byte(fmt.Sprintf("%s %d", w, i+1)) expected = expected[:len(expected)/2] - actual, _ := g.NextUncompressed() + actual, offset := g.NextUncompressed() if bytes.Equal(expected, actual) { t.Errorf("expected %s, actual %s", expected, actual) } i++ + offsets = append(offsets, offset) } + + found := g.BinarySearch([]byte("ipsum"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) + fmt.Printf("found: %d\n", found) + g.Reset(found) + k, _ := g.Next(nil) + fmt.Printf("found: %s\n", k) + } func TestDecompressor_OpenCorrupted(t *testing.T) { @@ -461,12 +472,13 @@ func TestDecompressor_OpenCorrupted(t *testing.T) { }) } -const lorem = `Lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor incididunt ut labore et -dolore magna aliqua Ut enim ad minim veniam quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo -consequat Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur -Excepteur sint occaecat cupidatat non proident sunt in culpa qui officia deserunt mollit anim id est laborum` +const lorem = ` +lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor incididunt ut labore et +dolore magna aliqua ut enim ad minim veniam quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo +consequat duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur +excepteur sint occaecat cupidatat non proident sunt in culpa qui officia deserunt mollit anim id est laborum` -var loremStrings = strings.Split(lorem, " ") +var loremStrings = strings.Split(strings.ReplaceAll(strings.ReplaceAll(lorem, "\n", " "), "\r", ""), " ") func TestDecompressTorrent(t *testing.T) { t.Skip()