save

core/state/dump.go

@@ -136,6 +136,10 @@ func NewDumper(db kv.Tx, txNumsReader rawdbv3.TxNumsReader, blockNumber uint64)
 	}
 }
 
+var TooMuchIterations = fmt.Errorf("[rpc] Dumper: too much iterations protection triggered")
+
+const DumperIterationsHardLimit = 10_000_000
+
 func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bool, startAddress libcommon.Address, maxResults int) ([]byte, error) {
 	var emptyCodeHash = crypto.Keccak256Hash(nil)
 	var emptyHash = libcommon.Hash{}
@@ -160,6 +164,8 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo
 		return nil, err
 	}
 
+	var hardLimit = DumperIterationsHardLimit
+
 	t := time.Now()
 	var nextKey []byte
 	it, err := ttx.DomainRange(kv.AccountsDomain, startAddress[:], nil, txNum, order.Asc, maxResults)
@@ -168,7 +174,6 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo
 	}
 	fmt.Printf("[dbg] after DomainRange: %s\n", time.Since(t))
 	defer it.Close()
-	i := 0
 	for it.HasNext() {
 		k, v, err := it.Next()
 		if err != nil {
@@ -181,8 +186,6 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo
 		if len(v) == 0 {
 			continue
 		}
-		i++
-		fmt.Printf("[dbg] iter: %d\n", i)
 
 		if e := accounts.DeserialiseV3(&acc, v); e != nil {
 			return nil, fmt.Errorf("decoding %x for %x: %w", v, k, e)
@@ -211,12 +214,15 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo
 		addrList = append(addrList, libcommon.BytesToAddress(k))
 
 		numberOfResults++
+
+		if hardLimit--; hardLimit < 0 {
+			return nil, TooMuchIterations
+		}
 	}
 	it.Close()
 
 	fmt.Printf("[dbg] first loop done: %d\n", len(addrList))
 
-	j := 0
 	for i, addr := range addrList {
 		account := accountList[i]
 
@@ -229,8 +235,6 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo
 			}
 			defer r.Close()
 			for r.HasNext() {
-				j++
-				fmt.Printf("[dbg] iter j: %d\n", j)
 				k, vs, err := r.Next()
 				if err != nil {
 					return nil, fmt.Errorf("walking over storage for %x: %w", addr, err)
@@ -242,6 +246,10 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo
 				account.Storage[libcommon.BytesToHash(loc).String()] = common.Bytes2Hex(vs)
 				h, _ := libcommon.HashData(loc)
 				t.Update(h.Bytes(), libcommon.Copy(vs))
+
+				if hardLimit--; hardLimit < 0 {
+					return nil, TooMuchIterations
+				}
 			}
 			r.Close()
 

erigon-lib/seg/decompress.go

@@ -1066,25 +1066,35 @@ func (g *Getter) FastNext(buf []byte) ([]byte, uint64) {
 	return buf[:wordLen], postLoopPos
 }
 
+// BinarySearch - doesn't know if file is sorted
 func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (offset uint64)) (foundOffset uint64) {
 	//TODO: unit tests with asserts
 	foundItem := sort.Search(count, func(i int) bool {
 		offset := f(uint64(i))
 		g.Reset(offset)
 		if g.HasNext() {
 			key, _ := g.Next(nil)
-			return bytes.Compare(fromPrefix, key) >= 0
+			fmt.Printf("ee: %s, %s,  %d, %d\n", key, fromPrefix, i, bytes.Compare(key, fromPrefix))
+			return bytes.Compare(key, fromPrefix) >= 0
+		} else {
+			panic(1)
 		}
 		return false
 	})
+	fmt.Printf("found:  %d, %d\n", foundItem, count)
 	if foundItem < count {
 		foundOffset = f(uint64(foundItem))
 	}
-
+	fmt.Printf("found offset: %d\n", foundOffset)
+	g.Reset(foundOffset)
+	if g.HasNext() {
+		key, _ := g.Next(nil)
+		fmt.Printf("found key: %s\n", key)
+	}
 	if dbg.AssertEnabled && foundItem > 2 {
 		g.Reset(f(uint64(foundItem - 2))) // prev key
 		prevKey, _ := g.Next(nil)
-		if bytes.Compare(fromPrefix, prevKey) < 0 {
+		if bytes.Compare(prevKey, fromPrefix) >= 0 {
 			panic(fmt.Errorf("see smaller key: fromPrefix=%x, prevKey=%x", fromPrefix, prevKey))
 		}
 	}

erigon-lib/seg/decompress_test.go

@@ -24,6 +24,7 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -261,6 +262,7 @@ func prepareLoremDictUncompressed(t *testing.T) *Decompressor {
 		t.Fatal(err)
 	}
 	defer c.Close()
+	slices.Sort(loremStrings)
 	for k, w := range loremStrings {
 		if err = c.AddUncompressedWord([]byte(fmt.Sprintf("%s %d", w, k))); err != nil {
 			t.Fatal(err)
@@ -281,16 +283,25 @@ func TestUncompressed(t *testing.T) {
 	defer d.Close()
 	g := d.MakeGetter()
 	i := 0
+	var offsets []uint64
 	for g.HasNext() {
 		w := loremStrings[i]
 		expected := []byte(fmt.Sprintf("%s %d", w, i+1))
 		expected = expected[:len(expected)/2]
-		actual, _ := g.NextUncompressed()
+		actual, offset := g.NextUncompressed()
 		if bytes.Equal(expected, actual) {
 			t.Errorf("expected %s, actual %s", expected, actual)
 		}
 		i++
+		offsets = append(offsets, offset)
 	}
+
+	found := g.BinarySearch([]byte("ipsum"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] })
+	fmt.Printf("found: %d\n", found)
+	g.Reset(found)
+	k, _ := g.Next(nil)
+	fmt.Printf("found: %s\n", k)
+
 }
 
 func TestDecompressor_OpenCorrupted(t *testing.T) {
@@ -461,12 +472,13 @@ func TestDecompressor_OpenCorrupted(t *testing.T) {
 	})
 }
 
-const lorem = `Lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor incididunt ut labore et
-dolore magna aliqua Ut enim ad minim veniam quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
-consequat Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur
-Excepteur sint occaecat cupidatat non proident sunt in culpa qui officia deserunt mollit anim id est laborum`
+const lorem = `
+lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor incididunt ut labore et
+dolore magna aliqua ut enim ad minim veniam quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+consequat duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur
+excepteur sint occaecat cupidatat non proident sunt in culpa qui officia deserunt mollit anim id est laborum`
 
-var loremStrings = strings.Split(lorem, " ")
+var loremStrings = strings.Split(strings.ReplaceAll(strings.ReplaceAll(lorem, "\n", " "), "\r", ""), " ")
 
 func TestDecompressTorrent(t *testing.T) {
 	t.Skip()