From 7e31d31cc000ded414b6402982edd31933d98e4e Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 18:20:28 +0700 Subject: [PATCH 01/91] save --- core/state/dump.go | 1 + turbo/jsonrpc/debug_api.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/core/state/dump.go b/core/state/dump.go index e34118392f8..e0d713c14ef 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -148,6 +148,7 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo c.OnRoot(emptyHash) // We do not calculate the root + fmt.Printf("[dbg] DumpToCollector limits: %t, %t, %d\n", excludeCode, excludeStorage, maxResults) ttx := d.db.(kv.TemporalTx) txNum, err := d.txNumsReader.Min(ttx, d.blockNumber+1) if err != nil { diff --git a/turbo/jsonrpc/debug_api.go b/turbo/jsonrpc/debug_api.go index d932b50ccf9..64e5f82fba9 100644 --- a/turbo/jsonrpc/debug_api.go +++ b/turbo/jsonrpc/debug_api.go @@ -111,6 +111,8 @@ func (api *PrivateDebugAPIImpl) AccountRange(ctx context.Context, blockNrOrHash } defer tx.Rollback() + fmt.Printf("[dbg] AccountRange limits: %t, %t, %d\n", excludeCode, excludeStorage, maxResults) + var blockNumber uint64 if number, ok := blockNrOrHash.Number(); ok { From f4746f91d8a0d2cbf2f02cf55d74dfb4a06eefc7 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 18:21:55 +0700 Subject: [PATCH 02/91] save --- core/state/dump.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/state/dump.go b/core/state/dump.go index e0d713c14ef..1713078378e 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -165,6 +165,7 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo return nil, err } defer it.Close() + i := 0 for it.HasNext() { k, v, err := it.Next() if err != nil { @@ -177,6 +178,8 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo if len(v) == 0 { continue } + i++ + fmt.Printf("[dbg] iter: %d\n", i) if e := accounts.DeserialiseV3(&acc, v); e != nil { return nil, fmt.Errorf("decoding %x for %x: %w", v, k, e) From fe2cab2961b00da01047c9cb089a54ca86ffc594 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 18:25:06 +0700 Subject: [PATCH 03/91] save --- core/state/dump.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/state/dump.go b/core/state/dump.go index 1713078378e..0c3974f003d 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -210,6 +210,7 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo numberOfResults++ } + j := 0 for i, addr := range addrList { account := accountList[i] @@ -222,6 +223,8 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo } defer r.Close() for r.HasNext() { + j++ + fmt.Printf("[dbg] iter j: %d\n", j) k, vs, err := r.Next() if err != nil { return nil, fmt.Errorf("walking over storage for %x: %w", addr, err) @@ -234,6 +237,7 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo h, _ := libcommon.HashData(loc) t.Update(h.Bytes(), libcommon.Copy(vs)) } + r.Close() account.Root = t.Hash().Bytes() } From 24a1b8e0d826cbbd7f41518ed5c2875081612604 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 18:29:31 +0700 Subject: [PATCH 04/91] save --- core/state/dump.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/state/dump.go b/core/state/dump.go index 0c3974f003d..2fb1649abe2 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -22,6 +22,7 @@ package state import ( "encoding/json" "fmt" + "time" libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/hexutility" @@ -159,11 +160,14 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo return nil, err } + fmt.Printf("[dbg] before DomainRange\n") + t := time.Now() var nextKey []byte it, err := ttx.DomainRange(kv.AccountsDomain, startAddress[:], nil, txNum, order.Asc, maxResults) if err != nil { return nil, err } + fmt.Printf("[dbg] after DomainRange: %s\n", time.Since(t)) defer it.Close() i := 0 for it.HasNext() { @@ -209,6 +213,7 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo numberOfResults++ } + it.Close() j := 0 for i, addr := range addrList { From 3f489fbdcda8af4171c8b3d3cadd5efd3478b3a0 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 18:29:56 +0700 Subject: [PATCH 05/91] save --- core/state/dump.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/state/dump.go b/core/state/dump.go index 2fb1649abe2..50699fd7740 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -215,6 +215,8 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo } it.Close() + fmt.Printf("[dbg] first loop done: %d\n", len(addrList)) + j := 0 for i, addr := range addrList { account := accountList[i] From eeb7d17e8432585ba618ca362d42f8e95fc8f54f Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 18:34:44 +0700 Subject: [PATCH 06/91] save --- erigon-lib/state/history.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 31adb3cf01d..94dc6450ea1 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1353,6 +1353,7 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to ctx: ctx, } + fmt.Printf("[dbg] ht.iit.files: %d\n", len(ht.iit.files)) for _, item := range ht.iit.files { if item.endTxNum <= startTxNum { continue @@ -1418,6 +1419,7 @@ func (hi *StateAsOfIterF) advanceInFiles() error { var idxVal []byte //if hi.compressVals { idxVal, _ = top.g.Next(nil) + fmt.Printf("[dbg] StateAsOfIterF advanceInFiles: %s, %x\n", top.g.FileName(), key) //} else { // idxVal, _ = top.g.NextUncompressed() //} From 9580d178dd0a3244a0c3298620de1766bdf5bbf2 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 18:38:24 +0700 Subject: [PATCH 07/91] save --- erigon-lib/state/domain.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index a2dd099e924..de58239307b 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1952,6 +1952,7 @@ func (dt *DomainRoTx) GetLatest(key1, key2 []byte, roTx kv.Tx) ([]byte, uint64, func (dt *DomainRoTx) DomainRange(ctx context.Context, tx kv.Tx, fromKey, toKey []byte, ts uint64, asc order.By, limit int) (it stream.KV, err error) { if !asc { + fmt.Printf("[dbg] what 1?\n") panic("implement me") } //histStateIt, err := tx.aggTx.AccountHistoricalStateRange(asOfTs, fromKey, toKey, limit, tx.MdbxTx) @@ -1962,6 +1963,7 @@ func (dt *DomainRoTx) DomainRange(ctx context.Context, tx kv.Tx, fromKey, toKey //if err != nil { // return nil, err //} + fmt.Printf("[dbg] DomainRange 1: %s\n", dt.d.name) histStateIt, err := dt.ht.WalkAsOf(ctx, ts, fromKey, toKey, tx, limit) if err != nil { return nil, err From 998f1c93545ba69e7dd029c0792bcb1e1f694086 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 18:38:52 +0700 Subject: [PATCH 08/91] save --- erigon-lib/state/domain.go | 1 + 1 file changed, 1 insertion(+) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index de58239307b..be1604a0075 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1968,6 +1968,7 @@ func (dt *DomainRoTx) DomainRange(ctx context.Context, tx kv.Tx, fromKey, toKey if err != nil { return nil, err } + fmt.Printf("[dbg] DomainRange 2: %s\n", dt.d.name) lastestStateIt, err := dt.DomainRangeLatest(tx, fromKey, toKey, limit) if err != nil { return nil, err From b9bc6d2edfec9c2714cd2aaf46e4f8293b7c7a3c Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 18:40:23 +0700 Subject: [PATCH 09/91] save --- erigon-lib/state/domain.go | 1 + 1 file changed, 1 insertion(+) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index be1604a0075..ee77f3eef44 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1973,6 +1973,7 @@ func (dt *DomainRoTx) DomainRange(ctx context.Context, tx kv.Tx, fromKey, toKey if err != nil { return nil, err } + fmt.Printf("[dbg] DomainRange 3: %s\n", dt.d.name) return stream.UnionKV(histStateIt, lastestStateIt, limit), nil } From 8e396c2fa7e2722e2fc32e02ec2f434a3a4075ab Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 18:53:28 +0700 Subject: [PATCH 10/91] save --- erigon-lib/state/history.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 94dc6450ea1..eefbc2b6a8d 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1419,7 +1419,7 @@ func (hi *StateAsOfIterF) advanceInFiles() error { var idxVal []byte //if hi.compressVals { idxVal, _ = top.g.Next(nil) - fmt.Printf("[dbg] StateAsOfIterF advanceInFiles: %s, %x\n", top.g.FileName(), key) + fmt.Printf("[dbg] StateAsOfIterF advanceInFiles: %s, %x, to: %x\n", top.g.FileName(), key, hi.to) //} else { // idxVal, _ = top.g.NextUncompressed() //} From 4bbd90305232e9508bdd7e380649e6fa0cc6b927 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 18:55:48 +0700 Subject: [PATCH 11/91] save --- core/state/dump.go | 3 ++- erigon-lib/state/domain.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/state/dump.go b/core/state/dump.go index 50699fd7740..6010717bd42 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -163,7 +163,8 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo fmt.Printf("[dbg] before DomainRange\n") t := time.Now() var nextKey []byte - it, err := ttx.DomainRange(kv.AccountsDomain, startAddress[:], nil, txNum, order.Asc, maxResults) + nextAcc, _ := kv.NextSubtree(startAddress[:]) + it, err := ttx.DomainRange(kv.AccountsDomain, startAddress[:], nextAcc, txNum, order.Asc, maxResults) if err != nil { return nil, err } diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index ee77f3eef44..cc322b677a0 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1963,7 +1963,7 @@ func (dt *DomainRoTx) DomainRange(ctx context.Context, tx kv.Tx, fromKey, toKey //if err != nil { // return nil, err //} - fmt.Printf("[dbg] DomainRange 1: %s\n", dt.d.name) + fmt.Printf("[dbg] DomainRange 1: %s, %x, %x\n", dt.d.name, fromKey, toKey) histStateIt, err := dt.ht.WalkAsOf(ctx, ts, fromKey, toKey, tx, limit) if err != nil { return nil, err From c420190e99516dd2e2b0ee392e44c641fa64b23c Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 18:57:14 +0700 Subject: [PATCH 12/91] save --- .github/workflows/qa-rpc-integration-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qa-rpc-integration-tests.yml b/.github/workflows/qa-rpc-integration-tests.yml index 3f3a9eea273..13f661abbfd 100644 --- a/.github/workflows/qa-rpc-integration-tests.yml +++ b/.github/workflows/qa-rpc-integration-tests.yml @@ -86,7 +86,7 @@ jobs: # Run RPC integration test runner via http python3 ./run_tests.py -p 8545 --continue -f --json-diff -x \ - debug_accountAt,debug_accountRange,debug_getModifiedAccountsByHash,debug_getModifiedAccountsByNumber,debug_storageRangeAt,debug_traceBlockByHash,\ + debug_accountAt,debug_getModifiedAccountsByHash,debug_getModifiedAccountsByNumber,debug_storageRangeAt,debug_traceBlockByHash,\ debug_traceCallMany/test_02.tar,debug_traceCallMany/test_04.tar,debug_traceCallMany/test_05.tar,debug_traceCallMany/test_06.tar,debug_traceCallMany/test_07.tar,debug_traceCallMany/test_09.json,debug_traceCallMany/test_10.tar,\ debug_traceBlockByNumber/test_05.tar,debug_traceBlockByNumber/test_08.tar,debug_traceBlockByNumber/test_09.tar,debug_traceBlockByNumber/test_10.tar,debug_traceBlockByNumber/test_11.tar,debug_traceBlockByNumber/test_12.tar,\ debug_traceTransaction,\ From fa66df41f88db0d8bfa23295f1d1cac65a52ea80 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 18:58:30 +0700 Subject: [PATCH 13/91] save --- core/state/dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state/dump.go b/core/state/dump.go index 6010717bd42..fb686d01a8c 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -160,10 +160,10 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo return nil, err } - fmt.Printf("[dbg] before DomainRange\n") t := time.Now() var nextKey []byte nextAcc, _ := kv.NextSubtree(startAddress[:]) + fmt.Printf("[dbg] before DomainRange: %x, %x\n", startAddress[:], nextAcc) it, err := ttx.DomainRange(kv.AccountsDomain, startAddress[:], nextAcc, txNum, order.Asc, maxResults) if err != nil { return nil, err From b044a4e4f5eb73f4502621dbd5209cb2ecc568b4 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 19:30:57 +0700 Subject: [PATCH 14/91] save --- erigon-lib/state/history.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index eefbc2b6a8d..03d8cf1b769 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1353,7 +1353,7 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to ctx: ctx, } - fmt.Printf("[dbg] ht.iit.files: %d\n", len(ht.iit.files)) + fmt.Printf("[dbg] ht.iit.files: %d, %x, %x\n", len(ht.iit.files), from, to) for _, item := range ht.iit.files { if item.endTxNum <= startTxNum { continue From 49452725d3f3d047bbf96384b130d15f08c1e6b9 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 19:33:51 +0700 Subject: [PATCH 15/91] save --- erigon-lib/state/history.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 03d8cf1b769..53b5ccda5d9 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1360,7 +1360,16 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to } // TODO: seek(from) g := seg.NewReader(item.src.decompressor.MakeGetter(), ht.h.compression) - g.Reset(0) + + var offset uint64 + if len(from) > 0 { + var ok bool + offset, ok = item.reader.Lookup(from) + if !ok { + fmt.Printf("[dbg] ht.iit.files1 !ok, %x\n", from) + } + } + g.Reset(offset) if g.HasNext() { key, offset := g.Next(nil) heap.Push(&hi.h, &ReconItem{g: g, key: key, startTxNum: item.startTxNum, endTxNum: item.endTxNum, txNum: item.endTxNum, startOffset: offset, lastOffset: offset}) From 346ad3c279c7d515f5eb2c114edc209e0fe2c0e7 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 19:35:58 +0700 Subject: [PATCH 16/91] save --- erigon-lib/state/history.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 53b5ccda5d9..68ca96b04ae 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1354,18 +1354,20 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to ctx: ctx, } fmt.Printf("[dbg] ht.iit.files: %d, %x, %x\n", len(ht.iit.files), from, to) - for _, item := range ht.iit.files { + for i, item := range ht.iit.files { if item.endTxNum <= startTxNum { continue } // TODO: seek(from) g := seg.NewReader(item.src.decompressor.MakeGetter(), ht.h.compression) + idx := ht.iit.statelessIdxReader(i) var offset uint64 if len(from) > 0 { var ok bool - offset, ok = item.reader.Lookup(from) + offset, ok = idx.Lookup(from) if !ok { + // TODO: seek(from) - to support prefix - by binary-search fmt.Printf("[dbg] ht.iit.files1 !ok, %x\n", from) } } From dc4b274799b5834b080b8b447ab35af8285dd131 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 19:39:29 +0700 Subject: [PATCH 17/91] save --- erigon-lib/state/history.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 68ca96b04ae..1c024a59b26 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1368,7 +1368,9 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to offset, ok = idx.Lookup(from) if !ok { // TODO: seek(from) - to support prefix - by binary-search - fmt.Printf("[dbg] ht.iit.files1 !ok, %x\n", from) + fmt.Printf("[dbg] ht.iit.files1 !ok, %x, %d\n", from, offset) + } else { + fmt.Printf("[dbg] ht.iit.files2 ok, %x, %d\n", from, offset) } } g.Reset(offset) From 8eed43769e8a77df3f314a48b02d5da7e7c57410 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 19:59:42 +0700 Subject: [PATCH 18/91] save --- core/state/dump.go | 4 +--- erigon-lib/state/history.go | 1 + turbo/jsonrpc/debug_api_test.go | 10 +++++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/core/state/dump.go b/core/state/dump.go index fb686d01a8c..6a52cd49f2f 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -162,9 +162,7 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo t := time.Now() var nextKey []byte - nextAcc, _ := kv.NextSubtree(startAddress[:]) - fmt.Printf("[dbg] before DomainRange: %x, %x\n", startAddress[:], nextAcc) - it, err := ttx.DomainRange(kv.AccountsDomain, startAddress[:], nextAcc, txNum, order.Asc, maxResults) + it, err := ttx.DomainRange(kv.AccountsDomain, startAddress[:], nil, txNum, order.Asc, maxResults) if err != nil { return nil, err } diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 1c024a59b26..a4e0b2c887e 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1376,6 +1376,7 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to g.Reset(offset) if g.HasNext() { key, offset := g.Next(nil) + fmt.Printf("[dbg] ht.iit.files seek(%x) -> %x\n", from, key) heap.Push(&hi.h, &ReconItem{g: g, key: key, startTxNum: item.startTxNum, endTxNum: item.endTxNum, txNum: item.endTxNum, startOffset: offset, lastOffset: offset}) } } diff --git a/turbo/jsonrpc/debug_api_test.go b/turbo/jsonrpc/debug_api_test.go index 667f921728f..70681b58c1c 100644 --- a/turbo/jsonrpc/debug_api_test.go +++ b/turbo/jsonrpc/debug_api_test.go @@ -305,12 +305,12 @@ func TestAccountRange(t *testing.T) { t.Run("valid account", func(t *testing.T) { addr := common.HexToAddress("0x537e697c7ab75a26f9ecf0ce810e3154dfcaaf55") n := rpc.BlockNumber(1) - result, err := api.AccountRange(m.Ctx, rpc.BlockNumberOrHash{BlockNumber: &n}, addr[:], 10, true, true) - require.NoError(t, err) - require.Equal(t, 2, len(result.Accounts)) - + //result, err := api.AccountRange(m.Ctx, rpc.BlockNumberOrHash{BlockNumber: &n}, addr[:], 10, true, true) + //require.NoError(t, err) + //require.Equal(t, 2, len(result.Accounts)) + // n = rpc.BlockNumber(7) - result, err = api.AccountRange(m.Ctx, rpc.BlockNumberOrHash{BlockNumber: &n}, addr[:], 10, true, true) + result, err := api.AccountRange(m.Ctx, rpc.BlockNumberOrHash{BlockNumber: &n}, addr[:], 10, true, true) require.NoError(t, err) require.Equal(t, 3, len(result.Accounts)) }) From ca40ad63582fd025b54c8f528bea32d946f610cb Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 20:06:57 +0700 Subject: [PATCH 19/91] save --- erigon-lib/state/history.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index a4e0b2c887e..44a7d70b08d 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -26,6 +26,7 @@ import ( "math" "path/filepath" "regexp" + "sort" "strconv" "sync" "time" @@ -1364,13 +1365,22 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to idx := ht.iit.statelessIdxReader(i) var offset uint64 if len(from) > 0 { - var ok bool - offset, ok = idx.Lookup(from) - if !ok { - // TODO: seek(from) - to support prefix - by binary-search - fmt.Printf("[dbg] ht.iit.files1 !ok, %x, %d\n", from, offset) + n := item.src.decompressor.Count() / 2 + found := sort.Search(n, func(i int) bool { + offset = idx.OrdinalLookup(uint64(i)) + g.Reset(offset) + if g.HasNext() { + key, _ := g.Next(nil) + fmt.Printf("bs: %x, %x\n", from, key) + return bytes.Compare(from, key) < 0 + } + return false + }) + if found < n { + fmt.Printf("[dbg] bs1 !ok, %x, %d\n", from, offset) } else { - fmt.Printf("[dbg] ht.iit.files2 ok, %x, %d\n", from, offset) + fmt.Printf("[dbg] bs2 !ok, %x, %d\n", from, offset) + offset = 0 } } g.Reset(offset) From 056026ca7bc0feef0cbdef6ce4b2b4f18865d5a3 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 20:11:35 +0700 Subject: [PATCH 20/91] save --- erigon-lib/state/history.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 44a7d70b08d..8880e15dfa6 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1372,7 +1372,7 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to if g.HasNext() { key, _ := g.Next(nil) fmt.Printf("bs: %x, %x\n", from, key) - return bytes.Compare(from, key) < 0 + return bytes.Compare(from, key) >= 0 } return false }) From f991aa89ab04027577b32ef7e3327224beb6855b Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 20:17:34 +0700 Subject: [PATCH 21/91] save --- erigon-lib/seg/decompress.go | 21 +++++++++++++++++++++ erigon-lib/state/history.go | 14 ++------------ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/erigon-lib/seg/decompress.go b/erigon-lib/seg/decompress.go index 44b890b064e..916b8a10db7 100644 --- a/erigon-lib/seg/decompress.go +++ b/erigon-lib/seg/decompress.go @@ -23,6 +23,7 @@ import ( "fmt" "os" "path/filepath" + "sort" "strconv" "sync/atomic" "time" @@ -1064,3 +1065,23 @@ func (g *Getter) FastNext(buf []byte) ([]byte, uint64) { g.dataBit = 0 return buf[:wordLen], postLoopPos } + +func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (offset uint64)) (foundOffset uint64) { + //n := item.src.decompressor.Count() / 2 + foundItem := sort.Search(count, func(i int) bool { + offset := f(uint64(i)) + g.Reset(offset) + if g.HasNext() { + key, _ := g.Next(nil) + fmt.Printf("bs: %x, %x\n", fromPrefix, key) + return bytes.Compare(fromPrefix, key) >= 0 + } + return false + }) + if foundItem < count { + fmt.Printf("[dbg] bs1 !ok, %x, %d\n", fromPrefix, f(uint64(foundItem))) + return f(uint64(foundItem)) + } + fmt.Printf("[dbg] bs2 !ok, %x, %d\n", fromPrefix, 0) + return 0 +} diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 8880e15dfa6..5988db11157 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -26,7 +26,6 @@ import ( "math" "path/filepath" "regexp" - "sort" "strconv" "sync" "time" @@ -1366,17 +1365,8 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to var offset uint64 if len(from) > 0 { n := item.src.decompressor.Count() / 2 - found := sort.Search(n, func(i int) bool { - offset = idx.OrdinalLookup(uint64(i)) - g.Reset(offset) - if g.HasNext() { - key, _ := g.Next(nil) - fmt.Printf("bs: %x, %x\n", from, key) - return bytes.Compare(from, key) >= 0 - } - return false - }) - if found < n { + found := item.getter.BinarySearch(from, n, idx.OrdinalLookup) + if int(found) < n { fmt.Printf("[dbg] bs1 !ok, %x, %d\n", from, offset) } else { fmt.Printf("[dbg] bs2 !ok, %x, %d\n", from, offset) From 7cf18e5d5f0453f984add6a2de8ee11388377a77 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 20:19:02 +0700 Subject: [PATCH 22/91] save --- erigon-lib/seg/decompress.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/seg/decompress.go b/erigon-lib/seg/decompress.go index 916b8a10db7..2db6d5bd3da 100644 --- a/erigon-lib/seg/decompress.go +++ b/erigon-lib/seg/decompress.go @@ -1073,7 +1073,7 @@ func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (of g.Reset(offset) if g.HasNext() { key, _ := g.Next(nil) - fmt.Printf("bs: %x, %x\n", fromPrefix, key) + fmt.Printf("bs: %x, %x, %t\n", fromPrefix, key, bytes.Compare(fromPrefix, key) >= 0) return bytes.Compare(fromPrefix, key) >= 0 } return false From 654073634f1a23384f35f909170f5edf853db2fd Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 20:21:42 +0700 Subject: [PATCH 23/91] save --- erigon-lib/seg/decompress.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/erigon-lib/seg/decompress.go b/erigon-lib/seg/decompress.go index 2db6d5bd3da..ab26c6952ea 100644 --- a/erigon-lib/seg/decompress.go +++ b/erigon-lib/seg/decompress.go @@ -1070,10 +1070,11 @@ func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (of //n := item.src.decompressor.Count() / 2 foundItem := sort.Search(count, func(i int) bool { offset := f(uint64(i)) + fmt.Printf("bs_0: %d, %d; %d, %d\n", count, i, len(g.data), offset) g.Reset(offset) if g.HasNext() { key, _ := g.Next(nil) - fmt.Printf("bs: %x, %x, %t\n", fromPrefix, key, bytes.Compare(fromPrefix, key) >= 0) + fmt.Printf("bs_1: %x, %x, %t\n", fromPrefix, key, bytes.Compare(fromPrefix, key) >= 0) return bytes.Compare(fromPrefix, key) >= 0 } return false From ce530c04da683caa73d8b0fbe1a2ad595c5e73be Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 20:23:24 +0700 Subject: [PATCH 24/91] save --- erigon-lib/state/history.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 5988db11157..d3c04d47ce0 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1365,7 +1365,7 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to var offset uint64 if len(from) > 0 { n := item.src.decompressor.Count() / 2 - found := item.getter.BinarySearch(from, n, idx.OrdinalLookup) + found := g.BinarySearch(from, n, idx.OrdinalLookup) if int(found) < n { fmt.Printf("[dbg] bs1 !ok, %x, %d\n", from, offset) } else { From 329ed1ea5cbfbd57f22bbe0d7059ca55e51e336f Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 20:27:21 +0700 Subject: [PATCH 25/91] save --- erigon-lib/seg/decompress.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/erigon-lib/seg/decompress.go b/erigon-lib/seg/decompress.go index ab26c6952ea..80f705a30ef 100644 --- a/erigon-lib/seg/decompress.go +++ b/erigon-lib/seg/decompress.go @@ -1081,8 +1081,16 @@ func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (of }) if foundItem < count { fmt.Printf("[dbg] bs1 !ok, %x, %d\n", fromPrefix, f(uint64(foundItem))) - return f(uint64(foundItem)) + foundOffset = f(uint64(foundItem)) + } + + if foundItem > 2 { + g.Reset(f(uint64(foundItem))) + key, _ := g.Next(nil) + if bytes.Compare(fromPrefix, key) > 0 { + panic(fmt.Errorf("see smaller key: %x, %x", fromPrefix, key)) + } } fmt.Printf("[dbg] bs2 !ok, %x, %d\n", fromPrefix, 0) - return 0 + return foundOffset } From f7c44c89b7ac5f96148d62cff906a9b1e6983666 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 20:27:29 +0700 Subject: [PATCH 26/91] save --- erigon-lib/seg/decompress.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/seg/decompress.go b/erigon-lib/seg/decompress.go index 80f705a30ef..1cde03ebaf8 100644 --- a/erigon-lib/seg/decompress.go +++ b/erigon-lib/seg/decompress.go @@ -1085,7 +1085,7 @@ func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (of } if foundItem > 2 { - g.Reset(f(uint64(foundItem))) + g.Reset(f(uint64(foundItem - 2))) key, _ := g.Next(nil) if bytes.Compare(fromPrefix, key) > 0 { panic(fmt.Errorf("see smaller key: %x, %x", fromPrefix, key)) From 5ab67cf78d6a61b74f7d6f1dff744ac37987a2c7 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 20:27:36 +0700 Subject: [PATCH 27/91] save --- erigon-lib/seg/decompress.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/seg/decompress.go b/erigon-lib/seg/decompress.go index 1cde03ebaf8..c7be52be6aa 100644 --- a/erigon-lib/seg/decompress.go +++ b/erigon-lib/seg/decompress.go @@ -1085,7 +1085,7 @@ func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (of } if foundItem > 2 { - g.Reset(f(uint64(foundItem - 2))) + g.Reset(f(uint64(foundItem - 2))) // prev key key, _ := g.Next(nil) if bytes.Compare(fromPrefix, key) > 0 { panic(fmt.Errorf("see smaller key: %x, %x", fromPrefix, key)) From f2737e860efc0c90ce6fa8df58a7bce5506db9e2 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 20:28:09 +0700 Subject: [PATCH 28/91] save --- erigon-lib/seg/decompress.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/erigon-lib/seg/decompress.go b/erigon-lib/seg/decompress.go index c7be52be6aa..daa138e2a17 100644 --- a/erigon-lib/seg/decompress.go +++ b/erigon-lib/seg/decompress.go @@ -1086,9 +1086,9 @@ func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (of if foundItem > 2 { g.Reset(f(uint64(foundItem - 2))) // prev key - key, _ := g.Next(nil) - if bytes.Compare(fromPrefix, key) > 0 { - panic(fmt.Errorf("see smaller key: %x, %x", fromPrefix, key)) + prevKey, _ := g.Next(nil) + if bytes.Compare(fromPrefix, prevKey) < 0 { + panic(fmt.Errorf("see smaller key: fromPrefix=%x, prevKey=%x", fromPrefix, prevKey)) } } fmt.Printf("[dbg] bs2 !ok, %x, %d\n", fromPrefix, 0) From b3e4937e48191d9ccd53b370c91ac06970d1eab3 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 20:30:49 +0700 Subject: [PATCH 29/91] save --- erigon-lib/seg/decompress.go | 6 +----- erigon-lib/state/history.go | 8 +------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/erigon-lib/seg/decompress.go b/erigon-lib/seg/decompress.go index daa138e2a17..917e76dc098 100644 --- a/erigon-lib/seg/decompress.go +++ b/erigon-lib/seg/decompress.go @@ -1070,27 +1070,23 @@ func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (of //n := item.src.decompressor.Count() / 2 foundItem := sort.Search(count, func(i int) bool { offset := f(uint64(i)) - fmt.Printf("bs_0: %d, %d; %d, %d\n", count, i, len(g.data), offset) g.Reset(offset) if g.HasNext() { key, _ := g.Next(nil) - fmt.Printf("bs_1: %x, %x, %t\n", fromPrefix, key, bytes.Compare(fromPrefix, key) >= 0) return bytes.Compare(fromPrefix, key) >= 0 } return false }) if foundItem < count { - fmt.Printf("[dbg] bs1 !ok, %x, %d\n", fromPrefix, f(uint64(foundItem))) foundOffset = f(uint64(foundItem)) } - if foundItem > 2 { + if dbg.AssertEnabled && foundItem > 2 { g.Reset(f(uint64(foundItem - 2))) // prev key prevKey, _ := g.Next(nil) if bytes.Compare(fromPrefix, prevKey) < 0 { panic(fmt.Errorf("see smaller key: fromPrefix=%x, prevKey=%x", fromPrefix, prevKey)) } } - fmt.Printf("[dbg] bs2 !ok, %x, %d\n", fromPrefix, 0) return foundOffset } diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index d3c04d47ce0..483b572cd31 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1365,13 +1365,7 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to var offset uint64 if len(from) > 0 { n := item.src.decompressor.Count() / 2 - found := g.BinarySearch(from, n, idx.OrdinalLookup) - if int(found) < n { - fmt.Printf("[dbg] bs1 !ok, %x, %d\n", from, offset) - } else { - fmt.Printf("[dbg] bs2 !ok, %x, %d\n", from, offset) - offset = 0 - } + offset = g.BinarySearch(from, n, idx.OrdinalLookup) } g.Reset(offset) if g.HasNext() { From 405fb89ea560ecc087fc24a3fbc0ffe33dd77a79 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 20:32:46 +0700 Subject: [PATCH 30/91] save --- erigon-lib/seg/decompress.go | 1 + 1 file changed, 1 insertion(+) diff --git a/erigon-lib/seg/decompress.go b/erigon-lib/seg/decompress.go index 917e76dc098..faf7e0ea568 100644 --- a/erigon-lib/seg/decompress.go +++ b/erigon-lib/seg/decompress.go @@ -1067,6 +1067,7 @@ func (g *Getter) FastNext(buf []byte) ([]byte, uint64) { } func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (offset uint64)) (foundOffset uint64) { + //TODO: unit tests with asserts //n := item.src.decompressor.Count() / 2 foundItem := sort.Search(count, func(i int) bool { offset := f(uint64(i)) From 239b672e27dd3b6403d890d39cb68b1ff911fbc9 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 4 Nov 2024 20:33:05 +0700 Subject: [PATCH 31/91] save --- erigon-lib/seg/decompress.go | 1 - 1 file changed, 1 deletion(-) diff --git a/erigon-lib/seg/decompress.go b/erigon-lib/seg/decompress.go index faf7e0ea568..01c4bfec0d6 100644 --- a/erigon-lib/seg/decompress.go +++ b/erigon-lib/seg/decompress.go @@ -1068,7 +1068,6 @@ func (g *Getter) FastNext(buf []byte) ([]byte, uint64) { func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (offset uint64)) (foundOffset uint64) { //TODO: unit tests with asserts - //n := item.src.decompressor.Count() / 2 foundItem := sort.Search(count, func(i int) bool { offset := f(uint64(i)) g.Reset(offset) From 2764da84c1867dbb4c2aaa77467b05d36710fb41 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 09:36:21 +0700 Subject: [PATCH 32/91] save --- core/state/dump.go | 20 ++++++++++++++------ erigon-lib/seg/decompress.go | 16 +++++++++++++--- erigon-lib/seg/decompress_test.go | 24 ++++++++++++++++++------ 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/core/state/dump.go b/core/state/dump.go index 6a52cd49f2f..c684b6870ac 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -136,6 +136,10 @@ func NewDumper(db kv.Tx, txNumsReader rawdbv3.TxNumsReader, blockNumber uint64) } } +var TooMuchIterations = fmt.Errorf("[rpc] Dumper: too much iterations protection triggered") + +const DumperIterationsHardLimit = 10_000_000 + func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bool, startAddress libcommon.Address, maxResults int) ([]byte, error) { var emptyCodeHash = crypto.Keccak256Hash(nil) var emptyHash = libcommon.Hash{} @@ -160,6 +164,8 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo return nil, err } + var hardLimit = DumperIterationsHardLimit + t := time.Now() var nextKey []byte it, err := ttx.DomainRange(kv.AccountsDomain, startAddress[:], nil, txNum, order.Asc, maxResults) @@ -168,7 +174,6 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo } fmt.Printf("[dbg] after DomainRange: %s\n", time.Since(t)) defer it.Close() - i := 0 for it.HasNext() { k, v, err := it.Next() if err != nil { @@ -181,8 +186,6 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo if len(v) == 0 { continue } - i++ - fmt.Printf("[dbg] iter: %d\n", i) if e := accounts.DeserialiseV3(&acc, v); e != nil { return nil, fmt.Errorf("decoding %x for %x: %w", v, k, e) @@ -211,12 +214,15 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo addrList = append(addrList, libcommon.BytesToAddress(k)) numberOfResults++ + + if hardLimit--; hardLimit < 0 { + return nil, TooMuchIterations + } } it.Close() fmt.Printf("[dbg] first loop done: %d\n", len(addrList)) - j := 0 for i, addr := range addrList { account := accountList[i] @@ -229,8 +235,6 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo } defer r.Close() for r.HasNext() { - j++ - fmt.Printf("[dbg] iter j: %d\n", j) k, vs, err := r.Next() if err != nil { return nil, fmt.Errorf("walking over storage for %x: %w", addr, err) @@ -242,6 +246,10 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo account.Storage[libcommon.BytesToHash(loc).String()] = common.Bytes2Hex(vs) h, _ := libcommon.HashData(loc) t.Update(h.Bytes(), libcommon.Copy(vs)) + + if hardLimit--; hardLimit < 0 { + return nil, TooMuchIterations + } } r.Close() diff --git a/erigon-lib/seg/decompress.go b/erigon-lib/seg/decompress.go index 01c4bfec0d6..5c55c1d381a 100644 --- a/erigon-lib/seg/decompress.go +++ b/erigon-lib/seg/decompress.go @@ -1066,6 +1066,7 @@ func (g *Getter) FastNext(buf []byte) ([]byte, uint64) { return buf[:wordLen], postLoopPos } +// BinarySearch - doesn't know if file is sorted func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (offset uint64)) (foundOffset uint64) { //TODO: unit tests with asserts foundItem := sort.Search(count, func(i int) bool { @@ -1073,18 +1074,27 @@ func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (of g.Reset(offset) if g.HasNext() { key, _ := g.Next(nil) - return bytes.Compare(fromPrefix, key) >= 0 + fmt.Printf("ee: %s, %s, %d, %d\n", key, fromPrefix, i, bytes.Compare(key, fromPrefix)) + return bytes.Compare(key, fromPrefix) >= 0 + } else { + panic(1) } return false }) + fmt.Printf("found: %d, %d\n", foundItem, count) if foundItem < count { foundOffset = f(uint64(foundItem)) } - + fmt.Printf("found offset: %d\n", foundOffset) + g.Reset(foundOffset) + if g.HasNext() { + key, _ := g.Next(nil) + fmt.Printf("found key: %s\n", key) + } if dbg.AssertEnabled && foundItem > 2 { g.Reset(f(uint64(foundItem - 2))) // prev key prevKey, _ := g.Next(nil) - if bytes.Compare(fromPrefix, prevKey) < 0 { + if bytes.Compare(prevKey, fromPrefix) >= 0 { panic(fmt.Errorf("see smaller key: fromPrefix=%x, prevKey=%x", fromPrefix, prevKey)) } } diff --git a/erigon-lib/seg/decompress_test.go b/erigon-lib/seg/decompress_test.go index f43f0e08048..3ed0ecb2bdc 100644 --- a/erigon-lib/seg/decompress_test.go +++ b/erigon-lib/seg/decompress_test.go @@ -24,6 +24,7 @@ import ( "math/rand" "os" "path/filepath" + "slices" "strings" "testing" "time" @@ -261,6 +262,7 @@ func prepareLoremDictUncompressed(t *testing.T) *Decompressor { t.Fatal(err) } defer c.Close() + slices.Sort(loremStrings) for k, w := range loremStrings { if err = c.AddUncompressedWord([]byte(fmt.Sprintf("%s %d", w, k))); err != nil { t.Fatal(err) @@ -281,16 +283,25 @@ func TestUncompressed(t *testing.T) { defer d.Close() g := d.MakeGetter() i := 0 + var offsets []uint64 for g.HasNext() { w := loremStrings[i] expected := []byte(fmt.Sprintf("%s %d", w, i+1)) expected = expected[:len(expected)/2] - actual, _ := g.NextUncompressed() + actual, offset := g.NextUncompressed() if bytes.Equal(expected, actual) { t.Errorf("expected %s, actual %s", expected, actual) } i++ + offsets = append(offsets, offset) } + + found := g.BinarySearch([]byte("ipsum"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) + fmt.Printf("found: %d\n", found) + g.Reset(found) + k, _ := g.Next(nil) + fmt.Printf("found: %s\n", k) + } func TestDecompressor_OpenCorrupted(t *testing.T) { @@ -461,12 +472,13 @@ func TestDecompressor_OpenCorrupted(t *testing.T) { }) } -const lorem = `Lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor incididunt ut labore et -dolore magna aliqua Ut enim ad minim veniam quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo -consequat Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur -Excepteur sint occaecat cupidatat non proident sunt in culpa qui officia deserunt mollit anim id est laborum` +const lorem = ` +lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor incididunt ut labore et +dolore magna aliqua ut enim ad minim veniam quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo +consequat duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur +excepteur sint occaecat cupidatat non proident sunt in culpa qui officia deserunt mollit anim id est laborum` -var loremStrings = strings.Split(lorem, " ") +var loremStrings = strings.Split(strings.ReplaceAll(strings.ReplaceAll(lorem, "\n", " "), "\r", ""), " ") func TestDecompressTorrent(t *testing.T) { t.Skip() From edfdaec3c3a2bcead3ea15b89385d666643aa4e1 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 10:14:24 +0700 Subject: [PATCH 33/91] save --- erigon-lib/seg/decompress.go | 20 ++++----- erigon-lib/seg/decompress_test.go | 67 ++++++++++++++++++++++--------- 2 files changed, 54 insertions(+), 33 deletions(-) diff --git a/erigon-lib/seg/decompress.go b/erigon-lib/seg/decompress.go index 5c55c1d381a..08573e08f85 100644 --- a/erigon-lib/seg/decompress.go +++ b/erigon-lib/seg/decompress.go @@ -1066,30 +1066,24 @@ func (g *Getter) FastNext(buf []byte) ([]byte, uint64) { return buf[:wordLen], postLoopPos } -// BinarySearch - doesn't know if file is sorted -func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (offset uint64)) (foundOffset uint64) { - //TODO: unit tests with asserts +// BinarySearch - !expecting sorted file +func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (offset uint64)) (value []byte, ok bool) { foundItem := sort.Search(count, func(i int) bool { offset := f(uint64(i)) g.Reset(offset) if g.HasNext() { key, _ := g.Next(nil) - fmt.Printf("ee: %s, %s, %d, %d\n", key, fromPrefix, i, bytes.Compare(key, fromPrefix)) return bytes.Compare(key, fromPrefix) >= 0 - } else { - panic(1) } return false }) - fmt.Printf("found: %d, %d\n", foundItem, count) - if foundItem < count { - foundOffset = f(uint64(foundItem)) + if foundItem == count { + return nil, false } - fmt.Printf("found offset: %d\n", foundOffset) + foundOffset := f(uint64(foundItem)) g.Reset(foundOffset) if g.HasNext() { - key, _ := g.Next(nil) - fmt.Printf("found key: %s\n", key) + value, _ = g.Next(nil) } if dbg.AssertEnabled && foundItem > 2 { g.Reset(f(uint64(foundItem - 2))) // prev key @@ -1098,5 +1092,5 @@ func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (of panic(fmt.Errorf("see smaller key: fromPrefix=%x, prevKey=%x", fromPrefix, prevKey)) } } - return foundOffset + return value, true } diff --git a/erigon-lib/seg/decompress_test.go b/erigon-lib/seg/decompress_test.go index 3ed0ecb2bdc..0c71a41fcc2 100644 --- a/erigon-lib/seg/decompress_test.go +++ b/erigon-lib/seg/decompress_test.go @@ -258,23 +258,23 @@ func prepareLoremDictUncompressed(t *testing.T) *Decompressor { cfg.MinPatternScore = 1 cfg.Workers = 2 c, err := NewCompressor(context.Background(), t.Name(), file, tmpDir, cfg, log.LvlDebug, logger) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer c.Close() slices.Sort(loremStrings) for k, w := range loremStrings { - if err = c.AddUncompressedWord([]byte(fmt.Sprintf("%s %d", w, k))); err != nil { - t.Fatal(err) + if len(w) == 0 { + err = c.AddUncompressedWord([]byte(w)) + require.NoError(t, err) + continue } + err = c.AddUncompressedWord([]byte(fmt.Sprintf("%s %d", w, k))) + require.NoError(t, err) } - if err = c.Compress(); err != nil { - t.Fatal(err) - } - var d *Decompressor - if d, err = NewDecompressor(file); err != nil { - t.Fatal(err) - } + err = c.Compress() + require.NoError(t, err) + d, err := NewDecompressor(file) + require.NoError(t, err) + t.Cleanup(d.Close) return d } @@ -284,6 +284,7 @@ func TestUncompressed(t *testing.T) { g := d.MakeGetter() i := 0 var offsets []uint64 + offsets = append(offsets, 0) for g.HasNext() { w := loremStrings[i] expected := []byte(fmt.Sprintf("%s %d", w, i+1)) @@ -296,11 +297,35 @@ func TestUncompressed(t *testing.T) { offsets = append(offsets, offset) } - found := g.BinarySearch([]byte("ipsum"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) - fmt.Printf("found: %d\n", found) - g.Reset(found) - k, _ := g.Next(nil) - fmt.Printf("found: %s\n", k) + t.Run("BinarySearch", func(t *testing.T) { + require := require.New(t) + k, ok := g.BinarySearch([]byte("ipsum"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) + require.True(ok) + require.Equal("ipsum 38", string(k)) + k, ok = g.BinarySearch([]byte("ipsu"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) + require.True(ok) + require.Equal("ipsum 38", string(k)) + + //last word is `voluptate` + k, ok = g.BinarySearch([]byte("voluptate"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) + require.True(ok) + require.Equal("voluptate 69", string(k)) + k, ok = g.BinarySearch([]byte("voluptat"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) + require.True(ok) + require.Equal("voluptate 69", string(k)) + k, ok = g.BinarySearch([]byte("voluptatez"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) + require.False(ok) + require.Equal("", string(k)) + + //first word is `` + k, ok = g.BinarySearch([]byte(""), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) + require.True(ok) + require.Equal(" 0", string(k)) + + k, ok = g.BinarySearch(nil, d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) + require.True(ok) + require.Equal(" 0", string(k)) + }) } @@ -472,13 +497,15 @@ func TestDecompressor_OpenCorrupted(t *testing.T) { }) } -const lorem = ` -lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor incididunt ut labore et +const lorem = `lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor incididunt ut labore et dolore magna aliqua ut enim ad minim veniam quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur excepteur sint occaecat cupidatat non proident sunt in culpa qui officia deserunt mollit anim id est laborum` -var loremStrings = strings.Split(strings.ReplaceAll(strings.ReplaceAll(lorem, "\n", " "), "\r", ""), " ") +var loremStrings = append(strings.Split(rmNewLine(lorem), " "), "") // including emtpy string - to trigger corner cases +func rmNewLine(s string) string { + return strings.ReplaceAll(strings.ReplaceAll(s, "\n", " "), "\r", "") +} func TestDecompressTorrent(t *testing.T) { t.Skip() From 6a1d322fd5f82755d8476ebebdbd6c99daaac366 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 10:15:23 +0700 Subject: [PATCH 34/91] save --- erigon-lib/seg/decompress.go | 2 +- erigon-lib/seg/decompress_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/erigon-lib/seg/decompress.go b/erigon-lib/seg/decompress.go index 08573e08f85..8621b2a4ab8 100644 --- a/erigon-lib/seg/decompress.go +++ b/erigon-lib/seg/decompress.go @@ -1077,7 +1077,7 @@ func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (of } return false }) - if foundItem == count { + if foundItem == count { // `Search` returns `n` if not found return nil, false } foundOffset := f(uint64(foundItem)) diff --git a/erigon-lib/seg/decompress_test.go b/erigon-lib/seg/decompress_test.go index 0c71a41fcc2..f77dc51d325 100644 --- a/erigon-lib/seg/decompress_test.go +++ b/erigon-lib/seg/decompress_test.go @@ -320,11 +320,11 @@ func TestUncompressed(t *testing.T) { //first word is `` k, ok = g.BinarySearch([]byte(""), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) require.True(ok) - require.Equal(" 0", string(k)) + require.Equal("", string(k)) k, ok = g.BinarySearch(nil, d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) require.True(ok) - require.Equal(" 0", string(k)) + require.Equal("", string(k)) }) } From 61e3a99b54f3cb9f20b3cce4d9bb91c9a9ab796a Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 10:31:44 +0700 Subject: [PATCH 35/91] save --- erigon-lib/seg/decompress.go | 30 +++++++++++++----------------- erigon-lib/seg/decompress_test.go | 30 ++++++++++++++++++++---------- erigon-lib/state/history.go | 6 +++++- 3 files changed, 38 insertions(+), 28 deletions(-) diff --git a/erigon-lib/seg/decompress.go b/erigon-lib/seg/decompress.go index 8621b2a4ab8..ca257a11bae 100644 --- a/erigon-lib/seg/decompress.go +++ b/erigon-lib/seg/decompress.go @@ -1066,31 +1066,27 @@ func (g *Getter) FastNext(buf []byte) ([]byte, uint64) { return buf[:wordLen], postLoopPos } -// BinarySearch - !expecting sorted file -func (g *Getter) BinarySearch(fromPrefix []byte, count int, f func(i uint64) (offset uint64)) (value []byte, ok bool) { +// BinarySearch - !expecting sorted file - does Seek `g` to key which >= `fromPrefix` by using BinarySearch - means unoptimal and touching many places in file +// use `.Next` to read found +// at `ok = false` leaving `g` in unpredictible state +func (g *Getter) BinarySearch(seek []byte, count int, getOffset func(i uint64) (offset uint64)) (foundOffset uint64, ok bool) { + var key []byte foundItem := sort.Search(count, func(i int) bool { - offset := f(uint64(i)) + offset := getOffset(uint64(i)) g.Reset(offset) if g.HasNext() { - key, _ := g.Next(nil) - return bytes.Compare(key, fromPrefix) >= 0 + key, _ = g.Next(key[:0]) + return bytes.Compare(key, seek) >= 0 } return false }) if foundItem == count { // `Search` returns `n` if not found - return nil, false + return 0, false } - foundOffset := f(uint64(foundItem)) + foundOffset = getOffset(uint64(foundItem)) g.Reset(foundOffset) - if g.HasNext() { - value, _ = g.Next(nil) - } - if dbg.AssertEnabled && foundItem > 2 { - g.Reset(f(uint64(foundItem - 2))) // prev key - prevKey, _ := g.Next(nil) - if bytes.Compare(prevKey, fromPrefix) >= 0 { - panic(fmt.Errorf("see smaller key: fromPrefix=%x, prevKey=%x", fromPrefix, prevKey)) - } + if !g.HasNext() { + return 0, false } - return value, true + return foundOffset, true } diff --git a/erigon-lib/seg/decompress_test.go b/erigon-lib/seg/decompress_test.go index f77dc51d325..9e6ba470323 100644 --- a/erigon-lib/seg/decompress_test.go +++ b/erigon-lib/seg/decompress_test.go @@ -297,33 +297,43 @@ func TestUncompressed(t *testing.T) { offsets = append(offsets, offset) } - t.Run("BinarySearch", func(t *testing.T) { + t.Run("BinarySearch middle", func(t *testing.T) { require := require.New(t) - k, ok := g.BinarySearch([]byte("ipsum"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) + _, ok := g.BinarySearch([]byte("ipsum"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) require.True(ok) + k, _ := g.Next(nil) require.Equal("ipsum 38", string(k)) - k, ok = g.BinarySearch([]byte("ipsu"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) + _, ok = g.BinarySearch([]byte("ipsu"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) require.True(ok) + k, _ = g.Next(nil) require.Equal("ipsum 38", string(k)) - + }) + t.Run("BinarySearch end of file", func(t *testing.T) { + require := require.New(t) //last word is `voluptate` - k, ok = g.BinarySearch([]byte("voluptate"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) + _, ok := g.BinarySearch([]byte("voluptate"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) require.True(ok) + k, _ := g.Next(nil) require.Equal("voluptate 69", string(k)) - k, ok = g.BinarySearch([]byte("voluptat"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) + _, ok = g.BinarySearch([]byte("voluptat"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) require.True(ok) + k, _ = g.Next(nil) require.Equal("voluptate 69", string(k)) - k, ok = g.BinarySearch([]byte("voluptatez"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) + _, ok = g.BinarySearch([]byte("voluptatez"), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) require.False(ok) - require.Equal("", string(k)) + }) + t.Run("BinarySearch begin of file", func(t *testing.T) { + require := require.New(t) //first word is `` - k, ok = g.BinarySearch([]byte(""), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) + _, ok := g.BinarySearch([]byte(""), d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) require.True(ok) + k, _ := g.Next(nil) require.Equal("", string(k)) - k, ok = g.BinarySearch(nil, d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) + _, ok = g.BinarySearch(nil, d.Count(), func(i uint64) (offset uint64) { return offsets[i] }) require.True(ok) + k, _ = g.Next(nil) require.Equal("", string(k)) }) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 483b572cd31..8f2f85f2cf6 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1365,7 +1365,11 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to var offset uint64 if len(from) > 0 { n := item.src.decompressor.Count() / 2 - offset = g.BinarySearch(from, n, idx.OrdinalLookup) + var ok bool + offset, ok = g.BinarySearch(from, n, idx.OrdinalLookup) + if !ok { + offset = 0 + } } g.Reset(offset) if g.HasNext() { From ca6ab9257719508c5a5949c7f19bfb49a6175f8f Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 10:40:27 +0700 Subject: [PATCH 36/91] save --- erigon-lib/state/history.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 8f2f85f2cf6..798b5bcaae0 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1431,7 +1431,6 @@ func (hi *StateAsOfIterF) advanceInFiles() error { var idxVal []byte //if hi.compressVals { idxVal, _ = top.g.Next(nil) - fmt.Printf("[dbg] StateAsOfIterF advanceInFiles: %s, %x, to: %x\n", top.g.FileName(), key, hi.to) //} else { // idxVal, _ = top.g.NextUncompressed() //} @@ -1442,6 +1441,7 @@ func (hi *StateAsOfIterF) advanceInFiles() error { // top.key, _ = top.g.NextUncompressed() //} if hi.to == nil || bytes.Compare(top.key, hi.to) < 0 { + fmt.Printf("[dbg] StateAsOfIterF advanceInFiles keep: %s, %x, to: %x\n", top.g.FileName(), top.key, hi.to) heap.Push(&hi.h, top) } } From 98e95f1ab5faa077f42f3722d41c5992cd36fad5 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 10:42:17 +0700 Subject: [PATCH 37/91] save --- core/state/dump.go | 2 ++ turbo/jsonrpc/debug_api.go | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/core/state/dump.go b/core/state/dump.go index c684b6870ac..f9df246837f 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -186,6 +186,7 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo if len(v) == 0 { continue } + fmt.Printf("[dbg] dumper iter acc: %x\n", k) if e := accounts.DeserialiseV3(&acc, v); e != nil { return nil, fmt.Errorf("decoding %x for %x: %w", v, k, e) @@ -243,6 +244,7 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo continue // Skip deleted entries } loc := k[20:] + fmt.Printf("[dbg] dumper iter storage: %x, %x\n", k[:20], k[20:]) account.Storage[libcommon.BytesToHash(loc).String()] = common.Bytes2Hex(vs) h, _ := libcommon.HashData(loc) t.Update(h.Bytes(), libcommon.Copy(vs)) diff --git a/turbo/jsonrpc/debug_api.go b/turbo/jsonrpc/debug_api.go index 64e5f82fba9..05e33950b0f 100644 --- a/turbo/jsonrpc/debug_api.go +++ b/turbo/jsonrpc/debug_api.go @@ -110,7 +110,6 @@ func (api *PrivateDebugAPIImpl) AccountRange(ctx context.Context, blockNrOrHash return state.IteratorDump{}, err } defer tx.Rollback() - fmt.Printf("[dbg] AccountRange limits: %t, %t, %d\n", excludeCode, excludeStorage, maxResults) var blockNumber uint64 From 5325c474387a67d3a32a314e857df04c819b9d49 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 10:43:34 +0700 Subject: [PATCH 38/91] save --- erigon-lib/state/history.go | 1 - 1 file changed, 1 deletion(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 798b5bcaae0..3309dd54b67 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1374,7 +1374,6 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to g.Reset(offset) if g.HasNext() { key, offset := g.Next(nil) - fmt.Printf("[dbg] ht.iit.files seek(%x) -> %x\n", from, key) heap.Push(&hi.h, &ReconItem{g: g, key: key, startTxNum: item.startTxNum, endTxNum: item.endTxNum, txNum: item.endTxNum, startOffset: offset, lastOffset: offset}) } } From 2b52e7087cd153ddddad939b18fffbdf2421cd7d Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 10:45:58 +0700 Subject: [PATCH 39/91] save --- core/state/dump.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/state/dump.go b/core/state/dump.go index f9df246837f..d96551f1ddb 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -181,6 +181,7 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo } if maxResults > 0 && numberOfResults >= maxResults { nextKey = append(nextKey[:0], k...) + fmt.Printf("[dbg] dumper iter acc. break: nextKey %x\n", nextKey) break } if len(v) == 0 { From f1725ea2228659c121859ddd84dd5d2f7a47fb9d Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 10:48:25 +0700 Subject: [PATCH 40/91] save --- erigon-lib/state/history.go | 1 + 1 file changed, 1 insertion(+) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 3309dd54b67..d88092f0b36 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1487,6 +1487,7 @@ func (hi *StateAsOfIterF) Next() ([]byte, []byte, error) { return nil, nil, hi.ctx.Err() default: } + fmt.Printf("[dbg] StateAsOfIterF.next: limit=%d\n", hi.limit) hi.limit-- hi.k, hi.v = append(hi.k[:0], hi.nextKey...), append(hi.v[:0], hi.nextVal...) From 29be11ec8ebba7f1a731a05422f79a286768db14 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 10:53:32 +0700 Subject: [PATCH 41/91] save --- erigon-lib/state/history.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index d88092f0b36..22b650b4da5 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -30,6 +30,7 @@ import ( "sync" "time" + "github.com/erigontech/erigon-lib/common/dbg" btree2 "github.com/tidwall/btree" "golang.org/x/sync/errgroup" @@ -1478,7 +1479,7 @@ func (hi *StateAsOfIterF) advanceInFiles() error { } func (hi *StateAsOfIterF) HasNext() bool { - return hi.limit != 0 && hi.nextKey != nil + return hi.limit > 0 && hi.nextKey != nil } func (hi *StateAsOfIterF) Next() ([]byte, []byte, error) { @@ -1487,7 +1488,7 @@ func (hi *StateAsOfIterF) Next() ([]byte, []byte, error) { return nil, nil, hi.ctx.Err() default: } - fmt.Printf("[dbg] StateAsOfIterF.next: limit=%d\n", hi.limit) + fmt.Printf("[dbg] StateAsOfIterF.next: limit=%d, %s\n", hi.limit, dbg.Stack()) hi.limit-- hi.k, hi.v = append(hi.k[:0], hi.nextKey...), append(hi.v[:0], hi.nextVal...) @@ -1625,7 +1626,7 @@ func (hi *StateAsOfIterDB) HasNext() bool { if hi.err != nil { return true } - return hi.limit != 0 && hi.nextKey != nil + return hi.limit > 0 && hi.nextKey != nil } func (hi *StateAsOfIterDB) Next() ([]byte, []byte, error) { @@ -2041,7 +2042,7 @@ func (hi *HistoryChangesIterDB) HasNext() bool { if hi.err != nil { // always true, then .Next() call will return this error return true } - if hi.limit == 0 { // limit reached + if hi.limit <= 0 { // limit reached return false } if hi.nextKey == nil { // EndOfTable From fc3a6ff830c7d11bb4a017f6ff4924384acf4d8c Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 10:59:14 +0700 Subject: [PATCH 42/91] save --- erigon-lib/kv/mdbx/kv_mdbx.go | 4 +-- erigon-lib/state/domain.go | 3 +-- erigon-lib/state/domain_test.go | 4 +-- erigon-lib/state/history.go | 45 ++++++++++++++++++++++++++++----- 4 files changed, 43 insertions(+), 13 deletions(-) diff --git a/erigon-lib/kv/mdbx/kv_mdbx.go b/erigon-lib/kv/mdbx/kv_mdbx.go index 968a24e2fa7..71d297e68e3 100644 --- a/erigon-lib/kv/mdbx/kv_mdbx.go +++ b/erigon-lib/kv/mdbx/kv_mdbx.go @@ -1582,7 +1582,7 @@ func (s *cursor2iter) Close() { } func (s *cursor2iter) HasNext() bool { - if s.limit == 0 { // limit reached + if s.limit <= 0 { // limit reached return false } if s.nextK == nil { // EndOfTable @@ -1744,7 +1744,7 @@ func (s *cursorDup2iter) Close() { } } func (s *cursorDup2iter) HasNext() bool { - if s.limit == 0 { // limit reached + if s.limit <= 0 { // limit reached return false } if s.nextV == nil { // EndOfTable diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index cc322b677a0..34712b5458f 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1952,7 +1952,6 @@ func (dt *DomainRoTx) GetLatest(key1, key2 []byte, roTx kv.Tx) ([]byte, uint64, func (dt *DomainRoTx) DomainRange(ctx context.Context, tx kv.Tx, fromKey, toKey []byte, ts uint64, asc order.By, limit int) (it stream.KV, err error) { if !asc { - fmt.Printf("[dbg] what 1?\n") panic("implement me") } //histStateIt, err := tx.aggTx.AccountHistoricalStateRange(asOfTs, fromKey, toKey, limit, tx.MdbxTx) @@ -1964,7 +1963,7 @@ func (dt *DomainRoTx) DomainRange(ctx context.Context, tx kv.Tx, fromKey, toKey // return nil, err //} fmt.Printf("[dbg] DomainRange 1: %s, %x, %x\n", dt.d.name, fromKey, toKey) - histStateIt, err := dt.ht.WalkAsOf(ctx, ts, fromKey, toKey, tx, limit) + histStateIt, err := dt.ht.WalkAsOf(ctx, ts, fromKey, toKey, asc, limit, tx) if err != nil { return nil, err } diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index f8f0b8d0ee3..c5e7df8e62c 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -1975,10 +1975,10 @@ func TestDomain_Unwind(t *testing.T) { uc := d.BeginFilesRo() defer uc.Close() - et, err := ectx.ht.WalkAsOf(context.Background(), unwindTo-1, nil, nil, etx, -1) + et, err := ectx.ht.WalkAsOf(context.Background(), unwindTo-1, nil, nil, order.Asc, -1, etx) require.NoError(t, err) - ut, err := uc.ht.WalkAsOf(context.Background(), unwindTo-1, nil, nil, utx, -1) + ut, err := uc.ht.WalkAsOf(context.Background(), unwindTo-1, nil, nil, order.Asc, -1, utx) require.NoError(t, err) compareIterators(t, et, ut) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 22b650b4da5..08e918d3e3d 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1345,9 +1345,12 @@ func (ht *HistoryRoTx) historySeekInDB(key []byte, txNum uint64, tx kv.Tx) ([]by // `val == []byte{}` means key was created in this txNum and doesn't exist before. return val[8:], true, nil } -func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to []byte, roTx kv.Tx, limit int) (stream.KV, error) { +func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to []byte, asc order.By, limit int, roTx kv.Tx) (stream.KV, error) { + if !asc { + panic("implement me") + } hi := &StateAsOfIterF{ - from: from, to: to, limit: limit, + from: from, to: to, limit: limit, orderAscend: asc, hc: ht, startTxNum: startTxNum, @@ -1388,7 +1391,7 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to largeValues: ht.h.historyLargeValues, roTx: roTx, valsTable: ht.h.historyValsTable, - from: from, to: to, limit: limit, + from: from, to: to, limit: limit, orderAscend: asc, startTxNum: startTxNum, @@ -1417,6 +1420,7 @@ type StateAsOfIterF struct { txnKey [8]byte k, v, kBackup, vBackup []byte + orderAscend order.By ctx context.Context } @@ -1479,7 +1483,20 @@ func (hi *StateAsOfIterF) advanceInFiles() error { } func (hi *StateAsOfIterF) HasNext() bool { - return hi.limit > 0 && hi.nextKey != nil + if hi.limit <= 0 { // limit reached + return false + } + if hi.nextKey == nil { // EndOfTable + return false + } + if hi.to == nil { // s.nextK == nil check is above + return true + } + + //Asc: [from, to) AND from < to + //Desc: [from, to) AND from > to + cmp := bytes.Compare(hi.nextKey, hi.to) + return (bool(hi.orderAscend) && cmp < 0) || (!bool(hi.orderAscend) && cmp > 0) } func (hi *StateAsOfIterF) Next() ([]byte, []byte, error) { @@ -1509,8 +1526,9 @@ type StateAsOfIterDB struct { valsCDup kv.CursorDupSort valsTable string - from, to []byte - limit int + from, to []byte + orderAscend order.By + limit int nextKey, nextVal []byte @@ -1626,7 +1644,20 @@ func (hi *StateAsOfIterDB) HasNext() bool { if hi.err != nil { return true } - return hi.limit > 0 && hi.nextKey != nil + if hi.limit <= 0 { // limit reached + return false + } + if hi.nextKey == nil { // EndOfTable + return false + } + if hi.to == nil { // s.nextK == nil check is above + return true + } + + //Asc: [from, to) AND from < to + //Desc: [from, to) AND from > to + cmp := bytes.Compare(hi.nextKey, hi.to) + return (bool(hi.orderAscend) && cmp < 0) || (!bool(hi.orderAscend) && cmp > 0) } func (hi *StateAsOfIterDB) Next() ([]byte, []byte, error) { From a8be6b498dff5b3ba9e168e3afc5b7420af2d75c Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 11:03:57 +0700 Subject: [PATCH 43/91] save --- erigon-lib/kv/mdbx/kv_mdbx.go | 36 ++++++++++++++--------------------- erigon-lib/state/history.go | 5 +---- 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/erigon-lib/kv/mdbx/kv_mdbx.go b/erigon-lib/kv/mdbx/kv_mdbx.go index 71d297e68e3..5bc703ed29f 100644 --- a/erigon-lib/kv/mdbx/kv_mdbx.go +++ b/erigon-lib/kv/mdbx/kv_mdbx.go @@ -1582,20 +1582,16 @@ func (s *cursor2iter) Close() { } func (s *cursor2iter) HasNext() bool { - if s.limit <= 0 { // limit reached + if s.limit <= 0 || s.nextV == nil { // Limit or EndOfTable return false } - if s.nextK == nil { // EndOfTable - return false - } - if s.toPrefix == nil { // s.nextK == nil check is above - return true + if s.toPrefix != nil { + //Asc: [from, to) AND from < to + //Desc: [from, to) AND from > to + cmp := bytes.Compare(s.nextK, s.toPrefix) + return (bool(s.orderAscend) && cmp < 0) || (!bool(s.orderAscend) && cmp > 0) } - - //Asc: [from, to) AND from < to - //Desc: [from, to) AND from > to - cmp := bytes.Compare(s.nextK, s.toPrefix) - return (bool(s.orderAscend) && cmp < 0) || (!bool(s.orderAscend) && cmp > 0) + return true } func (s *cursor2iter) Next() (k, v []byte, err error) { @@ -1744,20 +1740,16 @@ func (s *cursorDup2iter) Close() { } } func (s *cursorDup2iter) HasNext() bool { - if s.limit <= 0 { // limit reached + if s.limit <= 0 || s.nextV == nil { // Limit or EndOfTable return false } - if s.nextV == nil { // EndOfTable - return false - } - if s.toPrefix == nil { // s.nextK == nil check is above - return true + if s.toPrefix != nil { + //Asc: [from, to) AND from < to + //Desc: [from, to) AND from > to + cmp := bytes.Compare(s.nextV, s.toPrefix) + return (s.orderAscend && cmp < 0) || (!s.orderAscend && cmp > 0) } - - //Asc: [from, to) AND from < to - //Desc: [from, to) AND from > to - cmp := bytes.Compare(s.nextV, s.toPrefix) - return (s.orderAscend && cmp < 0) || (!s.orderAscend && cmp > 0) + return true } func (s *cursorDup2iter) Next() (k, v []byte, err error) { select { diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 08e918d3e3d..806211ddf6f 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1644,10 +1644,7 @@ func (hi *StateAsOfIterDB) HasNext() bool { if hi.err != nil { return true } - if hi.limit <= 0 { // limit reached - return false - } - if hi.nextKey == nil { // EndOfTable + if hi.limit <= 0 || hi.nextKey == nil { // Limit or EndOfTable return false } if hi.to == nil { // s.nextK == nil check is above From 320a5f4aaef66ab622c23f7d1f72f9bbd79a7e38 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 11:06:02 +0700 Subject: [PATCH 44/91] save --- erigon-lib/state/history.go | 56 ++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 806211ddf6f..12539f42337 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1483,20 +1483,16 @@ func (hi *StateAsOfIterF) advanceInFiles() error { } func (hi *StateAsOfIterF) HasNext() bool { - if hi.limit <= 0 { // limit reached + if hi.limit <= 0 || hi.nextKey == nil { // Limit or EndOfTable return false } - if hi.nextKey == nil { // EndOfTable - return false + if hi.to != nil { + //Asc: [from, to) AND from < to + //Desc: [from, to) AND from > to + cmp := bytes.Compare(hi.nextKey, hi.to) + return (bool(hi.orderAscend) && cmp < 0) || (!bool(hi.orderAscend) && cmp > 0) } - if hi.to == nil { // s.nextK == nil check is above - return true - } - - //Asc: [from, to) AND from < to - //Desc: [from, to) AND from > to - cmp := bytes.Compare(hi.nextKey, hi.to) - return (bool(hi.orderAscend) && cmp < 0) || (!bool(hi.orderAscend) && cmp > 0) + return true } func (hi *StateAsOfIterF) Next() ([]byte, []byte, error) { @@ -1647,14 +1643,13 @@ func (hi *StateAsOfIterDB) HasNext() bool { if hi.limit <= 0 || hi.nextKey == nil { // Limit or EndOfTable return false } - if hi.to == nil { // s.nextK == nil check is above - return true + if hi.to != nil { + //Asc: [from, to) AND from < to + //Desc: [from, to) AND from > to + cmp := bytes.Compare(hi.nextKey, hi.to) + return (bool(hi.orderAscend) && cmp < 0) || (!bool(hi.orderAscend) && cmp > 0) } - - //Asc: [from, to) AND from < to - //Desc: [from, to) AND from > to - cmp := bytes.Compare(hi.nextKey, hi.to) - return (bool(hi.orderAscend) && cmp < 0) || (!bool(hi.orderAscend) && cmp > 0) + return true } func (hi *StateAsOfIterDB) Next() ([]byte, []byte, error) { @@ -1894,16 +1889,16 @@ func (hi *HistoryChangesIterFiles) HasNext() bool { if hi.err != nil { // always true, then .Next() call will return this error return true } - if hi.limit == 0 { // limit reached - return false - } - if hi.nextKey == nil { // EndOfTable + if hi.limit <= 0 || hi.nextKey == nil { // Limit or EndOfTable return false } - return true - //if hi.toPrefix == nil { // s.nextK == nil check is above - // return true + //if hi.to != nil { + // //Asc: [from, to) AND from < to + // //Desc: [from, to) AND from > to + // cmp := bytes.Compare(hi.nextKey, hi.to) + // return (bool(hi.orderAscend) && cmp < 0) || (!bool(hi.orderAscend) && cmp > 0) //} + return true } func (hi *HistoryChangesIterFiles) Next() ([]byte, []byte, error) { @@ -2070,12 +2065,15 @@ func (hi *HistoryChangesIterDB) HasNext() bool { if hi.err != nil { // always true, then .Next() call will return this error return true } - if hi.limit <= 0 { // limit reached - return false - } - if hi.nextKey == nil { // EndOfTable + if hi.limit <= 0 || hi.nextKey == nil { // Limit or EndOfTable return false } + //if hi.to != nil { + // //Asc: [from, to) AND from < to + // //Desc: [from, to) AND from > to + // cmp := bytes.Compare(hi.nextKey, hi.to) + // return (bool(hi.orderAscend) && cmp < 0) || (!bool(hi.orderAscend) && cmp > 0) + //} return true } From 6d7e2abc87c5faf2478ed996f2361e7d13bf76c3 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 13:19:59 +0700 Subject: [PATCH 45/91] save --- erigon-lib/kv/stream/{stream_exact.go => stream_impl.go} | 0 erigon-lib/state/history.go | 3 +-- 2 files changed, 1 insertion(+), 2 deletions(-) rename erigon-lib/kv/stream/{stream_exact.go => stream_impl.go} (100%) diff --git a/erigon-lib/kv/stream/stream_exact.go b/erigon-lib/kv/stream/stream_impl.go similarity index 100% rename from erigon-lib/kv/stream/stream_exact.go rename to erigon-lib/kv/stream/stream_impl.go diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 12539f42337..7b15d895476 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -30,7 +30,6 @@ import ( "sync" "time" - "github.com/erigontech/erigon-lib/common/dbg" btree2 "github.com/tidwall/btree" "golang.org/x/sync/errgroup" @@ -1501,7 +1500,6 @@ func (hi *StateAsOfIterF) Next() ([]byte, []byte, error) { return nil, nil, hi.ctx.Err() default: } - fmt.Printf("[dbg] StateAsOfIterF.next: limit=%d, %s\n", hi.limit, dbg.Stack()) hi.limit-- hi.k, hi.v = append(hi.k[:0], hi.nextKey...), append(hi.v[:0], hi.nextVal...) @@ -1511,6 +1509,7 @@ func (hi *StateAsOfIterF) Next() ([]byte, []byte, error) { if err := hi.advanceInFiles(); err != nil { return nil, nil, err } + fmt.Printf("[dbg] StateAsOfIterF.next: limit=%d, %x\n", hi.limit, hi.kBackup) return hi.kBackup, hi.vBackup, nil } From 0eb4604ffae483db49e570f7963652abe56fadfd Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 13:21:31 +0700 Subject: [PATCH 46/91] save --- erigon-lib/state/history.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 7b15d895476..f39f14c6b98 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1458,6 +1458,7 @@ func (hi *StateAsOfIterF) advanceInFiles() error { } n, ok := eliasfano32.Seek(idxVal, hi.startTxNum) if !ok { + fmt.Printf("[dbg] StateAsOfIterF advanceInFiles ef.seek not found. %s, %x\n", top.g.FileName(), key) continue } @@ -1470,6 +1471,7 @@ func (hi *StateAsOfIterF) advanceInFiles() error { reader := hi.hc.statelessIdxReader(historyItem.i) offset, ok := reader.Lookup2(hi.txnKey[:], hi.nextKey) if !ok { + fmt.Printf("[dbg] StateAsOfIterF advanceInFiles Lookup2 not found. %s, %x\n", top.g.FileName(), key) continue } g := hi.hc.statelessGetter(historyItem.i) From 8f263d7ca047754d62eabc8cb7a66ca1c0eaa245 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 13:21:44 +0700 Subject: [PATCH 47/91] save --- erigon-lib/state/history.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index f39f14c6b98..9a6d5c7a684 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1458,7 +1458,7 @@ func (hi *StateAsOfIterF) advanceInFiles() error { } n, ok := eliasfano32.Seek(idxVal, hi.startTxNum) if !ok { - fmt.Printf("[dbg] StateAsOfIterF advanceInFiles ef.seek not found. %s, %x\n", top.g.FileName(), key) + fmt.Printf("[dbg] StateAsOfIterF advanceInFiles ef.seek not found. %s, %x\n", top.g.FileName(), hi.nextKey) continue } @@ -1471,7 +1471,7 @@ func (hi *StateAsOfIterF) advanceInFiles() error { reader := hi.hc.statelessIdxReader(historyItem.i) offset, ok := reader.Lookup2(hi.txnKey[:], hi.nextKey) if !ok { - fmt.Printf("[dbg] StateAsOfIterF advanceInFiles Lookup2 not found. %s, %x\n", top.g.FileName(), key) + fmt.Printf("[dbg] StateAsOfIterF advanceInFiles Lookup2 not found. %s, %x\n", top.g.FileName(), hi.nextKey) continue } g := hi.hc.statelessGetter(historyItem.i) From d6bf3f7c43b48164033ab88287ff4639f80428bf Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 13:22:28 +0700 Subject: [PATCH 48/91] save --- erigon-lib/state/history.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 9a6d5c7a684..41b7fa92f52 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1444,7 +1444,6 @@ func (hi *StateAsOfIterF) advanceInFiles() error { // top.key, _ = top.g.NextUncompressed() //} if hi.to == nil || bytes.Compare(top.key, hi.to) < 0 { - fmt.Printf("[dbg] StateAsOfIterF advanceInFiles keep: %s, %x, to: %x\n", top.g.FileName(), top.key, hi.to) heap.Push(&hi.h, top) } } @@ -1461,6 +1460,7 @@ func (hi *StateAsOfIterF) advanceInFiles() error { fmt.Printf("[dbg] StateAsOfIterF advanceInFiles ef.seek not found. %s, %x\n", top.g.FileName(), hi.nextKey) continue } + fmt.Printf("[dbg] StateAsOfIterF advanceInFiles keep: %s, %x, to: %x\n", top.g.FileName(), top.key, hi.to) hi.nextKey = key binary.BigEndian.PutUint64(hi.txnKey[:], n) From 530bfebea75ac5e1f4f2e606dae1ee1e443d2874 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 13:28:16 +0700 Subject: [PATCH 49/91] save --- erigon-lib/state/domain.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index 34712b5458f..5e891bbf754 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1968,10 +1968,11 @@ func (dt *DomainRoTx) DomainRange(ctx context.Context, tx kv.Tx, fromKey, toKey return nil, err } fmt.Printf("[dbg] DomainRange 2: %s\n", dt.d.name) - lastestStateIt, err := dt.DomainRangeLatest(tx, fromKey, toKey, limit) - if err != nil { - return nil, err - } + //lastestStateIt, err := dt.DomainRangeLatest(tx, fromKey, toKey, limit) + //if err != nil { + // return nil, err + //} + lastestStateIt := stream.EmptyKV fmt.Printf("[dbg] DomainRange 3: %s\n", dt.d.name) return stream.UnionKV(histStateIt, lastestStateIt, limit), nil } From 3c2d682fa2e7698b75d2747a68c114d49afe3221 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 13:30:49 +0700 Subject: [PATCH 50/91] save --- erigon-lib/state/history.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 41b7fa92f52..26e299f4253 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1401,7 +1401,8 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to dbit.Close() //it's responsibility of constructor (our) to close resource on error return nil, err } - return stream.UnionKV(hi, dbit, limit), nil + //return stream.UnionKV(hi, dbit, limit), nil + return stream.UnionKV(hi, stream.EmptyKV, limit), nil } // StateAsOfIter - returns state range at given time in history From 939746432eb767648f90d151a5091d29bb797ed6 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 14:11:57 +0700 Subject: [PATCH 51/91] save --- erigon-lib/state/aggregator_test.go | 17 ++++---- erigon-lib/state/domain_shared_test.go | 2 +- erigon-lib/state/domain_test.go | 56 +++++++++++++++++--------- erigon-lib/state/history.go | 4 ++ 4 files changed, 50 insertions(+), 29 deletions(-) diff --git a/erigon-lib/state/aggregator_test.go b/erigon-lib/state/aggregator_test.go index 4a8e2a505d2..c5f5c0fbcce 100644 --- a/erigon-lib/state/aggregator_test.go +++ b/erigon-lib/state/aggregator_test.go @@ -22,7 +22,6 @@ import ( "encoding/binary" "encoding/hex" "fmt" - "github.com/erigontech/erigon-lib/commitment" "math" "math/rand" "os" @@ -33,6 +32,8 @@ import ( "testing" "time" + "github.com/erigontech/erigon-lib/commitment" + "github.com/erigontech/erigon-lib/common/background" "github.com/c2h5oh/datasize" @@ -456,7 +457,7 @@ func TestAggregatorV3_PruneSmallBatches(t *testing.T) { maxTx := aggStep * 5 t.Logf("step=%d tx_count=%d\n", aggStep, maxTx) - rnd := rand.New(rand.NewSource(0)) + rnd := newRnd(0) generateSharedDomainsUpdates(t, domains, maxTx, rnd, 20, 10, aggStep/2) @@ -636,7 +637,7 @@ func fillRawdbTxNumsIndexForSharedDomains(t *testing.T, rwTx kv.RwTx, maxTx, com } } -func generateSharedDomainsUpdates(t *testing.T, domains *SharedDomains, maxTxNum uint64, rnd *rand.Rand, keyMaxLen, keysCount, commitEvery uint64) map[string]struct{} { +func generateSharedDomainsUpdates(t *testing.T, domains *SharedDomains, maxTxNum uint64, rnd *rnd, keyMaxLen, keysCount, commitEvery uint64) map[string]struct{} { t.Helper() usedKeys := make(map[string]struct{}, keysCount*maxTxNum) for txNum := uint64(1); txNum <= maxTxNum; txNum++ { @@ -652,14 +653,14 @@ func generateSharedDomainsUpdates(t *testing.T, domains *SharedDomains, maxTxNum return usedKeys } -func generateSharedDomainsUpdatesForTx(t *testing.T, domains *SharedDomains, txNum uint64, rnd *rand.Rand, prevKeys map[string]struct{}, keyMaxLen, keysCount uint64) map[string]struct{} { +func generateSharedDomainsUpdatesForTx(t *testing.T, domains *SharedDomains, txNum uint64, rnd *rnd, prevKeys map[string]struct{}, keyMaxLen, keysCount uint64) map[string]struct{} { t.Helper() domains.SetTxNum(txNum) getKey := func() ([]byte, bool) { - r := rnd.Intn(100) + r := int(rnd.Uint64N(100)) if r < 50 && len(prevKeys) > 0 { - ri := rnd.Intn(len(prevKeys)) + ri := int(rnd.Uint64N(uint64(len(prevKeys)))) for k := range prevKeys { if ri == 0 { return []byte(k), true @@ -678,7 +679,7 @@ func generateSharedDomainsUpdatesForTx(t *testing.T, domains *SharedDomains, txN for j := uint64(0); j < keysCount; j++ { key, existed := getKey() - r := rnd.Intn(101) + r := int(rnd.Uint64N(101)) switch { case r <= 33: buf := types.EncodeAccountBytesV3(txNum, uint256.NewInt(txNum*100_000), nil, 0) @@ -691,7 +692,7 @@ func generateSharedDomainsUpdatesForTx(t *testing.T, domains *SharedDomains, txN require.NoError(t, err) case r > 33 && r <= 66: - codeUpd := make([]byte, rnd.Intn(24576)) + codeUpd := make([]byte, int(rnd.Uint64N(24576))) _, err := rnd.Read(codeUpd) require.NoError(t, err) for limit := 1000; len(key) > length.Addr && limit > 0; limit-- { diff --git a/erigon-lib/state/domain_shared_test.go b/erigon-lib/state/domain_shared_test.go index d9fcbd20201..208b69243a5 100644 --- a/erigon-lib/state/domain_shared_test.go +++ b/erigon-lib/state/domain_shared_test.go @@ -53,7 +53,7 @@ func TestSharedDomain_CommitmentKeyReplacement(t *testing.T) { require.NoError(t, err) defer domains.Close() - rnd := rand.New(rand.NewSource(2342)) + rnd := newRnd(2342) maxTx := stepSize * 8 // 1. generate data diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index c5e7df8e62c..046f200fd1c 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -24,7 +24,7 @@ import ( "fmt" "io/fs" "math" - "math/rand" + "math/rand/v2" "os" "path/filepath" "sort" @@ -51,6 +51,19 @@ import ( "github.com/erigontech/erigon-lib/types" ) +type rnd struct { + src *rand.ChaCha8 + gen *rand.Rand +} + +func newRnd(seed uint64) *rnd { + _ = seed + src := rand.NewChaCha8([32]byte{}) + return &rnd{src: src, gen: rand.New(src)} +} +func (r *rnd) Uint64N(n uint64) uint64 { return r.gen.Uint64N(n) } +func (r *rnd) Read(p []byte) (n int, err error) { return r.src.Read(p) } + func testDbAndDomain(t *testing.T, logger log.Logger) (kv.RwDB, *Domain) { t.Helper() return testDbAndDomainOfStep(t, 16, logger) @@ -1260,12 +1273,13 @@ func filledDomainFixedSize(t *testing.T, keysCount, txCount, aggStep uint64, log func generateTestDataForDomainCommitment(tb testing.TB, keySize1, keySize2, totalTx, keyTxsLimit, keyLimit uint64) map[string]map[string][]upd { tb.Helper() + defer func(t time.Time) { fmt.Printf("domain_test.go:1262: %s\n", time.Since(t)) }(time.Now()) doms := make(map[string]map[string][]upd) seed := 31 //seed := time.Now().Unix() defer tb.Logf("generated data with seed %d, keys %d", seed, keyLimit) - r := rand.New(rand.NewSource(0)) + r := newRnd(0) accs := make(map[string][]upd) stor := make(map[string][]upd) @@ -1293,11 +1307,7 @@ func generateTestData(tb testing.TB, keySize1, keySize2, totalTx, keyTxsLimit, k tb.Helper() data := make(map[string][]upd) - //seed := time.Now().Unix() - seed := 31 - defer tb.Logf("generated data with seed %d, keys %d", seed, keyLimit) - - r := rand.New(rand.NewSource(0)) + r := newRnd(32) if keyLimit == 1 { key1 := generateRandomKey(r, keySize1) data[key1] = generateUpdates(r, totalTx, keyTxsLimit) @@ -1313,24 +1323,23 @@ func generateTestData(tb testing.TB, keySize1, keySize2, totalTx, keyTxsLimit, k return data } -func generateRandomKey(r *rand.Rand, size uint64) string { +func generateRandomKey(r *rnd, size uint64) string { return string(generateRandomKeyBytes(r, size)) } -func generateRandomKeyBytes(r *rand.Rand, size uint64) []byte { +func generateRandomKeyBytes(r *rnd, size uint64) []byte { key := make([]byte, size) r.Read(key) - return key } -func generateAccountUpdates(r *rand.Rand, totalTx, keyTxsLimit uint64) []upd { +func generateAccountUpdates(r *rnd, totalTx, keyTxsLimit uint64) []upd { updates := make([]upd, 0) usedTxNums := make(map[uint64]bool) for i := uint64(0); i < keyTxsLimit; i++ { txNum := generateRandomTxNum(r, totalTx, usedTxNums) - jitter := r.Intn(10e7) + jitter := int(r.Uint64N(10e7)) value := types.EncodeAccountBytesV3(i, uint256.NewInt(i*10e4+uint64(jitter)), nil, 0) updates = append(updates, upd{txNum: txNum, value: value}) @@ -1341,7 +1350,7 @@ func generateAccountUpdates(r *rand.Rand, totalTx, keyTxsLimit uint64) []upd { return updates } -func generateArbitraryValueUpdates(r *rand.Rand, totalTx, keyTxsLimit, maxSize uint64) []upd { +func generateArbitraryValueUpdates(r *rnd, totalTx, keyTxsLimit, maxSize uint64) []upd { updates := make([]upd, 0) usedTxNums := make(map[uint64]bool) //maxStorageSize := 24 * (1 << 10) // limit on contract code @@ -1349,7 +1358,7 @@ func generateArbitraryValueUpdates(r *rand.Rand, totalTx, keyTxsLimit, maxSize u for i := uint64(0); i < keyTxsLimit; i++ { txNum := generateRandomTxNum(r, totalTx, usedTxNums) - value := make([]byte, r.Intn(int(maxSize))) + value := make([]byte, r.Uint64N(maxSize)) r.Read(value) updates = append(updates, upd{txNum: txNum, value: value}) @@ -1360,7 +1369,7 @@ func generateArbitraryValueUpdates(r *rand.Rand, totalTx, keyTxsLimit, maxSize u return updates } -func generateUpdates(r *rand.Rand, totalTx, keyTxsLimit uint64) []upd { +func generateUpdates(r *rnd, totalTx, keyTxsLimit uint64) []upd { updates := make([]upd, 0) usedTxNums := make(map[uint64]bool) @@ -1377,10 +1386,10 @@ func generateUpdates(r *rand.Rand, totalTx, keyTxsLimit uint64) []upd { return updates } -func generateRandomTxNum(r *rand.Rand, maxTxNum uint64, usedTxNums map[uint64]bool) uint64 { - txNum := uint64(r.Intn(int(maxTxNum))) +func generateRandomTxNum(r *rnd, maxTxNum uint64, usedTxNums map[uint64]bool) uint64 { + txNum := r.Uint64N(maxTxNum) for usedTxNums[txNum] { - txNum = uint64(r.Intn(int(maxTxNum))) + txNum = r.Uint64N(maxTxNum) } return txNum @@ -1395,8 +1404,8 @@ func TestDomain_GetAfterAggregation(t *testing.T) { defer tx.Rollback() d.historyLargeValues = false - d.History.compression = seg.CompressKeys | seg.CompressVals - d.compression = seg.CompressKeys | seg.CompressVals + d.History.compression = seg.CompressNone // seg.CompressKeys | seg.CompressVals + d.compression = seg.CompressNone // seg.CompressKeys | seg.CompressVals d.filenameBase = kv.FileCommitmentDomain dc := d.BeginFilesRo() @@ -1454,6 +1463,13 @@ func TestDomain_GetAfterAggregation(t *testing.T) { require.EqualValuesf(updates[len(updates)-1].value, v, "key %x latest", []byte(key)) require.True(ok) } + + it, err := dc.DomainRange(context.Background(), tx, nil, nil, 1000, order.Asc, -1) + require.NoError(err) + keys, vals, err := stream.ToArrayKV(it) + require.NoError(err) + fmt.Printf("keys: %x\n", keys) + fmt.Printf("vals: %x\n", vals) } func TestDomain_CanPruneAfterAggregation(t *testing.T) { diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 26e299f4253..9612c4268c9 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -600,6 +600,7 @@ func (c HistoryCollation) Close() { // [txFrom; txTo) func (h *History) collate(ctx context.Context, step, txFrom, txTo uint64, roTx kv.Tx) (HistoryCollation, error) { + if h.snapshotsDisabled { return HistoryCollation{}, nil } @@ -632,6 +633,9 @@ func (h *History) collate(ctx context.Context, step, txFrom, txTo uint64, roTx k return HistoryCollation{}, fmt.Errorf("create %s history compressor: %w", h.filenameBase, err) } historyComp = seg.NewWriter(comp, h.compression) + if h.compression != seg.CompressNone { + panic(1) + } keysCursor, err := roTx.CursorDupSort(h.indexKeysTable) if err != nil { From 1d57b1a191e3629aa58960b50b9ac801c925fe89 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 14:15:51 +0700 Subject: [PATCH 52/91] save --- erigon-lib/state/aggregator_test.go | 17 +++++----- erigon-lib/state/domain_shared_test.go | 2 +- erigon-lib/state/domain_test.go | 47 +++++++++++++++----------- 3 files changed, 37 insertions(+), 29 deletions(-) diff --git a/erigon-lib/state/aggregator_test.go b/erigon-lib/state/aggregator_test.go index 4a8e2a505d2..d25bcdd24c5 100644 --- a/erigon-lib/state/aggregator_test.go +++ b/erigon-lib/state/aggregator_test.go @@ -22,7 +22,6 @@ import ( "encoding/binary" "encoding/hex" "fmt" - "github.com/erigontech/erigon-lib/commitment" "math" "math/rand" "os" @@ -33,6 +32,8 @@ import ( "testing" "time" + "github.com/erigontech/erigon-lib/commitment" + "github.com/erigontech/erigon-lib/common/background" "github.com/c2h5oh/datasize" @@ -456,7 +457,7 @@ func TestAggregatorV3_PruneSmallBatches(t *testing.T) { maxTx := aggStep * 5 t.Logf("step=%d tx_count=%d\n", aggStep, maxTx) - rnd := rand.New(rand.NewSource(0)) + rnd := newRnd(0) generateSharedDomainsUpdates(t, domains, maxTx, rnd, 20, 10, aggStep/2) @@ -636,7 +637,7 @@ func fillRawdbTxNumsIndexForSharedDomains(t *testing.T, rwTx kv.RwTx, maxTx, com } } -func generateSharedDomainsUpdates(t *testing.T, domains *SharedDomains, maxTxNum uint64, rnd *rand.Rand, keyMaxLen, keysCount, commitEvery uint64) map[string]struct{} { +func generateSharedDomainsUpdates(t *testing.T, domains *SharedDomains, maxTxNum uint64, rnd *rnd, keyMaxLen, keysCount, commitEvery uint64) map[string]struct{} { t.Helper() usedKeys := make(map[string]struct{}, keysCount*maxTxNum) for txNum := uint64(1); txNum <= maxTxNum; txNum++ { @@ -652,14 +653,14 @@ func generateSharedDomainsUpdates(t *testing.T, domains *SharedDomains, maxTxNum return usedKeys } -func generateSharedDomainsUpdatesForTx(t *testing.T, domains *SharedDomains, txNum uint64, rnd *rand.Rand, prevKeys map[string]struct{}, keyMaxLen, keysCount uint64) map[string]struct{} { +func generateSharedDomainsUpdatesForTx(t *testing.T, domains *SharedDomains, txNum uint64, rnd *rnd, prevKeys map[string]struct{}, keyMaxLen, keysCount uint64) map[string]struct{} { t.Helper() domains.SetTxNum(txNum) getKey := func() ([]byte, bool) { - r := rnd.Intn(100) + r := rnd.IntN(100) if r < 50 && len(prevKeys) > 0 { - ri := rnd.Intn(len(prevKeys)) + ri := rnd.IntN(len(prevKeys)) for k := range prevKeys { if ri == 0 { return []byte(k), true @@ -678,7 +679,7 @@ func generateSharedDomainsUpdatesForTx(t *testing.T, domains *SharedDomains, txN for j := uint64(0); j < keysCount; j++ { key, existed := getKey() - r := rnd.Intn(101) + r := rnd.IntN(101) switch { case r <= 33: buf := types.EncodeAccountBytesV3(txNum, uint256.NewInt(txNum*100_000), nil, 0) @@ -691,7 +692,7 @@ func generateSharedDomainsUpdatesForTx(t *testing.T, domains *SharedDomains, txN require.NoError(t, err) case r > 33 && r <= 66: - codeUpd := make([]byte, rnd.Intn(24576)) + codeUpd := make([]byte, rnd.IntN(24576)) _, err := rnd.Read(codeUpd) require.NoError(t, err) for limit := 1000; len(key) > length.Addr && limit > 0; limit-- { diff --git a/erigon-lib/state/domain_shared_test.go b/erigon-lib/state/domain_shared_test.go index d9fcbd20201..208b69243a5 100644 --- a/erigon-lib/state/domain_shared_test.go +++ b/erigon-lib/state/domain_shared_test.go @@ -53,7 +53,7 @@ func TestSharedDomain_CommitmentKeyReplacement(t *testing.T) { require.NoError(t, err) defer domains.Close() - rnd := rand.New(rand.NewSource(2342)) + rnd := newRnd(2342) maxTx := stepSize * 8 // 1. generate data diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index f8f0b8d0ee3..900dfb9dfa3 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -24,7 +24,7 @@ import ( "fmt" "io/fs" "math" - "math/rand" + "math/rand/v2" "os" "path/filepath" "sort" @@ -51,6 +51,20 @@ import ( "github.com/erigontech/erigon-lib/types" ) +type rnd struct { + src *rand.ChaCha8 + gen *rand.Rand +} + +func newRnd(seed uint64) *rnd { + _ = seed + src := rand.NewChaCha8([32]byte{}) + return &rnd{src: src, gen: rand.New(src)} +} +func (r *rnd) Uint64N(n uint64) uint64 { return r.gen.Uint64N(n) } +func (r *rnd) IntN(n int) int { return int(r.Uint64N(uint64(n))) } +func (r *rnd) Read(p []byte) (n int, err error) { return r.src.Read(p) } + func testDbAndDomain(t *testing.T, logger log.Logger) (kv.RwDB, *Domain) { t.Helper() return testDbAndDomainOfStep(t, 16, logger) @@ -1262,10 +1276,7 @@ func generateTestDataForDomainCommitment(tb testing.TB, keySize1, keySize2, tota tb.Helper() doms := make(map[string]map[string][]upd) - seed := 31 - //seed := time.Now().Unix() - defer tb.Logf("generated data with seed %d, keys %d", seed, keyLimit) - r := rand.New(rand.NewSource(0)) + r := newRnd(31) accs := make(map[string][]upd) stor := make(map[string][]upd) @@ -1293,11 +1304,7 @@ func generateTestData(tb testing.TB, keySize1, keySize2, totalTx, keyTxsLimit, k tb.Helper() data := make(map[string][]upd) - //seed := time.Now().Unix() - seed := 31 - defer tb.Logf("generated data with seed %d, keys %d", seed, keyLimit) - - r := rand.New(rand.NewSource(0)) + r := newRnd(31) if keyLimit == 1 { key1 := generateRandomKey(r, keySize1) data[key1] = generateUpdates(r, totalTx, keyTxsLimit) @@ -1313,24 +1320,24 @@ func generateTestData(tb testing.TB, keySize1, keySize2, totalTx, keyTxsLimit, k return data } -func generateRandomKey(r *rand.Rand, size uint64) string { +func generateRandomKey(r *rnd, size uint64) string { return string(generateRandomKeyBytes(r, size)) } -func generateRandomKeyBytes(r *rand.Rand, size uint64) []byte { +func generateRandomKeyBytes(r *rnd, size uint64) []byte { key := make([]byte, size) r.Read(key) return key } -func generateAccountUpdates(r *rand.Rand, totalTx, keyTxsLimit uint64) []upd { +func generateAccountUpdates(r *rnd, totalTx, keyTxsLimit uint64) []upd { updates := make([]upd, 0) usedTxNums := make(map[uint64]bool) for i := uint64(0); i < keyTxsLimit; i++ { txNum := generateRandomTxNum(r, totalTx, usedTxNums) - jitter := r.Intn(10e7) + jitter := r.IntN(10e7) value := types.EncodeAccountBytesV3(i, uint256.NewInt(i*10e4+uint64(jitter)), nil, 0) updates = append(updates, upd{txNum: txNum, value: value}) @@ -1341,7 +1348,7 @@ func generateAccountUpdates(r *rand.Rand, totalTx, keyTxsLimit uint64) []upd { return updates } -func generateArbitraryValueUpdates(r *rand.Rand, totalTx, keyTxsLimit, maxSize uint64) []upd { +func generateArbitraryValueUpdates(r *rnd, totalTx, keyTxsLimit, maxSize uint64) []upd { updates := make([]upd, 0) usedTxNums := make(map[uint64]bool) //maxStorageSize := 24 * (1 << 10) // limit on contract code @@ -1349,7 +1356,7 @@ func generateArbitraryValueUpdates(r *rand.Rand, totalTx, keyTxsLimit, maxSize u for i := uint64(0); i < keyTxsLimit; i++ { txNum := generateRandomTxNum(r, totalTx, usedTxNums) - value := make([]byte, r.Intn(int(maxSize))) + value := make([]byte, r.IntN(int(maxSize))) r.Read(value) updates = append(updates, upd{txNum: txNum, value: value}) @@ -1360,7 +1367,7 @@ func generateArbitraryValueUpdates(r *rand.Rand, totalTx, keyTxsLimit, maxSize u return updates } -func generateUpdates(r *rand.Rand, totalTx, keyTxsLimit uint64) []upd { +func generateUpdates(r *rnd, totalTx, keyTxsLimit uint64) []upd { updates := make([]upd, 0) usedTxNums := make(map[uint64]bool) @@ -1377,10 +1384,10 @@ func generateUpdates(r *rand.Rand, totalTx, keyTxsLimit uint64) []upd { return updates } -func generateRandomTxNum(r *rand.Rand, maxTxNum uint64, usedTxNums map[uint64]bool) uint64 { - txNum := uint64(r.Intn(int(maxTxNum))) +func generateRandomTxNum(r *rnd, maxTxNum uint64, usedTxNums map[uint64]bool) uint64 { + txNum := uint64(r.IntN(int(maxTxNum))) for usedTxNums[txNum] { - txNum = uint64(r.Intn(int(maxTxNum))) + txNum = uint64(r.IntN(int(maxTxNum))) } return txNum From d800dabd78d5aa49b197c4291c8aa2286739c35d Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 14:23:20 +0700 Subject: [PATCH 53/91] save --- erigon-lib/state/aggregator_bench_test.go | 6 +++--- erigon-lib/state/aggregator_test.go | 10 +++++----- erigon-lib/state/domain_shared_bench_test.go | 6 ++---- erigon-lib/state/domain_shared_test.go | 5 ++--- erigon-lib/state/domain_test.go | 7 ++----- 5 files changed, 14 insertions(+), 20 deletions(-) diff --git a/erigon-lib/state/aggregator_bench_test.go b/erigon-lib/state/aggregator_bench_test.go index 894563f9eef..2402494ac9f 100644 --- a/erigon-lib/state/aggregator_bench_test.go +++ b/erigon-lib/state/aggregator_bench_test.go @@ -109,7 +109,7 @@ func BenchmarkAggregator_Processing(b *testing.B) { } func queueKeys(ctx context.Context, seed, ofSize uint64) <-chan []byte { - rnd := rand.New(rand.NewSource(int64(seed))) + rnd := newRnd(seed) keys := make(chan []byte, 1) go func() { for { @@ -127,10 +127,10 @@ func queueKeys(ctx context.Context, seed, ofSize uint64) <-chan []byte { } func Benchmark_BtreeIndex_Allocation(b *testing.B) { - rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + rnd := newRnd(0) for i := 0; i < b.N; i++ { now := time.Now() - count := rnd.Intn(1000000000) + count := rnd.IntN(1000000000) bt := newBtAlloc(uint64(count), uint64(1<<12), true, nil, nil) bt.traverseDfs() fmt.Printf("alloc %v\n", time.Since(now)) diff --git a/erigon-lib/state/aggregator_test.go b/erigon-lib/state/aggregator_test.go index d25bcdd24c5..b696635e4be 100644 --- a/erigon-lib/state/aggregator_test.go +++ b/erigon-lib/state/aggregator_test.go @@ -71,7 +71,7 @@ func TestAggregatorV3_Merge(t *testing.T) { defer domains.Close() txs := uint64(1000) - rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + rnd := newRnd(0) var ( commKey1 = []byte("someCommKey") @@ -187,7 +187,7 @@ func TestAggregatorV3_MergeValTransform(t *testing.T) { defer domains.Close() txs := uint64(1000) - rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + rnd := newRnd(0) agg.commitmentValuesTransform = true @@ -317,7 +317,7 @@ func aggregatorV3_RestartOnDatadir(t *testing.T, rc runCfg) { defer domains.Close() var latestCommitTxNum uint64 - rnd := rand.New(rand.NewSource(time.Now().Unix())) + rnd := newRnd(0) someKey := []byte("somekey") txs := (aggStep / 2) * 19 @@ -777,7 +777,7 @@ func TestAggregatorV3_RestartOnFiles(t *testing.T) { txs := aggStep * 5 t.Logf("step=%d tx_count=%d\n", aggStep, txs) - rnd := rand.New(rand.NewSource(0)) + rnd := newRnd(0) keys := make([][]byte, txs) for txNum := uint64(1); txNum <= txs; txNum++ { @@ -916,7 +916,7 @@ func TestAggregatorV3_ReplaceCommittedKeys(t *testing.T) { txs := (aggStep) * StepsInColdFile t.Logf("step=%d tx_count=%d", aggStep, txs) - rnd := rand.New(rand.NewSource(0)) + rnd := newRnd(0) keys := make([][]byte, txs/2) var prev1, prev2 []byte diff --git a/erigon-lib/state/domain_shared_bench_test.go b/erigon-lib/state/domain_shared_bench_test.go index 927255bbaba..2c8700e1734 100644 --- a/erigon-lib/state/domain_shared_bench_test.go +++ b/erigon-lib/state/domain_shared_bench_test.go @@ -19,7 +19,6 @@ package state import ( "context" "encoding/binary" - "math/rand" "testing" "github.com/stretchr/testify/require" @@ -46,8 +45,7 @@ func Benchmark_SharedDomains_GetLatest(t *testing.B) { defer domains.Close() maxTx := stepSize * 258 - seed := int64(4500) - rnd := rand.New(rand.NewSource(seed)) + rnd := newRnd(4500) keys := make([][]byte, 8) for i := 0; i < len(keys); i++ { @@ -104,7 +102,7 @@ func Benchmark_SharedDomains_GetLatest(t *testing.B) { for ik := 0; ik < t.N; ik++ { for i := 0; i < len(keys); i++ { - ts := uint64(rnd.Intn(int(maxTx))) + ts := uint64(rnd.IntN(int(maxTx))) v, ok, err := ac2.HistorySeek(kv.AccountsHistory, keys[i], ts, rwTx) require.True(t, ok) diff --git a/erigon-lib/state/domain_shared_test.go b/erigon-lib/state/domain_shared_test.go index 208b69243a5..17606175321 100644 --- a/erigon-lib/state/domain_shared_test.go +++ b/erigon-lib/state/domain_shared_test.go @@ -20,7 +20,6 @@ import ( "context" "encoding/binary" "fmt" - "math/rand" "testing" "time" @@ -134,7 +133,7 @@ func TestSharedDomain_Unwind(t *testing.T) { maxTx := stepSize hashes := make([][]byte, maxTx) count := 10 - rnd := rand.New(rand.NewSource(0)) + rnd := newRnd(0) ac.Close() err = rwTx.Commit() require.NoError(t, err) @@ -180,7 +179,7 @@ Loop: err = domains.Flush(ctx, rwTx) require.NoError(t, err) - unwindTo := uint64(commitStep * rnd.Intn(int(maxTx)/commitStep)) + unwindTo := uint64(commitStep * rnd.IntN(int(maxTx)/commitStep)) domains.currentChangesAccumulator = nil acu := agg.BeginFilesRo() diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index 900dfb9dfa3..c300ba2975c 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -52,16 +52,15 @@ import ( ) type rnd struct { + *rand.Rand src *rand.ChaCha8 - gen *rand.Rand } func newRnd(seed uint64) *rnd { _ = seed src := rand.NewChaCha8([32]byte{}) - return &rnd{src: src, gen: rand.New(src)} + return &rnd{Rand: rand.New(src), src: src} } -func (r *rnd) Uint64N(n uint64) uint64 { return r.gen.Uint64N(n) } func (r *rnd) IntN(n int) int { return int(r.Uint64N(uint64(n))) } func (r *rnd) Read(p []byte) (n int, err error) { return r.src.Read(p) } @@ -1468,8 +1467,6 @@ func TestDomain_CanPruneAfterAggregation(t *testing.T) { aggStep := uint64(25) db, d := testDbAndDomainOfStep(t, aggStep, log.New()) - defer db.Close() - defer d.Close() tx, err := db.BeginRw(context.Background()) require.NoError(t, err) From 3277c6806a1e3fd4af48e5ca3d5f7ded91282c57 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 14:24:21 +0700 Subject: [PATCH 54/91] save --- erigon-lib/state/aggregator_bench_test.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/erigon-lib/state/aggregator_bench_test.go b/erigon-lib/state/aggregator_bench_test.go index 2402494ac9f..e722f179210 100644 --- a/erigon-lib/state/aggregator_bench_test.go +++ b/erigon-lib/state/aggregator_bench_test.go @@ -20,7 +20,6 @@ import ( "bytes" "context" "fmt" - "math/rand" "os" "path" "path/filepath" @@ -127,7 +126,7 @@ func queueKeys(ctx context.Context, seed, ofSize uint64) <-chan []byte { } func Benchmark_BtreeIndex_Allocation(b *testing.B) { - rnd := newRnd(0) + rnd := newRnd(uint64(time.Now().UnixNano())) for i := 0; i < b.N; i++ { now := time.Now() count := rnd.IntN(1000000000) @@ -139,7 +138,7 @@ func Benchmark_BtreeIndex_Allocation(b *testing.B) { func Benchmark_BtreeIndex_Search(b *testing.B) { logger := log.New() - rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + rnd := newRnd(uint64(time.Now().UnixNano())) tmp := b.TempDir() defer os.RemoveAll(tmp) dataPath := "../../data/storage.256-288.kv" @@ -193,12 +192,12 @@ func Benchmark_BTree_Seek(b *testing.B) { M := uint64(1024) compress := seg.CompressNone kv, bt, keys, _ := benchInitBtreeIndex(b, M, compress) - rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + rnd := newRnd(uint64(time.Now().UnixNano())) getter := seg.NewReader(kv.MakeGetter(), compress) b.Run("seek_only", func(b *testing.B) { for i := 0; i < b.N; i++ { - p := rnd.Intn(len(keys)) + p := rnd.IntN(len(keys)) cur, err := bt.Seek(getter, keys[p]) require.NoError(b, err) @@ -249,7 +248,7 @@ func Benchmark_Recsplit_Find_ExternalFile(b *testing.B) { b.Skip("requires existing KV index file at ../../data/storage.kv") } - rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + rnd := newRnd(uint64(time.Now().UnixNano())) tmp := b.TempDir() defer os.RemoveAll(tmp) From 33e3aaa4b4528592f86615e98a170c6672045c7f Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 14:25:05 +0700 Subject: [PATCH 55/91] save --- erigon-lib/state/aggregator_bench_test.go | 6 +++--- erigon-lib/state/aggregator_test.go | 4 ++-- erigon-lib/state/domain_test.go | 22 +++++++++++----------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/erigon-lib/state/aggregator_bench_test.go b/erigon-lib/state/aggregator_bench_test.go index e722f179210..089884db80a 100644 --- a/erigon-lib/state/aggregator_bench_test.go +++ b/erigon-lib/state/aggregator_bench_test.go @@ -158,7 +158,7 @@ func Benchmark_BtreeIndex_Search(b *testing.B) { getter := seg.NewReader(kv.MakeGetter(), comp) for i := 0; i < b.N; i++ { - p := rnd.Intn(len(keys)) + p := rnd.IntN(len(keys)) cur, err := bt.Seek(getter, keys[p]) require.NoErrorf(b, err, "i=%d", i) require.EqualValues(b, keys[p], cur.Key()) @@ -208,7 +208,7 @@ func Benchmark_BTree_Seek(b *testing.B) { b.Run("seek_then_next", func(b *testing.B) { for i := 0; i < b.N; i++ { - p := rnd.Intn(len(keys)) + p := rnd.IntN(len(keys)) cur, err := bt.Seek(getter, keys[p]) require.NoError(b, err) @@ -268,7 +268,7 @@ func Benchmark_Recsplit_Find_ExternalFile(b *testing.B) { require.NoError(b, err) for i := 0; i < b.N; i++ { - p := rnd.Intn(len(keys)) + p := rnd.IntN(len(keys)) offset, _ := idxr.Lookup(keys[p]) getter.Reset(offset) diff --git a/erigon-lib/state/aggregator_test.go b/erigon-lib/state/aggregator_test.go index b696635e4be..6087d20aee0 100644 --- a/erigon-lib/state/aggregator_test.go +++ b/erigon-lib/state/aggregator_test.go @@ -637,7 +637,7 @@ func fillRawdbTxNumsIndexForSharedDomains(t *testing.T, rwTx kv.RwTx, maxTx, com } } -func generateSharedDomainsUpdates(t *testing.T, domains *SharedDomains, maxTxNum uint64, rnd *rnd, keyMaxLen, keysCount, commitEvery uint64) map[string]struct{} { +func generateSharedDomainsUpdates(t *testing.T, domains *SharedDomains, maxTxNum uint64, rnd *rndGen, keyMaxLen, keysCount, commitEvery uint64) map[string]struct{} { t.Helper() usedKeys := make(map[string]struct{}, keysCount*maxTxNum) for txNum := uint64(1); txNum <= maxTxNum; txNum++ { @@ -653,7 +653,7 @@ func generateSharedDomainsUpdates(t *testing.T, domains *SharedDomains, maxTxNum return usedKeys } -func generateSharedDomainsUpdatesForTx(t *testing.T, domains *SharedDomains, txNum uint64, rnd *rnd, prevKeys map[string]struct{}, keyMaxLen, keysCount uint64) map[string]struct{} { +func generateSharedDomainsUpdatesForTx(t *testing.T, domains *SharedDomains, txNum uint64, rnd *rndGen, prevKeys map[string]struct{}, keyMaxLen, keysCount uint64) map[string]struct{} { t.Helper() domains.SetTxNum(txNum) diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index c300ba2975c..e4849e63b8f 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -51,18 +51,18 @@ import ( "github.com/erigontech/erigon-lib/types" ) -type rnd struct { +type rndGen struct { *rand.Rand src *rand.ChaCha8 } -func newRnd(seed uint64) *rnd { +func newRnd(seed uint64) *rndGen { _ = seed src := rand.NewChaCha8([32]byte{}) - return &rnd{Rand: rand.New(src), src: src} + return &rndGen{Rand: rand.New(src), src: src} } -func (r *rnd) IntN(n int) int { return int(r.Uint64N(uint64(n))) } -func (r *rnd) Read(p []byte) (n int, err error) { return r.src.Read(p) } +func (r *rndGen) IntN(n int) int { return int(r.Uint64N(uint64(n))) } +func (r *rndGen) Read(p []byte) (n int, err error) { return r.src.Read(p) } func testDbAndDomain(t *testing.T, logger log.Logger) (kv.RwDB, *Domain) { t.Helper() @@ -1319,18 +1319,18 @@ func generateTestData(tb testing.TB, keySize1, keySize2, totalTx, keyTxsLimit, k return data } -func generateRandomKey(r *rnd, size uint64) string { +func generateRandomKey(r *rndGen, size uint64) string { return string(generateRandomKeyBytes(r, size)) } -func generateRandomKeyBytes(r *rnd, size uint64) []byte { +func generateRandomKeyBytes(r *rndGen, size uint64) []byte { key := make([]byte, size) r.Read(key) return key } -func generateAccountUpdates(r *rnd, totalTx, keyTxsLimit uint64) []upd { +func generateAccountUpdates(r *rndGen, totalTx, keyTxsLimit uint64) []upd { updates := make([]upd, 0) usedTxNums := make(map[uint64]bool) @@ -1347,7 +1347,7 @@ func generateAccountUpdates(r *rnd, totalTx, keyTxsLimit uint64) []upd { return updates } -func generateArbitraryValueUpdates(r *rnd, totalTx, keyTxsLimit, maxSize uint64) []upd { +func generateArbitraryValueUpdates(r *rndGen, totalTx, keyTxsLimit, maxSize uint64) []upd { updates := make([]upd, 0) usedTxNums := make(map[uint64]bool) //maxStorageSize := 24 * (1 << 10) // limit on contract code @@ -1366,7 +1366,7 @@ func generateArbitraryValueUpdates(r *rnd, totalTx, keyTxsLimit, maxSize uint64) return updates } -func generateUpdates(r *rnd, totalTx, keyTxsLimit uint64) []upd { +func generateUpdates(r *rndGen, totalTx, keyTxsLimit uint64) []upd { updates := make([]upd, 0) usedTxNums := make(map[uint64]bool) @@ -1383,7 +1383,7 @@ func generateUpdates(r *rnd, totalTx, keyTxsLimit uint64) []upd { return updates } -func generateRandomTxNum(r *rnd, maxTxNum uint64, usedTxNums map[uint64]bool) uint64 { +func generateRandomTxNum(r *rndGen, maxTxNum uint64, usedTxNums map[uint64]bool) uint64 { txNum := uint64(r.IntN(int(maxTxNum))) for usedTxNums[txNum] { txNum = uint64(r.IntN(int(maxTxNum))) From c26246621d6db1c8a501e5fe7cb26c4d3e86bb1e Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 14:25:34 +0700 Subject: [PATCH 56/91] save --- erigon-lib/state/aggregator_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/erigon-lib/state/aggregator_test.go b/erigon-lib/state/aggregator_test.go index 6087d20aee0..c39e333ec5b 100644 --- a/erigon-lib/state/aggregator_test.go +++ b/erigon-lib/state/aggregator_test.go @@ -1030,7 +1030,7 @@ func pivotKeysFromKV(dataPath string) ([][]byte, error) { func generateKV(tb testing.TB, tmp string, keySize, valueSize, keyCount int, logger log.Logger, compressFlags seg.FileCompression) string { tb.Helper() - rnd := rand.New(rand.NewSource(0)) + rnd := newRnd(0) values := make([]byte, valueSize) dataPath := path.Join(tmp, fmt.Sprintf("%dk.kv", keyCount/1000)) @@ -1112,7 +1112,7 @@ func testDbAndAggregatorv3(t *testing.T, aggStep uint64) (kv.RwDB, *Aggregator) func generateInputData(tb testing.TB, keySize, valueSize, keyCount int) ([][]byte, [][]byte) { tb.Helper() - rnd := rand.New(rand.NewSource(0)) + rnd := newRnd(0) values := make([][]byte, keyCount) keys := make([][]byte, keyCount) @@ -1123,7 +1123,7 @@ func generateInputData(tb testing.TB, keySize, valueSize, keyCount int) ([][]byt require.NoError(tb, err) keys[i] = common.Copy(bk[:n]) - n, err = rnd.Read(bv[:rnd.Intn(valueSize)+1]) + n, err = rnd.Read(bv[:rnd.IntN(valueSize)+1]) require.NoError(tb, err) values[i] = common.Copy(bv[:n]) From bfe7a997911f52ea589e2697a7f634a67e656aff Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 14:26:29 +0700 Subject: [PATCH 57/91] save --- erigon-lib/state/bps_tree.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/erigon-lib/state/bps_tree.go b/erigon-lib/state/bps_tree.go index d039f8ef5dd..188d7723590 100644 --- a/erigon-lib/state/bps_tree.go +++ b/erigon-lib/state/bps_tree.go @@ -25,14 +25,13 @@ import ( "time" "unsafe" - "github.com/erigontech/erigon-lib/common/dbg" - "github.com/c2h5oh/datasize" - "github.com/erigontech/erigon-lib/seg" "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/common/dbg" "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/recsplit/eliasfano32" + "github.com/erigontech/erigon-lib/seg" ) // nolint From 1ea4559a0e0a9b18eaed8694a92af29790de2b23 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 14:51:55 +0700 Subject: [PATCH 58/91] save --- erigon-lib/state/domain.go | 5 ++-- erigon-lib/state/domain_test.go | 46 +++++++++++++++++---------------- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index 5e891bbf754..9ff610134a5 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1972,9 +1972,8 @@ func (dt *DomainRoTx) DomainRange(ctx context.Context, tx kv.Tx, fromKey, toKey //if err != nil { // return nil, err //} - lastestStateIt := stream.EmptyKV - fmt.Printf("[dbg] DomainRange 3: %s\n", dt.d.name) - return stream.UnionKV(histStateIt, lastestStateIt, limit), nil + //return stream.UnionKV(histStateIt, lastestStateIt, limit), nil + return stream.UnionKV(histStateIt, stream.EmptyKV, limit), nil } func (dt *DomainRoTx) DomainRangeLatest(roTx kv.Tx, fromKey, toKey []byte, limit int) (stream.KV, error) { diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index 046f200fd1c..262f07a1d2e 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -1413,9 +1413,9 @@ func TestDomain_GetAfterAggregation(t *testing.T) { writer := dc.NewWriter() defer writer.close() - keySize1 := uint64(length.Addr) - keySize2 := uint64(length.Addr + length.Hash) - totalTx := uint64(3000) + keySize1 := uint64(length.Addr) / 4 + keySize2 := uint64(length.Addr+length.Hash) / 4 + totalTx := uint64(300) keyTxsLimit := uint64(50) keyLimit := uint64(200) @@ -1425,7 +1425,8 @@ func TestDomain_GetAfterAggregation(t *testing.T) { p := []byte{} for i := 0; i < len(updates); i++ { writer.SetTxNum(updates[i].txNum) - writer.PutWithPrev([]byte(key), nil, updates[i].value, p, 0) + err = writer.PutWithPrev([]byte(key), nil, updates[i].value, p, 0) + require.NoError(err) p = common.Copy(updates[i].value) } } @@ -1446,25 +1447,26 @@ func TestDomain_GetAfterAggregation(t *testing.T) { dc = d.BeginFilesRo() defer dc.Close() - kc := 0 - for key, updates := range data { - kc++ - for i := 1; i < len(updates); i++ { - v, ok, err := dc.GetAsOf([]byte(key), updates[i].txNum, tx) - require.NoError(err) - require.True(ok) - require.EqualValuesf(updates[i-1].value, v, "(%d/%d) key %x, txn %d", kc, len(data), []byte(key), updates[i-1].txNum) - } - if len(updates) == 0 { - continue - } - v, _, ok, err := dc.GetLatest([]byte(key), nil, tx) - require.NoError(err) - require.EqualValuesf(updates[len(updates)-1].value, v, "key %x latest", []byte(key)) - require.True(ok) - } + //kc := 0 + //for key, updates := range data { + // kc++ + // for i := 1; i < len(updates); i++ { + // v, ok, err := dc.GetAsOf([]byte(key), updates[i].txNum, tx) + // require.NoError(err) + // require.True(ok) + // require.EqualValuesf(updates[i-1].value, v, "(%d/%d) key %x, txn %d", kc, len(data), []byte(key), updates[i-1].txNum) + // } + // if len(updates) == 0 { + // continue + // } + // v, _, ok, err := dc.GetLatest([]byte(key), nil, tx) + // require.NoError(err) + // require.EqualValuesf(updates[len(updates)-1].value, v, "key %x latest", []byte(key)) + // require.True(ok) + //} - it, err := dc.DomainRange(context.Background(), tx, nil, nil, 1000, order.Asc, -1) + //it, err := dc.DomainRangeLatest(tx, nil, nil, -1) + it, err := dc.DomainRange(context.Background(), tx, nil, nil, 54, order.Asc, -1) require.NoError(err) keys, vals, err := stream.ToArrayKV(it) require.NoError(err) From 9b2f9a69d6a2e4fb036860dda95300504b490b7d Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 15:12:11 +0700 Subject: [PATCH 59/91] save --- erigon-lib/kv/mdbx/kv_mdbx.go | 36 +++++++------ erigon-lib/seg/decompress.go | 5 +- erigon-lib/state/aggregator_test.go | 7 ++- erigon-lib/state/domain_shared_test.go | 3 +- erigon-lib/state/domain_test.go | 4 +- erigon-lib/state/history.go | 70 ++++++++++++++------------ turbo/jsonrpc/eth_call.go | 1 + 7 files changed, 68 insertions(+), 58 deletions(-) diff --git a/erigon-lib/kv/mdbx/kv_mdbx.go b/erigon-lib/kv/mdbx/kv_mdbx.go index 5bc703ed29f..968a24e2fa7 100644 --- a/erigon-lib/kv/mdbx/kv_mdbx.go +++ b/erigon-lib/kv/mdbx/kv_mdbx.go @@ -1582,16 +1582,20 @@ func (s *cursor2iter) Close() { } func (s *cursor2iter) HasNext() bool { - if s.limit <= 0 || s.nextV == nil { // Limit or EndOfTable + if s.limit == 0 { // limit reached return false } - if s.toPrefix != nil { - //Asc: [from, to) AND from < to - //Desc: [from, to) AND from > to - cmp := bytes.Compare(s.nextK, s.toPrefix) - return (bool(s.orderAscend) && cmp < 0) || (!bool(s.orderAscend) && cmp > 0) + if s.nextK == nil { // EndOfTable + return false + } + if s.toPrefix == nil { // s.nextK == nil check is above + return true } - return true + + //Asc: [from, to) AND from < to + //Desc: [from, to) AND from > to + cmp := bytes.Compare(s.nextK, s.toPrefix) + return (bool(s.orderAscend) && cmp < 0) || (!bool(s.orderAscend) && cmp > 0) } func (s *cursor2iter) Next() (k, v []byte, err error) { @@ -1740,16 +1744,20 @@ func (s *cursorDup2iter) Close() { } } func (s *cursorDup2iter) HasNext() bool { - if s.limit <= 0 || s.nextV == nil { // Limit or EndOfTable + if s.limit == 0 { // limit reached return false } - if s.toPrefix != nil { - //Asc: [from, to) AND from < to - //Desc: [from, to) AND from > to - cmp := bytes.Compare(s.nextV, s.toPrefix) - return (s.orderAscend && cmp < 0) || (!s.orderAscend && cmp > 0) + if s.nextV == nil { // EndOfTable + return false + } + if s.toPrefix == nil { // s.nextK == nil check is above + return true } - return true + + //Asc: [from, to) AND from < to + //Desc: [from, to) AND from > to + cmp := bytes.Compare(s.nextV, s.toPrefix) + return (s.orderAscend && cmp < 0) || (!s.orderAscend && cmp > 0) } func (s *cursorDup2iter) Next() (k, v []byte, err error) { select { diff --git a/erigon-lib/seg/decompress.go b/erigon-lib/seg/decompress.go index ca257a11bae..3475ebdd6e8 100644 --- a/erigon-lib/seg/decompress.go +++ b/erigon-lib/seg/decompress.go @@ -29,12 +29,11 @@ import ( "time" "unsafe" - "github.com/erigontech/erigon-lib/common/assert" - "github.com/erigontech/erigon-lib/log/v3" - "github.com/c2h5oh/datasize" + "github.com/erigontech/erigon-lib/common/assert" "github.com/erigontech/erigon-lib/common/dbg" + "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/mmap" ) diff --git a/erigon-lib/state/aggregator_test.go b/erigon-lib/state/aggregator_test.go index c5f5c0fbcce..922a80bc08e 100644 --- a/erigon-lib/state/aggregator_test.go +++ b/erigon-lib/state/aggregator_test.go @@ -32,12 +32,11 @@ import ( "testing" "time" - "github.com/erigontech/erigon-lib/commitment" - - "github.com/erigontech/erigon-lib/common/background" - "github.com/c2h5oh/datasize" + + "github.com/erigontech/erigon-lib/commitment" "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/common/background" "github.com/erigontech/erigon-lib/common/datadir" "github.com/erigontech/erigon-lib/common/length" "github.com/erigontech/erigon-lib/etl" diff --git a/erigon-lib/state/domain_shared_test.go b/erigon-lib/state/domain_shared_test.go index 208b69243a5..5aa8e026278 100644 --- a/erigon-lib/state/domain_shared_test.go +++ b/erigon-lib/state/domain_shared_test.go @@ -27,11 +27,10 @@ import ( "github.com/holiman/uint256" "github.com/stretchr/testify/require" + "github.com/erigontech/erigon-lib/common/length" "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/kv/rawdbv3" "github.com/erigontech/erigon-lib/log/v3" - - "github.com/erigontech/erigon-lib/common/length" "github.com/erigontech/erigon-lib/types" ) diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index 262f07a1d2e..c271103891c 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -1406,7 +1406,7 @@ func TestDomain_GetAfterAggregation(t *testing.T) { d.historyLargeValues = false d.History.compression = seg.CompressNone // seg.CompressKeys | seg.CompressVals d.compression = seg.CompressNone // seg.CompressKeys | seg.CompressVals - d.filenameBase = kv.FileCommitmentDomain + d.filenameBase = kv.FileAccountDomain dc := d.BeginFilesRo() defer d.Close() @@ -1454,7 +1454,7 @@ func TestDomain_GetAfterAggregation(t *testing.T) { // v, ok, err := dc.GetAsOf([]byte(key), updates[i].txNum, tx) // require.NoError(err) // require.True(ok) - // require.EqualValuesf(updates[i-1].value, v, "(%d/%d) key %x, txn %d", kc, len(data), []byte(key), updates[i-1].txNum) + // require.EqualValuesf(updates[i-1].value, v¬, "(%d/%d) key %x, txn %d", kc, len(data), []byte(key), updates[i-1].txNum) // } // if len(updates) == 0 { // continue diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 9612c4268c9..2bd7fbe0cbc 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -600,7 +600,6 @@ func (c HistoryCollation) Close() { // [txFrom; txTo) func (h *History) collate(ctx context.Context, step, txFrom, txTo uint64, roTx kv.Tx) (HistoryCollation, error) { - if h.snapshotsDisabled { return HistoryCollation{}, nil } @@ -633,9 +632,6 @@ func (h *History) collate(ctx context.Context, step, txFrom, txTo uint64, roTx k return HistoryCollation{}, fmt.Errorf("create %s history compressor: %w", h.filenameBase, err) } historyComp = seg.NewWriter(comp, h.compression) - if h.compression != seg.CompressNone { - panic(1) - } keysCursor, err := roTx.CursorDupSort(h.indexKeysTable) if err != nil { @@ -1353,18 +1349,19 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to panic("implement me") } hi := &StateAsOfIterF{ - from: from, to: to, limit: limit, orderAscend: asc, + from: from, toPrefix: to, limit: limit, orderAscend: asc, hc: ht, startTxNum: startTxNum, ctx: ctx, } - fmt.Printf("[dbg] ht.iit.files: %d, %x, %x\n", len(ht.iit.files), from, to) for i, item := range ht.iit.files { if item.endTxNum <= startTxNum { + fmt.Printf("[dbg] ht.iit.files skip file: %s, %d, %d\n", item.getter.FileName(), item.endTxNum, startTxNum) continue } + fmt.Printf("[dbg] ht.iit.files use file: %s\n", item.src.decompressor.FileName()) // TODO: seek(from) g := seg.NewReader(item.src.decompressor.MakeGetter(), ht.h.compression) @@ -1394,7 +1391,7 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to largeValues: ht.h.historyLargeValues, roTx: roTx, valsTable: ht.h.historyValsTable, - from: from, to: to, limit: limit, orderAscend: asc, + from: from, toPrefix: to, limit: limit, orderAscend: asc, startTxNum: startTxNum, @@ -1405,8 +1402,7 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to dbit.Close() //it's responsibility of constructor (our) to close resource on error return nil, err } - //return stream.UnionKV(hi, dbit, limit), nil - return stream.UnionKV(hi, stream.EmptyKV, limit), nil + return stream.UnionKV(hi, dbit, limit), nil } // StateAsOfIter - returns state range at given time in history @@ -1414,9 +1410,9 @@ type StateAsOfIterF struct { hc *HistoryRoTx limit int - from, to []byte - nextVal []byte - nextKey []byte + from, toPrefix []byte + nextVal []byte + nextKey []byte h ReconHeap startTxNum uint64 @@ -1448,7 +1444,7 @@ func (hi *StateAsOfIterF) advanceInFiles() error { //} else { // top.key, _ = top.g.NextUncompressed() //} - if hi.to == nil || bytes.Compare(top.key, hi.to) < 0 { + if hi.toPrefix == nil || bytes.Compare(top.key, hi.toPrefix) < 0 { heap.Push(&hi.h, top) } } @@ -1465,7 +1461,7 @@ func (hi *StateAsOfIterF) advanceInFiles() error { fmt.Printf("[dbg] StateAsOfIterF advanceInFiles ef.seek not found. %s, %x\n", top.g.FileName(), hi.nextKey) continue } - fmt.Printf("[dbg] StateAsOfIterF advanceInFiles keep: %s, %x, to: %x\n", top.g.FileName(), top.key, hi.to) + fmt.Printf("[dbg] StateAsOfIterF advanceInFiles keep: %s, %x, to: %x\n", top.g.FileName(), top.key, hi.toPrefix) hi.nextKey = key binary.BigEndian.PutUint64(hi.txnKey[:], n) @@ -1489,16 +1485,20 @@ func (hi *StateAsOfIterF) advanceInFiles() error { } func (hi *StateAsOfIterF) HasNext() bool { - if hi.limit <= 0 || hi.nextKey == nil { // Limit or EndOfTable + if hi.limit == 0 { // limit reached return false } - if hi.to != nil { - //Asc: [from, to) AND from < to - //Desc: [from, to) AND from > to - cmp := bytes.Compare(hi.nextKey, hi.to) - return (bool(hi.orderAscend) && cmp < 0) || (!bool(hi.orderAscend) && cmp > 0) + if hi.nextKey == nil { // EndOfTable + return false } - return true + if hi.toPrefix == nil { // s.nextK == nil check is above + return true + } + + //Asc: [from, to) AND from < to + //Desc: [from, to) AND from > to + cmp := bytes.Compare(hi.nextKey, hi.toPrefix) + return (bool(hi.orderAscend) && cmp < 0) || (!bool(hi.orderAscend) && cmp > 0) } func (hi *StateAsOfIterF) Next() ([]byte, []byte, error) { @@ -1528,9 +1528,9 @@ type StateAsOfIterDB struct { valsCDup kv.CursorDupSort valsTable string - from, to []byte - orderAscend order.By - limit int + from, toPrefix []byte + orderAscend order.By + limit int nextKey, nextVal []byte @@ -1590,7 +1590,7 @@ func (hi *StateAsOfIterDB) advanceLargeVals() error { if err != nil { return err } - if hi.to != nil && bytes.Compare(k[:len(k)-8], hi.to) >= 0 { + if hi.toPrefix != nil && bytes.Compare(k[:len(k)-8], hi.toPrefix) >= 0 { break } if !bytes.Equal(seek[:len(k)-8], k[:len(k)-8]) { @@ -1624,7 +1624,7 @@ func (hi *StateAsOfIterDB) advanceSmallVals() error { if err != nil { return err } - if hi.to != nil && bytes.Compare(k, hi.to) >= 0 { + if hi.toPrefix != nil && bytes.Compare(k, hi.toPrefix) >= 0 { break } v, err := hi.valsCDup.SeekBothRange(k, hi.startTxKey[:]) @@ -1646,16 +1646,20 @@ func (hi *StateAsOfIterDB) HasNext() bool { if hi.err != nil { return true } - if hi.limit <= 0 || hi.nextKey == nil { // Limit or EndOfTable + if hi.limit == 0 { // limit reached return false } - if hi.to != nil { - //Asc: [from, to) AND from < to - //Desc: [from, to) AND from > to - cmp := bytes.Compare(hi.nextKey, hi.to) - return (bool(hi.orderAscend) && cmp < 0) || (!bool(hi.orderAscend) && cmp > 0) + if hi.nextKey == nil { // EndOfTable + return false } - return true + if hi.toPrefix == nil { // s.nextK == nil check is above + return true + } + + //Asc: [from, to) AND from < to + //Desc: [from, to) AND from > to + cmp := bytes.Compare(hi.nextKey, hi.toPrefix) + return (bool(hi.orderAscend) && cmp < 0) || (!bool(hi.orderAscend) && cmp > 0) } func (hi *StateAsOfIterDB) Next() ([]byte, []byte, error) { diff --git a/turbo/jsonrpc/eth_call.go b/turbo/jsonrpc/eth_call.go index 7cd29b7ef28..ea921e63175 100644 --- a/turbo/jsonrpc/eth_call.go +++ b/turbo/jsonrpc/eth_call.go @@ -90,6 +90,7 @@ func (api *APIImpl) Call(ctx context.Context, args ethapi2.CallArgs, blockNrOrHa header := block.HeaderNoCopy() result, err := transactions.DoCall(ctx, engine, args, tx, blockNrOrHash, header, overrides, api.GasCap, chainConfig, stateReader, api._blockReader, api.evmCallTimeout) if err != nil { + panic(err) return nil, err } From fe2a7144eb77afd95cf56a9a3cbaeaaab17b2ced Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 15:19:32 +0700 Subject: [PATCH 60/91] save --- erigon-lib/state/domain_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index c271103891c..1d0f71afde1 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -1413,8 +1413,8 @@ func TestDomain_GetAfterAggregation(t *testing.T) { writer := dc.NewWriter() defer writer.close() - keySize1 := uint64(length.Addr) / 4 - keySize2 := uint64(length.Addr+length.Hash) / 4 + keySize1 := uint64(4) + keySize2 := uint64(4) totalTx := uint64(300) keyTxsLimit := uint64(50) keyLimit := uint64(200) @@ -1466,7 +1466,7 @@ func TestDomain_GetAfterAggregation(t *testing.T) { //} //it, err := dc.DomainRangeLatest(tx, nil, nil, -1) - it, err := dc.DomainRange(context.Background(), tx, nil, nil, 54, order.Asc, -1) + it, err := dc.DomainRange(context.Background(), tx, nil, nil, 190, order.Asc, -1) require.NoError(err) keys, vals, err := stream.ToArrayKV(it) require.NoError(err) From 50c468d2b10971f40bfe7cec84eba0c95f6cb9a6 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 15:21:06 +0700 Subject: [PATCH 61/91] save --- erigon-lib/state/domain.go | 11 +++++------ erigon-lib/state/domain_test.go | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index 9ff610134a5..96b770101dc 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1968,12 +1968,11 @@ func (dt *DomainRoTx) DomainRange(ctx context.Context, tx kv.Tx, fromKey, toKey return nil, err } fmt.Printf("[dbg] DomainRange 2: %s\n", dt.d.name) - //lastestStateIt, err := dt.DomainRangeLatest(tx, fromKey, toKey, limit) - //if err != nil { - // return nil, err - //} - //return stream.UnionKV(histStateIt, lastestStateIt, limit), nil - return stream.UnionKV(histStateIt, stream.EmptyKV, limit), nil + lastestStateIt, err := dt.DomainRangeLatest(tx, fromKey, toKey, limit) + if err != nil { + return nil, err + } + return stream.UnionKV(histStateIt, lastestStateIt, limit), nil } func (dt *DomainRoTx) DomainRangeLatest(roTx kv.Tx, fromKey, toKey []byte, limit int) (stream.KV, error) { diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index 1d0f71afde1..b0a3293e71c 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -1416,8 +1416,8 @@ func TestDomain_GetAfterAggregation(t *testing.T) { keySize1 := uint64(4) keySize2 := uint64(4) totalTx := uint64(300) - keyTxsLimit := uint64(50) - keyLimit := uint64(200) + keyTxsLimit := uint64(2) + keyLimit := uint64(10) // put some kvs data := generateTestData(t, keySize1, keySize2, totalTx, keyTxsLimit, keyLimit) From 6029d89bec256afab23960d5f46689a2fcceaaad Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 15:25:15 +0700 Subject: [PATCH 62/91] save --- erigon-lib/state/aggregator_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/state/aggregator_test.go b/erigon-lib/state/aggregator_test.go index c39e333ec5b..fb6bd933179 100644 --- a/erigon-lib/state/aggregator_test.go +++ b/erigon-lib/state/aggregator_test.go @@ -1050,7 +1050,7 @@ func generateKV(tb testing.TB, tmp string, keySize, valueSize, keyCount int, log binary.BigEndian.PutUint64(key[keySize-8:], uint64(i)) require.NoError(tb, err) - n, err = rnd.Read(values[:rnd.Intn(valueSize)+1]) + n, err = rnd.Read(values[:rnd.IntN(valueSize)+1]) require.NoError(tb, err) err = collector.Collect(key, values[:n]) From b79682efc8f4b65d5fc2dd63533adef4fd0f7150 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 15:27:00 +0700 Subject: [PATCH 63/91] save --- .github/workflows/lint.yml | 2 +- erigon-lib/tools/golangci_lint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 86502cf83d8..3efd756d7e5 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -32,7 +32,7 @@ jobs: if: runner.os == 'Linux' uses: golangci/golangci-lint-action@v6 with: - version: v1.59.1 + version: v1.61.0 args: --help - name: Lint diff --git a/erigon-lib/tools/golangci_lint.sh b/erigon-lib/tools/golangci_lint.sh index ada4234150d..4c812bc72b9 100755 --- a/erigon-lib/tools/golangci_lint.sh +++ b/erigon-lib/tools/golangci_lint.sh @@ -2,7 +2,7 @@ scriptDir=$(dirname "${BASH_SOURCE[0]}") scriptName=$(basename "${BASH_SOURCE[0]}") -version="v1.59.1" +version="v1.60.0" if [[ "$1" == "--install-deps" ]] then From be6699d0e51157319c781656773ada62a9b74e3a Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 15:30:40 +0700 Subject: [PATCH 64/91] save --- erigon-lib/state/domain_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index e4849e63b8f..64ad17ea773 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -62,7 +62,7 @@ func newRnd(seed uint64) *rndGen { return &rndGen{Rand: rand.New(src), src: src} } func (r *rndGen) IntN(n int) int { return int(r.Uint64N(uint64(n))) } -func (r *rndGen) Read(p []byte) (n int, err error) { return r.src.Read(p) } +func (r *rndGen) Read(p []byte) (n int, err error) { return r.src.Read(p) } //nolint:typecheck func testDbAndDomain(t *testing.T, logger log.Logger) (kv.RwDB, *Domain) { t.Helper() From e03b745ae1ed302b988123afd9ff84741179bafe Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 15:33:01 +0700 Subject: [PATCH 65/91] save --- erigon-lib/state/domain_test.go | 88 ++++++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 18 deletions(-) diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index b0a3293e71c..a1262bd20a0 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -1403,6 +1403,76 @@ func TestDomain_GetAfterAggregation(t *testing.T) { require.NoError(err) defer tx.Rollback() + d.historyLargeValues = false + d.History.compression = seg.CompressKeys | seg.CompressVals + d.compression = seg.CompressKeys | seg.CompressVals + d.filenameBase = kv.FileCommitmentDomain + + dc := d.BeginFilesRo() + defer d.Close() + writer := dc.NewWriter() + defer writer.close() + + keySize1 := uint64(length.Addr) + keySize2 := uint64(length.Addr + length.Hash) + totalTx := uint64(3000) + keyTxsLimit := uint64(50) + keyLimit := uint64(200) + + // put some kvs + data := generateTestData(t, keySize1, keySize2, totalTx, keyTxsLimit, keyLimit) + for key, updates := range data { + p := []byte{} + for i := 0; i < len(updates); i++ { + writer.SetTxNum(updates[i].txNum) + writer.PutWithPrev([]byte(key), nil, updates[i].value, p, 0) + p = common.Copy(updates[i].value) + } + } + writer.SetTxNum(totalTx) + + err = writer.Flush(context.Background(), tx) + require.NoError(err) + + // aggregate + collateAndMerge(t, db, tx, d, totalTx) + require.NoError(tx.Commit()) + + tx, err = db.BeginRw(context.Background()) + require.NoError(err) + defer tx.Rollback() + dc.Close() + + dc = d.BeginFilesRo() + defer dc.Close() + + kc := 0 + for key, updates := range data { + kc++ + for i := 1; i < len(updates); i++ { + v, ok, err := dc.GetAsOf([]byte(key), updates[i].txNum, tx) + require.NoError(err) + require.True(ok) + require.EqualValuesf(updates[i-1].value, v, "(%d/%d) key %x, txn %d", kc, len(data), []byte(key), updates[i-1].txNum) + } + if len(updates) == 0 { + continue + } + v, _, ok, err := dc.GetLatest([]byte(key), nil, tx) + require.NoError(err) + require.EqualValuesf(updates[len(updates)-1].value, v, "key %x latest", []byte(key)) + require.True(ok) + } +} + +func TestDomainRange(t *testing.T) { + db, d := testDbAndDomainOfStep(t, 25, log.New()) + require := require.New(t) + + tx, err := db.BeginRw(context.Background()) + require.NoError(err) + defer tx.Rollback() + d.historyLargeValues = false d.History.compression = seg.CompressNone // seg.CompressKeys | seg.CompressVals d.compression = seg.CompressNone // seg.CompressKeys | seg.CompressVals @@ -1447,24 +1517,6 @@ func TestDomain_GetAfterAggregation(t *testing.T) { dc = d.BeginFilesRo() defer dc.Close() - //kc := 0 - //for key, updates := range data { - // kc++ - // for i := 1; i < len(updates); i++ { - // v, ok, err := dc.GetAsOf([]byte(key), updates[i].txNum, tx) - // require.NoError(err) - // require.True(ok) - // require.EqualValuesf(updates[i-1].value, v¬, "(%d/%d) key %x, txn %d", kc, len(data), []byte(key), updates[i-1].txNum) - // } - // if len(updates) == 0 { - // continue - // } - // v, _, ok, err := dc.GetLatest([]byte(key), nil, tx) - // require.NoError(err) - // require.EqualValuesf(updates[len(updates)-1].value, v, "key %x latest", []byte(key)) - // require.True(ok) - //} - //it, err := dc.DomainRangeLatest(tx, nil, nil, -1) it, err := dc.DomainRange(context.Background(), tx, nil, nil, 190, order.Asc, -1) require.NoError(err) From 963dbcabacfc2f9a1f72ccd3548b226dc1c10c71 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 15:33:39 +0700 Subject: [PATCH 66/91] save --- erigon-lib/state/domain_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index a1262bd20a0..a63350377c9 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -1404,8 +1404,8 @@ func TestDomain_GetAfterAggregation(t *testing.T) { defer tx.Rollback() d.historyLargeValues = false - d.History.compression = seg.CompressKeys | seg.CompressVals - d.compression = seg.CompressKeys | seg.CompressVals + d.History.compression = seg.CompressNone //seg.CompressKeys | seg.CompressVals + d.compression = seg.CompressNone //seg.CompressKeys | seg.CompressVals d.filenameBase = kv.FileCommitmentDomain dc := d.BeginFilesRo() From 5f44189455779e40e752544d1c295a27f41b9a90 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 15:37:13 +0700 Subject: [PATCH 67/91] save --- erigon-lib/state/history.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 2bd7fbe0cbc..cd8b065742a 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1358,7 +1358,7 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to } for i, item := range ht.iit.files { if item.endTxNum <= startTxNum { - fmt.Printf("[dbg] ht.iit.files skip file: %s, %d, %d\n", item.getter.FileName(), item.endTxNum, startTxNum) + fmt.Printf("[dbg] ht.iit.files skip file: %s, %d, %d\n", item.src.decompressor.FileName(), item.endTxNum, startTxNum) continue } fmt.Printf("[dbg] ht.iit.files use file: %s\n", item.src.decompressor.FileName()) From 3b88d2dbf2546e2f0f092d640e71948cb9cc723e Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 15:39:31 +0700 Subject: [PATCH 68/91] save --- erigon-lib/state/domain_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index 64ad17ea773..4b8a4ef81ab 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -61,8 +61,12 @@ func newRnd(seed uint64) *rndGen { src := rand.NewChaCha8([32]byte{}) return &rndGen{Rand: rand.New(src), src: src} } -func (r *rndGen) IntN(n int) int { return int(r.Uint64N(uint64(n))) } -func (r *rndGen) Read(p []byte) (n int, err error) { return r.src.Read(p) } //nolint:typecheck +func (r *rndGen) IntN(n int) int { return int(r.Uint64N(uint64(n))) } + +// nolin +func (r *rndGen) Read(p []byte) (n int, err error) { + return r.src.Read(p) //nolint +} func testDbAndDomain(t *testing.T, logger log.Logger) (kv.RwDB, *Domain) { t.Helper() From fb32f84140352592307ba3ed89b9480cc33feef8 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 15:39:39 +0700 Subject: [PATCH 69/91] save --- erigon-lib/state/domain_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index 4b8a4ef81ab..bf88e12a098 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -63,7 +63,7 @@ func newRnd(seed uint64) *rndGen { } func (r *rndGen) IntN(n int) int { return int(r.Uint64N(uint64(n))) } -// nolin +// nolint func (r *rndGen) Read(p []byte) (n int, err error) { return r.src.Read(p) //nolint } From 394fbf3e58998fbcedb305097707d118577103b1 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 15:39:58 +0700 Subject: [PATCH 70/91] save --- erigon-lib/state/domain_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index bf88e12a098..f13bd3ec572 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -63,8 +63,7 @@ func newRnd(seed uint64) *rndGen { } func (r *rndGen) IntN(n int) int { return int(r.Uint64N(uint64(n))) } -// nolint -func (r *rndGen) Read(p []byte) (n int, err error) { +func (r *rndGen) Read(p []byte) (n int, err error) { //nolint return r.src.Read(p) //nolint } From 15d7195285d69956e49deaaa52294ef25701215c Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 15:43:27 +0700 Subject: [PATCH 71/91] save --- erigon-lib/state/history.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index cd8b065742a..f82e06d4630 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1358,10 +1358,8 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to } for i, item := range ht.iit.files { if item.endTxNum <= startTxNum { - fmt.Printf("[dbg] ht.iit.files skip file: %s, %d, %d\n", item.src.decompressor.FileName(), item.endTxNum, startTxNum) continue } - fmt.Printf("[dbg] ht.iit.files use file: %s\n", item.src.decompressor.FileName()) // TODO: seek(from) g := seg.NewReader(item.src.decompressor.MakeGetter(), ht.h.compression) From 4b67a52039badeb7ea6ab7a043b30b2c65c5892b Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 16:04:20 +0700 Subject: [PATCH 72/91] save --- erigon-lib/.golangci.yml | 1 + erigon-lib/state/domain_test.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/erigon-lib/.golangci.yml b/erigon-lib/.golangci.yml index d66cd898232..3af106f0c95 100644 --- a/erigon-lib/.golangci.yml +++ b/erigon-lib/.golangci.yml @@ -22,6 +22,7 @@ linters: - testifylint #TODO: enable me - perfsprint #TODO: enable me - protogetter + - typecheck enable: - unconvert - predeclared diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index f13bd3ec572..0b3c2922704 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -64,7 +64,8 @@ func newRnd(seed uint64) *rndGen { func (r *rndGen) IntN(n int) int { return int(r.Uint64N(uint64(n))) } func (r *rndGen) Read(p []byte) (n int, err error) { //nolint - return r.src.Read(p) //nolint + //nolint:typecheck + return r.src.Read(p) //nolint:typecheck } func testDbAndDomain(t *testing.T, logger log.Logger) (kv.RwDB, *Domain) { From 6b06087adfd51cde934f32e465c9e0809afe2991 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 5 Nov 2024 16:16:14 +0700 Subject: [PATCH 73/91] save --- erigon-lib/.golangci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/erigon-lib/.golangci.yml b/erigon-lib/.golangci.yml index 3af106f0c95..7f4a485356d 100644 --- a/erigon-lib/.golangci.yml +++ b/erigon-lib/.golangci.yml @@ -112,6 +112,7 @@ issues: - unused - gocritic - perfsprint + - typecheck - path: hack\.go linters: - gosec From ceb153fb52aa80cc327b3e6b9dddb9e225c01add Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 6 Nov 2024 09:07:46 +0700 Subject: [PATCH 74/91] save --- erigon-lib/state/domain_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index 0b3c2922704..dd273cb0fc9 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -22,6 +22,7 @@ import ( "encoding/binary" "encoding/hex" "fmt" + "io" "io/fs" "math" "math/rand/v2" @@ -63,9 +64,9 @@ func newRnd(seed uint64) *rndGen { } func (r *rndGen) IntN(n int) int { return int(r.Uint64N(uint64(n))) } -func (r *rndGen) Read(p []byte) (n int, err error) { //nolint - //nolint:typecheck - return r.src.Read(p) //nolint:typecheck +func (r *rndGen) Read(p []byte) (n int, err error) { + var a io.Reader = r.src + return a.Read(p) } func testDbAndDomain(t *testing.T, logger log.Logger) (kv.RwDB, *Domain) { From cf43ae6302e0db48bd1de6eed0b15132589a3d3e Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 6 Nov 2024 09:19:50 +0700 Subject: [PATCH 75/91] save --- erigon-lib/state/domain_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index dd273cb0fc9..c43c43bc6c5 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -22,7 +22,6 @@ import ( "encoding/binary" "encoding/hex" "fmt" - "io" "io/fs" "math" "math/rand/v2" @@ -65,8 +64,7 @@ func newRnd(seed uint64) *rndGen { func (r *rndGen) IntN(n int) int { return int(r.Uint64N(uint64(n))) } func (r *rndGen) Read(p []byte) (n int, err error) { - var a io.Reader = r.src - return a.Read(p) + return r.src.Read(p) } func testDbAndDomain(t *testing.T, logger log.Logger) (kv.RwDB, *Domain) { From 7b1806c8371f336e8a403af2f020b66ae88550a7 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 6 Nov 2024 09:28:23 +0700 Subject: [PATCH 76/91] save --- erigon-lib/state/domain_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index c43c43bc6c5..7a2a4e13a2e 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -24,6 +24,7 @@ import ( "fmt" "io/fs" "math" + randOld "math/rand" "math/rand/v2" "os" "path/filepath" @@ -53,18 +54,18 @@ import ( type rndGen struct { *rand.Rand - src *rand.ChaCha8 + oldGen *randOld.Rand } func newRnd(seed uint64) *rndGen { - _ = seed - src := rand.NewChaCha8([32]byte{}) - return &rndGen{Rand: rand.New(src), src: src} + src := rand.NewChaCha8([32]byte{byte(seed)}) + oldSrc := randOld.NewSource(int64(seed)) + return &rndGen{Rand: rand.New(src), oldGen: randOld.New(oldSrc)} } func (r *rndGen) IntN(n int) int { return int(r.Uint64N(uint64(n))) } func (r *rndGen) Read(p []byte) (n int, err error) { - return r.src.Read(p) + return r.oldGen.Read(p) // seems `go1.22` doesn't have `Read` method on `math/v2` generator } func testDbAndDomain(t *testing.T, logger log.Logger) (kv.RwDB, *Domain) { From 8dd0c27121167456bf87ec804c3040b4e0477a12 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 6 Nov 2024 09:29:13 +0700 Subject: [PATCH 77/91] save --- erigon-lib/state/domain_test.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index 7a2a4e13a2e..5f29a48b56e 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -58,15 +58,13 @@ type rndGen struct { } func newRnd(seed uint64) *rndGen { - src := rand.NewChaCha8([32]byte{byte(seed)}) - oldSrc := randOld.NewSource(int64(seed)) - return &rndGen{Rand: rand.New(src), oldGen: randOld.New(oldSrc)} -} -func (r *rndGen) IntN(n int) int { return int(r.Uint64N(uint64(n))) } - -func (r *rndGen) Read(p []byte) (n int, err error) { - return r.oldGen.Read(p) // seems `go1.22` doesn't have `Read` method on `math/v2` generator + return &rndGen{ + Rand: rand.New(rand.NewChaCha8([32]byte{byte(seed)})), + oldGen: randOld.New(randOld.NewSource(int64(seed))), + } } +func (r *rndGen) IntN(n int) int { return int(r.Uint64N(uint64(n))) } +func (r *rndGen) Read(p []byte) (n int, err error) { return r.oldGen.Read(p) } // seems `go1.22` doesn't have `Read` method on `math/v2` generator func testDbAndDomain(t *testing.T, logger log.Logger) (kv.RwDB, *Domain) { t.Helper() From 46e4da72c9434129ccaef2202a9cbcef5af8ec23 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 6 Nov 2024 14:26:10 +0700 Subject: [PATCH 78/91] save --- core/state/dump.go | 3 ++- turbo/jsonrpc/debug_api_test.go | 4 ++-- turbo/jsonrpc/eth_call.go | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/state/dump.go b/core/state/dump.go index d96551f1ddb..41c5728b143 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -21,6 +21,7 @@ package state import ( "encoding/json" + "errors" "fmt" "time" @@ -136,7 +137,7 @@ func NewDumper(db kv.Tx, txNumsReader rawdbv3.TxNumsReader, blockNumber uint64) } } -var TooMuchIterations = fmt.Errorf("[rpc] Dumper: too much iterations protection triggered") +var TooMuchIterations = errors.New("[rpc] dumper: too much iterations protection triggered") const DumperIterationsHardLimit = 10_000_000 diff --git a/turbo/jsonrpc/debug_api_test.go b/turbo/jsonrpc/debug_api_test.go index 70681b58c1c..3112177cbf8 100644 --- a/turbo/jsonrpc/debug_api_test.go +++ b/turbo/jsonrpc/debug_api_test.go @@ -304,12 +304,12 @@ func TestAccountRange(t *testing.T) { t.Run("valid account", func(t *testing.T) { addr := common.HexToAddress("0x537e697c7ab75a26f9ecf0ce810e3154dfcaaf55") - n := rpc.BlockNumber(1) + //n := rpc.BlockNumber(1) //result, err := api.AccountRange(m.Ctx, rpc.BlockNumberOrHash{BlockNumber: &n}, addr[:], 10, true, true) //require.NoError(t, err) //require.Equal(t, 2, len(result.Accounts)) // - n = rpc.BlockNumber(7) + n := rpc.BlockNumber(7) result, err := api.AccountRange(m.Ctx, rpc.BlockNumberOrHash{BlockNumber: &n}, addr[:], 10, true, true) require.NoError(t, err) require.Equal(t, 3, len(result.Accounts)) diff --git a/turbo/jsonrpc/eth_call.go b/turbo/jsonrpc/eth_call.go index ea921e63175..7cd29b7ef28 100644 --- a/turbo/jsonrpc/eth_call.go +++ b/turbo/jsonrpc/eth_call.go @@ -90,7 +90,6 @@ func (api *APIImpl) Call(ctx context.Context, args ethapi2.CallArgs, blockNrOrHa header := block.HeaderNoCopy() result, err := transactions.DoCall(ctx, engine, args, tx, blockNrOrHash, header, overrides, api.GasCap, chainConfig, stateReader, api._blockReader, api.evmCallTimeout) if err != nil { - panic(err) return nil, err } From 1021f835f9ef401ebd6b2f1de0409f4793539391 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 6 Nov 2024 14:58:30 +0700 Subject: [PATCH 79/91] save --- erigon-lib/state/history.go | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index f82e06d4630..3963003be89 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1897,15 +1897,12 @@ func (hi *HistoryChangesIterFiles) HasNext() bool { if hi.err != nil { // always true, then .Next() call will return this error return true } - if hi.limit <= 0 || hi.nextKey == nil { // Limit or EndOfTable + if hi.limit == 0 { // limit reached + return false + } + if hi.nextKey == nil { // EndOfTable return false } - //if hi.to != nil { - // //Asc: [from, to) AND from < to - // //Desc: [from, to) AND from > to - // cmp := bytes.Compare(hi.nextKey, hi.to) - // return (bool(hi.orderAscend) && cmp < 0) || (!bool(hi.orderAscend) && cmp > 0) - //} return true } @@ -2073,15 +2070,12 @@ func (hi *HistoryChangesIterDB) HasNext() bool { if hi.err != nil { // always true, then .Next() call will return this error return true } - if hi.limit <= 0 || hi.nextKey == nil { // Limit or EndOfTable + if hi.limit == 0 { // limit reached + return false + } + if hi.nextKey == nil { // EndOfTable return false } - //if hi.to != nil { - // //Asc: [from, to) AND from < to - // //Desc: [from, to) AND from > to - // cmp := bytes.Compare(hi.nextKey, hi.to) - // return (bool(hi.orderAscend) && cmp < 0) || (!bool(hi.orderAscend) && cmp > 0) - //} return true } From 8e8a0e8aa7c80bb3964416f016ca702967ff58cf Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 6 Nov 2024 15:05:25 +0700 Subject: [PATCH 80/91] save --- erigon-lib/state/domain_test.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index 59dbba6d072..67e24daddb5 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -1275,7 +1275,6 @@ func filledDomainFixedSize(t *testing.T, keysCount, txCount, aggStep uint64, log func generateTestDataForDomainCommitment(tb testing.TB, keySize1, keySize2, totalTx, keyTxsLimit, keyLimit uint64) map[string]map[string][]upd { tb.Helper() - defer func(t time.Time) { fmt.Printf("domain_test.go:1262: %s\n", time.Since(t)) }(time.Now()) doms := make(map[string]map[string][]upd) r := newRnd(31) @@ -1482,8 +1481,8 @@ func TestDomainRange(t *testing.T) { writer := dc.NewWriter() defer writer.close() - keySize1 := uint64(4) - keySize2 := uint64(4) + keySize1 := uint64(2) + keySize2 := uint64(2) totalTx := uint64(300) keyTxsLimit := uint64(2) keyLimit := uint64(10) @@ -1516,13 +1515,22 @@ func TestDomainRange(t *testing.T) { dc = d.BeginFilesRo() defer dc.Close() - //it, err := dc.DomainRangeLatest(tx, nil, nil, -1) it, err := dc.DomainRange(context.Background(), tx, nil, nil, 190, order.Asc, -1) require.NoError(err) keys, vals, err := stream.ToArrayKV(it) require.NoError(err) + require.Equal(5, len(keys)) + require.Equal(5, len(vals)) fmt.Printf("keys: %x\n", keys) - fmt.Printf("vals: %x\n", vals) + + //it, err := dc.DomainRange(context.Background(), tx, []byte{""}, nil, 190, order.Asc, -1) + //require.NoError(err) + //keys, vals, err := stream.ToArrayKV(it) + //require.NoError(err) + //require.Equal(5, len(keys)) + //require.Equal(5, len(vals)) + + t.Fail() } func TestDomain_CanPruneAfterAggregation(t *testing.T) { From 8d5f89af0ef7298f88cd2ad2a0c153679ff378bd Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 7 Nov 2024 10:10:20 +0700 Subject: [PATCH 81/91] save --- erigon-lib/state/history.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 3963003be89..d7373b9ec7d 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1456,10 +1456,8 @@ func (hi *StateAsOfIterF) advanceInFiles() error { } n, ok := eliasfano32.Seek(idxVal, hi.startTxNum) if !ok { - fmt.Printf("[dbg] StateAsOfIterF advanceInFiles ef.seek not found. %s, %x\n", top.g.FileName(), hi.nextKey) continue } - fmt.Printf("[dbg] StateAsOfIterF advanceInFiles keep: %s, %x, to: %x\n", top.g.FileName(), top.key, hi.toPrefix) hi.nextKey = key binary.BigEndian.PutUint64(hi.txnKey[:], n) From 9c828f3d0bd2e36ee07bb96966f6549ade58897b Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 7 Nov 2024 10:30:35 +0700 Subject: [PATCH 82/91] save --- erigon-lib/state/domain.go | 17 ++++++++--- erigon-lib/state/domain_test.go | 26 +++++++++++----- erigon-lib/state/history.go | 54 ++++++++++++++++++--------------- 3 files changed, 60 insertions(+), 37 deletions(-) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index 96b770101dc..0ff092ccc78 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1968,11 +1968,12 @@ func (dt *DomainRoTx) DomainRange(ctx context.Context, tx kv.Tx, fromKey, toKey return nil, err } fmt.Printf("[dbg] DomainRange 2: %s\n", dt.d.name) - lastestStateIt, err := dt.DomainRangeLatest(tx, fromKey, toKey, limit) - if err != nil { - return nil, err - } - return stream.UnionKV(histStateIt, lastestStateIt, limit), nil + //lastestStateIt, err := dt.DomainRangeLatest(tx, fromKey, toKey, limit) + //if err != nil { + // return nil, err + //} + //return stream.UnionKV(histStateIt, lastestStateIt, limit), nil + return stream.UnionKV(histStateIt, stream.EmptyKV, limit), nil } func (dt *DomainRoTx) DomainRangeLatest(roTx kv.Tx, fromKey, toKey []byte, limit int) (stream.KV, error) { @@ -2264,6 +2265,7 @@ func (hi *DomainLatestIterFile) init(dc *DomainRoTx) error { if key, value, err = valsCursor.Seek(hi.from); err != nil { return err } + fmt.Printf("[dbg] rangeLatest: %x\n", key) if key != nil && (hi.to == nil || bytes.Compare(key[:len(key)-8], hi.to) < 0) { k := key[:len(key)-8] stepBytes := key[len(key)-8:] @@ -2281,6 +2283,7 @@ func (hi *DomainLatestIterFile) init(dc *DomainRoTx) error { if key, value, err = valsCursor.Seek(hi.from); err != nil { return err } + fmt.Printf("[dbg] rangeLatest2: %x\n", key) if key != nil && (hi.to == nil || bytes.Compare(key, hi.to) < 0) { stepBytes := value[:8] value = value[8:] @@ -2302,6 +2305,7 @@ func (hi *DomainLatestIterFile) init(dc *DomainRoTx) error { } key := btCursor.Key() + fmt.Printf("[dbg] rangeLatest3: %x\n", key) if key != nil && (hi.to == nil || bytes.Compare(key, hi.to) < 0) { val := btCursor.Value() txNum := item.endTxNum - 1 // !important: .kv files have semantic [from, t) @@ -2324,6 +2328,7 @@ func (hi *DomainLatestIterFile) advanceInFiles() error { if ci1.btCursor.Next() { ci1.key = ci1.btCursor.Key() ci1.val = ci1.btCursor.Value() + fmt.Printf("[dbg] advInFiles1: %x\n", ci1.key) if ci1.key != nil && (hi.to == nil || bytes.Compare(ci1.key, hi.to) < 0) { heap.Push(hi.h, ci1) } @@ -2346,6 +2351,7 @@ func (hi *DomainLatestIterFile) advanceInFiles() error { } } + fmt.Printf("[dbg] advInFiles2: %x\n", k) if len(k) > 0 && (hi.to == nil || bytes.Compare(k[:len(k)-8], hi.to) < 0) { stepBytes := k[len(k)-8:] k = k[:len(k)-8] @@ -2364,6 +2370,7 @@ func (hi *DomainLatestIterFile) advanceInFiles() error { return err } + fmt.Printf("[dbg] advInFiles3: %x\n", k) if len(k) > 0 && (hi.to == nil || bytes.Compare(k, hi.to) < 0) { stepBytes := stepBytesWithValue[:8] v := stepBytesWithValue[8:] diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index 67e24daddb5..1b88d3e0355 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -1515,13 +1515,25 @@ func TestDomainRange(t *testing.T) { dc = d.BeginFilesRo() defer dc.Close() - it, err := dc.DomainRange(context.Background(), tx, nil, nil, 190, order.Asc, -1) - require.NoError(err) - keys, vals, err := stream.ToArrayKV(it) - require.NoError(err) - require.Equal(5, len(keys)) - require.Equal(5, len(vals)) - fmt.Printf("keys: %x\n", keys) + { + it, err := dc.ht.WalkAsOf(context.Background(), 190, nil, nil, order.Asc, -1, tx) + require.NoError(err) + keys, vals, err := stream.ToArrayKV(it) + fmt.Printf("keys: %x\n", keys) + require.NoError(err) + require.Equal(3, len(keys)) + require.Equal(3, len(vals)) + } + + { + it, err := dc.DomainRangeLatest(tx, nil, nil, -1) + require.NoError(err) + keys, vals, err := stream.ToArrayKV(it) + fmt.Printf("keys: %x\n", keys) + require.NoError(err) + require.Equal(3, len(keys)) + require.Equal(3, len(vals)) + } //it, err := dc.DomainRange(context.Background(), tx, []byte{""}, nil, 190, order.Asc, -1) //require.NoError(err) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index d7373b9ec7d..4e8047616e7 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1356,31 +1356,7 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to ctx: ctx, } - for i, item := range ht.iit.files { - if item.endTxNum <= startTxNum { - continue - } - // TODO: seek(from) - g := seg.NewReader(item.src.decompressor.MakeGetter(), ht.h.compression) - - idx := ht.iit.statelessIdxReader(i) - var offset uint64 - if len(from) > 0 { - n := item.src.decompressor.Count() / 2 - var ok bool - offset, ok = g.BinarySearch(from, n, idx.OrdinalLookup) - if !ok { - offset = 0 - } - } - g.Reset(offset) - if g.HasNext() { - key, offset := g.Next(nil) - heap.Push(&hi.h, &ReconItem{g: g, key: key, startTxNum: item.startTxNum, endTxNum: item.endTxNum, txNum: item.endTxNum, startOffset: offset, lastOffset: offset}) - } - } - binary.BigEndian.PutUint64(hi.startTxKey[:], startTxNum) - if err := hi.advanceInFiles(); err != nil { + if err := hi.init(ht.iit.files); err != nil { hi.Close() //it's responsibility of constructor (our) to close resource on error return nil, err } @@ -1426,6 +1402,34 @@ type StateAsOfIterF struct { func (hi *StateAsOfIterF) Close() { } +func (hi *StateAsOfIterF) init(files visibleFiles) error { + for i, item := range files { + if item.endTxNum <= hi.startTxNum { + continue + } + // TODO: seek(from) + g := seg.NewReader(item.src.decompressor.MakeGetter(), hi.hc.h.compression) + + idx := hi.hc.iit.statelessIdxReader(i) + var offset uint64 + if len(hi.from) > 0 { + n := item.src.decompressor.Count() / 2 + var ok bool + offset, ok = g.BinarySearch(hi.from, n, idx.OrdinalLookup) + if !ok { + offset = 0 + } + } + g.Reset(offset) + if g.HasNext() { + key, offset := g.Next(nil) + heap.Push(&hi.h, &ReconItem{g: g, key: key, startTxNum: item.startTxNum, endTxNum: item.endTxNum, txNum: item.endTxNum, startOffset: offset, lastOffset: offset}) + } + } + binary.BigEndian.PutUint64(hi.startTxKey[:], hi.startTxNum) + return hi.advanceInFiles() +} + func (hi *StateAsOfIterF) advanceInFiles() error { for hi.h.Len() > 0 { top := heap.Pop(&hi.h).(*ReconItem) From ccb381f64a35371d33548456189268278efa46be Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 7 Nov 2024 10:30:51 +0700 Subject: [PATCH 83/91] save --- erigon-lib/state/history.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 4e8047616e7..07e4d78389b 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1353,8 +1353,7 @@ func (ht *HistoryRoTx) WalkAsOf(ctx context.Context, startTxNum uint64, from, to hc: ht, startTxNum: startTxNum, - - ctx: ctx, + ctx: ctx, } if err := hi.init(ht.iit.files); err != nil { hi.Close() //it's responsibility of constructor (our) to close resource on error From 906ccfc7fb53ed27ed78fa7e04de637472432ba7 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 7 Nov 2024 10:49:09 +0700 Subject: [PATCH 84/91] save --- erigon-lib/common/dbg/dbg_env.go | 19 +++++++++++++------ erigon-lib/kv/order/order.go | 25 +++++++++++++++++++++++++ erigon-lib/state/domain.go | 1 + 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/erigon-lib/common/dbg/dbg_env.go b/erigon-lib/common/dbg/dbg_env.go index 2c2d337ef08..4a0e364b741 100644 --- a/erigon-lib/common/dbg/dbg_env.go +++ b/erigon-lib/common/dbg/dbg_env.go @@ -19,6 +19,7 @@ package dbg import ( "os" "strconv" + "strings" "time" "github.com/c2h5oh/datasize" @@ -29,7 +30,7 @@ import ( func EnvString(envVarName string, defaultVal string) string { v, _ := os.LookupEnv(envVarName) if v != "" { - log.Warn("[env] please use ERIGON_ prefix for env variables of erigon", "var", envVarName) + WarnOnErigonPrefix(envVarName) log.Info("[env]", envVarName, v) return v } @@ -44,12 +45,12 @@ func EnvString(envVarName string, defaultVal string) string { func EnvBool(envVarName string, defaultVal bool) bool { v, _ := os.LookupEnv(envVarName) if v == "true" { - log.Warn("[env] please use ERIGON_ prefix for env variables of erigon", "var", envVarName) + WarnOnErigonPrefix(envVarName) log.Info("[env]", envVarName, true) return true } if v == "false" { - log.Warn("[env] please use ERIGON_ prefix for env variables of erigon", "var", envVarName) + WarnOnErigonPrefix(envVarName) log.Info("[env]", envVarName, false) return false } @@ -68,7 +69,7 @@ func EnvBool(envVarName string, defaultVal bool) bool { func EnvInt(envVarName string, defaultVal int) int { v, _ := os.LookupEnv(envVarName) if v != "" { - log.Warn("[env] please use ERIGON_ prefix for env variables of erigon", "var", envVarName) + WarnOnErigonPrefix(envVarName) i, err := strconv.Atoi(v) if err != nil { panic(err) @@ -91,7 +92,7 @@ func EnvInt(envVarName string, defaultVal int) int { func EnvDataSize(envVarName string, defaultVal datasize.ByteSize) datasize.ByteSize { v, _ := os.LookupEnv(envVarName) if v != "" { - log.Warn("[env] please use ERIGON_ prefix for env variables of erigon", "var", envVarName) + WarnOnErigonPrefix(envVarName) val, err := datasize.ParseString(v) if err != nil { panic(err) @@ -115,7 +116,7 @@ func EnvDataSize(envVarName string, defaultVal datasize.ByteSize) datasize.ByteS func EnvDuration(envVarName string, defaultVal time.Duration) time.Duration { v, _ := os.LookupEnv(envVarName) if v != "" { - log.Warn("[env] please use ERIGON_ prefix for env variables of erigon", "var", envVarName) + WarnOnErigonPrefix(envVarName) log.Info("[env]", envVarName, v) val, err := time.ParseDuration(v) if err != nil { @@ -134,3 +135,9 @@ func EnvDuration(envVarName string, defaultVal time.Duration) time.Duration { } return defaultVal } + +func WarnOnErigonPrefix(envVarName string) { + if !strings.HasPrefix(envVarName, "ERIGON_") { + log.Warn("[env] please use ERIGON_ prefix for env variables of erigon", "var", envVarName) + } +} diff --git a/erigon-lib/kv/order/order.go b/erigon-lib/kv/order/order.go index fa00afb97fb..f75b9bceaa8 100644 --- a/erigon-lib/kv/order/order.go +++ b/erigon-lib/kv/order/order.go @@ -16,6 +16,13 @@ package order +import ( + "bytes" + "fmt" + + "github.com/erigontech/erigon-lib/common/dbg" +) + type By bool const ( @@ -29,3 +36,21 @@ func FromBool(v bool) By { } return Desc } + +func Must(asc By, k1, k2 []byte) { + if !dbg.AssertEnabled { + return + } + if k1 == nil || k2 == nil { + return + } + if asc { + if bytes.Compare(k1, k2) <= 0 { + panic(fmt.Sprintf("epect: %x <= %x", k1, k2)) + } + } else { + if bytes.Compare(k1, k2) >= 0 { + panic(fmt.Sprintf("epect: %x >= %x", k1, k2)) + } + } +} diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index 0ff092ccc78..c148a8dca59 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -2408,6 +2408,7 @@ func (hi *DomainLatestIterFile) Next() ([]byte, []byte, error) { if err := hi.advanceInFiles(); err != nil { return nil, nil, err } + order.Must(order.Asc, hi.kBackup, hi.nextKey) return hi.kBackup, hi.vBackup, nil } From 0c4e0adb65ff3c66a25c9e4a2a2bc2fdf034422d Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 7 Nov 2024 10:52:35 +0700 Subject: [PATCH 85/91] save --- erigon-lib/kv/order/order.go | 10 +++++----- erigon-lib/state/history.go | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/erigon-lib/kv/order/order.go b/erigon-lib/kv/order/order.go index f75b9bceaa8..59f4665f779 100644 --- a/erigon-lib/kv/order/order.go +++ b/erigon-lib/kv/order/order.go @@ -45,12 +45,12 @@ func Must(asc By, k1, k2 []byte) { return } if asc { - if bytes.Compare(k1, k2) <= 0 { + if bytes.Compare(k1, k2) > 0 { panic(fmt.Sprintf("epect: %x <= %x", k1, k2)) } - } else { - if bytes.Compare(k1, k2) >= 0 { - panic(fmt.Sprintf("epect: %x >= %x", k1, k2)) - } + return + } + if bytes.Compare(k1, k2) < 0 { + panic(fmt.Sprintf("epect: %x >= %x", k1, k2)) } } diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 07e4d78389b..b32c9049d20 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1515,7 +1515,7 @@ func (hi *StateAsOfIterF) Next() ([]byte, []byte, error) { if err := hi.advanceInFiles(); err != nil { return nil, nil, err } - fmt.Printf("[dbg] StateAsOfIterF.next: limit=%d, %x\n", hi.limit, hi.kBackup) + order.Must(hi.orderAscend, hi.kBackup, hi.nextKey) return hi.kBackup, hi.vBackup, nil } @@ -1679,6 +1679,7 @@ func (hi *StateAsOfIterDB) Next() ([]byte, []byte, error) { if err := hi.advance(); err != nil { return nil, nil, err } + order.Must(hi.orderAscend, hi.kBackup, hi.nextKey) return hi.kBackup, hi.vBackup, nil } From 49f96cde2552d25fc1708176695fcadb2c3c8bb3 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 7 Nov 2024 11:08:22 +0700 Subject: [PATCH 86/91] save --- erigon-lib/kv/order/order.go | 14 +++++++++++++- erigon-lib/state/domain.go | 7 +++---- erigon-lib/state/domain_test.go | 4 ++-- erigon-lib/state/history.go | 4 ++-- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/erigon-lib/kv/order/order.go b/erigon-lib/kv/order/order.go index 59f4665f779..afbd8ed520c 100644 --- a/erigon-lib/kv/order/order.go +++ b/erigon-lib/kv/order/order.go @@ -37,7 +37,7 @@ func FromBool(v bool) By { return Desc } -func Must(asc By, k1, k2 []byte) { +func (asc By) Assert(k1, k2 []byte) { if !dbg.AssertEnabled { return } @@ -54,3 +54,15 @@ func Must(asc By, k1, k2 []byte) { panic(fmt.Sprintf("epect: %x >= %x", k1, k2)) } } + +func (asc By) AssertList(keys [][]byte) { + if !dbg.AssertEnabled { + return + } + if len(keys) < 2 { + return + } + for i := 0; i < len(keys)-2; i++ { + asc.Assert(keys[i], keys[i+1]) + } +} diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index c148a8dca59..2059b8e3b25 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -2351,7 +2351,6 @@ func (hi *DomainLatestIterFile) advanceInFiles() error { } } - fmt.Printf("[dbg] advInFiles2: %x\n", k) if len(k) > 0 && (hi.to == nil || bytes.Compare(k[:len(k)-8], hi.to) < 0) { stepBytes := k[len(k)-8:] k = k[:len(k)-8] @@ -2370,7 +2369,6 @@ func (hi *DomainLatestIterFile) advanceInFiles() error { return err } - fmt.Printf("[dbg] advInFiles3: %x\n", k) if len(k) > 0 && (hi.to == nil || bytes.Compare(k, hi.to) < 0) { stepBytes := stepBytesWithValue[:8] v := stepBytesWithValue[8:] @@ -2408,8 +2406,9 @@ func (hi *DomainLatestIterFile) Next() ([]byte, []byte, error) { if err := hi.advanceInFiles(); err != nil { return nil, nil, err } - order.Must(order.Asc, hi.kBackup, hi.nextKey) - return hi.kBackup, hi.vBackup, nil + order.Assert(order.Asc, hi.kBackup, hi.nextKey) + // TODO: remove `common.Copy`. it protecting from some existing bug. + return common.Copy(hi.kBackup), hi.vBackup, nil } func (d *Domain) stepsRangeInDBAsStr(tx kv.Tx) string { diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index 1b88d3e0355..2b36847b7b8 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -1519,8 +1519,8 @@ func TestDomainRange(t *testing.T) { it, err := dc.ht.WalkAsOf(context.Background(), 190, nil, nil, order.Asc, -1, tx) require.NoError(err) keys, vals, err := stream.ToArrayKV(it) - fmt.Printf("keys: %x\n", keys) require.NoError(err) + order.Asc.AssertList(keys) require.Equal(3, len(keys)) require.Equal(3, len(vals)) } @@ -1529,8 +1529,8 @@ func TestDomainRange(t *testing.T) { it, err := dc.DomainRangeLatest(tx, nil, nil, -1) require.NoError(err) keys, vals, err := stream.ToArrayKV(it) - fmt.Printf("keys: %x\n", keys) require.NoError(err) + order.Asc.AssertList(keys) require.Equal(3, len(keys)) require.Equal(3, len(vals)) } diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index b32c9049d20..5d7a5080b46 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1515,7 +1515,7 @@ func (hi *StateAsOfIterF) Next() ([]byte, []byte, error) { if err := hi.advanceInFiles(); err != nil { return nil, nil, err } - order.Must(hi.orderAscend, hi.kBackup, hi.nextKey) + order.Assert(hi.orderAscend, hi.kBackup, hi.nextKey) return hi.kBackup, hi.vBackup, nil } @@ -1679,7 +1679,7 @@ func (hi *StateAsOfIterDB) Next() ([]byte, []byte, error) { if err := hi.advance(); err != nil { return nil, nil, err } - order.Must(hi.orderAscend, hi.kBackup, hi.nextKey) + hi.orderAscend.Assert(hi.kBackup, hi.nextKey) return hi.kBackup, hi.vBackup, nil } From 5ef790f0139def511f9214cdcdd9ce0e212a62e2 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 7 Nov 2024 11:09:05 +0700 Subject: [PATCH 87/91] save --- erigon-lib/state/domain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index 2059b8e3b25..de0f5929428 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -2406,7 +2406,7 @@ func (hi *DomainLatestIterFile) Next() ([]byte, []byte, error) { if err := hi.advanceInFiles(); err != nil { return nil, nil, err } - order.Assert(order.Asc, hi.kBackup, hi.nextKey) + order.Asc.Assert(hi.kBackup, hi.nextKey) // TODO: remove `common.Copy`. it protecting from some existing bug. return common.Copy(hi.kBackup), hi.vBackup, nil } From 1be1647697b201631bd344ab33f6762e98aacf6a Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 7 Nov 2024 11:09:14 +0700 Subject: [PATCH 88/91] save --- erigon-lib/state/history.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 5d7a5080b46..8ca037b9c39 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -1515,7 +1515,7 @@ func (hi *StateAsOfIterF) Next() ([]byte, []byte, error) { if err := hi.advanceInFiles(); err != nil { return nil, nil, err } - order.Assert(hi.orderAscend, hi.kBackup, hi.nextKey) + hi.orderAscend.Assert(hi.kBackup, hi.nextKey) return hi.kBackup, hi.vBackup, nil } From 92f174f51e3ebdc538fd8a777de3f982fe847db2 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 7 Nov 2024 11:11:55 +0700 Subject: [PATCH 89/91] save --- erigon-lib/state/domain_test.go | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index 2b36847b7b8..66b03fd5c6c 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -1465,9 +1465,9 @@ func TestDomain_GetAfterAggregation(t *testing.T) { func TestDomainRange(t *testing.T) { db, d := testDbAndDomainOfStep(t, 25, log.New()) - require := require.New(t) + require, ctx := require.New(t), context.Background() - tx, err := db.BeginRw(context.Background()) + tx, err := db.BeginRw(ctx) require.NoError(err) defer tx.Rollback() @@ -1500,14 +1500,14 @@ func TestDomainRange(t *testing.T) { } writer.SetTxNum(totalTx) - err = writer.Flush(context.Background(), tx) + err = writer.Flush(ctx, tx) require.NoError(err) // aggregate collateAndMerge(t, db, tx, d, totalTx) require.NoError(tx.Commit()) - tx, err = db.BeginRw(context.Background()) + tx, err = db.BeginRw(ctx) require.NoError(err) defer tx.Rollback() dc.Close() @@ -1516,7 +1516,7 @@ func TestDomainRange(t *testing.T) { defer dc.Close() { - it, err := dc.ht.WalkAsOf(context.Background(), 190, nil, nil, order.Asc, -1, tx) + it, err := dc.ht.WalkAsOf(ctx, 190, nil, nil, order.Asc, -1, tx) require.NoError(err) keys, vals, err := stream.ToArrayKV(it) require.NoError(err) @@ -1535,12 +1535,15 @@ func TestDomainRange(t *testing.T) { require.Equal(3, len(vals)) } - //it, err := dc.DomainRange(context.Background(), tx, []byte{""}, nil, 190, order.Asc, -1) - //require.NoError(err) - //keys, vals, err := stream.ToArrayKV(it) - //require.NoError(err) - //require.Equal(5, len(keys)) - //require.Equal(5, len(vals)) + { + it, err := dc.DomainRange(ctx, tx, []byte(""), nil, 190, order.Asc, -1) + require.NoError(err) + keys, vals, err := stream.ToArrayKV(it) + require.NoError(err) + order.Asc.AssertList(keys) + require.Equal(5, len(keys)) + require.Equal(5, len(vals)) + } t.Fail() } @@ -1548,10 +1551,10 @@ func TestDomainRange(t *testing.T) { func TestDomain_CanPruneAfterAggregation(t *testing.T) { t.Parallel() - aggStep := uint64(25) + aggStep, ctx := uint64(25), context.Background() db, d := testDbAndDomainOfStep(t, aggStep, log.New()) - tx, err := db.BeginRw(context.Background()) + tx, err := db.BeginRw(ctx) require.NoError(t, err) defer tx.Rollback() From dad88ab75e4d7c07fe4fda3b3b3aef7a0ac27e95 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 7 Nov 2024 11:12:42 +0700 Subject: [PATCH 90/91] save --- erigon-lib/state/domain.go | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index de0f5929428..085f40ec165 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1954,26 +1954,15 @@ func (dt *DomainRoTx) DomainRange(ctx context.Context, tx kv.Tx, fromKey, toKey if !asc { panic("implement me") } - //histStateIt, err := tx.aggTx.AccountHistoricalStateRange(asOfTs, fromKey, toKey, limit, tx.MdbxTx) - //if err != nil { - // return nil, err - //} - //lastestStateIt, err := tx.aggTx.DomainRangeLatest(tx.MdbxTx, kv.AccountDomain, fromKey, toKey, limit) - //if err != nil { - // return nil, err - //} - fmt.Printf("[dbg] DomainRange 1: %s, %x, %x\n", dt.d.name, fromKey, toKey) histStateIt, err := dt.ht.WalkAsOf(ctx, ts, fromKey, toKey, asc, limit, tx) if err != nil { return nil, err } - fmt.Printf("[dbg] DomainRange 2: %s\n", dt.d.name) - //lastestStateIt, err := dt.DomainRangeLatest(tx, fromKey, toKey, limit) - //if err != nil { - // return nil, err - //} - //return stream.UnionKV(histStateIt, lastestStateIt, limit), nil - return stream.UnionKV(histStateIt, stream.EmptyKV, limit), nil + lastestStateIt, err := dt.DomainRangeLatest(tx, fromKey, toKey, limit) + if err != nil { + return nil, err + } + return stream.UnionKV(histStateIt, lastestStateIt, limit), nil } func (dt *DomainRoTx) DomainRangeLatest(roTx kv.Tx, fromKey, toKey []byte, limit int) (stream.KV, error) { From ca5891260c27562470d1c6243a7580a3e794913c Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 7 Nov 2024 11:14:13 +0700 Subject: [PATCH 91/91] save --- erigon-lib/state/domain_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index 66b03fd5c6c..dd2f83685ec 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -1531,8 +1531,8 @@ func TestDomainRange(t *testing.T) { keys, vals, err := stream.ToArrayKV(it) require.NoError(err) order.Asc.AssertList(keys) - require.Equal(3, len(keys)) - require.Equal(3, len(vals)) + require.Equal(5, len(keys)) + require.Equal(5, len(vals)) } {