From cd2ce5bc0792b5555fe0bfe0b143de34a53ca870 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Nov 2024 13:24:51 +0100 Subject: [PATCH 01/15] build(deps): Bump github.com/stretchr/testify from 1.9.0 to 1.10.0 (#1008) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 176bd0d73..52627c24b 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/emicklei/dot v1.6.2 github.com/gogo/protobuf v1.3.2 github.com/google/btree v1.1.3 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 go.uber.org/mock v0.5.0 ) diff --git a/go.sum b/go.sum index a553fe0ae..6bf10da55 100644 --- a/go.sum +++ b/go.sum @@ -58,8 +58,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= From 28b9124b576fc2d10a54f94bbdd13c2e90b8c3b7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Nov 2024 10:11:00 +0100 Subject: [PATCH 02/15] build(deps): Bump github.com/emicklei/dot from 1.6.2 to 1.6.3 (#1005) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 52627c24b..8c5f61720 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.23.1 require ( cosmossdk.io/core v1.0.0-alpha.6 github.com/cosmos/ics23/go v0.11.0 - github.com/emicklei/dot v1.6.2 + github.com/emicklei/dot v1.6.3 github.com/gogo/protobuf v1.3.2 github.com/google/btree v1.1.3 github.com/stretchr/testify v1.10.0 diff --git a/go.sum b/go.sum index 6bf10da55..c8a5b411c 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/cosmos/ics23/go v0.11.0/go.mod h1:A8OjxPE67hHST4Icw94hOxxFEJMBG031xIG github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A= -github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= +github.com/emicklei/dot v1.6.3 h1:MW9eLeJWaN+QZVSPlrargGd/l92IA6d4fmo39/YD2cQ= +github.com/emicklei/dot v1.6.3/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= From cf74234d76cd92d9234121658b63dbc941743268 Mon Sep 17 00:00:00 2001 From: cool-developer <51834436+cool-develope@users.noreply.github.com> Date: Tue, 26 Nov 2024 04:12:29 -0500 Subject: [PATCH 03/15] fix: Add Extra Check for Reformatted Root Node in GetNode (#1007) --- CHANGELOG.md | 4 ++++ mutable_tree_test.go | 24 ++++++++++++++++++++++++ nodedb.go | 14 ++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ffeea775..aae7b5f0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +### Bug Fixes + +- [#1007](https://github.com/cosmos/iavl/pull/1007) Add the extra check for the reformatted root node in `GetNode` + ### Improvements - [#952](https://github.com/cosmos/iavl/pull/952) Add `DeleteVersionsFrom(int64)` API. diff --git a/mutable_tree_test.go b/mutable_tree_test.go index 3d92f888c..24ecbc7da 100644 --- a/mutable_tree_test.go +++ b/mutable_tree_test.go @@ -1482,6 +1482,30 @@ func TestMutableTreeClose(t *testing.T) { require.NoError(t, tree.Close()) } +func TestReferenceRootPruning(t *testing.T) { + memDB := dbm.NewMemDB() + tree := NewMutableTree(memDB, 0, true, NewNopLogger()) + + _, err := tree.Set([]byte("foo"), []byte("bar")) + require.NoError(t, err) + _, _, err = tree.SaveVersion() + require.NoError(t, err) + + _, _, err = tree.SaveVersion() + require.NoError(t, err) + + _, err = tree.Set([]byte("foo1"), []byte("bar")) + require.NoError(t, err) + _, _, err = tree.SaveVersion() + require.NoError(t, err) + + err = tree.DeleteVersionsTo(1) + require.NoError(t, err) + + _, err = tree.Set([]byte("foo"), []byte("bar*")) + require.NoError(t, err) +} + func TestMutableTree_InitialVersionZero(t *testing.T) { db := dbm.NewMemDB() diff --git a/nodedb.go b/nodedb.go index 0eb7003c7..096fafeac 100644 --- a/nodedb.go +++ b/nodedb.go @@ -159,6 +159,20 @@ func (ndb *nodeDB) GetNode(nk []byte) (*Node, error) { if err != nil { return nil, fmt.Errorf("can't get node %v: %v", nk, err) } + if buf == nil && !isLegcyNode { + // if the node is reformatted by pruning, check against (version, 0) + nKey := GetNodeKey(nk) + if nKey.nonce == 1 { + nodeKey = ndb.nodeKey((&NodeKey{ + version: nKey.version, + nonce: 0, + }).GetKey()) + buf, err = ndb.db.Get(nodeKey) + if err != nil { + return nil, fmt.Errorf("can't get the reformatted node %v: %v", nk, err) + } + } + } if buf == nil { return nil, fmt.Errorf("Value missing for key %v corresponding to nodeKey %x", nk, nodeKey) } From 1aaca64f3fc70647ac34286d638fd02c25e6e6b8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Dec 2024 12:56:30 +0100 Subject: [PATCH 04/15] build(deps): Bump github.com/emicklei/dot from 1.6.3 to 1.6.4 (#1011) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8c5f61720..8a655ec37 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.23.1 require ( cosmossdk.io/core v1.0.0-alpha.6 github.com/cosmos/ics23/go v0.11.0 - github.com/emicklei/dot v1.6.3 + github.com/emicklei/dot v1.6.4 github.com/gogo/protobuf v1.3.2 github.com/google/btree v1.1.3 github.com/stretchr/testify v1.10.0 diff --git a/go.sum b/go.sum index c8a5b411c..e1247cece 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/cosmos/ics23/go v0.11.0/go.mod h1:A8OjxPE67hHST4Icw94hOxxFEJMBG031xIG github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/dot v1.6.3 h1:MW9eLeJWaN+QZVSPlrargGd/l92IA6d4fmo39/YD2cQ= -github.com/emicklei/dot v1.6.3/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= +github.com/emicklei/dot v1.6.4 h1:cG9ycT67d9Yw22G+mAb4XiuUz6E6H1S0zePp/5Cwe/c= +github.com/emicklei/dot v1.6.4/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= From a989ed4ed0e4ab2d9f6a7e5b8341a8b5c97706f1 Mon Sep 17 00:00:00 2001 From: cuiyourong <166382782+cuiyourong@users.noreply.github.com> Date: Sat, 7 Dec 2024 04:57:20 +0800 Subject: [PATCH 05/15] chore: fix some function names in comment (#1012) Signed-off-by: cuiyourong --- internal/encoding/encoding.go | 4 ++-- mutable_tree.go | 2 +- mutable_tree_test.go | 2 +- v2/internal/encoding.go | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/encoding/encoding.go b/internal/encoding/encoding.go index 6e390c36c..40ff6f667 100644 --- a/internal/encoding/encoding.go +++ b/internal/encoding/encoding.go @@ -113,7 +113,7 @@ func Encode32BytesHash(w io.Writer, bz []byte) error { return err } -// encodeBytesSlice length-prefixes the byte slice and returns it. +// EncodeBytesSlice length-prefixes the byte slice and returns it. func EncodeBytesSlice(bz []byte) ([]byte, error) { buf := bufPool.Get().(*bytes.Buffer) buf.Reset() @@ -127,7 +127,7 @@ func EncodeBytesSlice(bz []byte) ([]byte, error) { return bytesCopy, err } -// encodeBytesSize returns the byte size of the given slice including length-prefixing. +// EncodeBytesSize returns the byte size of the given slice including length-prefixing. func EncodeBytesSize(bz []byte) int { return EncodeUvarintSize(uint64(len(bz))) + len(bz) } diff --git a/mutable_tree.go b/mutable_tree.go index ae7dc6280..a64be8e06 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -514,7 +514,7 @@ func (tree *MutableTree) LoadVersion(targetVersion int64) (int64, error) { return latestVersion, nil } -// loadVersionForOverwriting attempts to load a tree at a previously committed +// LoadVersionForOverwriting attempts to load a tree at a previously committed // version, or the latest version below it. Any versions greater than targetVersion will be deleted. func (tree *MutableTree) LoadVersionForOverwriting(targetVersion int64) error { if _, err := tree.LoadVersion(targetVersion); err != nil { diff --git a/mutable_tree_test.go b/mutable_tree_test.go index 24ecbc7da..9545fc1bf 100644 --- a/mutable_tree_test.go +++ b/mutable_tree_test.go @@ -60,7 +60,7 @@ func TestIterateConcurrency(t *testing.T) { wg.Wait() } -// TestConcurrency throws "fatal error: concurrent map iteration and map write" and +// TestIteratorConcurrency throws "fatal error: concurrent map iteration and map write" and // also sometimes "fatal error: concurrent map writes" when fast node is enabled func TestIteratorConcurrency(t *testing.T) { if testing.Short() { diff --git a/v2/internal/encoding.go b/v2/internal/encoding.go index 17a994bc1..65d3ca3c4 100644 --- a/v2/internal/encoding.go +++ b/v2/internal/encoding.go @@ -97,7 +97,7 @@ func EncodeBytes(w io.Writer, bz []byte) error { return err } -// encodeBytesSlice length-prefixes the byte slice and returns it. +// EncodeBytesSlice length-prefixes the byte slice and returns it. func EncodeBytesSlice(bz []byte) ([]byte, error) { buf := bufPool.Get().(*bytes.Buffer) buf.Reset() @@ -111,7 +111,7 @@ func EncodeBytesSlice(bz []byte) ([]byte, error) { return bytesCopy, err } -// encodeBytesSize returns the byte size of the given slice including length-prefixing. +// EncodeBytesSize returns the byte size of the given slice including length-prefixing. func EncodeBytesSize(bz []byte) int { return EncodeUvarintSize(uint64(len(bz))) + len(bz) } From 456159f12b9c8ce466236aa9975952c6d52cd1e6 Mon Sep 17 00:00:00 2001 From: Andi <36215014+ChengenH@users.noreply.github.com> Date: Sun, 8 Dec 2024 20:36:37 +0800 Subject: [PATCH 06/15] chore: use errors.New to replace fmt.Errorf with no parameters (#1013) --- cmd/iaviewer/main.go | 3 ++- mutable_tree.go | 4 ++-- nodedb.go | 2 +- proof.go | 6 +++--- proof_ics23.go | 7 +++---- v2/export.go | 6 ++++-- v2/migrate/v0/migrate_v0.go | 3 ++- v2/node.go | 6 +++--- v2/sqlite.go | 3 ++- v2/sqlite_metadata.go | 3 ++- v2/tree.go | 3 ++- 11 files changed, 26 insertions(+), 20 deletions(-) diff --git a/cmd/iaviewer/main.go b/cmd/iaviewer/main.go index ecc704e6d..acab613f9 100644 --- a/cmd/iaviewer/main.go +++ b/cmd/iaviewer/main.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/sha256" "encoding/hex" + "errors" "fmt" "os" "path/filepath" @@ -67,7 +68,7 @@ func OpenDB(dir string) (corestore.KVStoreWithBatch, error) { case strings.HasSuffix(dir, ".db/"): dir = dir[:len(dir)-4] default: - return nil, fmt.Errorf("database directory must end with .db") + return nil, errors.New("database directory must end with .db") } dir, err := filepath.Abs(dir) diff --git a/mutable_tree.go b/mutable_tree.go index a64be8e06..87d7dfd78 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -960,7 +960,7 @@ func (tree *MutableTree) rotateLeft(node *Node) (*Node, error) { // TODO: optimize balance & rotate func (tree *MutableTree) balance(node *Node) (newSelf *Node, err error) { if node.nodeKey != nil { - return nil, fmt.Errorf("unexpected balance() call on persisted node") + return nil, errors.New("unexpected balance() call on persisted node") } balance, err := node.calcBalance(tree.ImmutableTree) if err != nil { @@ -1084,7 +1084,7 @@ func (tree *MutableTree) saveNewNodes(version int64) error { func (tree *MutableTree) SaveChangeSet(cs *ChangeSet) (int64, error) { // if the tree has uncommitted changes, return error if tree.root != nil && tree.root.nodeKey == nil { - return 0, fmt.Errorf("cannot save changeset with uncommitted changes") + return 0, errors.New("cannot save changeset with uncommitted changes") } for _, pair := range cs.Pairs { if pair.Delete { diff --git a/nodedb.go b/nodedb.go index 096fafeac..10c5546d1 100644 --- a/nodedb.go +++ b/nodedb.go @@ -360,7 +360,7 @@ func (ndb *nodeDB) shouldForceFastStorageUpgrade() (bool, error) { // saveFastNodeUnlocked saves a FastNode to disk. func (ndb *nodeDB) saveFastNodeUnlocked(node *fastnode.Node, shouldAddToCache bool) error { if node.GetKey() == nil { - return fmt.Errorf("cannot have FastNode with a nil value for key") + return errors.New("cannot have FastNode with a nil value for key") } // Save node bytes to db. diff --git a/proof.go b/proof.go index 4fdbc35e0..bced6ff4e 100644 --- a/proof.go +++ b/proof.go @@ -19,13 +19,13 @@ var bufPool = &sync.Pool{ var ( // ErrInvalidProof is returned by Verify when a proof cannot be validated. - ErrInvalidProof = fmt.Errorf("invalid proof") + ErrInvalidProof = errors.New("invalid proof") // ErrInvalidInputs is returned when the inputs passed to the function are invalid. - ErrInvalidInputs = fmt.Errorf("invalid inputs") + ErrInvalidInputs = errors.New("invalid inputs") // ErrInvalidRoot is returned when the root passed in does not match the proof's. - ErrInvalidRoot = fmt.Errorf("invalid root") + ErrInvalidRoot = errors.New("invalid root") ) //---------------------------------------- diff --git a/proof_ics23.go b/proof_ics23.go index 913b6f7ec..2ee331ce1 100644 --- a/proof_ics23.go +++ b/proof_ics23.go @@ -2,8 +2,7 @@ package iavl import ( "encoding/binary" - "fmt" - + "errors" ics23 "github.com/cosmos/ics23/go" ) @@ -48,7 +47,7 @@ func (t *ImmutableTree) GetNonMembershipProof(key []byte) (*ics23.CommitmentProo } if val != nil { - return nil, fmt.Errorf("cannot create NonExistanceProof when Key in State") + return nil, errors.New("cannot create NonExistanceProof when Key in State") } nonexist := &ics23.NonExistenceProof{ @@ -177,7 +176,7 @@ func convertVarIntToBytes(orig int64, buf [binary.MaxVarintLen64]byte) []byte { // GetProof gets the proof for the given key. func (t *ImmutableTree) GetProof(key []byte) (*ics23.CommitmentProof, error) { if t.root == nil { - return nil, fmt.Errorf("cannot generate the proof with nil root") + return nil, errors.New("cannot generate the proof with nil root") } exist, err := t.Has(key) diff --git a/v2/export.go b/v2/export.go index a1f7e0570..0993f3dc8 100644 --- a/v2/export.go +++ b/v2/export.go @@ -1,6 +1,8 @@ package iavl -import "fmt" +import ( + "errors" +) // TraverseOrderType is the type of the order in which the tree is traversed. type TraverseOrderType uint8 @@ -98,4 +100,4 @@ func (e *Exporter) Next() (*SnapshotNode, error) { } } -var ErrorExportDone = fmt.Errorf("export done") +var ErrorExportDone = errors.New("export done") diff --git a/v2/migrate/v0/migrate_v0.go b/v2/migrate/v0/migrate_v0.go index eef61de61..9b305b738 100644 --- a/v2/migrate/v0/migrate_v0.go +++ b/v2/migrate/v0/migrate_v0.go @@ -2,6 +2,7 @@ package v0 import ( "bytes" + "errors" "fmt" "sync" @@ -111,7 +112,7 @@ func latestVersionCommand() *cobra.Command { return err } if set && version == -1 { - return fmt.Errorf("version must be set") + return errors.New("version must be set") } if set { diff --git a/v2/node.go b/v2/node.go index f27c208ec..31c8a1bdd 100644 --- a/v2/node.go +++ b/v2/node.go @@ -102,7 +102,7 @@ func (node *Node) right(t *Tree) *Node { // getLeftNode will never be called on leaf nodes. all tree nodes have 2 children. func (node *Node) getLeftNode(t *Tree) (*Node, error) { if node.isLeaf() { - return nil, fmt.Errorf("leaf node has no left node") + return nil, errors.New("leaf node has no left node") } if node.leftNode != nil { return node.leftNode, nil @@ -117,7 +117,7 @@ func (node *Node) getLeftNode(t *Tree) (*Node, error) { func (node *Node) getRightNode(t *Tree) (*Node, error) { if node.isLeaf() { - return nil, fmt.Errorf("leaf node has no right node") + return nil, errors.New("leaf node has no right node") } if node.rightNode != nil { return node.rightNode, nil @@ -158,7 +158,7 @@ func maxInt8(a, b int8) int8 { // TODO: optimize balance & rotate func (tree *Tree) balance(node *Node) (newSelf *Node, err error) { if node.hash != nil { - return nil, fmt.Errorf("unexpected balance() call on persisted node") + return nil, errors.New("unexpected balance() call on persisted node") } balance, err := node.calcBalance(tree) if err != nil { diff --git a/v2/sqlite.go b/v2/sqlite.go index 78feff512..7aef44576 100644 --- a/v2/sqlite.go +++ b/v2/sqlite.go @@ -2,6 +2,7 @@ package iavl import ( "bytes" + "errors" "fmt" "os" "strconv" @@ -99,7 +100,7 @@ func (opts SqliteDbOptions) EstimateMmapSize() (uint64, error) { return 0, err } if !hasRow { - return 0, fmt.Errorf("no row") + return 0, errors.New("no row") } var leafSize int64 err = q.Scan(&leafSize) diff --git a/v2/sqlite_metadata.go b/v2/sqlite_metadata.go index 2fd849645..f67faedcb 100644 --- a/v2/sqlite_metadata.go +++ b/v2/sqlite_metadata.go @@ -1,6 +1,7 @@ package iavl import ( + "errors" "fmt" "os" "sync" @@ -19,7 +20,7 @@ type SqliteKVStore struct { func NewSqliteKVStore(opts SqliteDbOptions) (kv *SqliteKVStore, err error) { if opts.Path == "" { - return nil, fmt.Errorf("path cannot be empty") + return nil, errors.New("path cannot be empty") } if opts.WalSize == 0 { opts.WalSize = 50 * 1024 * 1024 diff --git a/v2/tree.go b/v2/tree.go index e8111990a..89263585c 100644 --- a/v2/tree.go +++ b/v2/tree.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/sha256" + "errors" "fmt" "os" "time" @@ -102,7 +103,7 @@ func NewTree(sql *SqliteDb, pool *NodePool, opts TreeOptions) *Tree { func (tree *Tree) LoadVersion(version int64) (err error) { if tree.sql == nil { - return fmt.Errorf("sql is nil") + return errors.New("sql is nil") } tree.workingBytes = 0 From 73b21dd6d5dda5f584eb87ab0fd2f0fac8b20f35 Mon Sep 17 00:00:00 2001 From: RiceChuan Date: Thu, 12 Dec 2024 17:24:46 +0800 Subject: [PATCH 07/15] chore: use errors.New to replace fmt.Errorf with no parameters (#1015) Signed-off-by: RiceChuan --- nodedb.go | 4 ++-- nodedb_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nodedb.go b/nodedb.go index 10c5546d1..f997596a8 100644 --- a/nodedb.go +++ b/nodedb.go @@ -204,7 +204,7 @@ func (ndb *nodeDB) GetFastNode(key []byte) (*fastnode.Node, error) { defer ndb.mtx.Unlock() if len(key) == 0 { - return nil, fmt.Errorf("nodeDB.GetFastNode() requires key, len(key) equals 0") + return nil, errors.New("nodeDB.GetFastNode() requires key, len(key) equals 0") } if cachedFastNode := ndb.fastNodeCache.Get(key); cachedFastNode != nil { @@ -1338,4 +1338,4 @@ func (ndb *nodeDB) String() (string, error) { return "-" + "\n" + buf.String() + "-", nil } -var ErrNodeMissingNodeKey = fmt.Errorf("node does not have a nodeKey") +var ErrNodeMissingNodeKey = errors.New("node does not have a nodeKey") diff --git a/nodedb_test.go b/nodedb_test.go index 170261110..f87103025 100644 --- a/nodedb_test.go +++ b/nodedb_test.go @@ -278,7 +278,7 @@ func TestTraverseNodes(t *testing.T) { return err } if actualNode.String() != node.String() { - return fmt.Errorf("found unexpected node") + return errors.New("found unexpected node") } count++ return nil From d89d5d22030e8fea42ce5406743ce45b75ad86ea Mon Sep 17 00:00:00 2001 From: Andi <36215014+ChengenH@users.noreply.github.com> Date: Fri, 13 Dec 2024 18:37:32 +0800 Subject: [PATCH 08/15] refactor: unify the error handling methods in the crypto package that are different from the project style (#1016) Signed-off-by: ChengenH --- benchmarks/cosmos-exim/main.go | 3 ++- export_test.go | 9 +++++---- import_test.go | 5 +++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/benchmarks/cosmos-exim/main.go b/benchmarks/cosmos-exim/main.go index 34ba70266..5f62c4305 100644 --- a/benchmarks/cosmos-exim/main.go +++ b/benchmarks/cosmos-exim/main.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "os" "time" @@ -128,7 +129,7 @@ func runExport(dbPath string) (int64, map[string][]*iavl.ExportNode, error) { defer exporter.Close() for { node, err := exporter.Next() - if err == iavl.ErrorExportDone { + if errors.Is(err, iavl.ErrorExportDone) { break } else if err != nil { return 0, nil, err diff --git a/export_test.go b/export_test.go index 9772e2ed1..de83a4c98 100644 --- a/export_test.go +++ b/export_test.go @@ -1,6 +1,7 @@ package iavl import ( + "errors" "math" "math/rand" "testing" @@ -180,7 +181,7 @@ func TestExporter(t *testing.T) { defer exporter.Close() for { node, err := exporter.Next() - if err == ErrorExportDone { + if errors.Is(err, ErrorExportDone) { break } require.NoError(t, err) @@ -215,7 +216,7 @@ func TestExporterCompress(t *testing.T) { exporter := NewCompressExporter(innerExporter) for { node, err := exporter.Next() - if err == ErrorExportDone { + if errors.Is(err, ErrorExportDone) { break } require.NoError(t, err) @@ -266,7 +267,7 @@ func TestExporter_Import(t *testing.T) { for { item, err := exporter.Next() - if err == ErrorExportDone { + if errors.Is(err, ErrorExportDone) { err = innerImporter.Commit() require.NoError(t, err) break @@ -365,7 +366,7 @@ func BenchmarkExport(b *testing.B) { require.NoError(b, err) for { _, err := exporter.Next() - if err == ErrorExportDone { + if errors.Is(err, ErrorExportDone) { break } else if err != nil { b.Error(err) diff --git a/import_test.go b/import_test.go index 9f4d49aee..fd3621c1d 100644 --- a/import_test.go +++ b/import_test.go @@ -1,6 +1,7 @@ package iavl import ( + "errors" "testing" "github.com/stretchr/testify/assert" @@ -43,7 +44,7 @@ func ExampleImporter() { for { var node *ExportNode node, err = exporter.Next() - if err == ErrorExportDone { + if errors.Is(err, ErrorExportDone) { break } else if err != nil { panic(err) @@ -247,7 +248,7 @@ func benchmarkImport(b *testing.B, nodes int) { require.NoError(b, err) for { item, err := exporter.Next() - if err == ErrorExportDone { + if errors.Is(err, ErrorExportDone) { break } else if err != nil { b.Error(err) From e60601d37e18f069363faf9af48f105abee9cc35 Mon Sep 17 00:00:00 2001 From: yihuang Date: Tue, 17 Dec 2024 01:05:51 +0800 Subject: [PATCH 09/15] cache first version for legacy versions (#1018) --- CHANGELOG.md | 1 + nodedb.go | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aae7b5f0e..b018c8799 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - [#965](https://github.com/cosmos/iavl/pull/965) Use expected interface for expected IAVL `Logger`. - [#970](https://github.com/cosmos/iavl/pull/970) Close the pruning process when the nodeDB is closed. - [#980](https://github.com/cosmos/iavl/pull/980) Use the `sdk/core/store.KVStoreWithBatch` interface instead of `iavl/db.DB` interface +- [#1018](https://github.com/cosmos/iavl/pull/1018) Cache first version for legacy versions, fix performance regression after upgrade. ## v1.2.0 May 13, 2024 diff --git a/nodedb.go b/nodedb.go index f997596a8..1dad7e23f 100644 --- a/nodedb.go +++ b/nodedb.go @@ -738,6 +738,7 @@ func (ndb *nodeDB) getFirstVersion() (int64, error) { if itr.Valid() { var version int64 legacyRootKeyFormat.Scan(itr.Key(), &version) + ndb.resetFirstVersion(version) return version, nil } // Find the first version From f9c402d40752b2e16e72a7cca80d5d34cf5a2a46 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 19:30:19 +0000 Subject: [PATCH 10/15] build(deps): Bump golang.org/x/crypto from 0.23.0 to 0.31.0 in /cmd (#1014) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- cmd/go.mod | 17 ++++++++--------- cmd/go.sum | 42 ++++++++++++++++++++---------------------- 2 files changed, 28 insertions(+), 31 deletions(-) diff --git a/cmd/go.mod b/cmd/go.mod index 70265e9cb..5dcc81bc0 100644 --- a/cmd/go.mod +++ b/cmd/go.mod @@ -1,30 +1,29 @@ module github.com/cosmos/iavl/cmd go 1.21 +toolchain go1.23.4 require ( - cosmossdk.io/core v0.12.1-0.20240813134434-072a29c838a5 + cosmossdk.io/core v1.0.0-alpha.6 cosmossdk.io/log v1.3.1 github.com/cosmos/iavl v1.2.0 ) require ( - github.com/cosmos/gogoproto v1.6.0 // indirect - github.com/cosmos/ics23/go v0.10.0 // indirect - github.com/emicklei/dot v1.6.2 // indirect + github.com/cosmos/gogoproto v1.7.0 // indirect + github.com/cosmos/ics23/go v0.11.0 // indirect + github.com/emicklei/dot v1.6.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/btree v1.1.2 // indirect + github.com/google/btree v1.1.3 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/rs/zerolog v1.32.0 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect - golang.org/x/crypto v0.23.0 // indirect - golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/sys v0.22.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/sys v0.28.0 // indirect google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/cmd/go.sum b/cmd/go.sum index 4f963b1c8..3801668ca 100644 --- a/cmd/go.sum +++ b/cmd/go.sum @@ -1,16 +1,16 @@ -cosmossdk.io/core v0.12.1-0.20240813134434-072a29c838a5 h1:aIfrfJUronk2qHPH4N7M5nGzijdSWkRwHHqqPGnQBrk= -cosmossdk.io/core v0.12.1-0.20240813134434-072a29c838a5/go.mod h1:B8JQN1vmGCPSVFlmRb/22n1T736P4C2qfEsrSKDX1iM= +cosmossdk.io/core v1.0.0-alpha.6 h1:5ukC4JcQKmemLQXcAgu/QoOvJI50hpBkIIg4ZT2EN8E= +cosmossdk.io/core v1.0.0-alpha.6/go.mod h1:3u9cWq1FAVtiiCrDPpo4LhR+9V6k/ycSG4/Y/tREWCY= cosmossdk.io/log v1.3.1 h1:UZx8nWIkfbbNEWusZqzAx3ZGvu54TZacWib3EzUYmGI= cosmossdk.io/log v1.3.1/go.mod h1:2/dIomt8mKdk6vl3OWJcPk2be3pGOS8OQaLUM/3/tCM= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cosmos/gogoproto v1.6.0 h1:Xm0F/96O5Ox4g6xGgjA41rWaaPjYtOdTi59uBcV2qEE= -github.com/cosmos/gogoproto v1.6.0/go.mod h1:Y+g956rcUf2vr4uwtCcK/1Xx9BWVluCtcI9vsh0GHmk= -github.com/cosmos/ics23/go v0.10.0 h1:iXqLLgp2Lp+EdpIuwXTYIQU+AiHj9mOC2X9ab++bZDM= -github.com/cosmos/ics23/go v0.10.0/go.mod h1:ZfJSmng/TBNTBkFemHHHj5YY7VAU/MBU980F4VU1NG0= +github.com/cosmos/gogoproto v1.7.0 h1:79USr0oyXAbxg3rspGh/m4SWNyoz/GLaAh0QlCe2fro= +github.com/cosmos/gogoproto v1.7.0/go.mod h1:yWChEv5IUEYURQasfyBW5ffkMHR/90hiHgbNgrtp4j0= +github.com/cosmos/ics23/go v0.11.0 h1:jk5skjT0TqX5e5QJbEnwXIS2yI2vnmLOgpQPeM5RtnU= +github.com/cosmos/ics23/go v0.11.0/go.mod h1:A8OjxPE67hHST4Icw94hOxxFEJMBG031xIGF/JHNIY0= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A= -github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= +github.com/emicklei/dot v1.6.4 h1:cG9ycT67d9Yw22G+mAb4XiuUz6E6H1S0zePp/5Cwe/c= +github.com/emicklei/dot v1.6.4/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= @@ -29,8 +29,8 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= -github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -62,21 +62,19 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= -go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= +go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -105,13 +103,13 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= From f7bbb9dafb048fd259843bbba1fd1d1595ca5342 Mon Sep 17 00:00:00 2001 From: Alexander Peters Date: Tue, 17 Dec 2024 15:03:23 +0100 Subject: [PATCH 11/15] fix: Better lock handling on close (#1024) --- nodedb.go | 9 +++++---- nodedb_test.go | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/nodedb.go b/nodedb.go index 1dad7e23f..7ad69af28 100644 --- a/nodedb.go +++ b/nodedb.go @@ -600,7 +600,7 @@ func (ndb *nodeDB) startPruning() { for { select { case <-ndb.ctx.Done(): - ndb.done <- struct{}{} + close(ndb.done) return default: ndb.mtx.Lock() @@ -1121,14 +1121,15 @@ func (ndb *nodeDB) traverseOrphans(prevVersion, curVersion int64, fn func(*Node) // Close the nodeDB. func (ndb *nodeDB) Close() error { - ndb.mtx.Lock() - defer ndb.mtx.Unlock() - ndb.cancel() + if ndb.opts.AsyncPruning { <-ndb.done // wait for the pruning process to finish } + ndb.mtx.Lock() + defer ndb.mtx.Unlock() + if ndb.batch != nil { if err := ndb.batch.Close(); err != nil { return err diff --git a/nodedb_test.go b/nodedb_test.go index f87103025..ace9b052e 100644 --- a/nodedb_test.go +++ b/nodedb_test.go @@ -444,4 +444,5 @@ func TestCloseNodeDB(t *testing.T) { opts.AsyncPruning = true ndb := newNodeDB(db, 0, opts, NewNopLogger()) require.NoError(t, ndb.Close()) + require.NoError(t, ndb.Close()) // must not block or fail on second call } From f38a9197eacd1a7d8e3f591286803493eac206b8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:32:47 +0000 Subject: [PATCH 12/15] build(deps): Bump golang.org/x/crypto from 0.26.0 to 0.31.0 (#1022) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 8a655ec37..b37c6c7cc 100644 --- a/go.mod +++ b/go.mod @@ -24,8 +24,8 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/onsi/gomega v1.26.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/crypto v0.26.0 // indirect - golang.org/x/sys v0.23.0 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/sys v0.28.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index e1247cece..9db5d09da 100644 --- a/go.sum +++ b/go.sum @@ -69,8 +69,8 @@ go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -97,13 +97,13 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= From c20afc17599c540ebe7963ff5ca68756514c8dcc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:42:57 +0000 Subject: [PATCH 13/15] build(deps): Bump golang.org/x/crypto from 0.21.0 to 0.31.0 in /cmd/legacydump (#1021) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- cmd/legacydump/go.mod | 4 ++-- cmd/legacydump/go.sum | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/legacydump/go.mod b/cmd/legacydump/go.mod index 7fbfe9407..b673696d1 100644 --- a/cmd/legacydump/go.mod +++ b/cmd/legacydump/go.mod @@ -24,8 +24,8 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca // indirect github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect go.etcd.io/bbolt v1.3.6 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.18.0 // indirect + golang.org/x/sys v0.28.0 // indirect google.golang.org/protobuf v1.33.0 // indirect ) diff --git a/cmd/legacydump/go.sum b/cmd/legacydump/go.sum index 14ce3ac30..66fd82821 100644 --- a/cmd/legacydump/go.sum +++ b/cmd/legacydump/go.sum @@ -115,8 +115,8 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -145,12 +145,12 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= From 0f9b92f89a687973c8043d1cf29d73867bd80595 Mon Sep 17 00:00:00 2001 From: Dev Ojha Date: Fri, 27 Dec 2024 16:45:11 +0900 Subject: [PATCH 14/15] perf: remove duplicated rootkey fetch inpruning (9% pruning speedup on osmosis) (#1026) --- nodedb.go | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/nodedb.go b/nodedb.go index 7ad69af28..99323c122 100644 --- a/nodedb.go +++ b/nodedb.go @@ -416,10 +416,32 @@ func (ndb *nodeDB) saveNodeFromPruning(node *Node) error { return ndb.batch.Set(ndb.nodeKey(node.GetKey()), buf.Bytes()) } +type rootkeyCache struct { + version int64 + rootKey []byte +} + +func (rkc *rootkeyCache) getRootKey(ndb *nodeDB, version int64) ([]byte, error) { + if rkc.version == version { + return rkc.rootKey, nil + } + rootKey, err := ndb.GetRoot(version) + if err != nil { + return nil, err + } + rkc.setRootKey(version, rootKey) + return rootKey, nil +} + +func (rkc *rootkeyCache) setRootKey(version int64, rootKey []byte) { + rkc.version = version + rkc.rootKey = rootKey +} + // deleteVersion deletes a tree version from disk. // deletes orphans -func (ndb *nodeDB) deleteVersion(version int64) error { - rootKey, err := ndb.GetRoot(version) +func (ndb *nodeDB) deleteVersion(version int64, cache *rootkeyCache) error { + rootKey, err := cache.getRootKey(ndb, version) if err != nil { return err } @@ -457,7 +479,7 @@ func (ndb *nodeDB) deleteVersion(version int64) error { } // check if the version is referred by the next version - nextRootKey, err := ndb.GetRoot(version + 1) + nextRootKey, err := cache.getRootKey(ndb, version+1) if err != nil { return err } @@ -684,8 +706,9 @@ func (ndb *nodeDB) deleteVersionsTo(toVersion int64) error { ndb.resetLegacyLatestVersion(-1) } + rootkeyCache := &rootkeyCache{} for version := first; version <= toVersion; version++ { - if err := ndb.deleteVersion(version); err != nil { + if err := ndb.deleteVersion(version, rootkeyCache); err != nil { return err } ndb.resetFirstVersion(version + 1) From 1f9d525885791b96bb258d32f8ec90640a859f95 Mon Sep 17 00:00:00 2001 From: Dev Ojha Date: Tue, 31 Dec 2024 16:42:25 +0900 Subject: [PATCH 15/15] Improve rootkey cache in pruning: Reduce 4 rootkey fetches to 1 (#1029) --- nodedb.go | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/nodedb.go b/nodedb.go index 99323c122..00e903926 100644 --- a/nodedb.go +++ b/nodedb.go @@ -416,15 +416,22 @@ func (ndb *nodeDB) saveNodeFromPruning(node *Node) error { return ndb.batch.Set(ndb.nodeKey(node.GetKey()), buf.Bytes()) } +// rootkey cache of two elements, attempting to mimic a direct-mapped cache. type rootkeyCache struct { - version int64 - rootKey []byte + // initial value is set to {-1, -1}, which is an invalid version for a getrootkey call. + versions [2]int64 + rootKeys [2][]byte + next int } func (rkc *rootkeyCache) getRootKey(ndb *nodeDB, version int64) ([]byte, error) { - if rkc.version == version { - return rkc.rootKey, nil + // Check both cache entries + for i := 0; i < 2; i++ { + if rkc.versions[i] == version { + return rkc.rootKeys[i], nil + } } + rootKey, err := ndb.GetRoot(version) if err != nil { return nil, err @@ -434,8 +441,18 @@ func (rkc *rootkeyCache) getRootKey(ndb *nodeDB, version int64) ([]byte, error) } func (rkc *rootkeyCache) setRootKey(version int64, rootKey []byte) { - rkc.version = version - rkc.rootKey = rootKey + // Store in next available slot, cycling between 0 and 1 + rkc.versions[rkc.next] = version + rkc.rootKeys[rkc.next] = rootKey + rkc.next = (rkc.next + 1) % 2 +} + +func newRootkeyCache() *rootkeyCache { + return &rootkeyCache{ + versions: [2]int64{-1, -1}, + rootKeys: [2][]byte{}, + next: 0, + } } // deleteVersion deletes a tree version from disk. @@ -446,7 +463,7 @@ func (ndb *nodeDB) deleteVersion(version int64, cache *rootkeyCache) error { return err } - if err := ndb.traverseOrphans(version, version+1, func(orphan *Node) error { + if err := ndb.traverseOrphansWithRootkeyCache(cache, version, version+1, func(orphan *Node) error { if orphan.nodeKey.nonce == 0 && !orphan.isLegacy { // if the orphan is a reformatted root, it can be a legacy root // so it should be removed from the pruning process. @@ -706,7 +723,7 @@ func (ndb *nodeDB) deleteVersionsTo(toVersion int64) error { ndb.resetLegacyLatestVersion(-1) } - rootkeyCache := &rootkeyCache{} + rootkeyCache := newRootkeyCache() for version := first; version <= toVersion; version++ { if err := ndb.deleteVersion(version, rootkeyCache); err != nil { return err @@ -1095,7 +1112,12 @@ func isReferenceRoot(bz []byte) (bool, int) { // traverseOrphans traverses orphans which removed by the updates of the curVersion in the prevVersion. // NOTE: it is used for both legacy and new nodes. func (ndb *nodeDB) traverseOrphans(prevVersion, curVersion int64, fn func(*Node) error) error { - curKey, err := ndb.GetRoot(curVersion) + cache := newRootkeyCache() + return ndb.traverseOrphansWithRootkeyCache(cache, prevVersion, curVersion, fn) +} + +func (ndb *nodeDB) traverseOrphansWithRootkeyCache(cache *rootkeyCache, prevVersion, curVersion int64, fn func(*Node) error) error { + curKey, err := cache.getRootKey(ndb, curVersion) if err != nil { return err } @@ -1105,7 +1127,7 @@ func (ndb *nodeDB) traverseOrphans(prevVersion, curVersion int64, fn func(*Node) return err } - prevKey, err := ndb.GetRoot(prevVersion) + prevKey, err := cache.getRootKey(ndb, prevVersion) if err != nil { return err }