feat: iavl v2 (cosmos#872)

Co-authored-by: Marko <marbar3778@yahoo.com>

v2/.gitignore

@@ -0,0 +1,21 @@
+vendor
+.glide
+*.swp
+*.swo
+
+# created in test code
+test.db
+
+# profiling data
+*\.test
+cpu*.out
+mem*.out
+cpu*.pdf
+mem*.pdf
+
+# IDE files
+.idea/*
+.vscode/*
+
+go.work
+go.work.sum

v2/.golangci.yml

@@ -0,0 +1,66 @@
+run:
+  tests: true
+  # timeout for analysis, e.g. 30s, 5m, default is 1m
+  timeout: 5m
+
+linters:
+  disable-all: true
+  enable:
+    - bodyclose
+    - dogsled
+    - errcheck
+    - exportloopref
+    - goconst
+    - gocritic
+    - gofumpt
+    - gosec
+    - gosimple
+    - govet
+    - ineffassign
+    - misspell
+    - nakedret
+    - nolintlint
+    - prealloc
+    - revive
+    - staticcheck
+    - stylecheck
+    - typecheck
+    - unconvert
+    - unparam
+    - unused
+
+linters-settings:
+  nolintlint:
+    allow-leading-space: true
+    require-explanation: false
+    require-specific: true
+
+issues:
+  exclude-rules:
+    - text: "Use of weak random number generator"
+      linters:
+        - gosec
+    - text: "comment on exported var"
+      linters:
+        - golint
+    - text: "don't use an underscore in package name"
+      linters:
+        - golint
+    - text: "should be written without leading space as"
+      linters:
+        - nolintlint
+    - text: "ST1003:"
+      linters:
+        - stylecheck
+    # FIXME: Disabled until golangci-lint updates stylecheck with this fix:
+    # https://github.com/dominikh/go-tools/issues/389
+    - text: "ST1016:"
+      linters:
+        - stylecheck
+    - path: "migrations"
+      text: "SA1019:"
+      linters:
+        - staticcheck
+
+  max-issues-per-linter: 10000
+  max-same-issues: 10000

v2/cmd/gen/gen.go

@@ -0,0 +1,234 @@
+package gen
+
+import (
+	"fmt"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/cosmos/iavl-bench/bench"
+	"github.com/cosmos/iavl/v2"
+	"github.com/cosmos/iavl/v2/testutil"
+	"github.com/dustin/go-humanize"
+	"github.com/kocubinski/costor-api/compact"
+	"github.com/kocubinski/costor-api/core"
+	"github.com/rs/zerolog"
+	zlog "github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+)
+
+var log = zlog.Output(zerolog.ConsoleWriter{
+	Out:        os.Stderr,
+	TimeFormat: time.Stamp,
+})
+
+func Command() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "gen",
+		Short: "generate changesets",
+	}
+
+	cmd.AddCommand(emitCommand(), treeCommand())
+
+	return cmd
+}
+
+func getChangesetIterator(typ string) (bench.ChangesetIterator, error) {
+	switch typ {
+	case "osmo-like":
+		return testutil.OsmoLike().Iterator, nil
+	case "osmo-like-many":
+		return testutil.OsmoLikeManyTrees().Iterator, nil
+	case "height-zero":
+		return testutil.NewTreeBuildOptions().Iterator, nil
+	default:
+		return nil, fmt.Errorf("unknown generator type %s", typ)
+	}
+}
+
+func emitCommand() *cobra.Command {
+	var (
+		typ   string
+		out   string
+		start int
+		limit int
+	)
+	cmd := &cobra.Command{
+		Use:   "emit",
+		Short: "emit generated changesets to disk",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			itr, err := getChangesetIterator(typ)
+			if err != nil {
+				return err
+			}
+			ctx := core.Context{Context: cmd.Context()}
+
+			stream := compact.StreamingContext{
+				In:          make(chan compact.Sequenced),
+				Context:     ctx,
+				OutDir:      out,
+				MaxFileSize: 100 * 1024 * 1024,
+			}
+
+			var wg sync.WaitGroup
+			wg.Add(1)
+			go func() {
+				stats, err := stream.Compact()
+				if err != nil {
+					log.Fatal().Err(err).Msg("failed to compact")
+				}
+				log.Info().Msgf(stats.Report())
+				wg.Done()
+			}()
+
+			var cnt int64
+			for ; itr.Valid(); err = itr.Next() {
+				if err != nil {
+					return err
+				}
+				if limit > 0 && itr.Version() > int64(limit) {
+					break
+				}
+				nodes := itr.Nodes()
+				for ; nodes.Valid(); err = nodes.Next() {
+					cnt++
+
+					if itr.Version() < int64(start) {
+						if cnt%5_000_000 == 0 {
+							log.Info().Msgf("fast forward version=%d nodes=%s", itr.Version(), humanize.Comma(cnt))
+						}
+						continue
+					}
+
+					if cnt%500_000 == 0 {
+						log.Info().Msgf("version=%d nodes=%s", itr.Version(), humanize.Comma(cnt))
+					}
+
+					select {
+					case <-cmd.Context().Done():
+						close(stream.In)
+						wg.Wait()
+						return nil
+					default:
+					}
+
+					if err != nil {
+						return err
+					}
+					stream.In <- nodes.GetNode()
+				}
+			}
+			close(stream.In)
+			wg.Wait()
+
+			return nil
+		},
+	}
+
+	cmd.Flags().StringVar(&typ, "type", "", "the type of changeset to generate")
+	if err := cmd.MarkFlagRequired("type"); err != nil {
+		panic(err)
+	}
+	cmd.Flags().StringVar(&out, "out", "", "the directory to write changesets to")
+	if err := cmd.MarkFlagRequired("out"); err != nil {
+		panic(err)
+	}
+	cmd.Flags().IntVar(&limit, "limit", -1, "the version (inclusive) to halt generation at. -1 means no limit")
+	cmd.Flags().IntVar(&start, "start", 1, "the version (inclusive) to start generation at")
+
+	return cmd
+}
+
+func treeCommand() *cobra.Command {
+	var (
+		dbPath  string
+		genType string
+		limit   int64
+	)
+	cmd := &cobra.Command{
+		Use:   "tree",
+		Short: "build and save a Tree to disk, taking generated changesets as input",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			multiTree := iavl.NewMultiTree(dbPath, iavl.TreeOptions{StateStorage: true})
+			defer func(mt *iavl.MultiTree) {
+				err := mt.Close()
+				if err != nil {
+					log.Error().Err(err).Msg("failed to close db")
+				}
+			}(multiTree)
+
+			itr, err := getChangesetIterator(genType)
+			if err != nil {
+				return err
+			}
+
+			var i int64
+			var lastHash []byte
+			var lastVersion int64
+			start := time.Now()
+			for ; itr.Valid(); err = itr.Next() {
+				if err != nil {
+					return err
+				}
+				if limit > -1 && itr.Version() > limit {
+					break
+				}
+
+				changeset := itr.Nodes()
+				for ; changeset.Valid(); err = changeset.Next() {
+					if err != nil {
+						return err
+					}
+					node := changeset.GetNode()
+					key := node.Key
+
+					tree, ok := multiTree.Trees[node.StoreKey]
+					if !ok {
+						if err = multiTree.MountTree(node.StoreKey); err != nil {
+							return err
+						}
+						tree = multiTree.Trees[node.StoreKey]
+					}
+					if node.Delete {
+						_, _, err = tree.Remove(key)
+						if err != nil {
+							return err
+						}
+					} else {
+						_, err = tree.Set(key, node.Value)
+						if err != nil {
+							return err
+						}
+					}
+
+					i++
+					if i%100_000 == 0 {
+						log.Info().Msgf("leaves=%s dur=%s rate=%s version=%d",
+							humanize.Comma(i),
+							time.Since(start),
+							humanize.Comma(int64(100_000/time.Since(start).Seconds())),
+							itr.Version(),
+						)
+						start = time.Now()
+					}
+				}
+
+				lastHash, lastVersion, err = multiTree.SaveVersionConcurrently()
+				if err != nil {
+					return err
+				}
+			}
+
+			log.Info().Msgf("last version=%d hash=%x", lastVersion, lastHash)
+
+			return nil
+		},
+	}
+	cmd.Flags().StringVar(&genType, "type", "", "the type of changeset to generate")
+	if err := cmd.MarkFlagRequired("type"); err != nil {
+		panic(err)
+	}
+	cmd.Flags().StringVar(&dbPath, "db", "/tmp", "the path to the database")
+	cmd.Flags().Int64Var(&limit, "limit", -1, "the version (inclusive) to halt generation at. -1 means no limit")
+	return cmd
+}

v2/cmd/latest.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"github.com/cosmos/iavl/v2"
+	"github.com/spf13/cobra"
+)
+
+func latestCommand() *cobra.Command {
+	var (
+		dbPath  string
+		version int64
+	)
+	cmd := &cobra.Command{
+		Use:   "latest",
+		Short: "fill the latest table with the latest version of leaf nodes in a tree",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			paths, err := iavl.FindDbsInPath(dbPath)
+			if err != nil {
+				return err
+			}
+			var (
+				pool   = iavl.NewNodePool()
+				done   = make(chan struct{})
+				errors = make(chan error)
+				cnt    = 0
+			)
+			for _, path := range paths {
+				cnt++
+				sqlOpts := iavl.SqliteDbOptions{Path: path}
+				sql, err := iavl.NewSqliteDb(pool, sqlOpts)
+				if err != nil {
+					return err
+				}
+				tree := iavl.NewTree(sql, pool, iavl.TreeOptions{})
+				if err = tree.LoadVersion(version); err != nil {
+					return err
+				}
+				go func() {
+					fillErr := tree.WriteLatestLeaves()
+					if fillErr != nil {
+						errors <- fillErr
+					}
+					fillErr = tree.Close()
+					if fillErr != nil {
+						errors <- fillErr
+					}
+					done <- struct{}{}
+				}()
+			}
+			for i := 0; i < cnt; i++ {
+				select {
+				case <-done:
+					continue
+				case err := <-errors:
+					return err
+				}
+			}
+			return nil
+		},
+	}
+	cmd.Flags().StringVar(&dbPath, "db", "", "the path to the db to fill the latest table for")
+	if err := cmd.MarkFlagRequired("db"); err != nil {
+		panic(err)
+	}
+	cmd.Flags().Int64Var(&version, "version", 0, "version to fill from")
+	if err := cmd.MarkFlagRequired("version"); err != nil {
+		panic(err)
+	}
+	return cmd
+}