From ccdcaa44ef0279936859afbd1e39e9fe60b92136 Mon Sep 17 00:00:00 2001 From: Darioush Jalali Date: Tue, 25 Oct 2022 08:50:18 -0700 Subject: [PATCH] expose stats from fastcache (#313) --- cmd/simulator/go.mod | 3 + cmd/simulator/go.sum | 7 ++ core/blockchain.go | 18 ++-- core/state/snapshot/difflayer_test.go | 4 +- core/state/snapshot/disklayer.go | 4 +- core/state/snapshot/generate.go | 13 ++- core/state/snapshot/journal.go | 3 +- core/state/snapshot/snapshot.go | 6 +- trie/database.go | 23 ++--- utils/metered_cache.go | 120 ++++++++++++++++++++++++++ 10 files changed, 173 insertions(+), 28 deletions(-) create mode 100644 utils/metered_cache.go diff --git a/cmd/simulator/go.mod b/cmd/simulator/go.mod index f780295e1e..bc86766091 100644 --- a/cmd/simulator/go.mod +++ b/cmd/simulator/go.mod @@ -14,14 +14,17 @@ require ( replace github.com/ava-labs/subnet-evm => ../.. require ( + github.com/VictoriaMetrics/fastcache v1.10.0 // indirect github.com/ava-labs/avalanchego v1.9.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/deckarep/golang-set v1.8.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-stack/stack v1.8.0 // indirect github.com/golang/mock v1.6.0 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.2.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect diff --git a/cmd/simulator/go.sum b/cmd/simulator/go.sum index 109d54b6a0..b41d261434 100644 --- a/cmd/simulator/go.sum +++ b/cmd/simulator/go.sum @@ -1,4 +1,7 @@ github.com/VictoriaMetrics/fastcache v1.10.0 h1:5hDJnLsKLpnUEToub7ETuRu8RCkb40woBZAUiKonXzY= +github.com/VictoriaMetrics/fastcache v1.10.0/go.mod h1:tjiYeEfYXCqacuvYw/7UoDIeJaNxq6132xHICNP77w8= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/ava-labs/avalanchego v1.9.0 h1:9H6LZMVH9aSUJXviyBxn9owX8Dgs4azN7ojRsqLQpq0= github.com/ava-labs/avalanchego v1.9.0/go.mod h1:F0VZ+ukCllfz8VT79wZQlcRsTh1sbCCi5iWIhbLG8BQ= github.com/btcsuite/btcd v0.23.1 h1:IB8cVQcC2X5mHbnfirLG5IZnkWYNTPlLZVrxUYSotbE= @@ -7,6 +10,7 @@ github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFA github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -28,6 +32,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -62,6 +67,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/status-im/keycard-go v0.0.0-20200402102358-957c09536969 h1:Oo2KZNP70KE0+IUJSidPj/BFS/RXNHmKIJOdckzml2E= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -94,6 +100,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220727055044-e65921a090b8 h1:dyU22nBWzrmTQxtNrr4dzVOvaw35nUYE279vF9UmsI8= golang.org/x/sys v0.0.0-20220727055044-e65921a090b8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/core/blockchain.go b/core/blockchain.go index 710af1f35c..7a46f42819 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -104,6 +104,10 @@ const ( // statsReportLimit is the time limit during import and export after which we // always print out progress. This avoids the user wondering what's going on. statsReportLimit = 8 * time.Second + + // trieCleanCacheStatsNamespace is the namespace to surface stats from the trie + // clean cache's underlying fastcache. + trieCleanCacheStatsNamespace = "trie/memcache/clean/fastcache" ) // cacheableFeeConfig encapsulates fee configuration itself and the block number that it has changed at, @@ -265,9 +269,10 @@ func NewBlockChain( cacheConfig: cacheConfig, db: db, stateCache: state.NewDatabaseWithConfig(db, &trie.Config{ - Cache: cacheConfig.TrieCleanLimit, - Journal: cacheConfig.TrieCleanJournal, - Preimages: cacheConfig.Preimages, + Cache: cacheConfig.TrieCleanLimit, + Journal: cacheConfig.TrieCleanJournal, + Preimages: cacheConfig.Preimages, + StatsPrefix: trieCleanCacheStatsNamespace, }), bodyCache: bodyCache, receiptsCache: receiptsCache, @@ -1838,9 +1843,10 @@ func (bc *BlockChain) ResetState(block *types.Block) error { lastAcceptedHash := block.Hash() bc.stateCache = state.NewDatabaseWithConfig(bc.db, &trie.Config{ - Cache: bc.cacheConfig.TrieCleanLimit, - Journal: bc.cacheConfig.TrieCleanJournal, - Preimages: bc.cacheConfig.Preimages, + Cache: bc.cacheConfig.TrieCleanLimit, + Journal: bc.cacheConfig.TrieCleanJournal, + Preimages: bc.cacheConfig.Preimages, + StatsPrefix: trieCleanCacheStatsNamespace, }) if err := bc.loadLastState(lastAcceptedHash); err != nil { return err diff --git a/core/state/snapshot/difflayer_test.go b/core/state/snapshot/difflayer_test.go index 61b26f6872..523d1022c1 100644 --- a/core/state/snapshot/difflayer_test.go +++ b/core/state/snapshot/difflayer_test.go @@ -31,8 +31,8 @@ import ( "math/rand" "testing" - "github.com/VictoriaMetrics/fastcache" "github.com/ava-labs/subnet-evm/ethdb/memorydb" + "github.com/ava-labs/subnet-evm/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" ) @@ -243,7 +243,7 @@ func TestInsertAndMerge(t *testing.T) { func emptyLayer() *diskLayer { return &diskLayer{ diskdb: memorydb.New(), - cache: fastcache.New(500 * 1024), + cache: utils.NewMeteredCache(500*1024, "", "", 0), } } diff --git a/core/state/snapshot/disklayer.go b/core/state/snapshot/disklayer.go index 10afd79bbb..b2aca51793 100644 --- a/core/state/snapshot/disklayer.go +++ b/core/state/snapshot/disklayer.go @@ -31,10 +31,10 @@ import ( "sync" "time" - "github.com/VictoriaMetrics/fastcache" "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/trie" + "github.com/ava-labs/subnet-evm/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" ) @@ -43,7 +43,7 @@ import ( type diskLayer struct { diskdb ethdb.KeyValueStore // Key-value store containing the base snapshot triedb *trie.Database // Trie node cache for reconstruction purposes - cache *fastcache.Cache // Cache to avoid hitting the disk for direct access + cache *utils.MeteredCache // Cache to avoid hitting the disk for direct access blockHash common.Hash // Block hash of the base snapshot root common.Hash // Root hash of the base snapshot diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index f6f1a4ae87..112d523f72 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -33,10 +33,10 @@ import ( "math/big" "time" - "github.com/VictoriaMetrics/fastcache" "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/trie" + "github.com/ava-labs/subnet-evm/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/crypto" @@ -44,6 +44,11 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) +const ( + snapshotCacheNamespace = "state/snapshot/clean/fastcache" // prefix for detailed stats from the snapshot fastcache + snapshotCacheStatsUpdateFrequency = 1000 // update stats from the snapshot fastcache once per 1000 ops +) + var ( // emptyRoot is the known root hash of an empty trie. emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") @@ -155,7 +160,7 @@ func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache i triedb: triedb, blockHash: blockHash, root: root, - cache: fastcache.New(cache * 1024 * 1024), + cache: newMeteredSnapshotCache(cache * 1024 * 1024), genMarker: genMarker, genPending: make(chan struct{}), genAbort: make(chan chan struct{}), @@ -398,3 +403,7 @@ func (dl *diskLayer) generate(stats *generatorStats) { abort := <-dl.genAbort close(abort) } + +func newMeteredSnapshotCache(size int) *utils.MeteredCache { + return utils.NewMeteredCache(size, "", snapshotCacheNamespace, snapshotCacheStatsUpdateFrequency) +} diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 84b94d5488..4fa3b01fd6 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -32,7 +32,6 @@ import ( "fmt" "time" - "github.com/VictoriaMetrics/fastcache" "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/trie" @@ -91,7 +90,7 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, snapshot := &diskLayer{ diskdb: diskdb, triedb: triedb, - cache: fastcache.New(cache * 1024 * 1024), + cache: newMeteredSnapshotCache(cache * 1024 * 1024), root: baseRoot, blockHash: baseBlockHash, created: time.Now(), diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 32c5a886d0..c27f8782e6 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -35,11 +35,11 @@ import ( "sync/atomic" "time" - "github.com/VictoriaMetrics/fastcache" "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/metrics" "github.com/ava-labs/subnet-evm/trie" + "github.com/ava-labs/subnet-evm/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" ) @@ -933,7 +933,7 @@ func NewDiskLayer(diskdb ethdb.KeyValueStore) Snapshot { // state sync uses iterators to access data, so this cache is not used. // initializing it out of caution. - cache: fastcache.New(32 * 1024), + cache: utils.NewMeteredCache(32*1024, "", "", 0), } } @@ -943,7 +943,7 @@ func NewTestTree(diskdb ethdb.KeyValueStore, blockHash, root common.Hash) *Tree diskdb: diskdb, root: root, blockHash: blockHash, - cache: fastcache.New(128 * 256), + cache: utils.NewMeteredCache(128*256, "", "", 0), created: time.Now(), } return &Tree{ diff --git a/trie/database.go b/trie/database.go index 499bbe81e6..7451438bf6 100644 --- a/trie/database.go +++ b/trie/database.go @@ -35,16 +35,20 @@ import ( "sync" "time" - "github.com/VictoriaMetrics/fastcache" "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/metrics" + "github.com/ava-labs/subnet-evm/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) +const ( + cacheStatsUpdateFrequency = 1000 // update trie cache stats once per 1000 ops +) + var ( memcacheCleanHitMeter = metrics.NewRegisteredMeter("trie/memcache/clean/hit", nil) memcacheCleanMissMeter = metrics.NewRegisteredMeter("trie/memcache/clean/miss", nil) @@ -85,7 +89,7 @@ var ( type Database struct { diskdb ethdb.KeyValueStore // Persistent storage for matured trie nodes - cleans *fastcache.Cache // GC friendly memory cache of clean node RLPs + cleans *utils.MeteredCache // GC friendly memory cache of clean node RLPs dirties map[common.Hash]*cachedNode // Data and references relationships of dirty trie nodes oldest common.Hash // Oldest tracked node, flush-list head newest common.Hash // Newest tracked node, flush-list tail @@ -280,9 +284,10 @@ func expandNode(hash hashNode, n node) node { // Config defines all necessary options for database. type Config struct { - Cache int // Memory allowance (MB) to use for caching trie nodes in memory - Preimages bool // Flag whether the preimage of trie key is recorded - Journal string // File location to load trie clean cache from + Cache int // Memory allowance (MB) to use for caching trie nodes in memory + Preimages bool // Flag whether the preimage of trie key is recorded + Journal string // File location to load trie clean cache from + StatsPrefix string // Prefix for cache stats (disabled if empty) } // NewDatabase creates a new trie database to store ephemeral trie content before @@ -296,13 +301,9 @@ func NewDatabase(diskdb ethdb.KeyValueStore) *Database { // before its written out to disk or garbage collected. It also acts as a read cache // for nodes loaded from disk. func NewDatabaseWithConfig(diskdb ethdb.KeyValueStore, config *Config) *Database { - var cleans *fastcache.Cache + var cleans *utils.MeteredCache if config != nil && config.Cache > 0 { - if config.Journal == "" { - cleans = fastcache.New(config.Cache * 1024 * 1024) - } else { - cleans = fastcache.LoadFromFileOrNew(config.Journal, config.Cache*1024*1024) - } + cleans = utils.NewMeteredCache(config.Cache*1024*1024, config.Journal, config.StatsPrefix, cacheStatsUpdateFrequency) } var preimage *preimageStore if config != nil && config.Preimages { diff --git a/utils/metered_cache.go b/utils/metered_cache.go new file mode 100644 index 0000000000..16cadb81ec --- /dev/null +++ b/utils/metered_cache.go @@ -0,0 +1,120 @@ +// (c) 2022, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package utils + +import ( + "fmt" + "sync/atomic" + "time" + + "github.com/VictoriaMetrics/fastcache" + "github.com/ava-labs/subnet-evm/metrics" +) + +// MeteredCache wraps *fastcache.Cache and periodically pulls stats from it. +type MeteredCache struct { + *fastcache.Cache + namespace string + + // stats to be surfaced + entriesCount metrics.Gauge + bytesSize metrics.Gauge + collisions metrics.Gauge + gets metrics.Gauge + sets metrics.Gauge + misses metrics.Gauge + statsTime metrics.Gauge + + // count all operations to decide when to update stats + ops uint64 + updateFrequency uint64 +} + +// NewMeteredCache returns a new MeteredCache that will update stats to the +// provided namespace once per each [updateFrequency] operations. +// Note: if [updateFrequency] is passed as 0, it will be treated as 1. +func NewMeteredCache(size int, journal string, namespace string, updateFrequency uint64) *MeteredCache { + var cache *fastcache.Cache + if journal == "" { + cache = fastcache.New(size) + } else { + cache = fastcache.LoadFromFileOrNew(journal, size) + } + if updateFrequency == 0 { + updateFrequency = 1 // avoid division by zero + } + mc := &MeteredCache{ + Cache: cache, + namespace: namespace, + updateFrequency: updateFrequency, + } + if namespace != "" { + // only register stats if a namespace is provided. + mc.entriesCount = metrics.GetOrRegisterGauge(fmt.Sprintf("%s/entriesCount", namespace), nil) + mc.bytesSize = metrics.GetOrRegisterGauge(fmt.Sprintf("%s/bytesSize", namespace), nil) + mc.collisions = metrics.GetOrRegisterGauge(fmt.Sprintf("%s/collisions", namespace), nil) + mc.gets = metrics.GetOrRegisterGauge(fmt.Sprintf("%s/gets", namespace), nil) + mc.sets = metrics.GetOrRegisterGauge(fmt.Sprintf("%s/sets", namespace), nil) + mc.misses = metrics.GetOrRegisterGauge(fmt.Sprintf("%s/misses", namespace), nil) + mc.statsTime = metrics.GetOrRegisterGauge(fmt.Sprintf("%s/statsTime", namespace), nil) + } + return mc +} + +// updateStats updates metrics from fastcache +func (mc *MeteredCache) updateStatsIfNeeded() { + if mc.namespace == "" { + return + } + ops := atomic.AddUint64(&mc.ops, 1) + if ops%mc.updateFrequency != 0 { + return + } + + start := time.Now() + s := fastcache.Stats{} + mc.UpdateStats(&s) + mc.entriesCount.Update(int64(s.EntriesCount)) + mc.bytesSize.Update(int64(s.BytesSize)) + mc.collisions.Update(int64(s.Collisions)) + mc.gets.Update(int64(s.GetCalls)) + mc.sets.Update(int64(s.SetCalls)) + mc.misses.Update(int64(s.Misses)) + mc.statsTime.Inc(int64(time.Since(start))) // cumulative metric +} + +func (mc *MeteredCache) Del(k []byte) { + mc.updateStatsIfNeeded() + mc.Cache.Del(k) +} + +func (mc *MeteredCache) Get(dst, k []byte) []byte { + mc.updateStatsIfNeeded() + return mc.Cache.Get(dst, k) +} + +func (mc *MeteredCache) GetBig(dst, k []byte) []byte { + mc.updateStatsIfNeeded() + return mc.Cache.GetBig(dst, k) +} + +func (mc *MeteredCache) Has(k []byte) bool { + mc.updateStatsIfNeeded() + return mc.Cache.Has(k) +} + +func (mc *MeteredCache) HasGet(dst, k []byte) ([]byte, bool) { + mc.updateStatsIfNeeded() + return mc.Cache.HasGet(dst, k) +} + +func (mc *MeteredCache) Set(k, v []byte) { + mc.updateStatsIfNeeded() + mc.Cache.Set(k, v) +} + +func (mc *MeteredCache) SetBig(k, v []byte) { + mc.updateStatsIfNeeded() + mc.Cache.SetBig(k, v) +}