From 1de061a2e06ef73cdb67f74075dae3e526e53ea2 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 31 Jul 2023 12:30:19 -0400 Subject: [PATCH] Use native DB range queries for applying validator diffs (#1752) --- database/helpers.go | 25 +- scripts/build_fuzz.sh | 5 + scripts/mocks.mockgen.txt | 1 + vms/platformvm/config/execution_config.go | 2 - .../config/execution_config_test.go | 2 - .../state/disk_staker_diff_iterator.go | 97 +++ .../state/disk_staker_diff_iterator_test.go | 110 ++++ vms/platformvm/state/mock_state.go | 60 +- vms/platformvm/state/state.go | 506 +++++++++++----- vms/platformvm/state/state_test.go | 568 +++++------------- vms/platformvm/validators/manager.go | 322 +++++----- .../validators/manager_benchmark_test.go | 278 +++++++++ vms/platformvm/validators/manager_test.go | 452 -------------- 13 files changed, 1227 insertions(+), 1201 deletions(-) create mode 100644 vms/platformvm/state/disk_staker_diff_iterator.go create mode 100644 vms/platformvm/state/disk_staker_diff_iterator_test.go create mode 100644 vms/platformvm/validators/manager_benchmark_test.go delete mode 100644 vms/platformvm/validators/manager_test.go diff --git a/database/helpers.go b/database/helpers.go index e56245b1ed6..bf35b245549 100644 --- a/database/helpers.go +++ b/database/helpers.go @@ -14,11 +14,20 @@ import ( const ( Uint64Size = 8 // bytes + BoolSize = 1 // bytes + BoolFalse = 0x00 + BoolTrue = 0x01 + // kvPairOverhead is an estimated overhead for a kv pair in a database. kvPairOverhead = 8 // bytes ) -var errWrongSize = errors.New("value has unexpected size") +var ( + boolFalseKey = []byte{BoolFalse} + boolTrueKey = []byte{BoolTrue} + + errWrongSize = errors.New("value has unexpected size") +) func PutID(db KeyValueWriter, key []byte, val ids.ID) error { return db.Put(key, val[:]) @@ -114,9 +123,9 @@ func ParseTimestamp(b []byte) (time.Time, error) { func PutBool(db KeyValueWriter, key []byte, b bool) error { if b { - return db.Put(key, []byte{1}) + return db.Put(key, boolTrueKey) } - return db.Put(key, []byte{0}) + return db.Put(key, boolFalseKey) } func GetBool(db KeyValueReader, key []byte) (bool, error) { @@ -124,12 +133,12 @@ func GetBool(db KeyValueReader, key []byte) (bool, error) { switch { case err != nil: return false, err - case len(b) != 1: - return false, fmt.Errorf("length should be 1 but is %d", len(b)) - case b[0] != 0 && b[0] != 1: - return false, fmt.Errorf("should be 0 or 1 but is %v", b[0]) + case len(b) != BoolSize: + return false, fmt.Errorf("length should be %d but is %d", BoolSize, len(b)) + case b[0] != BoolFalse && b[0] != BoolTrue: + return false, fmt.Errorf("should be %d or %d but is %d", BoolFalse, BoolTrue, b[0]) } - return b[0] == 1, nil + return b[0] == BoolTrue, nil } func Count(db Iteratee) (int, error) { diff --git a/scripts/build_fuzz.sh b/scripts/build_fuzz.sh index c51f438f532..901cabf2dc4 100755 --- a/scripts/build_fuzz.sh +++ b/scripts/build_fuzz.sh @@ -2,6 +2,11 @@ # Mostly taken from https://github.com/golang/go/issues/46312#issuecomment-1153345129 +# Directory above this script +AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) +# Load the constants +source "$AVALANCHE_PATH"/scripts/constants.sh + fuzzTime=${1:-1} files=$(grep -r --include='**_test.go' --files-with-matches 'func Fuzz' .) failed=false diff --git a/scripts/mocks.mockgen.txt b/scripts/mocks.mockgen.txt index c09b6b97d72..7b40e0224c7 100644 --- a/scripts/mocks.mockgen.txt +++ b/scripts/mocks.mockgen.txt @@ -35,6 +35,7 @@ github.com/ava-labs/avalanchego/vms/platformvm/blocks=Block=vms/platformvm/block github.com/ava-labs/avalanchego/vms/platformvm/state=Chain=vms/platformvm/state/mock_chain.go github.com/ava-labs/avalanchego/vms/platformvm/state=Diff=vms/platformvm/state/mock_diff.go github.com/ava-labs/avalanchego/vms/platformvm/state=StakerIterator=vms/platformvm/state/mock_staker_iterator.go +github.com/ava-labs/avalanchego/vms/platformvm/state=State=vms/platformvm/state/mock_state.go github.com/ava-labs/avalanchego/vms/platformvm/state=Versions=vms/platformvm/state/mock_versions.go github.com/ava-labs/avalanchego/vms/platformvm/txs/builder=Builder=vms/platformvm/txs/builder/mock_builder.go github.com/ava-labs/avalanchego/vms/platformvm/txs/mempool=Mempool=vms/platformvm/txs/mempool/mock_mempool.go diff --git a/vms/platformvm/config/execution_config.go b/vms/platformvm/config/execution_config.go index 980a793fe3a..c083f82e83a 100644 --- a/vms/platformvm/config/execution_config.go +++ b/vms/platformvm/config/execution_config.go @@ -13,7 +13,6 @@ var DefaultExecutionConfig = ExecutionConfig{ BlockCacheSize: 64 * units.MiB, TxCacheSize: 128 * units.MiB, TransformedSubnetTxCacheSize: 4 * units.MiB, - ValidatorDiffsCacheSize: 2048, RewardUTXOsCacheSize: 2048, ChainCacheSize: 2048, ChainDBCacheSize: 2048, @@ -25,7 +24,6 @@ type ExecutionConfig struct { BlockCacheSize int `json:"block-cache-size"` TxCacheSize int `json:"tx-cache-size"` TransformedSubnetTxCacheSize int `json:"transformed-subnet-tx-cache-size"` - ValidatorDiffsCacheSize int `json:"validator-diffs-cache-size"` RewardUTXOsCacheSize int `json:"reward-utxos-cache-size"` ChainCacheSize int `json:"chain-cache-size"` ChainDBCacheSize int `json:"chain-db-cache-size"` diff --git a/vms/platformvm/config/execution_config_test.go b/vms/platformvm/config/execution_config_test.go index 28e49df4669..2b7397147e2 100644 --- a/vms/platformvm/config/execution_config_test.go +++ b/vms/platformvm/config/execution_config_test.go @@ -42,7 +42,6 @@ func TestExecutionConfigUnmarshal(t *testing.T) { "block-cache-size": 1, "tx-cache-size": 2, "transformed-subnet-tx-cache-size": 3, - "validator-diffs-cache-size": 4, "reward-utxos-cache-size": 5, "chain-cache-size": 6, "chain-db-cache-size": 7, @@ -54,7 +53,6 @@ func TestExecutionConfigUnmarshal(t *testing.T) { BlockCacheSize: 1, TxCacheSize: 2, TransformedSubnetTxCacheSize: 3, - ValidatorDiffsCacheSize: 4, RewardUTXOsCacheSize: 5, ChainCacheSize: 6, ChainDBCacheSize: 7, diff --git a/vms/platformvm/state/disk_staker_diff_iterator.go b/vms/platformvm/state/disk_staker_diff_iterator.go new file mode 100644 index 00000000000..44ee1ed8718 --- /dev/null +++ b/vms/platformvm/state/disk_staker_diff_iterator.go @@ -0,0 +1,97 @@ +// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package state + +import ( + "encoding/binary" + "fmt" + + "github.com/ava-labs/avalanchego/database" + "github.com/ava-labs/avalanchego/ids" +) + +const ( + // startDiffKey = [subnetID] + [inverseHeight] + startDiffKeyLength = ids.IDLen + database.Uint64Size + // diffKey = [subnetID] + [inverseHeight] + [nodeID] + diffKeyLength = startDiffKeyLength + ids.NodeIDLen + // diffKeyNodeIDOffset = [subnetIDLen] + [inverseHeightLen] + diffKeyNodeIDOffset = ids.IDLen + database.Uint64Size + + // weightValue = [isNegative] + [weight] + weightValueLength = database.BoolSize + database.Uint64Size +) + +var ( + errUnexpectedDiffKeyLength = fmt.Errorf("expected diff key length %d", diffKeyLength) + errUnexpectedWeightValueLength = fmt.Errorf("expected weight value length %d", weightValueLength) +) + +// marshalStartDiffKey is used to determine the starting key when iterating. +// +// Invariant: the result is a prefix of [marshalDiffKey] when called with the +// same arguments. +func marshalStartDiffKey(subnetID ids.ID, height uint64) []byte { + key := make([]byte, startDiffKeyLength) + copy(key, subnetID[:]) + packIterableHeight(key[ids.IDLen:], height) + return key +} + +func marshalDiffKey(subnetID ids.ID, height uint64, nodeID ids.NodeID) []byte { + key := make([]byte, diffKeyLength) + copy(key, subnetID[:]) + packIterableHeight(key[ids.IDLen:], height) + copy(key[diffKeyNodeIDOffset:], nodeID[:]) + return key +} + +func unmarshalDiffKey(key []byte) (ids.ID, uint64, ids.NodeID, error) { + if len(key) != diffKeyLength { + return ids.Empty, 0, ids.EmptyNodeID, errUnexpectedDiffKeyLength + } + var ( + subnetID ids.ID + nodeID ids.NodeID + ) + copy(subnetID[:], key) + height := unpackIterableHeight(key[ids.IDLen:]) + copy(nodeID[:], key[diffKeyNodeIDOffset:]) + return subnetID, height, nodeID, nil +} + +func marshalWeightDiff(diff *ValidatorWeightDiff) []byte { + value := make([]byte, weightValueLength) + if diff.Decrease { + value[0] = database.BoolTrue + } + binary.BigEndian.PutUint64(value[database.BoolSize:], diff.Amount) + return value +} + +func unmarshalWeightDiff(value []byte) (*ValidatorWeightDiff, error) { + if len(value) != weightValueLength { + return nil, errUnexpectedWeightValueLength + } + return &ValidatorWeightDiff{ + Decrease: value[0] == database.BoolTrue, + Amount: binary.BigEndian.Uint64(value[database.BoolSize:]), + }, nil +} + +// Note: [height] is encoded as a bit flipped big endian number so that +// iterating lexicographically results in iterating in decreasing heights. +// +// Invariant: [key] has sufficient length +func packIterableHeight(key []byte, height uint64) { + binary.BigEndian.PutUint64(key, ^height) +} + +// Because we bit flip the height when constructing the key, we must remember to +// bip flip again here. +// +// Invariant: [key] has sufficient length +func unpackIterableHeight(key []byte) uint64 { + return ^binary.BigEndian.Uint64(key) +} diff --git a/vms/platformvm/state/disk_staker_diff_iterator_test.go b/vms/platformvm/state/disk_staker_diff_iterator_test.go new file mode 100644 index 00000000000..9439428937b --- /dev/null +++ b/vms/platformvm/state/disk_staker_diff_iterator_test.go @@ -0,0 +1,110 @@ +// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package state + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/thepudds/fzgen/fuzzer" + + "github.com/ava-labs/avalanchego/database/memdb" + "github.com/ava-labs/avalanchego/ids" +) + +func FuzzMarshalDiffKey(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + require := require.New(t) + + var ( + subnetID ids.ID + height uint64 + nodeID ids.NodeID + ) + fz := fuzzer.NewFuzzer(data) + fz.Fill(&subnetID, &height, &nodeID) + + key := marshalDiffKey(subnetID, height, nodeID) + parsedSubnetID, parsedHeight, parsedNodeID, err := unmarshalDiffKey(key) + require.NoError(err) + require.Equal(subnetID, parsedSubnetID) + require.Equal(height, parsedHeight) + require.Equal(nodeID, parsedNodeID) + }) +} + +func FuzzUnmarshalDiffKey(f *testing.F) { + f.Fuzz(func(t *testing.T, key []byte) { + require := require.New(t) + + subnetID, height, nodeID, err := unmarshalDiffKey(key) + if err != nil { + require.ErrorIs(err, errUnexpectedDiffKeyLength) + return + } + + formattedKey := marshalDiffKey(subnetID, height, nodeID) + require.Equal(key, formattedKey) + }) +} + +func TestDiffIteration(t *testing.T) { + require := require.New(t) + + db := memdb.New() + + subnetID0 := ids.GenerateTestID() + subnetID1 := ids.GenerateTestID() + + nodeID0 := ids.NodeID{0x00} + nodeID1 := ids.NodeID{0x01} + + subnetID0Height0NodeID0 := marshalDiffKey(subnetID0, 0, nodeID0) + subnetID0Height1NodeID0 := marshalDiffKey(subnetID0, 1, nodeID0) + subnetID0Height1NodeID1 := marshalDiffKey(subnetID0, 1, nodeID1) + + subnetID1Height0NodeID0 := marshalDiffKey(subnetID1, 0, nodeID0) + subnetID1Height1NodeID0 := marshalDiffKey(subnetID1, 1, nodeID0) + subnetID1Height1NodeID1 := marshalDiffKey(subnetID1, 1, nodeID1) + + require.NoError(db.Put(subnetID0Height0NodeID0, nil)) + require.NoError(db.Put(subnetID0Height1NodeID0, nil)) + require.NoError(db.Put(subnetID0Height1NodeID1, nil)) + require.NoError(db.Put(subnetID1Height0NodeID0, nil)) + require.NoError(db.Put(subnetID1Height1NodeID0, nil)) + require.NoError(db.Put(subnetID1Height1NodeID1, nil)) + + { + it := db.NewIteratorWithStartAndPrefix(marshalStartDiffKey(subnetID0, 0), subnetID0[:]) + defer it.Release() + + expectedKeys := [][]byte{ + subnetID0Height0NodeID0, + } + for _, expectedKey := range expectedKeys { + require.True(it.Next()) + require.Equal(expectedKey, it.Key()) + } + require.False(it.Next()) + require.NoError(it.Error()) + } + + { + it := db.NewIteratorWithStartAndPrefix(marshalStartDiffKey(subnetID0, 1), subnetID0[:]) + defer it.Release() + + expectedKeys := [][]byte{ + subnetID0Height1NodeID0, + subnetID0Height1NodeID1, + subnetID0Height0NodeID0, + } + for _, expectedKey := range expectedKeys { + require.True(it.Next()) + require.Equal(expectedKey, it.Key()) + } + require.False(it.Next()) + require.NoError(it.Error()) + } +} diff --git a/vms/platformvm/state/mock_state.go b/vms/platformvm/state/mock_state.go index 73469862cfb..a150aecd251 100644 --- a/vms/platformvm/state/mock_state.go +++ b/vms/platformvm/state/mock_state.go @@ -8,13 +8,13 @@ package state import ( + context "context" reflect "reflect" time "time" database "github.com/ava-labs/avalanchego/database" ids "github.com/ava-labs/avalanchego/ids" validators "github.com/ava-labs/avalanchego/snow/validators" - bls "github.com/ava-labs/avalanchego/utils/crypto/bls" avax "github.com/ava-labs/avalanchego/vms/components/avax" blocks "github.com/ava-labs/avalanchego/vms/platformvm/blocks" status "github.com/ava-labs/avalanchego/vms/platformvm/status" @@ -141,6 +141,34 @@ func (mr *MockStateMockRecorder) AddUTXO(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddUTXO", reflect.TypeOf((*MockState)(nil).AddUTXO), arg0) } +// ApplyValidatorPublicKeyDiffs mocks base method. +func (m *MockState) ApplyValidatorPublicKeyDiffs(arg0 context.Context, arg1 map[ids.NodeID]*validators.GetValidatorOutput, arg2, arg3 uint64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ApplyValidatorPublicKeyDiffs", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// ApplyValidatorPublicKeyDiffs indicates an expected call of ApplyValidatorPublicKeyDiffs. +func (mr *MockStateMockRecorder) ApplyValidatorPublicKeyDiffs(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyValidatorPublicKeyDiffs", reflect.TypeOf((*MockState)(nil).ApplyValidatorPublicKeyDiffs), arg0, arg1, arg2, arg3) +} + +// ApplyValidatorWeightDiffs mocks base method. +func (m *MockState) ApplyValidatorWeightDiffs(arg0 context.Context, arg1 map[ids.NodeID]*validators.GetValidatorOutput, arg2, arg3 uint64, arg4 ids.ID) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ApplyValidatorWeightDiffs", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(error) + return ret0 +} + +// ApplyValidatorWeightDiffs indicates an expected call of ApplyValidatorWeightDiffs. +func (mr *MockStateMockRecorder) ApplyValidatorWeightDiffs(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyValidatorWeightDiffs", reflect.TypeOf((*MockState)(nil).ApplyValidatorWeightDiffs), arg0, arg1, arg2, arg3, arg4) +} + // Checksum mocks base method. func (m *MockState) Checksum() ids.ID { m.ctrl.T.Helper() @@ -543,36 +571,6 @@ func (mr *MockStateMockRecorder) GetUptime(arg0, arg1 interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUptime", reflect.TypeOf((*MockState)(nil).GetUptime), arg0, arg1) } -// GetValidatorPublicKeyDiffs mocks base method. -func (m *MockState) GetValidatorPublicKeyDiffs(arg0 uint64) (map[ids.NodeID]*bls.PublicKey, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetValidatorPublicKeyDiffs", arg0) - ret0, _ := ret[0].(map[ids.NodeID]*bls.PublicKey) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetValidatorPublicKeyDiffs indicates an expected call of GetValidatorPublicKeyDiffs. -func (mr *MockStateMockRecorder) GetValidatorPublicKeyDiffs(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetValidatorPublicKeyDiffs", reflect.TypeOf((*MockState)(nil).GetValidatorPublicKeyDiffs), arg0) -} - -// GetValidatorWeightDiffs mocks base method. -func (m *MockState) GetValidatorWeightDiffs(arg0 uint64, arg1 ids.ID) (map[ids.NodeID]*ValidatorWeightDiff, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetValidatorWeightDiffs", arg0, arg1) - ret0, _ := ret[0].(map[ids.NodeID]*ValidatorWeightDiff) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetValidatorWeightDiffs indicates an expected call of GetValidatorWeightDiffs. -func (mr *MockStateMockRecorder) GetValidatorWeightDiffs(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetValidatorWeightDiffs", reflect.TypeOf((*MockState)(nil).GetValidatorWeightDiffs), arg0, arg1) -} - // PutCurrentDelegator mocks base method. func (m *MockState) PutCurrentDelegator(arg0 *Staker) { m.ctrl.T.Helper() diff --git a/vms/platformvm/state/state.go b/vms/platformvm/state/state.go index 1743e766291..2086fe5f7e9 100644 --- a/vms/platformvm/state/state.go +++ b/vms/platformvm/state/state.go @@ -4,6 +4,7 @@ package state import ( + "context" "errors" "fmt" "time" @@ -49,29 +50,32 @@ var ( errValidatorSetAlreadyPopulated = errors.New("validator set already populated") errDuplicateValidatorSet = errors.New("duplicate validator set") - blockPrefix = []byte("block") - validatorsPrefix = []byte("validators") - currentPrefix = []byte("current") - pendingPrefix = []byte("pending") - validatorPrefix = []byte("validator") - delegatorPrefix = []byte("delegator") - subnetValidatorPrefix = []byte("subnetValidator") - subnetDelegatorPrefix = []byte("subnetDelegator") - validatorWeightDiffsPrefix = []byte("validatorDiffs") - validatorPublicKeyDiffsPrefix = []byte("publicKeyDiffs") - txPrefix = []byte("tx") - rewardUTXOsPrefix = []byte("rewardUTXOs") - utxoPrefix = []byte("utxo") - subnetPrefix = []byte("subnet") - transformedSubnetPrefix = []byte("transformedSubnet") - supplyPrefix = []byte("supply") - chainPrefix = []byte("chain") - singletonPrefix = []byte("singleton") - - timestampKey = []byte("timestamp") - currentSupplyKey = []byte("current supply") - lastAcceptedKey = []byte("last accepted") - initializedKey = []byte("initialized") + blockPrefix = []byte("block") + validatorsPrefix = []byte("validators") + currentPrefix = []byte("current") + pendingPrefix = []byte("pending") + validatorPrefix = []byte("validator") + delegatorPrefix = []byte("delegator") + subnetValidatorPrefix = []byte("subnetValidator") + subnetDelegatorPrefix = []byte("subnetDelegator") + nestedValidatorWeightDiffsPrefix = []byte("validatorDiffs") + nestedValidatorPublicKeyDiffsPrefix = []byte("publicKeyDiffs") + flatValidatorWeightDiffsPrefix = []byte("flatValidatorDiffs") + flatValidatorPublicKeyDiffsPrefix = []byte("flatPublicKeyDiffs") + txPrefix = []byte("tx") + rewardUTXOsPrefix = []byte("rewardUTXOs") + utxoPrefix = []byte("utxo") + subnetPrefix = []byte("subnet") + transformedSubnetPrefix = []byte("transformedSubnet") + supplyPrefix = []byte("supply") + chainPrefix = []byte("chain") + singletonPrefix = []byte("singleton") + + timestampKey = []byte("timestamp") + currentSupplyKey = []byte("current supply") + lastAcceptedKey = []byte("last accepted") + heightsIndexedKey = []byte("heights indexed") + initializedKey = []byte("initialized") ) // Chain collects all methods to manage the state of the chain for block @@ -121,11 +125,42 @@ type State interface { // [vdrs]. ValidatorSet(subnetID ids.ID, vdrs validators.Set) error - GetValidatorWeightDiffs(height uint64, subnetID ids.ID) (map[ids.NodeID]*ValidatorWeightDiff, error) - - // Returns a map of node ID --> BLS Public Key for all validators - // that left the Primary Network validator set. - GetValidatorPublicKeyDiffs(height uint64) (map[ids.NodeID]*bls.PublicKey, error) + // ApplyValidatorWeightDiffs iterates from [startHeight] towards the genesis + // block until it has applied all of the diffs up to and including + // [endHeight]. Applying the diffs modifies [validators]. + // + // Invariant: If attempting to generate the validator set for + // [endHeight - 1], [validators] must initially contain the validator + // weights for [startHeight]. + // + // Note: Because this function iterates towards the genesis, [startHeight] + // will typically be greater than or equal to [endHeight]. If [startHeight] + // is less than [endHeight], no diffs will be applied. + ApplyValidatorWeightDiffs( + ctx context.Context, + validators map[ids.NodeID]*validators.GetValidatorOutput, + startHeight uint64, + endHeight uint64, + subnetID ids.ID, + ) error + + // ApplyValidatorPublicKeyDiffs iterates from [startHeight] towards the + // genesis block until it has applied all of the diffs up to and including + // [endHeight]. Applying the diffs modifies [validators]. + // + // Invariant: If attempting to generate the validator set for + // [endHeight - 1], [validators] must initially contain the validator + // weights for [startHeight]. + // + // Note: Because this function iterates towards the genesis, [startHeight] + // will typically be greater than or equal to [endHeight]. If [startHeight] + // is less than [endHeight], no diffs will be applied. + ApplyValidatorPublicKeyDiffs( + ctx context.Context, + validators map[ids.NodeID]*validators.GetValidatorOutput, + startHeight uint64, + endHeight uint64, + ) error SetHeight(height uint64) @@ -180,14 +215,18 @@ type stateBlk struct { * | | '-. subnetDelegator * | | '-. list * | | '-- txID -> nil - * | |-. weight diffs + * | |-. nested weight diffs TODO: Remove once only the flat db is needed * | | '-. height+subnet * | | '-. list * | | '-- nodeID -> weightChange - * | '-. pub key diffs - * | '-. height - * | '-. list - * | '-- nodeID -> public key + * | |-. nested pub key diffs TODO: Remove once only the flat db is needed + * | | '-. height + * | | '-. list + * | | '-- nodeID -> compressed public key + * | |-. flat weight diffs + * | | '-- subnet+height+nodeID -> weightChange + * | '-. flat pub key diffs + * | '-- subnet+height+nodeID -> uncompressed public key or nil * |-. blocks * | '-- blockID -> block bytes * |-. txs @@ -209,7 +248,8 @@ type stateBlk struct { * |-- initializedKey -> nil * |-- timestampKey -> timestamp * |-- currentSupplyKey -> currentSupply - * '-- lastAcceptedKey -> lastAccepted + * |-- lastAcceptedKey -> lastAccepted + * '-- heightsIndexKey -> startIndexHeight + endIndexHeight */ type state struct { validatorState @@ -251,11 +291,10 @@ type state struct { pendingSubnetDelegatorBaseDB database.Database pendingSubnetDelegatorList linkeddb.LinkedDB - validatorWeightDiffsCache cache.Cacher[string, map[ids.NodeID]*ValidatorWeightDiff] // cache of heightWithSubnet -> map[ids.NodeID]*ValidatorWeightDiff - validatorWeightDiffsDB database.Database - - validatorPublicKeyDiffsCache cache.Cacher[uint64, map[ids.NodeID]*bls.PublicKey] // cache of height -> map[ids.NodeID]*bls.PublicKey - validatorPublicKeyDiffsDB database.Database + nestedValidatorWeightDiffsDB database.Database + nestedValidatorPublicKeyDiffsDB database.Database + flatValidatorWeightDiffsDB database.Database + flatValidatorPublicKeyDiffsDB database.Database addedTxs map[ids.ID]*txAndStatus // map of txID -> {*txs.Tx, Status} txCache cache.Cacher[ids.ID, *txAndStatus] // txID -> {*txs.Tx, Status}. If the entry is nil, it isn't in the database @@ -292,9 +331,20 @@ type state struct { currentSupply, persistedCurrentSupply uint64 // [lastAccepted] is the most recently accepted block. lastAccepted, persistedLastAccepted ids.ID + indexedHeights *heightRange singletonDB database.Database } +// heightRange is used to track which heights are safe to use the native DB +// iterator for querying validator diffs. +// +// TODO: Remove once we are guaranteed nodes can not rollback to not support the +// new indexing mechanism. +type heightRange struct { + LowerBound uint64 `serialize:"true"` + UpperBound uint64 `serialize:"true"` +} + type ValidatorWeightDiff struct { Decrease bool `serialize:"true"` Amount uint64 `serialize:"true"` @@ -363,7 +413,7 @@ func New( rewards reward.Calculator, bootstrapped *utils.Atomic[bool], ) (State, error) { - s, err := new( + s, err := newState( db, metrics, cfg, @@ -387,7 +437,7 @@ func New( return s, nil } -func new( +func newState( db database.Database, metrics metrics.Metrics, cfg *config.Config, @@ -422,25 +472,10 @@ func new( pendingSubnetValidatorBaseDB := prefixdb.New(subnetValidatorPrefix, pendingValidatorsDB) pendingSubnetDelegatorBaseDB := prefixdb.New(subnetDelegatorPrefix, pendingValidatorsDB) - validatorWeightDiffsDB := prefixdb.New(validatorWeightDiffsPrefix, validatorsDB) - validatorWeightDiffsCache, err := metercacher.New[string, map[ids.NodeID]*ValidatorWeightDiff]( - "validator_weight_diffs_cache", - metricsReg, - &cache.LRU[string, map[ids.NodeID]*ValidatorWeightDiff]{Size: execCfg.ValidatorDiffsCacheSize}, - ) - if err != nil { - return nil, err - } - - validatorPublicKeyDiffsDB := prefixdb.New(validatorPublicKeyDiffsPrefix, validatorsDB) - validatorPublicKeyDiffsCache, err := metercacher.New[uint64, map[ids.NodeID]*bls.PublicKey]( - "validator_pub_key_diffs_cache", - metricsReg, - &cache.LRU[uint64, map[ids.NodeID]*bls.PublicKey]{Size: execCfg.ValidatorDiffsCacheSize}, - ) - if err != nil { - return nil, err - } + nestedValidatorWeightDiffsDB := prefixdb.New(nestedValidatorWeightDiffsPrefix, validatorsDB) + nestedValidatorPublicKeyDiffsDB := prefixdb.New(nestedValidatorPublicKeyDiffsPrefix, validatorsDB) + flatValidatorWeightDiffsDB := prefixdb.New(flatValidatorWeightDiffsPrefix, validatorsDB) + flatValidatorPublicKeyDiffsDB := prefixdb.New(flatValidatorPublicKeyDiffsPrefix, validatorsDB) txCache, err := metercacher.New( "tx_cache", @@ -522,29 +557,29 @@ func new( currentStakers: newBaseStakers(), pendingStakers: newBaseStakers(), - validatorsDB: validatorsDB, - currentValidatorsDB: currentValidatorsDB, - currentValidatorBaseDB: currentValidatorBaseDB, - currentValidatorList: linkeddb.NewDefault(currentValidatorBaseDB), - currentDelegatorBaseDB: currentDelegatorBaseDB, - currentDelegatorList: linkeddb.NewDefault(currentDelegatorBaseDB), - currentSubnetValidatorBaseDB: currentSubnetValidatorBaseDB, - currentSubnetValidatorList: linkeddb.NewDefault(currentSubnetValidatorBaseDB), - currentSubnetDelegatorBaseDB: currentSubnetDelegatorBaseDB, - currentSubnetDelegatorList: linkeddb.NewDefault(currentSubnetDelegatorBaseDB), - pendingValidatorsDB: pendingValidatorsDB, - pendingValidatorBaseDB: pendingValidatorBaseDB, - pendingValidatorList: linkeddb.NewDefault(pendingValidatorBaseDB), - pendingDelegatorBaseDB: pendingDelegatorBaseDB, - pendingDelegatorList: linkeddb.NewDefault(pendingDelegatorBaseDB), - pendingSubnetValidatorBaseDB: pendingSubnetValidatorBaseDB, - pendingSubnetValidatorList: linkeddb.NewDefault(pendingSubnetValidatorBaseDB), - pendingSubnetDelegatorBaseDB: pendingSubnetDelegatorBaseDB, - pendingSubnetDelegatorList: linkeddb.NewDefault(pendingSubnetDelegatorBaseDB), - validatorWeightDiffsDB: validatorWeightDiffsDB, - validatorWeightDiffsCache: validatorWeightDiffsCache, - validatorPublicKeyDiffsCache: validatorPublicKeyDiffsCache, - validatorPublicKeyDiffsDB: validatorPublicKeyDiffsDB, + validatorsDB: validatorsDB, + currentValidatorsDB: currentValidatorsDB, + currentValidatorBaseDB: currentValidatorBaseDB, + currentValidatorList: linkeddb.NewDefault(currentValidatorBaseDB), + currentDelegatorBaseDB: currentDelegatorBaseDB, + currentDelegatorList: linkeddb.NewDefault(currentDelegatorBaseDB), + currentSubnetValidatorBaseDB: currentSubnetValidatorBaseDB, + currentSubnetValidatorList: linkeddb.NewDefault(currentSubnetValidatorBaseDB), + currentSubnetDelegatorBaseDB: currentSubnetDelegatorBaseDB, + currentSubnetDelegatorList: linkeddb.NewDefault(currentSubnetDelegatorBaseDB), + pendingValidatorsDB: pendingValidatorsDB, + pendingValidatorBaseDB: pendingValidatorBaseDB, + pendingValidatorList: linkeddb.NewDefault(pendingValidatorBaseDB), + pendingDelegatorBaseDB: pendingDelegatorBaseDB, + pendingDelegatorList: linkeddb.NewDefault(pendingDelegatorBaseDB), + pendingSubnetValidatorBaseDB: pendingSubnetValidatorBaseDB, + pendingSubnetValidatorList: linkeddb.NewDefault(pendingSubnetValidatorBaseDB), + pendingSubnetDelegatorBaseDB: pendingSubnetDelegatorBaseDB, + pendingSubnetDelegatorList: linkeddb.NewDefault(pendingSubnetDelegatorBaseDB), + nestedValidatorWeightDiffsDB: nestedValidatorWeightDiffsDB, + nestedValidatorPublicKeyDiffsDB: nestedValidatorPublicKeyDiffsDB, + flatValidatorWeightDiffsDB: flatValidatorWeightDiffsDB, + flatValidatorPublicKeyDiffsDB: flatValidatorPublicKeyDiffsDB, addedTxs: make(map[ids.ID]*txAndStatus), txDB: prefixdb.New(txPrefix, baseDB), @@ -944,74 +979,178 @@ func (s *state) ValidatorSet(subnetID ids.ID, vdrs validators.Set) error { return nil } -func (s *state) GetValidatorWeightDiffs(height uint64, subnetID ids.ID) (map[ids.NodeID]*ValidatorWeightDiff, error) { - prefixStruct := heightWithSubnet{ - Height: height, - SubnetID: subnetID, - } - prefixBytes, err := blocks.GenesisCodec.Marshal(blocks.Version, prefixStruct) - if err != nil { - return nil, err - } - prefixStr := string(prefixBytes) +func (s *state) ApplyValidatorWeightDiffs( + ctx context.Context, + validators map[ids.NodeID]*validators.GetValidatorOutput, + startHeight uint64, + endHeight uint64, + subnetID ids.ID, +) error { + diffIter := s.flatValidatorWeightDiffsDB.NewIteratorWithStartAndPrefix( + marshalStartDiffKey(subnetID, startHeight), + subnetID[:], + ) + defer diffIter.Release() - if weightDiffs, ok := s.validatorWeightDiffsCache.Get(prefixStr); ok { - return weightDiffs, nil - } + prevHeight := startHeight + 1 + // TODO: Remove the index continuity checks once we are guaranteed nodes can + // not rollback to not support the new indexing mechanism. + for diffIter.Next() && s.indexedHeights != nil && s.indexedHeights.LowerBound <= endHeight { + if err := ctx.Err(); err != nil { + return err + } - rawDiffDB := prefixdb.New(prefixBytes, s.validatorWeightDiffsDB) - diffDB := linkeddb.NewDefault(rawDiffDB) - diffIter := diffDB.NewIterator() - defer diffIter.Release() + _, parsedHeight, nodeID, err := unmarshalDiffKey(diffIter.Key()) + if err != nil { + return err + } + // If the parsedHeight is less than our target endHeight, then we have + // fully processed the diffs from startHeight through endHeight. + if parsedHeight < endHeight { + return diffIter.Error() + } - weightDiffs := make(map[ids.NodeID]*ValidatorWeightDiff) - for diffIter.Next() { - nodeID, err := ids.ToNodeID(diffIter.Key()) + prevHeight = parsedHeight + + weightDiff, err := unmarshalWeightDiff(diffIter.Value()) if err != nil { - return nil, err + return err + } + + if err := applyWeightDiff(validators, nodeID, weightDiff); err != nil { + return err + } + } + if err := diffIter.Error(); err != nil { + return err + } + + // TODO: Remove this once it is assumed that all subnet validators have + // adopted the new indexing. + for height := prevHeight - 1; height >= endHeight; height-- { + if err := ctx.Err(); err != nil { + return err } - weightDiff := ValidatorWeightDiff{} - _, err = blocks.GenesisCodec.Unmarshal(diffIter.Value(), &weightDiff) + prefixStruct := heightWithSubnet{ + Height: height, + SubnetID: subnetID, + } + prefixBytes, err := blocks.GenesisCodec.Marshal(blocks.Version, prefixStruct) if err != nil { - return nil, err + return err } - weightDiffs[nodeID] = &weightDiff + rawDiffDB := prefixdb.New(prefixBytes, s.nestedValidatorWeightDiffsDB) + diffDB := linkeddb.NewDefault(rawDiffDB) + diffIter := diffDB.NewIterator() + defer diffIter.Release() + + for diffIter.Next() { + nodeID, err := ids.ToNodeID(diffIter.Key()) + if err != nil { + return err + } + + weightDiff := ValidatorWeightDiff{} + _, err = blocks.GenesisCodec.Unmarshal(diffIter.Value(), &weightDiff) + if err != nil { + return err + } + + if err := applyWeightDiff(validators, nodeID, &weightDiff); err != nil { + return err + } + } } - s.validatorWeightDiffsCache.Put(prefixStr, weightDiffs) - return weightDiffs, diffIter.Error() + return nil } -func (s *state) GetValidatorPublicKeyDiffs(height uint64) (map[ids.NodeID]*bls.PublicKey, error) { - if publicKeyDiffs, ok := s.validatorPublicKeyDiffsCache.Get(height); ok { - return publicKeyDiffs, nil +func applyWeightDiff( + vdrs map[ids.NodeID]*validators.GetValidatorOutput, + nodeID ids.NodeID, + weightDiff *ValidatorWeightDiff, +) error { + vdr, ok := vdrs[nodeID] + if !ok { + // This node isn't in the current validator set. + vdr = &validators.GetValidatorOutput{ + NodeID: nodeID, + } + vdrs[nodeID] = vdr } - heightBytes := database.PackUInt64(height) - rawDiffDB := prefixdb.New(heightBytes, s.validatorPublicKeyDiffsDB) - diffDB := linkeddb.NewDefault(rawDiffDB) - diffIter := diffDB.NewIterator() + // The weight of this node changed at this block. + var err error + if weightDiff.Decrease { + // The validator's weight was decreased at this block, so in the + // prior block it was higher. + vdr.Weight, err = math.Add64(vdr.Weight, weightDiff.Amount) + } else { + // The validator's weight was increased at this block, so in the + // prior block it was lower. + vdr.Weight, err = math.Sub(vdr.Weight, weightDiff.Amount) + } + if err != nil { + return err + } + + if vdr.Weight == 0 { + // The validator's weight was 0 before this block so they weren't in the + // validator set. + delete(vdrs, nodeID) + } + return nil +} + +func (s *state) ApplyValidatorPublicKeyDiffs( + ctx context.Context, + validators map[ids.NodeID]*validators.GetValidatorOutput, + startHeight uint64, + endHeight uint64, +) error { + diffIter := s.flatValidatorPublicKeyDiffsDB.NewIteratorWithStartAndPrefix( + marshalStartDiffKey(constants.PrimaryNetworkID, startHeight), + constants.PrimaryNetworkID[:], + ) defer diffIter.Release() - pkDiffs := make(map[ids.NodeID]*bls.PublicKey) for diffIter.Next() { - nodeID, err := ids.ToNodeID(diffIter.Key()) + if err := ctx.Err(); err != nil { + return err + } + + _, parsedHeight, nodeID, err := unmarshalDiffKey(diffIter.Key()) if err != nil { - return nil, err + return err + } + // If the parsedHeight is less than our target endHeight, then we have + // fully processed the diffs from startHeight through endHeight. + if parsedHeight < endHeight { + break + } + + vdr, ok := validators[nodeID] + if !ok { + continue } pkBytes := diffIter.Value() - pk, err := bls.PublicKeyFromBytes(pkBytes) - if err != nil { - return nil, err + if len(pkBytes) == 0 { + vdr.PublicKey = nil + continue } - pkDiffs[nodeID] = pk + + vdr.PublicKey = new(bls.PublicKey).Deserialize(pkBytes) } - s.validatorPublicKeyDiffsCache.Put(height, pkDiffs) - return pkDiffs, diffIter.Error() + // Note: this does not fallback to the linkeddb index because the linkeddb + // index does not contain entries for when to remove the public key. + // + // Nodes may see inconsistent public keys for heights before the new public + // key index was populated. + return diffIter.Error() } func (s *state) syncGenesis(genesisBlk blocks.Block, genesis *genesis.State) error { @@ -1115,6 +1254,33 @@ func (s *state) loadMetadata() error { } s.persistedLastAccepted = lastAccepted s.lastAccepted = lastAccepted + + // Lookup the most recently indexed range on disk. If we haven't started + // indexing the weights, then we keep the indexed heights as nil. + indexedHeightsBytes, err := s.singletonDB.Get(heightsIndexedKey) + if err == database.ErrNotFound { + return nil + } + if err != nil { + return err + } + + indexedHeights := &heightRange{} + _, err = blocks.GenesisCodec.Unmarshal(indexedHeightsBytes, indexedHeights) + if err != nil { + return err + } + + // If the indexed range is not up to date, then we will act as if the range + // doesn't exist. + lastAcceptedBlock, err := s.GetStatelessBlock(lastAccepted) + if err != nil { + return err + } + if indexedHeights.UpperBound != lastAcceptedBlock.Height() { + return nil + } + s.indexedHeights = indexedHeights return nil } @@ -1487,6 +1653,16 @@ func (s *state) AddStatelessBlock(block blocks.Block) { } func (s *state) SetHeight(height uint64) { + if s.indexedHeights == nil { + // If indexedHeights hasn't been created yet, then we are newly tracking + // the range. This means we should initialize the LowerBound to the + // current height. + s.indexedHeights = &heightRange{ + LowerBound: height, + } + } + + s.indexedHeights.UpperBound = height s.currentHeight = height } @@ -1581,10 +1757,8 @@ func (s *state) GetStatelessBlock(blockID ids.ID) (blocks.Block, error) { func (s *state) writeCurrentStakers(updateValidators bool, height uint64) error { heightBytes := database.PackUInt64(height) - rawPublicKeyDiffDB := prefixdb.New(heightBytes, s.validatorPublicKeyDiffsDB) - pkDiffDB := linkeddb.NewDefault(rawPublicKeyDiffDB) - // Node ID --> BLS public key of node before it left the validator set. - pkDiffs := make(map[ids.NodeID]*bls.PublicKey) + rawNestedPublicKeyDiffDB := prefixdb.New(heightBytes, s.nestedValidatorPublicKeyDiffsDB) + nestedPKDiffDB := linkeddb.NewDefault(rawNestedPublicKeyDiffDB) for subnetID, validatorDiffs := range s.currentStakers.validatorDiffs { delete(s.currentStakers.validatorDiffs, subnetID) @@ -1605,9 +1779,8 @@ func (s *state) writeCurrentStakers(updateValidators bool, height uint64) error if err != nil { return fmt.Errorf("failed to create prefix bytes: %w", err) } - rawWeightDiffDB := prefixdb.New(prefixBytes, s.validatorWeightDiffsDB) - weightDiffDB := linkeddb.NewDefault(rawWeightDiffDB) - weightDiffs := make(map[ids.NodeID]*ValidatorWeightDiff) + rawNestedWeightDiffDB := prefixdb.New(prefixBytes, s.nestedValidatorWeightDiffsDB) + nestedWeightDiffDB := linkeddb.NewDefault(rawNestedWeightDiffDB) // Record the change in weight and/or public key for each validator. for nodeID, validatorDiff := range validatorDiffs { @@ -1622,6 +1795,21 @@ func (s *state) writeCurrentStakers(updateValidators bool, height uint64) error staker := validatorDiff.validator weightDiff.Amount = staker.Weight + // Invariant: Only the Primary Network contains non-nil public + // keys. + if staker.PublicKey != nil { + // Record that the public key for the validator is being + // added. This means the prior value for the public key was + // nil. + err := s.flatValidatorPublicKeyDiffsDB.Put( + marshalDiffKey(constants.PrimaryNetworkID, height, nodeID), + nil, + ) + if err != nil { + return err + } + } + // The validator is being added. // // Invariant: It's impossible for a delegator to have been @@ -1650,14 +1838,30 @@ func (s *state) writeCurrentStakers(updateValidators bool, height uint64) error staker := validatorDiff.validator weightDiff.Amount = staker.Weight - // Invariant: Only the Primary Network contains non-nil - // public keys. + // Invariant: Only the Primary Network contains non-nil public + // keys. if staker.PublicKey != nil { - // Record the public key of the validator being removed. - pkDiffs[nodeID] = staker.PublicKey + // Record that the public key for the validator is being + // removed. This means we must record the prior value of the + // public key. + // + // Note: We store the uncompressed public key here as it is + // significantly more efficient to parse when applying + // diffs. + err := s.flatValidatorPublicKeyDiffsDB.Put( + marshalDiffKey(constants.PrimaryNetworkID, height, nodeID), + staker.PublicKey.Serialize(), + ) + if err != nil { + return err + } + // TODO: Remove this once we no longer support version + // rollbacks. + // + // Note: We store the compressed public key here. pkBytes := bls.PublicKeyToBytes(staker.PublicKey) - if err := pkDiffDB.Put(nodeID[:], pkBytes); err != nil { + if err := nestedPKDiffDB.Put(nodeID[:], pkBytes); err != nil { return err } } @@ -1682,14 +1886,21 @@ func (s *state) writeCurrentStakers(updateValidators bool, height uint64) error // No weight change to record; go to next validator. continue } - weightDiffs[nodeID] = weightDiff + err = s.flatValidatorWeightDiffsDB.Put( + marshalDiffKey(subnetID, height, nodeID), + marshalWeightDiff(weightDiff), + ) + if err != nil { + return err + } + + // TODO: Remove this once we no longer support version rollbacks. weightDiffBytes, err := blocks.GenesisCodec.Marshal(blocks.Version, weightDiff) if err != nil { return fmt.Errorf("failed to serialize validator weight diff: %w", err) } - - if err := weightDiffDB.Put(nodeID[:], weightDiffBytes); err != nil { + if err := nestedWeightDiffDB.Put(nodeID[:], weightDiffBytes); err != nil { return err } @@ -1724,9 +1935,7 @@ func (s *state) writeCurrentStakers(updateValidators bool, height uint64) error return fmt.Errorf("failed to update validator weight: %w", err) } } - s.validatorWeightDiffsCache.Put(string(prefixBytes), weightDiffs) } - s.validatorPublicKeyDiffsCache.Put(height, pkDiffs) // TODO: Move validator set management out of the state package // @@ -1975,6 +2184,17 @@ func (s *state) writeMetadata() error { } s.persistedLastAccepted = s.lastAccepted } + + if s.indexedHeights != nil { + indexedHeightsBytes, err := blocks.GenesisCodec.Marshal(blocks.Version, s.indexedHeights) + if err != nil { + return err + } + if err := s.singletonDB.Put(heightsIndexedKey, indexedHeightsBytes); err != nil { + return fmt.Errorf("failed to write indexed range: %w", err) + } + } + return nil } diff --git a/vms/platformvm/state/state_test.go b/vms/platformvm/state/state_test.go index 71aa4e58ab5..65a6fc1bd4f 100644 --- a/vms/platformvm/state/state_test.go +++ b/vms/platformvm/state/state_test.go @@ -4,6 +4,7 @@ package state import ( + "context" "testing" "time" @@ -85,345 +86,6 @@ func TestStateSyncGenesis(t *testing.T) { assertIteratorsEqual(t, EmptyIterator, delegatorIterator) } -func TestGetValidatorWeightDiffs(t *testing.T) { - require := require.New(t) - stateIntf, _ := newInitializedState(require) - state := stateIntf.(*state) - - txID0 := ids.GenerateTestID() - txID1 := ids.GenerateTestID() - txID2 := ids.GenerateTestID() - txID3 := ids.GenerateTestID() - - nodeID0 := ids.GenerateTestNodeID() - - subnetID0 := ids.GenerateTestID() - - type stakerDiff struct { - validatorsToAdd []*Staker - delegatorsToAdd []*Staker - validatorsToRemove []*Staker - delegatorsToRemove []*Staker - - expectedValidatorWeightDiffs map[ids.ID]map[ids.NodeID]*ValidatorWeightDiff - } - stakerDiffs := []*stakerDiff{ - { - validatorsToAdd: []*Staker{ - { - TxID: txID0, - NodeID: nodeID0, - SubnetID: constants.PrimaryNetworkID, - Weight: 1, - }, - }, - expectedValidatorWeightDiffs: map[ids.ID]map[ids.NodeID]*ValidatorWeightDiff{ - constants.PrimaryNetworkID: { - nodeID0: { - Decrease: false, - Amount: 1, - }, - }, - }, - }, - { - validatorsToAdd: []*Staker{ - { - TxID: txID3, - NodeID: nodeID0, - SubnetID: subnetID0, - Weight: 10, - }, - }, - delegatorsToAdd: []*Staker{ - { - TxID: txID1, - NodeID: nodeID0, - SubnetID: constants.PrimaryNetworkID, - Weight: 5, - }, - }, - expectedValidatorWeightDiffs: map[ids.ID]map[ids.NodeID]*ValidatorWeightDiff{ - constants.PrimaryNetworkID: { - nodeID0: { - Decrease: false, - Amount: 5, - }, - }, - subnetID0: { - nodeID0: { - Decrease: false, - Amount: 10, - }, - }, - }, - }, - { - delegatorsToAdd: []*Staker{ - { - TxID: txID2, - NodeID: nodeID0, - SubnetID: constants.PrimaryNetworkID, - Weight: 15, - }, - }, - delegatorsToRemove: []*Staker{ - { - TxID: txID1, - NodeID: nodeID0, - SubnetID: constants.PrimaryNetworkID, - Weight: 5, - }, - }, - expectedValidatorWeightDiffs: map[ids.ID]map[ids.NodeID]*ValidatorWeightDiff{ - constants.PrimaryNetworkID: { - nodeID0: { - Decrease: false, - Amount: 10, - }, - }, - }, - }, - { - validatorsToRemove: []*Staker{ - { - TxID: txID0, - NodeID: nodeID0, - SubnetID: constants.PrimaryNetworkID, - Weight: 1, - }, - { - TxID: txID3, - NodeID: nodeID0, - SubnetID: subnetID0, - Weight: 10, - }, - }, - delegatorsToRemove: []*Staker{ - { - TxID: txID2, - NodeID: nodeID0, - SubnetID: constants.PrimaryNetworkID, - Weight: 15, - }, - }, - expectedValidatorWeightDiffs: map[ids.ID]map[ids.NodeID]*ValidatorWeightDiff{ - constants.PrimaryNetworkID: { - nodeID0: { - Decrease: true, - Amount: 16, - }, - }, - subnetID0: { - nodeID0: { - Decrease: true, - Amount: 10, - }, - }, - }, - }, - {}, - } - - for i, stakerDiff := range stakerDiffs { - for _, validator := range stakerDiff.validatorsToAdd { - state.PutCurrentValidator(validator) - } - for _, delegator := range stakerDiff.delegatorsToAdd { - state.PutCurrentDelegator(delegator) - } - for _, validator := range stakerDiff.validatorsToRemove { - state.DeleteCurrentValidator(validator) - } - for _, delegator := range stakerDiff.delegatorsToRemove { - state.DeleteCurrentDelegator(delegator) - } - state.SetHeight(uint64(i + 1)) - require.NoError(state.Commit()) - - // Calling write again should not change the state. - state.SetHeight(uint64(i + 1)) - require.NoError(state.Commit()) - - for j, stakerDiff := range stakerDiffs[:i+1] { - for subnetID, expectedValidatorWeightDiffs := range stakerDiff.expectedValidatorWeightDiffs { - validatorWeightDiffs, err := state.GetValidatorWeightDiffs(uint64(j+1), subnetID) - require.NoError(err) - require.Equal(expectedValidatorWeightDiffs, validatorWeightDiffs) - } - - state.validatorWeightDiffsCache.Flush() - } - } -} - -func TestGetValidatorPublicKeyDiffs(t *testing.T) { - require := require.New(t) - stateIntf, _ := newInitializedState(require) - state := stateIntf.(*state) - - var ( - numNodes = 6 - txIDs = make([]ids.ID, numNodes) - nodeIDs = make([]ids.NodeID, numNodes) - sks = make([]*bls.SecretKey, numNodes) - pks = make([]*bls.PublicKey, numNodes) - pkBytes = make([][]byte, numNodes) - err error - ) - for i := 0; i < numNodes; i++ { - txIDs[i] = ids.GenerateTestID() - nodeIDs[i] = ids.GenerateTestNodeID() - sks[i], err = bls.NewSecretKey() - require.NoError(err) - pks[i] = bls.PublicFromSecretKey(sks[i]) - pkBytes[i] = bls.PublicKeyToBytes(pks[i]) - } - - type stakerDiff struct { - validatorsToAdd []*Staker - validatorsToRemove []*Staker - expectedPublicKeyDiffs map[ids.NodeID]*bls.PublicKey - } - stakerDiffs := []*stakerDiff{ - { - // Add two validators - validatorsToAdd: []*Staker{ - { - TxID: txIDs[0], - NodeID: nodeIDs[0], - Weight: 1, - PublicKey: pks[0], - }, - { - TxID: txIDs[1], - NodeID: nodeIDs[1], - Weight: 10, - PublicKey: pks[1], - }, - }, - expectedPublicKeyDiffs: map[ids.NodeID]*bls.PublicKey{}, - }, - { - // Remove a validator - validatorsToRemove: []*Staker{ - { - TxID: txIDs[0], - NodeID: nodeIDs[0], - Weight: 1, - PublicKey: pks[0], - }, - }, - expectedPublicKeyDiffs: map[ids.NodeID]*bls.PublicKey{ - nodeIDs[0]: pks[0], - }, - }, - { - // Add 2 validators and remove a validator - validatorsToAdd: []*Staker{ - { - TxID: txIDs[2], - NodeID: nodeIDs[2], - Weight: 10, - PublicKey: pks[2], - }, - { - TxID: txIDs[3], - NodeID: nodeIDs[3], - Weight: 10, - PublicKey: pks[3], - }, - }, - validatorsToRemove: []*Staker{ - { - TxID: txIDs[1], - NodeID: nodeIDs[1], - Weight: 10, - PublicKey: pks[1], - }, - }, - expectedPublicKeyDiffs: map[ids.NodeID]*bls.PublicKey{ - nodeIDs[1]: pks[1], - }, - }, - { - // Remove 2 validators and add a validator - validatorsToAdd: []*Staker{ - { - TxID: txIDs[4], - NodeID: nodeIDs[4], - Weight: 10, - PublicKey: pks[4], - }, - }, - validatorsToRemove: []*Staker{ - { - TxID: txIDs[2], - NodeID: nodeIDs[2], - Weight: 10, - PublicKey: pks[2], - }, - { - TxID: txIDs[3], - NodeID: nodeIDs[3], - Weight: 10, - PublicKey: pks[3], - }, - }, - expectedPublicKeyDiffs: map[ids.NodeID]*bls.PublicKey{ - nodeIDs[2]: pks[2], - nodeIDs[3]: pks[3], - }, - }, - { - // Add a validator with no pub key - validatorsToAdd: []*Staker{ - { - TxID: txIDs[5], - NodeID: nodeIDs[5], - Weight: 10, - PublicKey: nil, - }, - }, - expectedPublicKeyDiffs: map[ids.NodeID]*bls.PublicKey{}, - }, - { - // Remove a validator with no pub key - validatorsToRemove: []*Staker{ - { - TxID: txIDs[5], - NodeID: nodeIDs[5], - Weight: 10, - PublicKey: nil, - }, - }, - expectedPublicKeyDiffs: map[ids.NodeID]*bls.PublicKey{}, - }, - } - - for i, stakerDiff := range stakerDiffs { - for _, validator := range stakerDiff.validatorsToAdd { - state.PutCurrentValidator(validator) - } - for _, validator := range stakerDiff.validatorsToRemove { - state.DeleteCurrentValidator(validator) - } - state.SetHeight(uint64(i + 1)) - require.NoError(state.Commit()) - - // Calling write again should not change the state. - state.SetHeight(uint64(i + 1)) - require.NoError(state.Commit()) - - for j, stakerDiff := range stakerDiffs[:i+1] { - pkDiffs, err := state.GetValidatorPublicKeyDiffs(uint64(j + 1)) - require.NoError(err) - require.Equal(stakerDiff.expectedPublicKeyDiffs, pkDiffs) - state.validatorPublicKeyDiffsCache.Flush() - } - } -} - func newInitializedState(require *require.Assertions) (State, database.Database) { s, db := newUninitializedState(require) @@ -497,8 +159,9 @@ func newStateFromDB(require *require.Assertions, db database.Database) State { vdrs := validators.NewManager() primaryVdrs := validators.NewSet() _ = vdrs.Add(constants.PrimaryNetworkID, primaryVdrs) + execCfg, _ := config.GetExecutionConfig(nil) - state, err := new( + state, err := newState( db, metrics.Noop, &config.Config{ @@ -661,7 +324,7 @@ func TestValidatorWeightDiff(t *testing.T) { } // Tests PutCurrentValidator, DeleteCurrentValidator, GetCurrentValidator, -// GetValidatorWeightDiffs, GetValidatorPublicKeyDiffs +// ApplyValidatorWeightDiffs, ApplyValidatorPublicKeyDiffs func TestStateAddRemoveValidator(t *testing.T) { require := require.New(t) @@ -694,148 +357,199 @@ func TestStateAddRemoveValidator(t *testing.T) { } type diff struct { - added []Staker - removed []Staker - expectedSubnetWeightDiff map[ids.NodeID]*ValidatorWeightDiff - expectedPrimaryNetworkWeightDiff map[ids.NodeID]*ValidatorWeightDiff - expectedPublicKeyDiff map[ids.NodeID]*bls.PublicKey + addedValidators []Staker + addedDelegators []Staker + removedDelegators []Staker + removedValidators []Staker + + expectedPrimaryValidatorSet map[ids.NodeID]*validators.GetValidatorOutput + expectedSubnetValidatorSet map[ids.NodeID]*validators.GetValidatorOutput } diffs := []diff{ + { + // Do nothing + expectedPrimaryValidatorSet: map[ids.NodeID]*validators.GetValidatorOutput{}, + expectedSubnetValidatorSet: map[ids.NodeID]*validators.GetValidatorOutput{}, + }, { // Add a subnet validator - added: []Staker{stakers[0]}, - expectedPrimaryNetworkWeightDiff: map[ids.NodeID]*ValidatorWeightDiff{}, - expectedSubnetWeightDiff: map[ids.NodeID]*ValidatorWeightDiff{ + addedValidators: []Staker{stakers[0]}, + expectedPrimaryValidatorSet: map[ids.NodeID]*validators.GetValidatorOutput{}, + expectedSubnetValidatorSet: map[ids.NodeID]*validators.GetValidatorOutput{ stakers[0].NodeID: { - Decrease: false, - Amount: stakers[0].Weight, + NodeID: stakers[0].NodeID, + Weight: stakers[0].Weight, }, }, - // No diff because this is a subnet validator - expectedPublicKeyDiff: map[ids.NodeID]*bls.PublicKey{}, }, { // Remove a subnet validator - removed: []Staker{stakers[0]}, - expectedPrimaryNetworkWeightDiff: map[ids.NodeID]*ValidatorWeightDiff{}, - expectedSubnetWeightDiff: map[ids.NodeID]*ValidatorWeightDiff{ - stakers[0].NodeID: { - Decrease: true, - Amount: stakers[0].Weight, - }, - }, - // No diff because this is a subnet validator - expectedPublicKeyDiff: map[ids.NodeID]*bls.PublicKey{}, + removedValidators: []Staker{stakers[0]}, + expectedPrimaryValidatorSet: map[ids.NodeID]*validators.GetValidatorOutput{}, + expectedSubnetValidatorSet: map[ids.NodeID]*validators.GetValidatorOutput{}, }, { // Add a primary network validator - added: []Staker{stakers[1]}, - expectedPrimaryNetworkWeightDiff: map[ids.NodeID]*ValidatorWeightDiff{ + addedValidators: []Staker{stakers[1]}, + expectedPrimaryValidatorSet: map[ids.NodeID]*validators.GetValidatorOutput{ stakers[1].NodeID: { - Decrease: false, - Amount: stakers[1].Weight, + NodeID: stakers[1].NodeID, + PublicKey: stakers[1].PublicKey, + Weight: stakers[1].Weight, }, }, - expectedSubnetWeightDiff: map[ids.NodeID]*ValidatorWeightDiff{}, - expectedPublicKeyDiff: map[ids.NodeID]*bls.PublicKey{}, + expectedSubnetValidatorSet: map[ids.NodeID]*validators.GetValidatorOutput{}, }, - { // Remove a primary network validator - removed: []Staker{stakers[1]}, - expectedPrimaryNetworkWeightDiff: map[ids.NodeID]*ValidatorWeightDiff{ + { + // Do nothing + expectedPrimaryValidatorSet: map[ids.NodeID]*validators.GetValidatorOutput{ stakers[1].NodeID: { - Decrease: true, - Amount: stakers[1].Weight, + NodeID: stakers[1].NodeID, + PublicKey: stakers[1].PublicKey, + Weight: stakers[1].Weight, }, }, - expectedSubnetWeightDiff: map[ids.NodeID]*ValidatorWeightDiff{}, - expectedPublicKeyDiff: map[ids.NodeID]*bls.PublicKey{ - stakers[1].NodeID: stakers[1].PublicKey, - }, + expectedSubnetValidatorSet: map[ids.NodeID]*validators.GetValidatorOutput{}, + }, + { // Remove a primary network validator + removedValidators: []Staker{stakers[1]}, + expectedPrimaryValidatorSet: map[ids.NodeID]*validators.GetValidatorOutput{}, + expectedSubnetValidatorSet: map[ids.NodeID]*validators.GetValidatorOutput{}, }, { // Add 2 subnet validators and a primary network validator - added: []Staker{stakers[0], stakers[1], stakers[2]}, - expectedPrimaryNetworkWeightDiff: map[ids.NodeID]*ValidatorWeightDiff{ + addedValidators: []Staker{stakers[0], stakers[1], stakers[2]}, + expectedPrimaryValidatorSet: map[ids.NodeID]*validators.GetValidatorOutput{ stakers[1].NodeID: { - Decrease: false, - Amount: stakers[1].Weight, + NodeID: stakers[1].NodeID, + PublicKey: stakers[1].PublicKey, + Weight: stakers[1].Weight, }, }, - expectedSubnetWeightDiff: map[ids.NodeID]*ValidatorWeightDiff{ + expectedSubnetValidatorSet: map[ids.NodeID]*validators.GetValidatorOutput{ stakers[0].NodeID: { - Decrease: false, - Amount: stakers[0].Weight, + NodeID: stakers[0].NodeID, + Weight: stakers[0].Weight, }, stakers[2].NodeID: { - Decrease: false, - Amount: stakers[2].Weight, + NodeID: stakers[2].NodeID, + Weight: stakers[2].Weight, }, }, - expectedPublicKeyDiff: map[ids.NodeID]*bls.PublicKey{}, }, { // Remove 2 subnet validators and a primary network validator. - removed: []Staker{stakers[0], stakers[1], stakers[2]}, - expectedPrimaryNetworkWeightDiff: map[ids.NodeID]*ValidatorWeightDiff{ - stakers[1].NodeID: { - Decrease: true, - Amount: stakers[1].Weight, - }, - }, - expectedSubnetWeightDiff: map[ids.NodeID]*ValidatorWeightDiff{ - stakers[0].NodeID: { - Decrease: true, - Amount: stakers[0].Weight, - }, - stakers[2].NodeID: { - Decrease: true, - Amount: stakers[2].Weight, - }, - }, - expectedPublicKeyDiff: map[ids.NodeID]*bls.PublicKey{ - stakers[1].NodeID: stakers[1].PublicKey, - }, + removedValidators: []Staker{stakers[0], stakers[1], stakers[2]}, + expectedPrimaryValidatorSet: map[ids.NodeID]*validators.GetValidatorOutput{}, + expectedSubnetValidatorSet: map[ids.NodeID]*validators.GetValidatorOutput{}, }, } - - for i, diff := range diffs { - for _, added := range diff.added { + for currentIndex, diff := range diffs { + for _, added := range diff.addedValidators { added := added state.PutCurrentValidator(&added) } - for _, removed := range diff.removed { + for _, added := range diff.addedDelegators { + added := added + state.PutCurrentDelegator(&added) + } + for _, removed := range diff.removedDelegators { + removed := removed + state.DeleteCurrentDelegator(&removed) + } + for _, removed := range diff.removedValidators { removed := removed state.DeleteCurrentValidator(&removed) } - newHeight := uint64(i + 1) - state.SetHeight(newHeight) + currentHeight := uint64(currentIndex + 1) + state.SetHeight(currentHeight) require.NoError(state.Commit()) - for _, added := range diff.added { + for _, added := range diff.addedValidators { gotValidator, err := state.GetCurrentValidator(added.SubnetID, added.NodeID) require.NoError(err) require.Equal(added, *gotValidator) } - for _, removed := range diff.removed { + for _, removed := range diff.removedValidators { _, err := state.GetCurrentValidator(removed.SubnetID, removed.NodeID) require.ErrorIs(err, database.ErrNotFound) } - // Assert that we get the expected weight diffs - gotSubnetWeightDiffs, err := state.GetValidatorWeightDiffs(newHeight, subnetID) - require.NoError(err) - require.Equal(diff.expectedSubnetWeightDiff, gotSubnetWeightDiffs) + for i := 0; i < currentIndex; i++ { + prevDiff := diffs[i] + prevHeight := uint64(i + 1) + + primaryValidatorSet := copyValidatorSet(diff.expectedPrimaryValidatorSet) + require.NoError(state.ApplyValidatorWeightDiffs( + context.Background(), + primaryValidatorSet, + currentHeight, + prevHeight+1, + constants.PrimaryNetworkID, + )) + requireEqualWeightsValidatorSet(require, prevDiff.expectedPrimaryValidatorSet, primaryValidatorSet) + + require.NoError(state.ApplyValidatorPublicKeyDiffs( + context.Background(), + primaryValidatorSet, + currentHeight, + prevHeight+1, + )) + requireEqualPublicKeysValidatorSet(require, prevDiff.expectedPrimaryValidatorSet, primaryValidatorSet) + + subnetValidatorSet := copyValidatorSet(diff.expectedSubnetValidatorSet) + require.NoError(state.ApplyValidatorWeightDiffs( + context.Background(), + subnetValidatorSet, + currentHeight, + prevHeight+1, + subnetID, + )) + requireEqualWeightsValidatorSet(require, prevDiff.expectedSubnetValidatorSet, subnetValidatorSet) + } + } +} - gotWeightDiffs, err := state.GetValidatorWeightDiffs(newHeight, constants.PrimaryNetworkID) - require.NoError(err) - require.Equal(diff.expectedPrimaryNetworkWeightDiff, gotWeightDiffs) +func copyValidatorSet( + input map[ids.NodeID]*validators.GetValidatorOutput, +) map[ids.NodeID]*validators.GetValidatorOutput { + result := make(map[ids.NodeID]*validators.GetValidatorOutput, len(input)) + for nodeID, vdr := range input { + vdrCopy := *vdr + result[nodeID] = &vdrCopy + } + return result +} - // Assert that we get the expected public key diff - gotPublicKeyDiffs, err := state.GetValidatorPublicKeyDiffs(newHeight) - require.NoError(err) - require.Equal(diff.expectedPublicKeyDiff, gotPublicKeyDiffs) +func requireEqualWeightsValidatorSet( + require *require.Assertions, + expected map[ids.NodeID]*validators.GetValidatorOutput, + actual map[ids.NodeID]*validators.GetValidatorOutput, +) { + require.Len(actual, len(expected)) + for nodeID, expectedVdr := range expected { + require.Contains(actual, nodeID) + + actualVdr := actual[nodeID] + require.Equal(expectedVdr.NodeID, actualVdr.NodeID) + require.Equal(expectedVdr.Weight, actualVdr.Weight) + } +} + +func requireEqualPublicKeysValidatorSet( + require *require.Assertions, + expected map[ids.NodeID]*validators.GetValidatorOutput, + actual map[ids.NodeID]*validators.GetValidatorOutput, +) { + require.Len(actual, len(expected)) + for nodeID, expectedVdr := range expected { + require.Contains(actual, nodeID) + + actualVdr := actual[nodeID] + require.Equal(expectedVdr.NodeID, actualVdr.NodeID) + require.Equal(expectedVdr.PublicKey, actualVdr.PublicKey) } } diff --git a/vms/platformvm/validators/manager.go b/vms/platformvm/validators/manager.go index c2287f9de14..16795b67da5 100644 --- a/vms/platformvm/validators/manager.go +++ b/vms/platformvm/validators/manager.go @@ -17,12 +17,12 @@ import ( "github.com/ava-labs/avalanchego/snow/validators" "github.com/ava-labs/avalanchego/utils/constants" "github.com/ava-labs/avalanchego/utils/logging" - "github.com/ava-labs/avalanchego/utils/math" "github.com/ava-labs/avalanchego/utils/timer/mockable" "github.com/ava-labs/avalanchego/utils/window" + "github.com/ava-labs/avalanchego/vms/platformvm/blocks" "github.com/ava-labs/avalanchego/vms/platformvm/config" "github.com/ava-labs/avalanchego/vms/platformvm/metrics" - "github.com/ava-labs/avalanchego/vms/platformvm/state" + "github.com/ava-labs/avalanchego/vms/platformvm/status" "github.com/ava-labs/avalanchego/vms/platformvm/txs" ) @@ -50,10 +50,56 @@ type Manager interface { OnAcceptedBlockID(blkID ids.ID) } +type State interface { + GetTx(txID ids.ID) (*txs.Tx, status.Status, error) + + GetLastAccepted() ids.ID + GetStatelessBlock(blockID ids.ID) (blocks.Block, error) + + // ValidatorSet adds all the validators and delegators of [subnetID] into + // [vdrs]. + ValidatorSet(subnetID ids.ID, vdrs validators.Set) error + + // ApplyValidatorWeightDiffs iterates from [startHeight] towards the genesis + // block until it has applied all of the diffs up to and including + // [endHeight]. Applying the diffs modifies [validators]. + // + // Invariant: If attempting to generate the validator set for + // [endHeight - 1], [validators] must initially contain the validator + // weights for [startHeight]. + // + // Note: Because this function iterates towards the genesis, [startHeight] + // should normally be greater than or equal to [endHeight]. + ApplyValidatorWeightDiffs( + ctx context.Context, + validators map[ids.NodeID]*validators.GetValidatorOutput, + startHeight uint64, + endHeight uint64, + subnetID ids.ID, + ) error + + // ApplyValidatorPublicKeyDiffs iterates from [startHeight] towards the + // genesis block until it has applied all of the diffs up to and including + // [endHeight]. Applying the diffs modifies [validators]. + // + // Invariant: If attempting to generate the validator set for + // [endHeight - 1], [validators] must initially contain the validator + // weights for [startHeight]. + // + // Note: Because this function iterates towards the genesis, [startHeight] + // should normally be greater than or equal to [endHeight]. + ApplyValidatorPublicKeyDiffs( + ctx context.Context, + validators map[ids.NodeID]*validators.GetValidatorOutput, + startHeight uint64, + endHeight uint64, + ) error +} + func NewManager( log logging.Logger, cfg config.Config, - state state.State, + state State, metrics metrics.Metrics, clk *mockable.Clock, ) Manager { @@ -75,10 +121,12 @@ func NewManager( } } +// TODO: Remove requirement for the P-chain's context lock to be held when +// calling exported functions. type manager struct { log logging.Logger cfg config.Config - state state.State + state State metrics metrics.Metrics clk *mockable.Clock @@ -110,12 +158,12 @@ type manager struct { // as the minimum. func (m *manager) GetMinimumHeight(ctx context.Context) (uint64, error) { if m.cfg.UseCurrentHeight { - return m.GetCurrentHeight(ctx) + return m.getCurrentHeight(ctx) } oldest, ok := m.recentlyAccepted.Oldest() if !ok { - return m.GetCurrentHeight(ctx) + return m.getCurrentHeight(ctx) } blk, err := m.state.GetStatelessBlock(oldest) @@ -131,7 +179,12 @@ func (m *manager) GetMinimumHeight(ctx context.Context) (uint64, error) { return blk.Height() - 1, nil } -func (m *manager) GetCurrentHeight(context.Context) (uint64, error) { +func (m *manager) GetCurrentHeight(ctx context.Context) (uint64, error) { + return m.getCurrentHeight(ctx) +} + +// TODO: Pass the context into the state. +func (m *manager) getCurrentHeight(context.Context) (uint64, error) { lastAcceptedID := m.state.GetLastAccepted() lastAccepted, err := m.state.GetStatelessBlock(lastAcceptedID) if err != nil { @@ -142,37 +195,10 @@ func (m *manager) GetCurrentHeight(context.Context) (uint64, error) { func (m *manager) GetValidatorSet( ctx context.Context, - height uint64, - subnetID ids.ID, -) (map[ids.NodeID]*validators.GetValidatorOutput, error) { - lastAcceptedHeight, err := m.GetCurrentHeight(ctx) - if err != nil { - return nil, err - } - if lastAcceptedHeight < height { - return nil, database.ErrNotFound - } - - return m.getValidatorSetFrom(lastAcceptedHeight, height, subnetID) -} - -// getValidatorSetFrom fetches the validator set of [subnetID] at [targetHeight] -// or builds it starting from [currentHeight]. -// -// Invariant: [m.cfg.Validators] contains the validator set at [currentHeight]. -func (m *manager) getValidatorSetFrom( - currentHeight uint64, targetHeight uint64, subnetID ids.ID, ) (map[ids.NodeID]*validators.GetValidatorOutput, error) { - validatorSetsCache, exists := m.caches[subnetID] - if !exists { - validatorSetsCache = &cache.LRU[uint64, map[ids.NodeID]*validators.GetValidatorOutput]{Size: validatorSetsCacheSize} - // Only cache tracked subnets - if subnetID == constants.PrimaryNetworkID || m.cfg.TrackedSubnets.Contains(subnetID) { - m.caches[subnetID] = validatorSetsCache - } - } + validatorSetsCache := m.getValidatorSetCache(subnetID) if validatorSet, ok := validatorSetsCache.Get(targetHeight); ok { m.metrics.IncValidatorSetsCached() @@ -183,13 +209,14 @@ func (m *manager) getValidatorSetFrom( startTime := m.clk.Time() var ( - validatorSet map[ids.NodeID]*validators.GetValidatorOutput - err error + validatorSet map[ids.NodeID]*validators.GetValidatorOutput + currentHeight uint64 + err error ) if subnetID == constants.PrimaryNetworkID { - validatorSet, err = m.makePrimaryNetworkValidatorSet(currentHeight, targetHeight) + validatorSet, currentHeight, err = m.makePrimaryNetworkValidatorSet(ctx, targetHeight) } else { - validatorSet, err = m.makeSubnetValidatorSet(currentHeight, targetHeight, subnetID) + validatorSet, currentHeight, err = m.makeSubnetValidatorSet(ctx, targetHeight, subnetID) } if err != nil { return nil, err @@ -198,146 +225,169 @@ func (m *manager) getValidatorSetFrom( // cache the validator set validatorSetsCache.Put(targetHeight, validatorSet) - endTime := m.clk.Time() + duration := m.clk.Time().Sub(startTime) m.metrics.IncValidatorSetsCreated() - m.metrics.AddValidatorSetsDuration(endTime.Sub(startTime)) + m.metrics.AddValidatorSetsDuration(duration) m.metrics.AddValidatorSetsHeightDiff(currentHeight - targetHeight) return validatorSet, nil } +func (m *manager) getValidatorSetCache(subnetID ids.ID) cache.Cacher[uint64, map[ids.NodeID]*validators.GetValidatorOutput] { + // Only cache tracked subnets + if subnetID != constants.PrimaryNetworkID && !m.cfg.TrackedSubnets.Contains(subnetID) { + return &cache.Empty[uint64, map[ids.NodeID]*validators.GetValidatorOutput]{} + } + + validatorSetsCache, exists := m.caches[subnetID] + if exists { + return validatorSetsCache + } + + validatorSetsCache = &cache.LRU[uint64, map[ids.NodeID]*validators.GetValidatorOutput]{ + Size: validatorSetsCacheSize, + } + m.caches[subnetID] = validatorSetsCache + return validatorSetsCache +} + func (m *manager) makePrimaryNetworkValidatorSet( - currentHeight uint64, + ctx context.Context, targetHeight uint64, -) (map[ids.NodeID]*validators.GetValidatorOutput, error) { +) (map[ids.NodeID]*validators.GetValidatorOutput, uint64, error) { + validatorSet, currentHeight, err := m.getCurrentPrimaryValidatorSet(ctx) + if err != nil { + return nil, 0, err + } + if currentHeight < targetHeight { + return nil, 0, database.ErrNotFound + } + + // Rebuild primary network validators at [targetHeight] + // + // Note: Since we are attempting to generate the validator set at + // [targetHeight], we want to apply the diffs from + // (targetHeight, currentHeight]. Because the state interface is implemented + // to be inclusive, we apply diffs in [targetHeight + 1, currentHeight]. + lastDiffHeight := targetHeight + 1 + err = m.state.ApplyValidatorWeightDiffs( + ctx, + validatorSet, + currentHeight, + lastDiffHeight, + constants.PlatformChainID, + ) + if err != nil { + return nil, 0, err + } + + err = m.state.ApplyValidatorPublicKeyDiffs( + ctx, + validatorSet, + currentHeight, + lastDiffHeight, + ) + return validatorSet, currentHeight, err +} + +func (m *manager) getCurrentPrimaryValidatorSet( + ctx context.Context, +) (map[ids.NodeID]*validators.GetValidatorOutput, uint64, error) { currentValidators, ok := m.cfg.Validators.Get(constants.PrimaryNetworkID) if !ok { // This should never happen m.log.Error(ErrMissingValidatorSet.Error(), zap.Stringer("subnetID", constants.PrimaryNetworkID), ) - return nil, ErrMissingValidatorSet + return nil, 0, ErrMissingValidatorSet } - // Node ID --> Validator information for the node validating the Primary - // Network. - validatorSet := currentValidators.Map() - - // Rebuild primary network validators at [height] - for diffHeight := currentHeight; diffHeight > targetHeight; diffHeight-- { - weightDiffs, err := m.state.GetValidatorWeightDiffs(diffHeight, constants.PlatformChainID) - if err != nil { - return nil, err - } - for nodeID, weightDiff := range weightDiffs { - if err := applyWeightDiff(validatorSet, nodeID, weightDiff); err != nil { - return nil, err - } - } - - pkDiffs, err := m.state.GetValidatorPublicKeyDiffs(diffHeight) - if err != nil { - return nil, err - } - for nodeID, pk := range pkDiffs { - if vdr, ok := validatorSet[nodeID]; ok { - // The validator's public key was removed at this block, so it - // was in the validator set before. - vdr.PublicKey = pk - } - } - } - return validatorSet, nil + currentHeight, err := m.getCurrentHeight(ctx) + return currentValidators.Map(), currentHeight, err } func (m *manager) makeSubnetValidatorSet( - currentHeight uint64, + ctx context.Context, targetHeight uint64, subnetID ids.ID, -) (map[ids.NodeID]*validators.GetValidatorOutput, error) { - currentValidators, ok := m.cfg.Validators.Get(subnetID) - if !ok { - currentValidators = validators.NewSet() - if err := m.state.ValidatorSet(subnetID, currentValidators); err != nil { - return nil, err - } +) (map[ids.NodeID]*validators.GetValidatorOutput, uint64, error) { + subnetValidatorSet, primaryValidatorSet, currentHeight, err := m.getCurrentValidatorSets(ctx, subnetID) + if err != nil { + return nil, 0, err } - - // Node ID --> Validator information for the node validating the Subnet. - subnetValidatorSet := currentValidators.Map() - - // Rebuild subnet validators at [targetHeight] - for diffHeight := currentHeight; diffHeight > targetHeight; diffHeight-- { - weightDiffs, err := m.state.GetValidatorWeightDiffs(diffHeight, subnetID) - if err != nil { - return nil, err - } - - for nodeID, weightDiff := range weightDiffs { - if err := applyWeightDiff(subnetValidatorSet, nodeID, weightDiff); err != nil { - return nil, err - } - } + if currentHeight < targetHeight { + return nil, 0, database.ErrNotFound } - // Get the public keys for all the validators at [targetHeight] - primarySet, err := m.getValidatorSetFrom(currentHeight, targetHeight, constants.PrimaryNetworkID) + // Rebuild subnet validators at [targetHeight] + // + // Note: Since we are attempting to generate the validator set at + // [targetHeight], we want to apply the diffs from + // (targetHeight, currentHeight]. Because the state interface is implemented + // to be inclusive, we apply diffs in [targetHeight + 1, currentHeight]. + lastDiffHeight := targetHeight + 1 + err = m.state.ApplyValidatorWeightDiffs( + ctx, + subnetValidatorSet, + currentHeight, + lastDiffHeight, + subnetID, + ) if err != nil { - return nil, err + return nil, 0, err } // Update the subnet validator set to include the public keys at - // [targetHeight]. - for nodeID, subnetValidator := range subnetValidatorSet { - primaryValidator, ok := primarySet[nodeID] + // [currentHeight]. When we apply the public key diffs, we will convert + // these keys to represent the public keys at [targetHeight]. + for nodeID, vdr := range subnetValidatorSet { + primaryVdr, ok := primaryValidatorSet[nodeID] if !ok { // This should never happen m.log.Error(ErrMissingValidator.Error(), - zap.Stringer("nodeID", nodeID), zap.Stringer("subnetID", subnetID), + zap.Stringer("nodeID", vdr.NodeID), ) - return nil, ErrMissingValidator + return nil, 0, ErrMissingValidator } - subnetValidator.PublicKey = primaryValidator.PublicKey + + vdr.PublicKey = primaryVdr.PublicKey } - return subnetValidatorSet, nil + err = m.state.ApplyValidatorPublicKeyDiffs( + ctx, + subnetValidatorSet, + currentHeight, + lastDiffHeight, + ) + return subnetValidatorSet, currentHeight, err } -func applyWeightDiff( - targetSet map[ids.NodeID]*validators.GetValidatorOutput, - nodeID ids.NodeID, - weightDiff *state.ValidatorWeightDiff, -) error { - vdr, ok := targetSet[nodeID] +func (m *manager) getCurrentValidatorSets( + ctx context.Context, + subnetID ids.ID, +) (map[ids.NodeID]*validators.GetValidatorOutput, map[ids.NodeID]*validators.GetValidatorOutput, uint64, error) { + currentSubnetValidators, ok := m.cfg.Validators.Get(subnetID) if !ok { - // This node isn't in the current validator set. - vdr = &validators.GetValidatorOutput{ - NodeID: nodeID, + // TODO: Require that the current validator set for all subnets is + // included in the validator manager. + currentSubnetValidators = validators.NewSet() + err := m.state.ValidatorSet(subnetID, currentSubnetValidators) + if err != nil { + return nil, nil, 0, err } - targetSet[nodeID] = vdr } - // The weight of this node changed at this block. - var err error - if weightDiff.Decrease { - // The validator's weight was decreased at this block, so in the - // prior block it was higher. - vdr.Weight, err = math.Add64(vdr.Weight, weightDiff.Amount) - } else { - // The validator's weight was increased at this block, so in the - // prior block it was lower. - vdr.Weight, err = math.Sub(vdr.Weight, weightDiff.Amount) - } - if err != nil { - return err + currentPrimaryValidators, ok := m.cfg.Validators.Get(constants.PrimaryNetworkID) + if !ok { + // This should never happen + m.log.Error(ErrMissingValidatorSet.Error(), + zap.Stringer("subnetID", constants.PrimaryNetworkID), + ) + return nil, nil, 0, ErrMissingValidatorSet } - if vdr.Weight == 0 { - // The validator's weight was 0 before this block so - // they weren't in the validator set. - delete(targetSet, nodeID) - } - return nil + currentHeight, err := m.getCurrentHeight(ctx) + return currentSubnetValidators.Map(), currentPrimaryValidators.Map(), currentHeight, err } func (m *manager) GetSubnetID(_ context.Context, chainID ids.ID) (ids.ID, error) { diff --git a/vms/platformvm/validators/manager_benchmark_test.go b/vms/platformvm/validators/manager_benchmark_test.go new file mode 100644 index 00000000000..65e2e9b9db7 --- /dev/null +++ b/vms/platformvm/validators/manager_benchmark_test.go @@ -0,0 +1,278 @@ +// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package validators + +import ( + "context" + "math/rand" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/stretchr/testify/require" + + "github.com/ava-labs/avalanchego/database/leveldb" + "github.com/ava-labs/avalanchego/ids" + "github.com/ava-labs/avalanchego/snow" + "github.com/ava-labs/avalanchego/snow/validators" + "github.com/ava-labs/avalanchego/utils" + "github.com/ava-labs/avalanchego/utils/constants" + "github.com/ava-labs/avalanchego/utils/crypto/bls" + "github.com/ava-labs/avalanchego/utils/formatting" + "github.com/ava-labs/avalanchego/utils/formatting/address" + "github.com/ava-labs/avalanchego/utils/json" + "github.com/ava-labs/avalanchego/utils/logging" + "github.com/ava-labs/avalanchego/utils/timer/mockable" + "github.com/ava-labs/avalanchego/utils/units" + "github.com/ava-labs/avalanchego/vms/platformvm/api" + "github.com/ava-labs/avalanchego/vms/platformvm/blocks" + "github.com/ava-labs/avalanchego/vms/platformvm/config" + "github.com/ava-labs/avalanchego/vms/platformvm/metrics" + "github.com/ava-labs/avalanchego/vms/platformvm/reward" + "github.com/ava-labs/avalanchego/vms/platformvm/state" + "github.com/ava-labs/avalanchego/vms/platformvm/txs" +) + +// BenchmarkGetValidatorSet generates 10k diffs and calculates the time to +// generate the genesis validator set by applying them. +// +// This generates a single diff for each height. In practice there could be +// multiple or zero diffs at a given height. +// +// Note: BenchmarkGetValidatorSet gets the validator set of a subnet rather than +// the primary network because the primary network performs caching that would +// interfere with the benchmark. +func BenchmarkGetValidatorSet(b *testing.B) { + require := require.New(b) + + db, err := leveldb.New( + b.TempDir(), + nil, + logging.NoLog{}, + "", + prometheus.NewRegistry(), + ) + require.NoError(err) + defer func() { + require.NoError(db.Close()) + }() + + avaxAssetID := ids.GenerateTestID() + genesisTime := time.Now().Truncate(time.Second) + genesisEndTime := genesisTime.Add(28 * 24 * time.Hour) + + addr, err := address.FormatBech32(constants.UnitTestHRP, ids.GenerateTestShortID().Bytes()) + require.NoError(err) + + genesisValidators := []api.PermissionlessValidator{{ + Staker: api.Staker{ + StartTime: json.Uint64(genesisTime.Unix()), + EndTime: json.Uint64(genesisEndTime.Unix()), + NodeID: ids.GenerateTestNodeID(), + }, + RewardOwner: &api.Owner{ + Threshold: 1, + Addresses: []string{addr}, + }, + Staked: []api.UTXO{{ + Amount: json.Uint64(2 * units.KiloAvax), + Address: addr, + }}, + DelegationFee: reward.PercentDenominator, + }} + + buildGenesisArgs := api.BuildGenesisArgs{ + NetworkID: json.Uint32(constants.UnitTestID), + AvaxAssetID: avaxAssetID, + UTXOs: nil, + Validators: genesisValidators, + Chains: nil, + Time: json.Uint64(genesisTime.Unix()), + InitialSupply: json.Uint64(360 * units.MegaAvax), + Encoding: formatting.Hex, + } + + buildGenesisResponse := api.BuildGenesisReply{} + platformvmSS := api.StaticService{} + require.NoError(platformvmSS.BuildGenesis(nil, &buildGenesisArgs, &buildGenesisResponse)) + + genesisBytes, err := formatting.Decode(buildGenesisResponse.Encoding, buildGenesisResponse.Bytes) + require.NoError(err) + + vdrs := validators.NewManager() + vdrs.Add(constants.PrimaryNetworkID, validators.NewSet()) + + execConfig, err := config.GetExecutionConfig(nil) + require.NoError(err) + + metrics, err := metrics.New("", prometheus.NewRegistry()) + require.NoError(err) + + s, err := state.New( + db, + genesisBytes, + prometheus.NewRegistry(), + &config.Config{ + Validators: vdrs, + }, + execConfig, + &snow.Context{ + NetworkID: constants.UnitTestID, + NodeID: ids.GenerateTestNodeID(), + Log: logging.NoLog{}, + }, + metrics, + reward.NewCalculator(reward.Config{ + MaxConsumptionRate: .12 * reward.PercentDenominator, + MinConsumptionRate: .10 * reward.PercentDenominator, + MintingPeriod: 365 * 24 * time.Hour, + SupplyCap: 720 * units.MegaAvax, + }), + new(utils.Atomic[bool]), + ) + require.NoError(err) + + m := NewManager( + logging.NoLog{}, + config.Config{ + Validators: vdrs, + }, + s, + metrics, + new(mockable.Clock), + ) + + var ( + nodeIDs []ids.NodeID + currentHeight uint64 + ) + for i := 0; i < 50; i++ { + currentHeight++ + nodeID, err := addPrimaryValidator(s, genesisTime, genesisEndTime, currentHeight) + require.NoError(err) + nodeIDs = append(nodeIDs, nodeID) + } + subnetID := ids.GenerateTestID() + for _, nodeID := range nodeIDs { + currentHeight++ + require.NoError(addSubnetValidator(s, subnetID, genesisTime, genesisEndTime, nodeID, currentHeight)) + } + for i := 0; i < 9900; i++ { + currentHeight++ + require.NoError(addSubnetDelegator(s, subnetID, genesisTime, genesisEndTime, nodeIDs, currentHeight)) + } + + ctx := context.Background() + height, err := m.GetCurrentHeight(ctx) + require.NoError(err) + require.Equal(currentHeight, height) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := m.GetValidatorSet(ctx, 0, subnetID) + require.NoError(err) + } + + b.StopTimer() +} + +func addPrimaryValidator( + s state.State, + startTime time.Time, + endTime time.Time, + height uint64, +) (ids.NodeID, error) { + sk, err := bls.NewSecretKey() + if err != nil { + return ids.EmptyNodeID, err + } + + nodeID := ids.GenerateTestNodeID() + s.PutCurrentValidator(&state.Staker{ + TxID: ids.GenerateTestID(), + NodeID: nodeID, + PublicKey: bls.PublicFromSecretKey(sk), + SubnetID: constants.PrimaryNetworkID, + Weight: 2 * units.MegaAvax, + StartTime: startTime, + EndTime: endTime, + PotentialReward: 0, + NextTime: endTime, + Priority: txs.PrimaryNetworkValidatorCurrentPriority, + }) + + blk, err := blocks.NewBanffStandardBlock(startTime, ids.GenerateTestID(), height, nil) + if err != nil { + return ids.EmptyNodeID, err + } + + s.AddStatelessBlock(blk) + s.SetHeight(height) + return nodeID, s.Commit() +} + +func addSubnetValidator( + s state.State, + subnetID ids.ID, + startTime time.Time, + endTime time.Time, + nodeID ids.NodeID, + height uint64, +) error { + s.PutCurrentValidator(&state.Staker{ + TxID: ids.GenerateTestID(), + NodeID: nodeID, + SubnetID: subnetID, + Weight: 1 * units.Avax, + StartTime: startTime, + EndTime: endTime, + PotentialReward: 0, + NextTime: endTime, + Priority: txs.SubnetPermissionlessValidatorCurrentPriority, + }) + + blk, err := blocks.NewBanffStandardBlock(startTime, ids.GenerateTestID(), height, nil) + if err != nil { + return err + } + + s.AddStatelessBlock(blk) + s.SetHeight(height) + return s.Commit() +} + +func addSubnetDelegator( + s state.State, + subnetID ids.ID, + startTime time.Time, + endTime time.Time, + nodeIDs []ids.NodeID, + height uint64, +) error { + i := rand.Intn(len(nodeIDs)) //#nosec G404 + nodeID := nodeIDs[i] + s.PutCurrentDelegator(&state.Staker{ + TxID: ids.GenerateTestID(), + NodeID: nodeID, + SubnetID: subnetID, + Weight: 1 * units.Avax, + StartTime: startTime, + EndTime: endTime, + PotentialReward: 0, + NextTime: endTime, + Priority: txs.SubnetPermissionlessDelegatorCurrentPriority, + }) + + blk, err := blocks.NewBanffStandardBlock(startTime, ids.GenerateTestID(), height, nil) + if err != nil { + return err + } + + s.AddStatelessBlock(blk) + s.SetLastAccepted(blk.ID()) + s.SetHeight(height) + return s.Commit() +} diff --git a/vms/platformvm/validators/manager_test.go b/vms/platformvm/validators/manager_test.go deleted file mode 100644 index c56f94fae6e..00000000000 --- a/vms/platformvm/validators/manager_test.go +++ /dev/null @@ -1,452 +0,0 @@ -// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved. -// See the file LICENSE for licensing terms. - -package validators - -import ( - "context" - "testing" - "time" - - "github.com/golang/mock/gomock" - - "github.com/prometheus/client_golang/prometheus" - - "github.com/stretchr/testify/require" - - "github.com/ava-labs/avalanchego/chains" - "github.com/ava-labs/avalanchego/database" - "github.com/ava-labs/avalanchego/ids" - "github.com/ava-labs/avalanchego/snow/uptime" - "github.com/ava-labs/avalanchego/snow/validators" - "github.com/ava-labs/avalanchego/utils/constants" - "github.com/ava-labs/avalanchego/utils/crypto/bls" - "github.com/ava-labs/avalanchego/utils/logging" - "github.com/ava-labs/avalanchego/utils/timer/mockable" - "github.com/ava-labs/avalanchego/utils/units" - "github.com/ava-labs/avalanchego/vms/platformvm/blocks" - "github.com/ava-labs/avalanchego/vms/platformvm/config" - "github.com/ava-labs/avalanchego/vms/platformvm/metrics" - "github.com/ava-labs/avalanchego/vms/platformvm/reward" - "github.com/ava-labs/avalanchego/vms/platformvm/state" -) - -var defaultRewardConfig = reward.Config{ - MaxConsumptionRate: .12 * reward.PercentDenominator, - MinConsumptionRate: .10 * reward.PercentDenominator, - MintingPeriod: 365 * 24 * time.Hour, - SupplyCap: 720 * units.MegaAvax, -} - -func TestVM_GetValidatorSet(t *testing.T) { - // Populate the validator set to use below - var ( - numVdrs = 4 - vdrBaseWeight = uint64(1_000) - testValidators []*validators.GetValidatorOutput - ) - - for i := 0; i < numVdrs; i++ { - sk, err := bls.NewSecretKey() - require.NoError(t, err) - - testValidators = append(testValidators, &validators.GetValidatorOutput{ - NodeID: ids.GenerateTestNodeID(), - PublicKey: bls.PublicFromSecretKey(sk), - Weight: vdrBaseWeight + uint64(i), - }) - } - - type test struct { - name string - // Height we're getting the diff at - height uint64 - lastAcceptedHeight uint64 - subnetID ids.ID - // Validator sets at tip - currentPrimaryNetworkValidators map[ids.NodeID]*validators.GetValidatorOutput - currentSubnetValidators map[ids.NodeID]*validators.GetValidatorOutput - - // height --> nodeID --> weightDiff - weightDiffs map[uint64]map[ids.NodeID]*state.ValidatorWeightDiff - - // height --> nodeID --> pkDiff - pkDiffs map[uint64]map[ids.NodeID]*bls.PublicKey - expectedVdrSet map[ids.NodeID]*validators.GetValidatorOutput - expectedErr error - } - - tests := []test{ - { - name: "after tip", - height: 1, - lastAcceptedHeight: 0, - expectedVdrSet: map[ids.NodeID]*validators.GetValidatorOutput{}, - expectedErr: database.ErrNotFound, - }, - { - name: "at tip", - subnetID: constants.PrimaryNetworkID, - height: 1, - lastAcceptedHeight: 1, - currentPrimaryNetworkValidators: map[ids.NodeID]*validators.GetValidatorOutput{ - testValidators[0].NodeID: copyPrimaryValidator(testValidators[0]), - }, - currentSubnetValidators: map[ids.NodeID]*validators.GetValidatorOutput{ - testValidators[0].NodeID: copySubnetValidator(testValidators[0]), - }, - expectedVdrSet: map[ids.NodeID]*validators.GetValidatorOutput{ - testValidators[0].NodeID: { - NodeID: testValidators[0].NodeID, - PublicKey: testValidators[0].PublicKey, - Weight: testValidators[0].Weight, - }, - }, - expectedErr: nil, - }, - { - name: "1 before tip", - height: 2, - lastAcceptedHeight: 3, - currentPrimaryNetworkValidators: map[ids.NodeID]*validators.GetValidatorOutput{ - testValidators[0].NodeID: copyPrimaryValidator(testValidators[0]), - testValidators[1].NodeID: copyPrimaryValidator(testValidators[1]), - }, - currentSubnetValidators: map[ids.NodeID]*validators.GetValidatorOutput{ - // At tip we have these 2 validators - testValidators[0].NodeID: copySubnetValidator(testValidators[0]), - testValidators[1].NodeID: copySubnetValidator(testValidators[1]), - }, - weightDiffs: map[uint64]map[ids.NodeID]*state.ValidatorWeightDiff{ - 3: { - // At the tip block vdrs[0] lost weight, vdrs[1] gained weight, - // and vdrs[2] left - testValidators[0].NodeID: { - Decrease: true, - Amount: 1, - }, - testValidators[1].NodeID: { - Decrease: false, - Amount: 1, - }, - testValidators[2].NodeID: { - Decrease: true, - Amount: testValidators[2].Weight, - }, - }, - }, - pkDiffs: map[uint64]map[ids.NodeID]*bls.PublicKey{ - 3: { - testValidators[2].NodeID: testValidators[2].PublicKey, - }, - }, - expectedVdrSet: map[ids.NodeID]*validators.GetValidatorOutput{ - testValidators[0].NodeID: { - NodeID: testValidators[0].NodeID, - PublicKey: testValidators[0].PublicKey, - Weight: testValidators[0].Weight + 1, - }, - testValidators[1].NodeID: { - NodeID: testValidators[1].NodeID, - PublicKey: testValidators[1].PublicKey, - Weight: testValidators[1].Weight - 1, - }, - testValidators[2].NodeID: { - NodeID: testValidators[2].NodeID, - PublicKey: testValidators[2].PublicKey, - Weight: testValidators[2].Weight, - }, - }, - expectedErr: nil, - }, - { - name: "2 before tip", - height: 3, - lastAcceptedHeight: 5, - currentPrimaryNetworkValidators: map[ids.NodeID]*validators.GetValidatorOutput{ - testValidators[0].NodeID: copyPrimaryValidator(testValidators[0]), - testValidators[1].NodeID: copyPrimaryValidator(testValidators[1]), - }, - currentSubnetValidators: map[ids.NodeID]*validators.GetValidatorOutput{ - // At tip we have these 2 validators - testValidators[0].NodeID: copySubnetValidator(testValidators[0]), - testValidators[1].NodeID: copySubnetValidator(testValidators[1]), - }, - weightDiffs: map[uint64]map[ids.NodeID]*state.ValidatorWeightDiff{ - 5: { - // At the tip block vdrs[0] lost weight, vdrs[1] gained weight, - // and vdrs[2] left - testValidators[0].NodeID: { - Decrease: true, - Amount: 1, - }, - testValidators[1].NodeID: { - Decrease: false, - Amount: 1, - }, - testValidators[2].NodeID: { - Decrease: true, - Amount: testValidators[2].Weight, - }, - }, - 4: { - // At the block before tip vdrs[0] lost weight, vdrs[1] gained weight, - // vdrs[2] joined - testValidators[0].NodeID: { - Decrease: true, - Amount: 1, - }, - testValidators[1].NodeID: { - Decrease: false, - Amount: 1, - }, - testValidators[2].NodeID: { - Decrease: false, - Amount: testValidators[2].Weight, - }, - }, - }, - pkDiffs: map[uint64]map[ids.NodeID]*bls.PublicKey{ - 5: { - testValidators[2].NodeID: testValidators[2].PublicKey, - }, - 4: {}, - }, - expectedVdrSet: map[ids.NodeID]*validators.GetValidatorOutput{ - testValidators[0].NodeID: { - NodeID: testValidators[0].NodeID, - PublicKey: testValidators[0].PublicKey, - Weight: testValidators[0].Weight + 2, - }, - testValidators[1].NodeID: { - NodeID: testValidators[1].NodeID, - PublicKey: testValidators[1].PublicKey, - Weight: testValidators[1].Weight - 2, - }, - }, - expectedErr: nil, - }, - { - name: "1 before tip; nil public key", - height: 4, - lastAcceptedHeight: 5, - currentPrimaryNetworkValidators: map[ids.NodeID]*validators.GetValidatorOutput{ - testValidators[0].NodeID: copyPrimaryValidator(testValidators[0]), - testValidators[1].NodeID: copyPrimaryValidator(testValidators[1]), - }, - currentSubnetValidators: map[ids.NodeID]*validators.GetValidatorOutput{ - // At tip we have these 2 validators - testValidators[0].NodeID: copySubnetValidator(testValidators[0]), - testValidators[1].NodeID: copySubnetValidator(testValidators[1]), - }, - weightDiffs: map[uint64]map[ids.NodeID]*state.ValidatorWeightDiff{ - 5: { - // At the tip block vdrs[0] lost weight, vdrs[1] gained weight, - // and vdrs[2] left - testValidators[0].NodeID: { - Decrease: true, - Amount: 1, - }, - testValidators[1].NodeID: { - Decrease: false, - Amount: 1, - }, - testValidators[2].NodeID: { - Decrease: true, - Amount: testValidators[2].Weight, - }, - }, - }, - pkDiffs: map[uint64]map[ids.NodeID]*bls.PublicKey{ - 5: {}, - }, - expectedVdrSet: map[ids.NodeID]*validators.GetValidatorOutput{ - testValidators[0].NodeID: { - NodeID: testValidators[0].NodeID, - PublicKey: testValidators[0].PublicKey, - Weight: testValidators[0].Weight + 1, - }, - testValidators[1].NodeID: { - NodeID: testValidators[1].NodeID, - PublicKey: testValidators[1].PublicKey, - Weight: testValidators[1].Weight - 1, - }, - testValidators[2].NodeID: { - NodeID: testValidators[2].NodeID, - Weight: testValidators[2].Weight, - }, - }, - expectedErr: nil, - }, - { - name: "1 before tip; subnet", - height: 5, - lastAcceptedHeight: 6, - subnetID: ids.GenerateTestID(), - currentPrimaryNetworkValidators: map[ids.NodeID]*validators.GetValidatorOutput{ - testValidators[0].NodeID: copyPrimaryValidator(testValidators[0]), - testValidators[1].NodeID: copyPrimaryValidator(testValidators[1]), - testValidators[3].NodeID: copyPrimaryValidator(testValidators[3]), - }, - currentSubnetValidators: map[ids.NodeID]*validators.GetValidatorOutput{ - // At tip we have these 2 validators - testValidators[0].NodeID: copySubnetValidator(testValidators[0]), - testValidators[1].NodeID: copySubnetValidator(testValidators[1]), - }, - weightDiffs: map[uint64]map[ids.NodeID]*state.ValidatorWeightDiff{ - 6: { - // At the tip block vdrs[0] lost weight, vdrs[1] gained weight, - // and vdrs[2] left - testValidators[0].NodeID: { - Decrease: true, - Amount: 1, - }, - testValidators[1].NodeID: { - Decrease: false, - Amount: 1, - }, - testValidators[2].NodeID: { - Decrease: true, - Amount: testValidators[2].Weight, - }, - }, - }, - pkDiffs: map[uint64]map[ids.NodeID]*bls.PublicKey{ - 6: {}, - }, - expectedVdrSet: map[ids.NodeID]*validators.GetValidatorOutput{ - testValidators[0].NodeID: { - NodeID: testValidators[0].NodeID, - PublicKey: testValidators[0].PublicKey, - Weight: testValidators[0].Weight + 1, - }, - testValidators[1].NodeID: { - NodeID: testValidators[1].NodeID, - PublicKey: testValidators[1].PublicKey, - Weight: testValidators[1].Weight - 1, - }, - testValidators[2].NodeID: { - NodeID: testValidators[2].NodeID, - Weight: testValidators[2].Weight, - }, - }, - expectedErr: nil, - }, - { - name: "unrelated primary network key removal on subnet lookup", - height: 4, - lastAcceptedHeight: 5, - subnetID: ids.GenerateTestID(), - currentPrimaryNetworkValidators: map[ids.NodeID]*validators.GetValidatorOutput{ - testValidators[0].NodeID: copyPrimaryValidator(testValidators[0]), - }, - currentSubnetValidators: map[ids.NodeID]*validators.GetValidatorOutput{ - testValidators[0].NodeID: copySubnetValidator(testValidators[0]), - }, - weightDiffs: map[uint64]map[ids.NodeID]*state.ValidatorWeightDiff{ - 5: {}, - }, - pkDiffs: map[uint64]map[ids.NodeID]*bls.PublicKey{ - 5: { - testValidators[1].NodeID: testValidators[1].PublicKey, - }, - }, - expectedVdrSet: map[ids.NodeID]*validators.GetValidatorOutput{ - testValidators[0].NodeID: { - NodeID: testValidators[0].NodeID, - PublicKey: testValidators[0].PublicKey, - Weight: testValidators[0].Weight, - }, - }, - expectedErr: nil, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - r := require.New(t) - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - // setup validators set - vdrs := validators.NewMockManager(ctrl) - cfg := config.Config{ - Chains: chains.TestManager, - UptimePercentage: .2, - RewardConfig: defaultRewardConfig, - Validators: vdrs, - UptimeLockedCalculator: uptime.NewLockedCalculator(), - BanffTime: mockable.MaxTime, - } - mockState := state.NewMockState(ctrl) - - metrics, err := metrics.New("", prometheus.NewRegistry()) - r.NoError(err) - - clk := &mockable.Clock{} - validatorSet := NewManager(logging.NoLog{}, cfg, mockState, metrics, clk) - - // Mock the VM's validators - mockPrimaryVdrSet := validators.NewMockSet(ctrl) - mockPrimaryVdrSet.EXPECT().Map().Return(tt.currentPrimaryNetworkValidators).AnyTimes() - vdrs.EXPECT().Get(constants.PrimaryNetworkID).Return(mockPrimaryVdrSet, true).AnyTimes() - - if tt.subnetID != constants.PrimaryNetworkID { - mockSubnetVdrSet := validators.NewMockSet(ctrl) - mockSubnetVdrSet.EXPECT().Map().Return(tt.currentSubnetValidators).AnyTimes() - vdrs.EXPECT().Get(tt.subnetID).Return(mockSubnetVdrSet, true).AnyTimes() - } - - for _, vdr := range testValidators { - _, current := tt.currentPrimaryNetworkValidators[vdr.NodeID] - if current { - mockPrimaryVdrSet.EXPECT().Get(vdr.NodeID).Return( - &validators.Validator{ - NodeID: vdr.NodeID, - PublicKey: vdr.PublicKey, - Weight: vdr.Weight, - }, - true, - ).AnyTimes() - } else { - mockPrimaryVdrSet.EXPECT().Get(vdr.NodeID).Return(nil, false).AnyTimes() - } - } - - // Tell state what diffs to report - for height, weightDiff := range tt.weightDiffs { - mockState.EXPECT().GetValidatorWeightDiffs(height, gomock.Any()).Return(weightDiff, nil).AnyTimes() - } - - for height, pkDiff := range tt.pkDiffs { - mockState.EXPECT().GetValidatorPublicKeyDiffs(height).Return(pkDiff, nil) - } - - // Tell state last accepted block to report - mockTip := blocks.NewMockBlock(ctrl) - mockTip.EXPECT().Height().Return(tt.lastAcceptedHeight) - mockTipID := ids.GenerateTestID() - mockState.EXPECT().GetLastAccepted().Return(mockTipID) - mockState.EXPECT().GetStatelessBlock(mockTipID).Return(mockTip, nil) - - // Compute validator set at previous height - gotVdrSet, err := validatorSet.GetValidatorSet(context.Background(), tt.height, tt.subnetID) - r.ErrorIs(err, tt.expectedErr) - if tt.expectedErr != nil { - return - } - r.Equal(tt.expectedVdrSet, gotVdrSet) - }) - } -} - -func copyPrimaryValidator(vdr *validators.GetValidatorOutput) *validators.GetValidatorOutput { - newVdr := *vdr - return &newVdr -} - -func copySubnetValidator(vdr *validators.GetValidatorOutput) *validators.GetValidatorOutput { - newVdr := *vdr - newVdr.PublicKey = nil - return &newVdr -}