From 864dbec3964dd4e0cc9080f19f1b5d1efa773685 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Sun, 14 Jan 2024 02:03:09 -0500 Subject: [PATCH] Replace `PeerListAck` with `GetPeerList` (#2580) --- config/config.go | 6 + config/flags.go | 2 + config/keys.go | 2 + genesis/bootstrappers.go | 9 +- message/messages_test.go | 87 ++- message/mock_outbound_message_builder.go | 40 +- message/ops.go | 22 +- message/outbound_msg_builder.go | 61 +- network/README.md | 69 +- network/certs_test.go | 24 + network/config.go | 12 +- network/ip_tracker.go | 364 +++++++++ network/ip_tracker_test.go | 690 +++++++++++++++++ network/network.go | 491 ++++-------- network/network_test.go | 92 +-- network/peer/gossip_tracker.go | 323 -------- network/peer/gossip_tracker_callback.go | 56 -- network/peer/gossip_tracker_metrics.go | 40 - network/peer/gossip_tracker_test.go | 620 --------------- network/peer/network.go | 25 +- network/peer/peer.go | 223 ++++-- network/peer/test_network.go | 16 +- network/test_network.go | 10 +- node/node.go | 13 - proto/p2p/p2p.proto | 45 +- proto/pb/p2p/p2p.pb.go | 943 ++++++++++++----------- utils/bloom/read_filter.go | 17 + utils/constants/networking.go | 2 + utils/ips/claimed_ip_port.go | 44 +- utils/ips/ip_port.go | 5 +- 30 files changed, 2142 insertions(+), 2211 deletions(-) create mode 100644 network/ip_tracker.go create mode 100644 network/ip_tracker_test.go delete mode 100644 network/peer/gossip_tracker.go delete mode 100644 network/peer/gossip_tracker_callback.go delete mode 100644 network/peer/gossip_tracker_metrics.go delete mode 100644 network/peer/gossip_tracker_test.go diff --git a/config/config.go b/config/config.go index e4ae0816c043..5de3e70777b4 100644 --- a/config/config.go +++ b/config/config.go @@ -427,6 +427,8 @@ func getNetworkConfig( PeerListNonValidatorGossipSize: v.GetUint32(NetworkPeerListNonValidatorGossipSizeKey), PeerListPeersGossipSize: v.GetUint32(NetworkPeerListPeersGossipSizeKey), PeerListGossipFreq: v.GetDuration(NetworkPeerListGossipFreqKey), + PeerListPullGossipFreq: v.GetDuration(NetworkPeerListPullGossipFreqKey), + PeerListBloomResetFreq: v.GetDuration(NetworkPeerListBloomResetFreqKey), }, DelayConfig: network.DelayConfig{ @@ -462,6 +464,10 @@ func getNetworkConfig( return network.Config{}, fmt.Errorf("%q must be >= 0", NetworkOutboundConnectionTimeoutKey) case config.PeerListGossipFreq < 0: return network.Config{}, fmt.Errorf("%s must be >= 0", NetworkPeerListGossipFreqKey) + case config.PeerListPullGossipFreq < 0: + return network.Config{}, fmt.Errorf("%s must be >= 0", NetworkPeerListPullGossipFreqKey) + case config.PeerListBloomResetFreq < 0: + return network.Config{}, fmt.Errorf("%s must be >= 0", NetworkPeerListBloomResetFreqKey) case config.ThrottlerConfig.InboundMsgThrottlerConfig.CPUThrottlerConfig.MaxRecheckDelay < constants.MinInboundThrottlerMaxRecheckDelay: return network.Config{}, fmt.Errorf("%s must be >= %d", InboundThrottlerCPUMaxRecheckDelayKey, constants.MinInboundThrottlerMaxRecheckDelay) case config.ThrottlerConfig.InboundMsgThrottlerConfig.DiskThrottlerConfig.MaxRecheckDelay < constants.MinInboundThrottlerMaxRecheckDelay: diff --git a/config/flags.go b/config/flags.go index e98f132da6a5..6f85fedfd33c 100644 --- a/config/flags.go +++ b/config/flags.go @@ -132,6 +132,8 @@ func addNodeFlags(fs *pflag.FlagSet) { fs.Uint(NetworkPeerListNonValidatorGossipSizeKey, constants.DefaultNetworkPeerListNonValidatorGossipSize, "Number of non-validators that the node will gossip peer list to") fs.Uint(NetworkPeerListPeersGossipSizeKey, constants.DefaultNetworkPeerListPeersGossipSize, "Number of total peers (including non-validators and validators) that the node will gossip peer list to") fs.Duration(NetworkPeerListGossipFreqKey, constants.DefaultNetworkPeerListGossipFreq, "Frequency to gossip peers to other nodes") + fs.Duration(NetworkPeerListPullGossipFreqKey, constants.DefaultNetworkPeerListPullGossipFreq, "Frequency to request peers from other nodes") + fs.Duration(NetworkPeerListBloomResetFreqKey, constants.DefaultNetworkPeerListBloomResetFreq, "Frequency to recalculate the bloom filter used to request new peers from other nodes") // Public IP Resolution fs.String(PublicIPKey, "", "Public IP of this node for P2P communication") diff --git a/config/keys.go b/config/keys.go index cdd90bf2b7f8..b2ccc16fc63d 100644 --- a/config/keys.go +++ b/config/keys.go @@ -94,6 +94,8 @@ const ( NetworkPeerListNonValidatorGossipSizeKey = "network-peer-list-non-validator-gossip-size" NetworkPeerListPeersGossipSizeKey = "network-peer-list-peers-gossip-size" NetworkPeerListGossipFreqKey = "network-peer-list-gossip-frequency" + NetworkPeerListPullGossipFreqKey = "network-peer-list-pull-gossip-frequency" + NetworkPeerListBloomResetFreqKey = "network-peer-list-bloom-reset-frequency" NetworkInitialReconnectDelayKey = "network-initial-reconnect-delay" NetworkReadHandshakeTimeoutKey = "network-read-handshake-timeout" NetworkPingTimeoutKey = "network-ping-timeout" diff --git a/genesis/bootstrappers.go b/genesis/bootstrappers.go index cac90bb50301..b3e3a2a24f0c 100644 --- a/genesis/bootstrappers.go +++ b/genesis/bootstrappers.go @@ -36,10 +36,15 @@ type Bootstrapper struct { IP ips.IPDesc `json:"ip"` } +// GetBootstrappers returns all default bootstrappers for the provided network. +func GetBootstrappers(networkID uint32) []Bootstrapper { + networkName := constants.NetworkIDToNetworkName[networkID] + return bootstrappersPerNetwork[networkName] +} + // SampleBootstrappers returns the some beacons this node should connect to func SampleBootstrappers(networkID uint32, count int) []Bootstrapper { - networkName := constants.NetworkIDToNetworkName[networkID] - bootstrappers := bootstrappersPerNetwork[networkName] + bootstrappers := GetBootstrappers(networkID) count = math.Min(count, len(bootstrappers)) s := sampler.NewUniform() diff --git a/message/messages_test.go b/message/messages_test.go index e164e8993007..b259cefa341e 100644 --- a/message/messages_test.go +++ b/message/messages_test.go @@ -142,12 +142,63 @@ func TestMessage(t *testing.T) { bypassThrottling: true, bytesSaved: false, }, + { + desc: "get_peer_list message with no compression", + op: GetPeerListOp, + msg: &p2p.Message{ + Message: &p2p.Message_GetPeerList{ + GetPeerList: &p2p.GetPeerList{ + KnownPeers: &p2p.BloomFilter{ + Filter: make([]byte, 2048), + Salt: make([]byte, 32), + }, + }, + }, + }, + compressionType: compression.TypeNone, + bypassThrottling: false, + bytesSaved: false, + }, + { + desc: "get_peer_list message with gzip compression", + op: GetPeerListOp, + msg: &p2p.Message{ + Message: &p2p.Message_GetPeerList{ + GetPeerList: &p2p.GetPeerList{ + KnownPeers: &p2p.BloomFilter{ + Filter: make([]byte, 2048), + Salt: make([]byte, 32), + }, + }, + }, + }, + compressionType: compression.TypeGzip, + bypassThrottling: false, + bytesSaved: true, + }, + { + desc: "get_peer_list message with zstd compression", + op: GetPeerListOp, + msg: &p2p.Message{ + Message: &p2p.Message_GetPeerList{ + GetPeerList: &p2p.GetPeerList{ + KnownPeers: &p2p.BloomFilter{ + Filter: make([]byte, 2048), + Salt: make([]byte, 32), + }, + }, + }, + }, + compressionType: compression.TypeZstd, + bypassThrottling: false, + bytesSaved: true, + }, { desc: "peer_list message with no compression", op: PeerListOp, msg: &p2p.Message{ - Message: &p2p.Message_PeerList{ - PeerList: &p2p.PeerList{ + Message: &p2p.Message_PeerList_{ + PeerList_: &p2p.PeerList{ ClaimedIpPorts: []*p2p.ClaimedIpPort{ { X509Certificate: testTLSCert.Certificate[0], @@ -168,8 +219,8 @@ func TestMessage(t *testing.T) { desc: "peer_list message with gzip compression", op: PeerListOp, msg: &p2p.Message{ - Message: &p2p.Message_PeerList{ - PeerList: &p2p.PeerList{ + Message: &p2p.Message_PeerList_{ + PeerList_: &p2p.PeerList{ ClaimedIpPorts: []*p2p.ClaimedIpPort{ { X509Certificate: testTLSCert.Certificate[0], @@ -190,8 +241,8 @@ func TestMessage(t *testing.T) { desc: "peer_list message with zstd compression", op: PeerListOp, msg: &p2p.Message{ - Message: &p2p.Message_PeerList{ - PeerList: &p2p.PeerList{ + Message: &p2p.Message_PeerList_{ + PeerList_: &p2p.PeerList{ ClaimedIpPorts: []*p2p.ClaimedIpPort{ { X509Certificate: testTLSCert.Certificate[0], @@ -208,25 +259,6 @@ func TestMessage(t *testing.T) { bypassThrottling: true, bytesSaved: true, }, - { - desc: "peer_list_ack message with no compression", - op: PeerListAckOp, - msg: &p2p.Message{ - Message: &p2p.Message_PeerListAck{ - PeerListAck: &p2p.PeerListAck{ - PeerAcks: []*p2p.PeerAck{ - { - TxId: testID[:], - Timestamp: 1, - }, - }, - }, - }, - }, - compressionType: compression.TypeNone, - bypassThrottling: false, - bytesSaved: false, - }, { desc: "get_state_summary_frontier message with no compression", op: GetStateSummaryFrontierOp, @@ -836,8 +868,9 @@ func TestMessage(t *testing.T) { require.Equal(tv.bypassThrottling, encodedMsg.BypassThrottling()) require.Equal(tv.op, encodedMsg.Op()) - bytesSaved := encodedMsg.BytesSavedCompression() - require.Equal(tv.bytesSaved, bytesSaved > 0) + if bytesSaved := encodedMsg.BytesSavedCompression(); tv.bytesSaved { + require.Greater(bytesSaved, 0) + } parsedMsg, err := mb.parseInbound(encodedMsg.Bytes(), ids.EmptyNodeID, func() {}) require.NoError(err) diff --git a/message/mock_outbound_message_builder.go b/message/mock_outbound_message_builder.go index 50af02669546..21cb518976ae 100644 --- a/message/mock_outbound_message_builder.go +++ b/message/mock_outbound_message_builder.go @@ -232,6 +232,21 @@ func (mr *MockOutboundMsgBuilderMockRecorder) GetAncestors(arg0, arg1, arg2, arg return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAncestors", reflect.TypeOf((*MockOutboundMsgBuilder)(nil).GetAncestors), arg0, arg1, arg2, arg3, arg4) } +// GetPeerList mocks base method. +func (m *MockOutboundMsgBuilder) GetPeerList(arg0, arg1 []byte) (OutboundMessage, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPeerList", arg0, arg1) + ret0, _ := ret[0].(OutboundMessage) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPeerList indicates an expected call of GetPeerList. +func (mr *MockOutboundMsgBuilderMockRecorder) GetPeerList(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPeerList", reflect.TypeOf((*MockOutboundMsgBuilder)(nil).GetPeerList), arg0, arg1) +} + // GetStateSummaryFrontier mocks base method. func (m *MockOutboundMsgBuilder) GetStateSummaryFrontier(arg0 ids.ID, arg1 uint32, arg2 time.Duration) (OutboundMessage, error) { m.ctrl.T.Helper() @@ -248,22 +263,22 @@ func (mr *MockOutboundMsgBuilderMockRecorder) GetStateSummaryFrontier(arg0, arg1 } // Handshake mocks base method. -func (m *MockOutboundMsgBuilder) Handshake(arg0 uint32, arg1 uint64, arg2 ips.IPPort, arg3, arg4 string, arg5, arg6, arg7 uint32, arg8 uint64, arg9 []byte, arg10 []ids.ID, arg11, arg12 []uint32) (OutboundMessage, error) { +func (m *MockOutboundMsgBuilder) Handshake(arg0 uint32, arg1 uint64, arg2 ips.IPPort, arg3, arg4 string, arg5, arg6, arg7 uint32, arg8 uint64, arg9 []byte, arg10 []ids.ID, arg11, arg12 []uint32, arg13, arg14 []byte) (OutboundMessage, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Handshake", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12) + ret := m.ctrl.Call(m, "Handshake", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12, arg13, arg14) ret0, _ := ret[0].(OutboundMessage) ret1, _ := ret[1].(error) return ret0, ret1 } // Handshake indicates an expected call of Handshake. -func (mr *MockOutboundMsgBuilderMockRecorder) Handshake(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12 interface{}) *gomock.Call { +func (mr *MockOutboundMsgBuilderMockRecorder) Handshake(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12, arg13, arg14 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Handshake", reflect.TypeOf((*MockOutboundMsgBuilder)(nil).Handshake), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Handshake", reflect.TypeOf((*MockOutboundMsgBuilder)(nil).Handshake), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12, arg13, arg14) } // PeerList mocks base method. -func (m *MockOutboundMsgBuilder) PeerList(arg0 []ips.ClaimedIPPort, arg1 bool) (OutboundMessage, error) { +func (m *MockOutboundMsgBuilder) PeerList(arg0 []*ips.ClaimedIPPort, arg1 bool) (OutboundMessage, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "PeerList", arg0, arg1) ret0, _ := ret[0].(OutboundMessage) @@ -277,21 +292,6 @@ func (mr *MockOutboundMsgBuilderMockRecorder) PeerList(arg0, arg1 interface{}) * return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PeerList", reflect.TypeOf((*MockOutboundMsgBuilder)(nil).PeerList), arg0, arg1) } -// PeerListAck mocks base method. -func (m *MockOutboundMsgBuilder) PeerListAck(arg0 []*p2p.PeerAck) (OutboundMessage, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "PeerListAck", arg0) - ret0, _ := ret[0].(OutboundMessage) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// PeerListAck indicates an expected call of PeerListAck. -func (mr *MockOutboundMsgBuilderMockRecorder) PeerListAck(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PeerListAck", reflect.TypeOf((*MockOutboundMsgBuilder)(nil).PeerListAck), arg0) -} - // Ping mocks base method. func (m *MockOutboundMsgBuilder) Ping(arg0 uint32, arg1 []*p2p.SubnetUptime) (OutboundMessage, error) { m.ctrl.T.Helper() diff --git a/message/ops.go b/message/ops.go index 2f496713e419..0c58eb60690b 100644 --- a/message/ops.go +++ b/message/ops.go @@ -22,8 +22,8 @@ const ( PingOp Op = iota PongOp HandshakeOp + GetPeerListOp PeerListOp - PeerListAckOp // State sync: GetStateSummaryFrontierOp GetStateSummaryFrontierFailedOp @@ -72,8 +72,8 @@ var ( PingOp, PongOp, HandshakeOp, + GetPeerListOp, PeerListOp, - PeerListAckOp, } // List of all consensus request message types @@ -211,10 +211,10 @@ func (op Op) String() string { return "pong" case HandshakeOp: return "handshake" + case GetPeerListOp: + return "get_peerlist" case PeerListOp: return "peerlist" - case PeerListAckOp: - return "peerlist_ack" // State sync case GetStateSummaryFrontierOp: return "get_state_summary_frontier" @@ -305,10 +305,10 @@ func Unwrap(m *p2p.Message) (fmt.Stringer, error) { return msg.Pong, nil case *p2p.Message_Handshake: return msg.Handshake, nil - case *p2p.Message_PeerList: - return msg.PeerList, nil - case *p2p.Message_PeerListAck: - return msg.PeerListAck, nil + case *p2p.Message_GetPeerList: + return msg.GetPeerList, nil + case *p2p.Message_PeerList_: + return msg.PeerList_, nil // State sync: case *p2p.Message_GetStateSummaryFrontier: return msg.GetStateSummaryFrontier, nil @@ -364,10 +364,10 @@ func ToOp(m *p2p.Message) (Op, error) { return PongOp, nil case *p2p.Message_Handshake: return HandshakeOp, nil - case *p2p.Message_PeerList: + case *p2p.Message_GetPeerList: + return GetPeerListOp, nil + case *p2p.Message_PeerList_: return PeerListOp, nil - case *p2p.Message_PeerListAck: - return PeerListAckOp, nil case *p2p.Message_GetStateSummaryFrontier: return GetStateSummaryFrontierOp, nil case *p2p.Message_StateSummaryFrontier_: diff --git a/message/outbound_msg_builder.go b/message/outbound_msg_builder.go index f50f5cf6ae5d..150c9f4f3a65 100644 --- a/message/outbound_msg_builder.go +++ b/message/outbound_msg_builder.go @@ -32,15 +32,18 @@ type OutboundMsgBuilder interface { trackedSubnets []ids.ID, supportedACPs []uint32, objectedACPs []uint32, + knownPeersFilter []byte, + knownPeersSalt []byte, ) (OutboundMessage, error) - PeerList( - peers []ips.ClaimedIPPort, - bypassThrottling bool, + GetPeerList( + knownPeersFilter []byte, + knownPeersSalt []byte, ) (OutboundMessage, error) - PeerListAck( - peerAcks []*p2p.PeerAck, + PeerList( + peers []*ips.ClaimedIPPort, + bypassThrottling bool, ) (OutboundMessage, error) Ping( @@ -244,6 +247,8 @@ func (b *outMsgBuilder) Handshake( trackedSubnets []ids.ID, supportedACPs []uint32, objectedACPs []uint32, + knownPeersFilter []byte, + knownPeersSalt []byte, ) (OutboundMessage, error) { subnetIDBytes := make([][]byte, len(trackedSubnets)) encodeIDs(trackedSubnets, subnetIDBytes) @@ -267,6 +272,10 @@ func (b *outMsgBuilder) Handshake( }, SupportedAcps: supportedACPs, ObjectedAcps: objectedACPs, + KnownPeers: &p2p.BloomFilter{ + Filter: knownPeersFilter, + Salt: knownPeersSalt, + }, }, }, }, @@ -275,7 +284,27 @@ func (b *outMsgBuilder) Handshake( ) } -func (b *outMsgBuilder) PeerList(peers []ips.ClaimedIPPort, bypassThrottling bool) (OutboundMessage, error) { +func (b *outMsgBuilder) GetPeerList( + knownPeersFilter []byte, + knownPeersSalt []byte, +) (OutboundMessage, error) { + return b.builder.createOutbound( + &p2p.Message{ + Message: &p2p.Message_GetPeerList{ + GetPeerList: &p2p.GetPeerList{ + KnownPeers: &p2p.BloomFilter{ + Filter: knownPeersFilter, + Salt: knownPeersSalt, + }, + }, + }, + }, + b.compressionType, + false, + ) +} + +func (b *outMsgBuilder) PeerList(peers []*ips.ClaimedIPPort, bypassThrottling bool) (OutboundMessage, error) { claimIPPorts := make([]*p2p.ClaimedIpPort, len(peers)) for i, p := range peers { claimIPPorts[i] = &p2p.ClaimedIpPort{ @@ -284,13 +313,13 @@ func (b *outMsgBuilder) PeerList(peers []ips.ClaimedIPPort, bypassThrottling boo IpPort: uint32(p.IPPort.Port), Timestamp: p.Timestamp, Signature: p.Signature, - TxId: p.TxID[:], + TxId: ids.Empty[:], } } return b.builder.createOutbound( &p2p.Message{ - Message: &p2p.Message_PeerList{ - PeerList: &p2p.PeerList{ + Message: &p2p.Message_PeerList_{ + PeerList_: &p2p.PeerList{ ClaimedIpPorts: claimIPPorts, }, }, @@ -300,20 +329,6 @@ func (b *outMsgBuilder) PeerList(peers []ips.ClaimedIPPort, bypassThrottling boo ) } -func (b *outMsgBuilder) PeerListAck(peerAcks []*p2p.PeerAck) (OutboundMessage, error) { - return b.builder.createOutbound( - &p2p.Message{ - Message: &p2p.Message_PeerListAck{ - PeerListAck: &p2p.PeerListAck{ - PeerAcks: peerAcks, - }, - }, - }, - compression.TypeNone, - false, - ) -} - func (b *outMsgBuilder) GetStateSummaryFrontier( chainID ids.ID, requestID uint32, diff --git a/network/README.md b/network/README.md index 5e66e2e25f4f..303d1f56ab82 100644 --- a/network/README.md +++ b/network/README.md @@ -46,7 +46,7 @@ When starting an Avalanche node, a node needs to be able to initiate some proces In Avalanche, nodes connect to an initial set of bootstrapper nodes known as **beacons** (this is user-configurable). Once connected to a set of beacons, a node is able to discover other nodes in the network. Over time, a node eventually discovers other peers in the network through `PeerList` messages it receives through: - The handshake initiated between two peers when attempting to connect to a peer (see [Connecting](#connecting)). -- Periodic `PeerList` gossip messages that every peer sends to the peers it's connected to (see [Connected](#connected)). +- Responses to periodically sent `GetPeerList` messages requesting a `PeerList` of unknown peers (see [Connected](#connected)). #### Connecting @@ -79,7 +79,6 @@ Note right of Peer: LGTM! Note over Node,Peer: PeerList message Peer->>Node: Peer-X, Peer-Y, Peer-Z Note over Node,Peer: Handshake Complete -Node->>Peer: ACK Peer-X, Peer-Y, Peer-Z ``` Once the node attempting to join the network receives this `PeerList` message, the handshake is complete and the node is now connected to the peer. The node attempts to connect to the new peers discovered in the `PeerList` message. Each connection results in another peer handshake, which results in the node incrementally discovering more and more peers in the network as more and more `PeerList` messages are exchanged. @@ -92,71 +91,53 @@ Some peers aren't discovered through the `PeerList` messages exchanged through p sequenceDiagram Node ->> Peer-1: Handshake - v1.9.5 Peer-1 ->> Node: PeerList - Peer-2 -Node ->> Peer-1: ACK - Peer-2 Note left of Node: Node is connected to Peer-1 and now tries to connect to Peer-2. Node ->> Peer-2: Handshake - v1.9.5 Peer-2 ->> Node: PeerList - Peer-1 -Node ->> Peer-2: ACK - Peer-1 Note left of Node: Peer-3 was never sampled, so we haven't connected yet! Node --> Peer-3: No connection ``` -To guarantee that a node can discover all peers, each node periodically gossips a sample of the peers it knows about to other peers. +To guarantee that a node can discover all peers, each node periodically sends a `GetPeerList` message to a random peer. ##### PeerList Gossip ###### Messages -A `PeerList` is the message that is used to communicate the presence of peers in the network. Each `PeerList` message contains networking-level metadata about the peer that provides the necessary information to connect to it, alongside the corresponding transaction id that added that peer to the validator set. Transaction ids are unique hashes that only add a single validator, so it is guaranteed that there is a 1:1 mapping between a validator and its associated transaction id. +A `GetPeerList` message requests that the peer sends a `PeerList` message. `GetPeerList` messages contain a bloom filter of already known peers to reduce useless bandwidth on `PeerList` messages. The bloom filter reduces bandwidth by enabling the `PeerList` message to only include peers that aren't already known. -`PeerListAck` messages are sent in response to `PeerList` messages to allow a peer to confirm which peers it will actually attempt to connect to. Because nodes only gossip peers they believe another peer doesn't already know about to optimize bandwidth, `PeerListAck` messages are important to confirm that a peer will attempt to connect to someone. Without this, a node might gossip a peer to another peer and assume a connection between the two is being established, and not re-gossip the peer in future gossip cycles. If the connection was never actually wanted by the peer being gossiped to due to a transient reason, that peer would never be able to re-discover the gossiped peer and could be isolated from a subset of the network. +A `PeerList` is the message that is used to communicate the presence of peers in the network. Each `PeerList` message contains signed networking-level metadata about a peer that provides the necessary information to connect to it. -Once a `PeerListAck` message is received from a peer, the node that sent the original `PeerList` message marks the corresponding acknowledged validators as already having been transmitted to the peer, so that it's excluded from subsequent iterations of `PeerList` gossip. +Once peer metadata is received, the node will add that data to its bloom filter to prevent learning about it again. ###### Gossip Handshake messages provide a node with some knowledge of peers in the network, but offers no guarantee that learning about a subset of peers from each peer the node connects with will result in the node learning about every peer in the network. -In order to provide a probabilistic guarantee that all peers in the network will eventually learn of one another, each node periodically gossips a sample of the peers that they're aware of to a sample of the peers that they're connected to. Over time, this probabilistically guarantees that every peer will eventually learn of every other peer. +To provide an eventual guarantee that all peers learn of one another, each node periodically requests peers from a random peer. -To optimize bandwidth usage, each node tracks which peers are guaranteed to know of which peers. A node learns this information by tracking both inbound and outbound `PeerList` gossip. +To optimize bandwidth, each node tracks the most recent IPs of validators. The validator's nodeID and timestamp are inserted into a bloom filter which is used to select only necessary IPs to gossip. -- Inbound - - If a node ever receives `PeerList` from a peer, that peer _must_ have known about the peers in that `PeerList` message in order to have gossiped them. -- Outbound - - If a node sends a `PeerList` to a peer and the peer replies with an `PeerListAck` message, then all peers in the `PeerListAck` must be known by the peer. +As the number of entries increases in the bloom filter, the probability of a false positive increases. False positives can cause recent IPs not to be gossiped when they otherwise should be, slowing down the rate of `PeerList` gossip. To prevent the bloom filter from having too many false positives, a new bloom filter is periodically generated and the number of entries a validator is allowed to have in the bloom filter is capped. Generating the new bloom filter both removes stale entries and modifies the hash functions to avoid persistent hash collisions. -To efficiently track which peers know of which peers, the peers that each peer is aware of is represented in a [bit set](https://en.wikipedia.org/wiki/Bit_array). A peer is represented by either a `0` if it isn't known by the peer yet, or a `1` if it is known by the peer. - -A node follows the following steps for every cycle of `PeerList` gossip: - -1. Get a sample of peers in the network that the node is connected to -2. For each peer: - 1. Figure out which peers the node hasn't gossiped to them yet. - 2. Take a random sample of these unknown peers. - 3. Send a message describing these peers to the peer. +A node follows the following steps for of `PeerList` gossip: ```mermaid sequenceDiagram -Note left of Node: Initialize gossip bit set for Peer-123 -Note left of Node: Peer-123: [0, 0, 0] -Node->>Peer-123: PeerList - Peer-1 -Peer-123->>Node: PeerListAck - Peer-1 -Note left of Node: Peer-123: [1, 0, 0] -Node->>Peer-123: PeerList - Peer-3 -Peer-123->>Node: PeerListAck - Peer-3 -Note left of Node: Peer-123: [1, 0, 1] -Node->>Peer-123: PeerList - Peer-2 -Peer-123->>Node: PeerListAck - Peer-2 -Note left of Node: Peer-123: [1, 1, 1] -Note left of Node: No more gossip left to send to Peer-123! +Note left of Node: Initialize bloom filter +Note left of Node: Bloom: [0, 0, 0] +Node->>Peer-123: GetPeerList [0, 0, 0] +Note right of Peer-123: Any peers can be sent. +Peer-123->>Node: PeerList - Peer-1 +Note left of Node: Bloom: [1, 0, 0] +Node->>Peer-123: GetPeerList [1, 0, 0] +Note right of Peer-123: Either Peer-2 or Peer-3 can be sent. +Peer-123->>Node: PeerList - Peer-3 +Note left of Node: Bloom: [1, 0, 1] +Node->>Peer-123: GetPeerList [1, 0, 1] +Note right of Peer-123: Only Peer-2 can be sent. +Peer-123->>Node: PeerList - Peer-2 +Note left of Node: Bloom: [1, 1, 1] +Node->>Peer-123: GetPeerList [1, 1, 1] +Note right of Peer-123: There are no more peers left to send! ``` - -Because network state is generally expected to be stable (i.e nodes are not continuously flickering online/offline), as more and more gossip messages are exchanged nodes eventually realize that the peers that they are connected to have learned about every other peer. - -A node eventually stops gossiping peers when there's no more new peers to gossip about. `PeerList` gossip only resumes once: - -1. a new peer joins -2. a peer disconnects and reconnects -3. a new validator joins the network -4. a validator's IP is updated diff --git a/network/certs_test.go b/network/certs_test.go index 7b59e11d800c..d172d68650fc 100644 --- a/network/certs_test.go +++ b/network/certs_test.go @@ -5,6 +5,7 @@ package network import ( "crypto/tls" + "net" "sync" "testing" @@ -15,6 +16,7 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/network/peer" "github.com/ava-labs/avalanchego/staking" + "github.com/ava-labs/avalanchego/utils/ips" ) var ( @@ -31,6 +33,9 @@ var ( //go:embed test_key_3.key testKeyBytes3 []byte + ip *ips.ClaimedIPPort + otherIP *ips.ClaimedIPPort + certLock sync.Mutex tlsCerts []*tls.Certificate tlsConfigs []*tls.Config @@ -52,6 +57,25 @@ func init() { tlsCerts = []*tls.Certificate{ cert1, cert2, cert3, } + + ip = ips.NewClaimedIPPort( + staking.CertificateFromX509(cert1.Leaf), + ips.IPPort{ + IP: net.IPv4(127, 0, 0, 1), + Port: 9651, + }, + 1, // timestamp + nil, // signature + ) + otherIP = ips.NewClaimedIPPort( + staking.CertificateFromX509(cert2.Leaf), + ips.IPPort{ + IP: net.IPv4(127, 0, 0, 1), + Port: 9651, + }, + 1, // timestamp + nil, // signature + ) } func getTLS(t *testing.T, index int) (ids.NodeID, *tls.Certificate, *tls.Config) { diff --git a/network/config.go b/network/config.go index eda7257fa62c..5cb014741f56 100644 --- a/network/config.go +++ b/network/config.go @@ -10,7 +10,6 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/network/dialer" - "github.com/ava-labs/avalanchego/network/peer" "github.com/ava-labs/avalanchego/network/throttling" "github.com/ava-labs/avalanchego/snow/networking/tracker" "github.com/ava-labs/avalanchego/snow/uptime" @@ -73,6 +72,14 @@ type PeerListGossipConfig struct { // PeerListGossipFreq is the frequency that this node will attempt to gossip // signed IPs to its peers. PeerListGossipFreq time.Duration `json:"peerListGossipFreq"` + + // PeerListPullGossipFreq is the frequency that this node will attempt to + // request signed IPs from its peers. + PeerListPullGossipFreq time.Duration `json:"peerListPullGossipFreq"` + + // PeerListBloomResetFreq is how frequently this node will recalculate the + // IP tracker's bloom filter. + PeerListBloomResetFreq time.Duration `json:"peerListBloomResetFreq"` } type TimeoutConfig struct { @@ -182,7 +189,4 @@ type Config struct { // Specifies how much disk usage each peer can cause before // we rate-limit them. DiskTargeter tracker.Targeter `json:"-"` - - // Tracks which validators have been sent to which peers - GossipTracker peer.GossipTracker `json:"-"` } diff --git a/network/ip_tracker.go b/network/ip_tracker.go new file mode 100644 index 000000000000..7bc6cb293810 --- /dev/null +++ b/network/ip_tracker.go @@ -0,0 +1,364 @@ +// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package network + +import ( + "crypto/rand" + "sync" + + "go.uber.org/zap" + + "golang.org/x/exp/maps" + + "github.com/ava-labs/avalanchego/ids" + "github.com/ava-labs/avalanchego/snow/validators" + "github.com/ava-labs/avalanchego/utils/bloom" + "github.com/ava-labs/avalanchego/utils/crypto/bls" + "github.com/ava-labs/avalanchego/utils/ips" + "github.com/ava-labs/avalanchego/utils/logging" + "github.com/ava-labs/avalanchego/utils/math" + "github.com/ava-labs/avalanchego/utils/sampler" + "github.com/ava-labs/avalanchego/utils/set" +) + +const ( + saltSize = 32 + minCountEstimate = 128 + targetFalsePositiveProbability = .001 + maxFalsePositiveProbability = .01 + // By setting maxIPEntriesPerValidator > 1, we allow validators to update + // their IP at least once per bloom filter reset. + maxIPEntriesPerValidator = 2 +) + +var _ validators.SetCallbackListener = (*ipTracker)(nil) + +func newIPTracker(log logging.Logger) (*ipTracker, error) { + tracker := &ipTracker{ + log: log, + connected: make(map[ids.NodeID]*ips.ClaimedIPPort), + mostRecentValidatorIPs: make(map[ids.NodeID]*ips.ClaimedIPPort), + gossipableIndicies: make(map[ids.NodeID]int), + bloomAdditions: make(map[ids.NodeID]int), + } + return tracker, tracker.resetBloom() +} + +type ipTracker struct { + log logging.Logger + + lock sync.RWMutex + // Manually tracked nodes are always treated like validators + manuallyTracked set.Set[ids.NodeID] + // Connected tracks the currently connected peers, including validators and + // non-validators. The IP is not necessarily the same IP as in + // mostRecentIPs. + connected map[ids.NodeID]*ips.ClaimedIPPort + mostRecentValidatorIPs map[ids.NodeID]*ips.ClaimedIPPort + validators set.Set[ids.NodeID] + + // An IP is marked as gossipable if: + // - The node is a validator + // - The node is connected + // - The IP the node connected with is its latest IP + gossipableIndicies map[ids.NodeID]int + gossipableIPs []*ips.ClaimedIPPort + + // The bloom filter contains the most recent validator IPs to avoid + // unnecessary IP gossip. + bloom *bloom.Filter + // To prevent validators from causing the bloom filter to have too many + // false positives, we limit each validator to maxIPEntriesPerValidator in + // the bloom filter. + bloomAdditions map[ids.NodeID]int // Number of IPs added to the bloom + bloomSalt []byte + maxBloomCount int +} + +func (i *ipTracker) ManuallyTrack(nodeID ids.NodeID) { + i.lock.Lock() + defer i.lock.Unlock() + + // We treat manually tracked nodes as if they were validators. + if !i.validators.Contains(nodeID) { + i.onValidatorAdded(nodeID) + } + // Now that the node is marked as a validator, freeze it's validation + // status. Future calls to OnValidatorAdded or OnValidatorRemoved will be + // treated as noops. + i.manuallyTracked.Add(nodeID) +} + +func (i *ipTracker) WantsConnection(nodeID ids.NodeID) bool { + i.lock.RLock() + defer i.lock.RUnlock() + + return i.validators.Contains(nodeID) +} + +func (i *ipTracker) ShouldVerifyIP(ip *ips.ClaimedIPPort) bool { + i.lock.RLock() + defer i.lock.RUnlock() + + if !i.validators.Contains(ip.NodeID) { + return false + } + + prevIP, ok := i.mostRecentValidatorIPs[ip.NodeID] + return !ok || // This would be the first IP + prevIP.Timestamp < ip.Timestamp // This would be a newer IP +} + +// AddIP returns true if the addition of the provided IP updated the most +// recently known IP of a validator. +func (i *ipTracker) AddIP(ip *ips.ClaimedIPPort) bool { + i.lock.Lock() + defer i.lock.Unlock() + + if !i.validators.Contains(ip.NodeID) { + return false + } + + prevIP, ok := i.mostRecentValidatorIPs[ip.NodeID] + if !ok { + // This is the first IP we've heard from the validator, so it is the + // most recent. + i.updateMostRecentValidatorIP(ip) + // Because we didn't previously have an IP, we know we aren't currently + // connected to them. + return true + } + + if prevIP.Timestamp >= ip.Timestamp { + // This IP is not newer than the previously known IP. + return false + } + + i.updateMostRecentValidatorIP(ip) + i.removeGossipableIP(ip.NodeID) + return true +} + +func (i *ipTracker) GetIP(nodeID ids.NodeID) (*ips.ClaimedIPPort, bool) { + i.lock.RLock() + defer i.lock.RUnlock() + + ip, ok := i.mostRecentValidatorIPs[nodeID] + return ip, ok +} + +func (i *ipTracker) Connected(ip *ips.ClaimedIPPort) { + i.lock.Lock() + defer i.lock.Unlock() + + i.connected[ip.NodeID] = ip + if !i.validators.Contains(ip.NodeID) { + return + } + + prevIP, ok := i.mostRecentValidatorIPs[ip.NodeID] + if !ok { + // This is the first IP we've heard from the validator, so it is the + // most recent. + i.updateMostRecentValidatorIP(ip) + i.addGossipableIP(ip) + return + } + + if prevIP.Timestamp > ip.Timestamp { + // There is a more up-to-date IP than the one that was used to connect. + return + } + + if prevIP.Timestamp < ip.Timestamp { + i.updateMostRecentValidatorIP(ip) + } + i.addGossipableIP(ip) +} + +func (i *ipTracker) Disconnected(nodeID ids.NodeID) { + i.lock.Lock() + defer i.lock.Unlock() + + delete(i.connected, nodeID) + i.removeGossipableIP(nodeID) +} + +func (i *ipTracker) OnValidatorAdded(nodeID ids.NodeID, _ *bls.PublicKey, _ ids.ID, _ uint64) { + i.lock.Lock() + defer i.lock.Unlock() + + i.onValidatorAdded(nodeID) +} + +func (i *ipTracker) onValidatorAdded(nodeID ids.NodeID) { + if i.manuallyTracked.Contains(nodeID) { + return + } + + i.validators.Add(nodeID) + ip, connected := i.connected[nodeID] + if !connected { + return + } + + // Because we only track validator IPs, the from the connection is + // guaranteed to be the most up-to-date IP that we know. + i.updateMostRecentValidatorIP(ip) + i.addGossipableIP(ip) +} + +func (*ipTracker) OnValidatorWeightChanged(ids.NodeID, uint64, uint64) {} + +func (i *ipTracker) OnValidatorRemoved(nodeID ids.NodeID, _ uint64) { + i.lock.Lock() + defer i.lock.Unlock() + + if i.manuallyTracked.Contains(nodeID) { + return + } + + delete(i.mostRecentValidatorIPs, nodeID) + i.validators.Remove(nodeID) + i.removeGossipableIP(nodeID) +} + +func (i *ipTracker) updateMostRecentValidatorIP(ip *ips.ClaimedIPPort) { + i.mostRecentValidatorIPs[ip.NodeID] = ip + oldCount := i.bloomAdditions[ip.NodeID] + if oldCount >= maxIPEntriesPerValidator { + return + } + + // If the validator set is growing rapidly, we should increase the size of + // the bloom filter. + if count := i.bloom.Count(); count >= i.maxBloomCount { + if err := i.resetBloom(); err != nil { + i.log.Error("failed to reset validator tracker bloom filter", + zap.Int("maxCount", i.maxBloomCount), + zap.Int("currentCount", count), + zap.Error(err), + ) + } else { + i.log.Info("reset validator tracker bloom filter", + zap.Int("currentCount", count), + ) + } + return + } + + i.bloomAdditions[ip.NodeID] = oldCount + 1 + bloom.Add(i.bloom, ip.GossipID[:], i.bloomSalt) +} + +func (i *ipTracker) addGossipableIP(ip *ips.ClaimedIPPort) { + i.gossipableIndicies[ip.NodeID] = len(i.gossipableIPs) + i.gossipableIPs = append(i.gossipableIPs, ip) +} + +func (i *ipTracker) removeGossipableIP(nodeID ids.NodeID) { + indexToRemove, wasGossipable := i.gossipableIndicies[nodeID] + if !wasGossipable { + return + } + + newNumGossipable := len(i.gossipableIPs) - 1 + if newNumGossipable != indexToRemove { + replacementIP := i.gossipableIPs[newNumGossipable] + i.gossipableIndicies[replacementIP.NodeID] = indexToRemove + i.gossipableIPs[indexToRemove] = replacementIP + } + + delete(i.gossipableIndicies, nodeID) + i.gossipableIPs[newNumGossipable] = nil + i.gossipableIPs = i.gossipableIPs[:newNumGossipable] +} + +// GetGossipableIPs returns the latest IPs of connected validators. The returned +// IPs will not contain [exceptNodeID] or any IPs contained in [exceptIPs]. If +// the number of eligible IPs to return low, it's possible that every IP will be +// iterated over while handling this call. +func (i *ipTracker) GetGossipableIPs( + exceptNodeID ids.NodeID, + exceptIPs *bloom.ReadFilter, + salt []byte, + maxNumIPs int, +) []*ips.ClaimedIPPort { + var ( + uniform = sampler.NewUniform() + ips = make([]*ips.ClaimedIPPort, 0, maxNumIPs) + ) + + i.lock.RLock() + defer i.lock.RUnlock() + + uniform.Initialize(uint64(len(i.gossipableIPs))) + for len(ips) < maxNumIPs { + index, err := uniform.Next() + if err != nil { + return ips + } + + ip := i.gossipableIPs[index] + if ip.NodeID == exceptNodeID { + continue + } + + if !bloom.Contains(exceptIPs, ip.GossipID[:], salt) { + ips = append(ips, ip) + } + } + return ips +} + +// ResetBloom prunes the current bloom filter. This must be called periodically +// to ensure that validators that change their IPs are updated correctly and +// that validators that left the validator set are removed. +func (i *ipTracker) ResetBloom() error { + i.lock.Lock() + defer i.lock.Unlock() + + return i.resetBloom() +} + +// Bloom returns the binary representation of the bloom filter along with the +// random salt. +func (i *ipTracker) Bloom() ([]byte, []byte) { + i.lock.RLock() + defer i.lock.RUnlock() + + return i.bloom.Marshal(), i.bloomSalt +} + +// resetBloom creates a new bloom filter with a reasonable size for the current +// validator set size. This function additionally populates the new bloom filter +// with the current most recently known IPs of validators. +func (i *ipTracker) resetBloom() error { + newSalt := make([]byte, saltSize) + _, err := rand.Reader.Read(newSalt) + if err != nil { + return err + } + + count := math.Max(maxIPEntriesPerValidator*i.validators.Len(), minCountEstimate) + numHashes, numEntries := bloom.OptimalParameters( + count, + targetFalsePositiveProbability, + ) + newFilter, err := bloom.New(numHashes, numEntries) + if err != nil { + return err + } + + i.bloom = newFilter + maps.Clear(i.bloomAdditions) + i.bloomSalt = newSalt + i.maxBloomCount = bloom.EstimateCount(numHashes, numEntries, maxFalsePositiveProbability) + + for nodeID, ip := range i.mostRecentValidatorIPs { + bloom.Add(newFilter, ip.GossipID[:], newSalt) + i.bloomAdditions[nodeID] = 1 + } + return nil +} diff --git a/network/ip_tracker_test.go b/network/ip_tracker_test.go new file mode 100644 index 000000000000..882c0a47ce8c --- /dev/null +++ b/network/ip_tracker_test.go @@ -0,0 +1,690 @@ +// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package network + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ava-labs/avalanchego/ids" + "github.com/ava-labs/avalanchego/utils/bloom" + "github.com/ava-labs/avalanchego/utils/ips" + "github.com/ava-labs/avalanchego/utils/logging" +) + +func newTestIPTracker(t *testing.T) *ipTracker { + tracker, err := newIPTracker(logging.NoLog{}) + require.NoError(t, err) + return tracker +} + +func newerTestIP(ip *ips.ClaimedIPPort) *ips.ClaimedIPPort { + return ips.NewClaimedIPPort( + ip.Cert, + ip.IPPort, + ip.Timestamp+1, + ip.Signature, + ) +} + +func requireEqual(t *testing.T, expected, actual *ipTracker) { + require := require.New(t) + require.Equal(expected.manuallyTracked, actual.manuallyTracked) + require.Equal(expected.connected, actual.connected) + require.Equal(expected.mostRecentValidatorIPs, actual.mostRecentValidatorIPs) + require.Equal(expected.validators, actual.validators) + require.Equal(expected.gossipableIndicies, actual.gossipableIndicies) + require.Equal(expected.gossipableIPs, actual.gossipableIPs) + require.Equal(expected.bloomAdditions, actual.bloomAdditions) + require.Equal(expected.maxBloomCount, actual.maxBloomCount) +} + +func TestIPTracker_ManuallyTrack(t *testing.T) { + tests := []struct { + name string + initialState *ipTracker + nodeID ids.NodeID + expectedState *ipTracker + }{ + { + name: "non-connected non-validator", + initialState: newTestIPTracker(t), + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.validators.Add(ip.NodeID) + tracker.manuallyTracked.Add(ip.NodeID) + return tracker + }(), + }, + { + name: "connected non-validator", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.Connected(ip) + return tracker + }(), + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.Connected(ip) + tracker.mostRecentValidatorIPs[ip.NodeID] = ip + tracker.bloomAdditions[ip.NodeID] = 1 + tracker.gossipableIndicies[ip.NodeID] = 0 + tracker.gossipableIPs = []*ips.ClaimedIPPort{ + ip, + } + tracker.validators.Add(ip.NodeID) + tracker.manuallyTracked.Add(ip.NodeID) + return tracker + }(), + }, + { + name: "non-connected validator", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + return tracker + }(), + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + tracker.manuallyTracked.Add(ip.NodeID) + return tracker + }(), + }, + { + name: "connected validator", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.Connected(ip) + tracker.onValidatorAdded(ip.NodeID) + return tracker + }(), + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.Connected(ip) + tracker.onValidatorAdded(ip.NodeID) + tracker.manuallyTracked.Add(ip.NodeID) + return tracker + }(), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + test.initialState.ManuallyTrack(test.nodeID) + requireEqual(t, test.expectedState, test.initialState) + }) + } +} + +func TestIPTracker_AddIP(t *testing.T) { + newerIP := newerTestIP(ip) + tests := []struct { + name string + initialState *ipTracker + ip *ips.ClaimedIPPort + expectedUpdated bool + expectedState *ipTracker + }{ + { + name: "non-validator", + initialState: newTestIPTracker(t), + ip: ip, + expectedUpdated: false, + expectedState: newTestIPTracker(t), + }, + { + name: "first known IP", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + return tracker + }(), + ip: ip, + expectedUpdated: true, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + tracker.mostRecentValidatorIPs[ip.NodeID] = ip + tracker.bloomAdditions[ip.NodeID] = 1 + return tracker + }(), + }, + { + name: "older IP", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(newerIP.NodeID) + require.True(t, tracker.AddIP(newerIP)) + return tracker + }(), + ip: ip, + expectedUpdated: false, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(newerIP.NodeID) + require.True(t, tracker.AddIP(newerIP)) + return tracker + }(), + }, + { + name: "same IP", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + require.True(t, tracker.AddIP(ip)) + return tracker + }(), + ip: ip, + expectedUpdated: false, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + require.True(t, tracker.AddIP(ip)) + return tracker + }(), + }, + { + name: "disconnected newer IP", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + require.True(t, tracker.AddIP(ip)) + return tracker + }(), + ip: newerIP, + expectedUpdated: true, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + require.True(t, tracker.AddIP(ip)) + tracker.mostRecentValidatorIPs[newerIP.NodeID] = newerIP + tracker.bloomAdditions[newerIP.NodeID] = 2 + return tracker + }(), + }, + { + name: "connected newer IP", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + tracker.Connected(ip) + return tracker + }(), + ip: newerIP, + expectedUpdated: true, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + tracker.Connected(ip) + tracker.mostRecentValidatorIPs[newerIP.NodeID] = newerIP + tracker.bloomAdditions[newerIP.NodeID] = 2 + delete(tracker.gossipableIndicies, newerIP.NodeID) + tracker.gossipableIPs = tracker.gossipableIPs[:0] + return tracker + }(), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + updated := test.initialState.AddIP(test.ip) + require.Equal(t, test.expectedUpdated, updated) + requireEqual(t, test.expectedState, test.initialState) + }) + } +} + +func TestIPTracker_Connected(t *testing.T) { + newerIP := newerTestIP(ip) + tests := []struct { + name string + initialState *ipTracker + ip *ips.ClaimedIPPort + expectedState *ipTracker + }{ + { + name: "non-validator", + initialState: newTestIPTracker(t), + ip: ip, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.connected[ip.NodeID] = ip + return tracker + }(), + }, + { + name: "first known IP", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + return tracker + }(), + ip: ip, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + tracker.connected[ip.NodeID] = ip + tracker.mostRecentValidatorIPs[ip.NodeID] = ip + tracker.bloomAdditions[ip.NodeID] = 1 + tracker.gossipableIndicies[ip.NodeID] = 0 + tracker.gossipableIPs = []*ips.ClaimedIPPort{ + ip, + } + return tracker + }(), + }, + { + name: "connected with older IP", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(newerIP.NodeID) + require.True(t, tracker.AddIP(newerIP)) + return tracker + }(), + ip: ip, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(newerIP.NodeID) + require.True(t, tracker.AddIP(newerIP)) + tracker.connected[ip.NodeID] = ip + return tracker + }(), + }, + { + name: "connected with newer IP", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + require.True(t, tracker.AddIP(ip)) + return tracker + }(), + ip: newerIP, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + require.True(t, tracker.AddIP(ip)) + tracker.connected[newerIP.NodeID] = newerIP + tracker.mostRecentValidatorIPs[newerIP.NodeID] = newerIP + tracker.bloomAdditions[newerIP.NodeID] = 2 + tracker.gossipableIndicies[newerIP.NodeID] = 0 + tracker.gossipableIPs = []*ips.ClaimedIPPort{ + newerIP, + } + return tracker + }(), + }, + { + name: "connected with same IP", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + require.True(t, tracker.AddIP(ip)) + return tracker + }(), + ip: ip, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + require.True(t, tracker.AddIP(ip)) + tracker.connected[ip.NodeID] = ip + tracker.gossipableIndicies[ip.NodeID] = 0 + tracker.gossipableIPs = []*ips.ClaimedIPPort{ + ip, + } + return tracker + }(), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + test.initialState.Connected(test.ip) + requireEqual(t, test.expectedState, test.initialState) + }) + } +} + +func TestIPTracker_Disconnected(t *testing.T) { + tests := []struct { + name string + initialState *ipTracker + nodeID ids.NodeID + expectedState *ipTracker + }{ + { + name: "not gossipable", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.Connected(ip) + return tracker + }(), + nodeID: ip.NodeID, + expectedState: newTestIPTracker(t), + }, + { + name: "latest gossipable", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + tracker.Connected(ip) + return tracker + }(), + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + tracker.Connected(ip) + delete(tracker.connected, ip.NodeID) + delete(tracker.gossipableIndicies, ip.NodeID) + tracker.gossipableIPs = tracker.gossipableIPs[:0] + return tracker + }(), + }, + { + name: "non-latest gossipable", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + tracker.Connected(ip) + tracker.onValidatorAdded(otherIP.NodeID) + tracker.Connected(otherIP) + return tracker + }(), + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + tracker.Connected(ip) + tracker.onValidatorAdded(otherIP.NodeID) + tracker.Connected(otherIP) + delete(tracker.connected, ip.NodeID) + tracker.gossipableIndicies = map[ids.NodeID]int{ + otherIP.NodeID: 0, + } + tracker.gossipableIPs = []*ips.ClaimedIPPort{ + otherIP, + } + return tracker + }(), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + test.initialState.Disconnected(test.nodeID) + requireEqual(t, test.expectedState, test.initialState) + }) + } +} + +func TestIPTracker_OnValidatorAdded(t *testing.T) { + tests := []struct { + name string + initialState *ipTracker + nodeID ids.NodeID + expectedState *ipTracker + }{ + { + name: "manually tracked", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.ManuallyTrack(ip.NodeID) + return tracker + }(), + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.ManuallyTrack(ip.NodeID) + return tracker + }(), + }, + { + name: "disconnected", + initialState: newTestIPTracker(t), + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.validators.Add(ip.NodeID) + return tracker + }(), + }, + { + name: "connected", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.Connected(ip) + return tracker + }(), + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.Connected(ip) + tracker.validators.Add(ip.NodeID) + tracker.mostRecentValidatorIPs[ip.NodeID] = ip + tracker.bloomAdditions[ip.NodeID] = 1 + tracker.gossipableIndicies[ip.NodeID] = 0 + tracker.gossipableIPs = []*ips.ClaimedIPPort{ + ip, + } + return tracker + }(), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + test.initialState.OnValidatorAdded(test.nodeID, nil, ids.Empty, 0) + requireEqual(t, test.expectedState, test.initialState) + }) + } +} + +func TestIPTracker_OnValidatorRemoved(t *testing.T) { + tests := []struct { + name string + initialState *ipTracker + nodeID ids.NodeID + expectedState *ipTracker + }{ + { + name: "manually tracked", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.ManuallyTrack(ip.NodeID) + tracker.onValidatorAdded(ip.NodeID) + tracker.Connected(ip) + return tracker + }(), + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.ManuallyTrack(ip.NodeID) + tracker.onValidatorAdded(ip.NodeID) + tracker.Connected(ip) + return tracker + }(), + }, + { + name: "not gossipable", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + require.True(t, tracker.AddIP(ip)) + return tracker + }(), + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + require.True(t, tracker.AddIP(ip)) + delete(tracker.mostRecentValidatorIPs, ip.NodeID) + tracker.validators.Remove(ip.NodeID) + return tracker + }(), + }, + { + name: "latest gossipable", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + tracker.Connected(ip) + return tracker + }(), + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + tracker.Connected(ip) + delete(tracker.mostRecentValidatorIPs, ip.NodeID) + tracker.validators.Remove(ip.NodeID) + delete(tracker.gossipableIndicies, ip.NodeID) + tracker.gossipableIPs = tracker.gossipableIPs[:0] + return tracker + }(), + }, + { + name: "non-latest gossipable", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + tracker.Connected(ip) + tracker.onValidatorAdded(otherIP.NodeID) + tracker.Connected(otherIP) + return tracker + }(), + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + tracker.Connected(ip) + tracker.onValidatorAdded(otherIP.NodeID) + tracker.Connected(otherIP) + delete(tracker.mostRecentValidatorIPs, ip.NodeID) + tracker.validators.Remove(ip.NodeID) + tracker.gossipableIndicies = map[ids.NodeID]int{ + otherIP.NodeID: 0, + } + tracker.gossipableIPs = []*ips.ClaimedIPPort{ + otherIP, + } + return tracker + }(), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + test.initialState.OnValidatorRemoved(test.nodeID, 0) + requireEqual(t, test.expectedState, test.initialState) + }) + } +} + +func TestIPTracker_GetGossipableIPs(t *testing.T) { + require := require.New(t) + + tracker := newTestIPTracker(t) + tracker.Connected(ip) + tracker.Connected(otherIP) + tracker.onValidatorAdded(ip.NodeID) + tracker.onValidatorAdded(otherIP.NodeID) + + gossipableIPs := tracker.GetGossipableIPs(ids.EmptyNodeID, bloom.EmptyFilter, nil, 2) + require.ElementsMatch([]*ips.ClaimedIPPort{ip, otherIP}, gossipableIPs) + + gossipableIPs = tracker.GetGossipableIPs(ip.NodeID, bloom.EmptyFilter, nil, 2) + require.Equal([]*ips.ClaimedIPPort{otherIP}, gossipableIPs) + + gossipableIPs = tracker.GetGossipableIPs(ids.EmptyNodeID, bloom.FullFilter, nil, 2) + require.Empty(gossipableIPs) + + filter, err := bloom.New(8, 1024) + require.NoError(err) + bloom.Add(filter, ip.GossipID[:], nil) + + readFilter, err := bloom.Parse(filter.Marshal()) + require.NoError(err) + + gossipableIPs = tracker.GetGossipableIPs(ip.NodeID, readFilter, nil, 2) + require.Equal([]*ips.ClaimedIPPort{otherIP}, gossipableIPs) +} + +func TestIPTracker_BloomFiltersEverything(t *testing.T) { + require := require.New(t) + + tracker := newTestIPTracker(t) + tracker.Connected(ip) + tracker.Connected(otherIP) + tracker.onValidatorAdded(ip.NodeID) + tracker.onValidatorAdded(otherIP.NodeID) + + bloomBytes, salt := tracker.Bloom() + readFilter, err := bloom.Parse(bloomBytes) + require.NoError(err) + + gossipableIPs := tracker.GetGossipableIPs(ids.EmptyNodeID, readFilter, salt, 2) + require.Empty(gossipableIPs) + + require.NoError(tracker.ResetBloom()) +} + +func TestIPTracker_BloomGrowsWithValidatorSet(t *testing.T) { + require := require.New(t) + + tracker := newTestIPTracker(t) + initialMaxBloomCount := tracker.maxBloomCount + for i := 0; i < 2048; i++ { + tracker.onValidatorAdded(ids.GenerateTestNodeID()) + } + + require.NoError(tracker.ResetBloom()) + require.Greater(tracker.maxBloomCount, initialMaxBloomCount) +} + +func TestIPTracker_BloomResetsDynamically(t *testing.T) { + require := require.New(t) + + tracker := newTestIPTracker(t) + tracker.Connected(ip) + tracker.onValidatorAdded(ip.NodeID) + tracker.OnValidatorRemoved(ip.NodeID, 0) + tracker.maxBloomCount = 1 + tracker.Connected(otherIP) + tracker.onValidatorAdded(otherIP.NodeID) + + bloomBytes, salt := tracker.Bloom() + readFilter, err := bloom.Parse(bloomBytes) + require.NoError(err) + + require.False(bloom.Contains(readFilter, ip.GossipID[:], salt)) + require.True(bloom.Contains(readFilter, otherIP.GossipID[:], salt)) +} + +func TestIPTracker_PreventBloomFilterAddition(t *testing.T) { + require := require.New(t) + + newerIP := newerTestIP(ip) + newestIP := newerTestIP(newerIP) + + tracker := newTestIPTracker(t) + tracker.onValidatorAdded(ip.NodeID) + require.True(tracker.AddIP(ip)) + require.True(tracker.AddIP(newerIP)) + require.True(tracker.AddIP(newestIP)) + require.Equal(maxIPEntriesPerValidator, tracker.bloomAdditions[ip.NodeID]) +} + +func TestIPTracker_ShouldVerifyIP(t *testing.T) { + require := require.New(t) + + newerIP := newerTestIP(ip) + + tracker := newTestIPTracker(t) + require.False(tracker.ShouldVerifyIP(ip)) + tracker.onValidatorAdded(ip.NodeID) + require.True(tracker.ShouldVerifyIP(ip)) + require.True(tracker.AddIP(ip)) + require.False(tracker.ShouldVerifyIP(ip)) + require.True(tracker.ShouldVerifyIP(newerIP)) +} diff --git a/network/network.go b/network/network.go index 2d178cb93488..374038883ba7 100644 --- a/network/network.go +++ b/network/network.go @@ -20,22 +20,20 @@ import ( "go.uber.org/zap" - "golang.org/x/exp/maps" - "github.com/ava-labs/avalanchego/api/health" + "github.com/ava-labs/avalanchego/genesis" "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/message" "github.com/ava-labs/avalanchego/network/dialer" "github.com/ava-labs/avalanchego/network/peer" "github.com/ava-labs/avalanchego/network/throttling" - "github.com/ava-labs/avalanchego/proto/pb/p2p" "github.com/ava-labs/avalanchego/snow/networking/router" "github.com/ava-labs/avalanchego/snow/networking/sender" "github.com/ava-labs/avalanchego/subnets" + "github.com/ava-labs/avalanchego/utils/bloom" "github.com/ava-labs/avalanchego/utils/constants" "github.com/ava-labs/avalanchego/utils/ips" "github.com/ava-labs/avalanchego/utils/logging" - "github.com/ava-labs/avalanchego/utils/sampler" "github.com/ava-labs/avalanchego/utils/set" "github.com/ava-labs/avalanchego/utils/wrappers" "github.com/ava-labs/avalanchego/version" @@ -79,12 +77,6 @@ type Network interface { // or the network is closed. Dispatch() error - // WantsConnection returns true if this node is willing to attempt to - // connect to the provided nodeID. If the node is attempting to connect to - // the minimum number of peers, then it should only connect if the peer is a - // validator or beacon. - WantsConnection(ids.NodeID) bool - // Attempt to connect to this IP. The network will never stop attempting to // connect to this ID. ManuallyTrack(nodeID ids.NodeID, ip ips.IPPort) @@ -151,13 +143,8 @@ type network struct { sendFailRateCalculator safemath.Averager // Tracks which peers know about which peers - gossipTracker peer.GossipTracker - peersLock sync.RWMutex - // peerIPs contains the most up to date set of signed IPs for nodes we are - // currently connected or attempting to connect to. - // Note: The txID provided inside of a claimed IP is not verified and should - // not be accessed from this map. - peerIPs map[ids.NodeID]*ips.ClaimedIPPort + ipTracker *ipTracker + peersLock sync.RWMutex // trackedIPs contains the set of IPs that we are currently attempting to // connect to. An entry is added to this set when we first start attempting // to connect to the peer. An entry is deleted from this set once we have @@ -167,10 +154,6 @@ type network struct { connectedPeers peer.Set closing bool - // Tracks special peers that the network should always track - manuallyTrackedIDsLock sync.RWMutex - manuallyTrackedIDs set.Set[ids.NodeID] - // router is notified about all peer [Connected] and [Disconnected] events // as well as all non-handshake peer messages. // @@ -254,6 +237,18 @@ func NewNetwork( return nil, fmt.Errorf("initializing network metrics failed with: %w", err) } + ipTracker, err := newIPTracker(log) + if err != nil { + return nil, fmt.Errorf("initializing ip tracker failed with: %w", err) + } + config.Validators.RegisterCallbackListener(constants.PrimaryNetworkID, ipTracker) + + // Track all default bootstrappers to ensure their current IPs are gossiped + // like validator IPs. + for _, bootstrapper := range genesis.GetBootstrappers(config.NetworkID) { + ipTracker.ManuallyTrack(bootstrapper.ID) + } + peerConfig := &peer.Config{ ReadBufferSize: config.PeerReadBufferSize, WriteBufferSize: config.PeerWriteBufferSize, @@ -300,9 +295,8 @@ func NewNetwork( time.Now(), )), - peerIPs: make(map[ids.NodeID]*ips.ClaimedIPPort), trackedIPs: make(map[ids.NodeID]*trackedIP), - gossipTracker: config.GossipTracker, + ipTracker: ipTracker, connectingPeers: peer.NewSet(), connectedPeers: peer.NewSet(), router: router, @@ -424,32 +418,6 @@ func (n *network) Connected(nodeID ids.NodeID) { return } - peerIP := peer.IP() - newIP := &ips.ClaimedIPPort{ - Cert: peer.Cert(), - IPPort: peerIP.IPPort, - Timestamp: peerIP.Timestamp, - Signature: peerIP.Signature, - } - prevIP, ok := n.peerIPs[nodeID] - if !ok { - // If the IP wasn't previously tracked, then we never could have - // gossiped it. This means we don't need to reset the validator's - // tracked set. - n.peerIPs[nodeID] = newIP - } else if prevIP.Timestamp < newIP.Timestamp { - // The previous IP was stale, so we should gossip the newer IP. - n.peerIPs[nodeID] = newIP - - if !prevIP.IPPort.Equal(newIP.IPPort) { - // This IP is actually different, so we should gossip it. - n.peerConfig.Log.Debug("resetting gossip due to ip change", - zap.Stringer("nodeID", nodeID), - ) - _ = n.gossipTracker.ResetValidator(nodeID) - } - } - if tracked, ok := n.trackedIPs[nodeID]; ok { tracked.stopTracking() delete(n.trackedIPs, nodeID) @@ -458,6 +426,15 @@ func (n *network) Connected(nodeID ids.NodeID) { n.connectedPeers.Add(peer) n.peersLock.Unlock() + peerIP := peer.IP() + newIP := ips.NewClaimedIPPort( + peer.Cert(), + peerIP.IPPort, + peerIP.Timestamp, + peerIP.Signature, + ) + n.ipTracker.Connected(newIP) + n.metrics.markConnected(peer) peerVersion := peer.Version() @@ -475,177 +452,15 @@ func (n *network) AllowConnection(nodeID ids.NodeID) bool { if !n.config.RequireValidatorToConnect { return true } - _, isValidator := n.config.Validators.GetValidator(constants.PrimaryNetworkID, n.config.MyNodeID) - return isValidator || n.WantsConnection(nodeID) -} - -func (n *network) Track(peerID ids.NodeID, claimedIPPorts []*ips.ClaimedIPPort) ([]*p2p.PeerAck, error) { - // Perform all signature verification and hashing before grabbing the peer - // lock. - // Note: Avoiding signature verification when the IP isn't needed is a - // **significant** performance optimization. - // Note: To avoid signature verification when the IP isn't needed, we - // optimistically filter out IPs. This can result in us not tracking an IP - // that we otherwise would have. This case can only happen if the node - // became a validator between the time we verified the signature and when we - // processed the IP; which should be very rare. - ipAuths, err := n.authenticateIPs(claimedIPPorts) - if err != nil { - n.peerConfig.Log.Debug("authenticating claimed IPs failed", - zap.Stringer("nodeID", peerID), - zap.Error(err), - ) - return nil, err - } - - // Information for them to update about us - ipLen := len(claimedIPPorts) - newestTimestamp := make(map[ids.ID]uint64, ipLen) - // Information for us to update about them - txIDsWithUpToDateIP := make([]ids.ID, 0, ipLen) - - // Atomically modify peer data - n.peersLock.Lock() - defer n.peersLock.Unlock() - for i, ip := range claimedIPPorts { - ipAuth := ipAuths[i] - nodeID := ipAuth.nodeID - // Invariant: [ip] is only used to modify local node state if - // [verifiedIP] is true. - // Note: modifying peer-level state is allowed regardless of - // [verifiedIP]. - verifiedIP := ipAuth.verified - - // Re-fetch latest info for a [nodeID] in case it changed since we last - // held [peersLock]. - prevIP, previouslyTracked, shouldUpdateOurIP, shouldDial := n.peerIPStatus(nodeID, ip) - tracked, isTracked := n.trackedIPs[nodeID] - - // Evaluate if the gossiped IP is useful to us or to the peer that - // shared it with us. - switch { - case previouslyTracked && prevIP.Timestamp > ip.Timestamp: - // Our previous IP was more up to date. We should tell the peer - // not to gossip their IP to us. We should still gossip our IP to - // them. - newestTimestamp[ip.TxID] = prevIP.Timestamp - - n.metrics.numUselessPeerListBytes.Add(float64(ip.BytesLen())) - case previouslyTracked && prevIP.Timestamp == ip.Timestamp: - // Our previous IP was equally fresh. We should tell the peer - // not to gossip this IP to us. We should not gossip our IP to them. - newestTimestamp[ip.TxID] = prevIP.Timestamp - txIDsWithUpToDateIP = append(txIDsWithUpToDateIP, ip.TxID) - - n.metrics.numUselessPeerListBytes.Add(float64(ip.BytesLen())) - case verifiedIP && shouldUpdateOurIP: - // This IP is more up to date. We should tell the peer not to gossip - // this IP to us. We should not gossip our IP to them. - newestTimestamp[ip.TxID] = ip.Timestamp - txIDsWithUpToDateIP = append(txIDsWithUpToDateIP, ip.TxID) - - // In the future, we should gossip this IP rather than the old IP. - n.peerIPs[nodeID] = ip - - // If the new IP is equal to the old IP, there is no reason to - // refresh the references to it. This can happen when a node - // restarts but does not change their IP. - if prevIP.IPPort.Equal(ip.IPPort) { - continue - } - - // We should gossip this new IP to all our peers. - n.peerConfig.Log.Debug("resetting gossip due to ip change", - zap.Stringer("nodeID", nodeID), - ) - _ = n.gossipTracker.ResetValidator(nodeID) - - // We should update any existing outbound connection attempts. - if isTracked { - // Stop tracking the old IP and start tracking the new one. - tracked := tracked.trackNewIP(ip.IPPort) - n.trackedIPs[nodeID] = tracked - n.dial(nodeID, tracked) - } - case verifiedIP && shouldDial: - // Invariant: [isTracked] is false here. - - // This is the first we've heard of this IP and we want to connect - // to it. We should tell the peer not to gossip this IP to us again. - newestTimestamp[ip.TxID] = ip.Timestamp - // We should not gossip this IP back to them. - txIDsWithUpToDateIP = append(txIDsWithUpToDateIP, ip.TxID) - - // We don't need to reset gossip about this validator because - // we've never gossiped it before. - n.peerIPs[nodeID] = ip - - tracked := newTrackedIP(ip.IPPort) - n.trackedIPs[nodeID] = tracked - n.dial(nodeID, tracked) - default: - // This IP isn't desired - n.metrics.numUselessPeerListBytes.Add(float64(ip.BytesLen())) - } - } - - txIDsToAck := maps.Keys(newestTimestamp) - txIDsToAck, ok := n.gossipTracker.AddKnown(peerID, txIDsWithUpToDateIP, txIDsToAck) - if !ok { - n.peerConfig.Log.Error("failed to update known peers", - zap.Stringer("nodeID", peerID), - ) - return nil, nil - } - - peerAcks := make([]*p2p.PeerAck, len(txIDsToAck)) - for i, txID := range txIDsToAck { - txID := txID - peerAcks[i] = &p2p.PeerAck{ - TxId: txID[:], - // By responding with the highest timestamp, not just the timestamp - // the peer provided us, we may be able to avoid some unnecessary - // gossip in the case that the peer is about to update this - // validator's IP. - Timestamp: newestTimestamp[txID], - } - } - return peerAcks, nil + _, iAmAValidator := n.config.Validators.GetValidator(constants.PrimaryNetworkID, n.config.MyNodeID) + return iAmAValidator || n.ipTracker.WantsConnection(nodeID) } -func (n *network) MarkTracked(peerID ids.NodeID, ips []*p2p.PeerAck) error { - txIDs := make([]ids.ID, 0, len(ips)) - - n.peersLock.RLock() - defer n.peersLock.RUnlock() - - for _, ip := range ips { - txID, err := ids.ToID(ip.TxId) - if err != nil { +func (n *network) Track(claimedIPPorts []*ips.ClaimedIPPort) error { + for _, ip := range claimedIPPorts { + if err := n.track(ip); err != nil { return err } - - // If [txID]'s corresponding nodeID isn't known, then they must no - // longer be a validator. Therefore we wouldn't gossip their IP anyways. - nodeID, ok := n.gossipTracker.GetNodeID(txID) - if !ok { - continue - } - - // If the peer returns a lower timestamp than I currently have, then I - // have updated the IP since I sent the PeerList message this is in - // response to. That means that I should re-gossip this node's IP to the - // peer. - myIP, previouslyTracked := n.peerIPs[nodeID] - if previouslyTracked && myIP.Timestamp <= ip.Timestamp { - txIDs = append(txIDs, txID) - } - } - - if _, ok := n.gossipTracker.AddKnown(peerID, txIDs, nil); !ok { - n.peerConfig.Log.Error("failed to update known peers", - zap.Stringer("nodeID", peerID), - ) } return nil } @@ -656,13 +471,6 @@ func (n *network) MarkTracked(peerID ids.NodeID, ips []*p2p.PeerAck) error { // call. Note that this is from the perspective of a single peer object, because // a peer with the same ID can reconnect to this network instance. func (n *network) Disconnected(nodeID ids.NodeID) { - if !n.gossipTracker.StopTrackingPeer(nodeID) { - n.peerConfig.Log.Error( - "stopped non-existent peer tracker", - zap.Stringer("nodeID", nodeID), - ) - } - n.peersLock.RLock() _, connecting := n.connectingPeers.GetByID(nodeID) peer, connected := n.connectedPeers.GetByID(nodeID) @@ -676,57 +484,17 @@ func (n *network) Disconnected(nodeID ids.NodeID) { } } -func (n *network) Peers(peerID ids.NodeID) ([]ips.ClaimedIPPort, error) { - // Only select validators that we haven't already sent to this peer - unknownValidators, ok := n.gossipTracker.GetUnknown(peerID) - if !ok { - n.peerConfig.Log.Debug( - "unable to find peer to gossip to", - zap.Stringer("nodeID", peerID), - ) - return nil, nil - } - - // We select a random sample of validators to gossip to avoid starving out a - // validator from being gossiped for an extended period of time. - s := sampler.NewUniform() - s.Initialize(uint64(len(unknownValidators))) - - // Calculate the unknown information we need to send to this peer. - validatorIPs := make([]ips.ClaimedIPPort, 0, int(n.config.PeerListNumValidatorIPs)) - for i := 0; i < len(unknownValidators) && len(validatorIPs) < int(n.config.PeerListNumValidatorIPs); i++ { - drawn, err := s.Next() - if err != nil { - return nil, err - } - - validator := unknownValidators[drawn] - n.peersLock.RLock() - _, isConnected := n.connectedPeers.GetByID(validator.NodeID) - peerIP := n.peerIPs[validator.NodeID] - n.peersLock.RUnlock() - if !isConnected { - n.peerConfig.Log.Verbo( - "unable to find validator in connected peers", - zap.Stringer("nodeID", validator.NodeID), - ) - continue - } - - // Note: peerIP isn't used directly here because the TxID may be - // incorrect. - validatorIPs = append(validatorIPs, - ips.ClaimedIPPort{ - Cert: peerIP.Cert, - IPPort: peerIP.IPPort, - Timestamp: peerIP.Timestamp, - Signature: peerIP.Signature, - TxID: validator.TxID, - }, - ) - } +func (n *network) KnownPeers() ([]byte, []byte) { + return n.ipTracker.Bloom() +} - return validatorIPs, nil +func (n *network) Peers(except ids.NodeID, knownPeers *bloom.ReadFilter, salt []byte) []*ips.ClaimedIPPort { + return n.ipTracker.GetGossipableIPs( + except, + knownPeers, + salt, + int(n.config.PeerListNumValidatorIPs), + ) } // Dispatch starts accepting connections from other nodes attempting to connect @@ -806,21 +574,8 @@ func (n *network) Dispatch() error { return errs.Err } -func (n *network) WantsConnection(nodeID ids.NodeID) bool { - if _, ok := n.config.Validators.GetValidator(constants.PrimaryNetworkID, nodeID); ok { - return true - } - - n.manuallyTrackedIDsLock.RLock() - defer n.manuallyTrackedIDsLock.RUnlock() - - return n.manuallyTrackedIDs.Contains(nodeID) -} - func (n *network) ManuallyTrack(nodeID ids.NodeID, ip ips.IPPort) { - n.manuallyTrackedIDsLock.Lock() - n.manuallyTrackedIDs.Add(nodeID) - n.manuallyTrackedIDsLock.Unlock() + n.ipTracker.ManuallyTrack(nodeID) n.peersLock.Lock() defer n.peersLock.Unlock() @@ -841,6 +596,58 @@ func (n *network) ManuallyTrack(nodeID ids.NodeID, ip ips.IPPort) { } } +func (n *network) track(ip *ips.ClaimedIPPort) error { + // To avoid signature verification when the IP isn't needed, we + // optimistically filter out IPs. This can result in us not tracking an IP + // that we otherwise would have. This case can only happen if the node + // became a validator between the time we verified the signature and when we + // processed the IP; which should be very rare. + // + // Note: Avoiding signature verification when the IP isn't needed is a + // **significant** performance optimization. + if !n.ipTracker.ShouldVerifyIP(ip) { + n.metrics.numUselessPeerListBytes.Add(float64(ip.Size())) + return nil + } + + // Perform all signature verification and hashing before grabbing the peer + // lock. + signedIP := peer.SignedIP{ + UnsignedIP: peer.UnsignedIP{ + IPPort: ip.IPPort, + Timestamp: ip.Timestamp, + }, + Signature: ip.Signature, + } + if err := signedIP.Verify(ip.Cert); err != nil { + return err + } + + n.peersLock.Lock() + defer n.peersLock.Unlock() + + if !n.ipTracker.AddIP(ip) { + return nil + } + + if _, connected := n.connectedPeers.GetByID(ip.NodeID); connected { + // If I'm currently connected to [nodeID] then I'll attempt to dial them + // when we disconnect. + return nil + } + + tracked, isTracked := n.trackedIPs[ip.NodeID] + if isTracked { + // Stop tracking the old IP and start tracking the new one. + tracked = tracked.trackNewIP(ip.IPPort) + } else { + tracked = newTrackedIP(ip.IPPort) + } + n.trackedIPs[ip.NodeID] = tracked + n.dial(ip.NodeID, tracked) + return nil +} + // getPeers returns a slice of connected peers from a set of [nodeIDs]. // // - [nodeIDs] the IDs of the peers that should be returned if they are @@ -965,13 +772,12 @@ func (n *network) disconnectedFromConnecting(nodeID ids.NodeID) { // The peer that is disconnecting from us didn't finish the handshake tracked, ok := n.trackedIPs[nodeID] if ok { - if n.WantsConnection(nodeID) { + if n.ipTracker.WantsConnection(nodeID) { tracked := tracked.trackNewIP(tracked.ip) n.trackedIPs[nodeID] = tracked n.dial(nodeID, tracked) } else { tracked.stopTracking() - delete(n.peerIPs, nodeID) delete(n.trackedIPs, nodeID) } } @@ -980,6 +786,7 @@ func (n *network) disconnectedFromConnecting(nodeID ids.NodeID) { } func (n *network) disconnectedFromConnected(peer peer.Peer, nodeID ids.NodeID) { + n.ipTracker.Disconnected(nodeID) n.router.Disconnected(nodeID) n.peersLock.Lock() @@ -988,66 +795,15 @@ func (n *network) disconnectedFromConnected(peer peer.Peer, nodeID ids.NodeID) { n.connectedPeers.Remove(nodeID) // The peer that is disconnecting from us finished the handshake - if n.WantsConnection(nodeID) { - prevIP := n.peerIPs[nodeID] - tracked := newTrackedIP(prevIP.IPPort) + if ip, wantsConnection := n.ipTracker.GetIP(nodeID); wantsConnection { + tracked := newTrackedIP(ip.IPPort) n.trackedIPs[nodeID] = tracked n.dial(nodeID, tracked) - } else { - delete(n.peerIPs, nodeID) } n.metrics.markDisconnected(peer) } -// ipAuth is a helper struct used to convey information about an -// [*ips.ClaimedIPPort]. -type ipAuth struct { - nodeID ids.NodeID - verified bool -} - -func (n *network) authenticateIPs(ips []*ips.ClaimedIPPort) ([]*ipAuth, error) { - ipAuths := make([]*ipAuth, len(ips)) - for i, ip := range ips { - nodeID := ids.NodeIDFromCert(ip.Cert) - n.peersLock.RLock() - _, _, shouldUpdateOurIP, shouldDial := n.peerIPStatus(nodeID, ip) - n.peersLock.RUnlock() - if !shouldUpdateOurIP && !shouldDial { - ipAuths[i] = &ipAuth{ - nodeID: nodeID, - } - continue - } - - // Verify signature if needed - signedIP := peer.SignedIP{ - UnsignedIP: peer.UnsignedIP{ - IPPort: ip.IPPort, - Timestamp: ip.Timestamp, - }, - Signature: ip.Signature, - } - if err := signedIP.Verify(ip.Cert); err != nil { - return nil, err - } - ipAuths[i] = &ipAuth{ - nodeID: nodeID, - verified: true, - } - } - return ipAuths, nil -} - -// peerIPStatus assumes the caller holds [peersLock] -func (n *network) peerIPStatus(nodeID ids.NodeID, ip *ips.ClaimedIPPort) (*ips.ClaimedIPPort, bool, bool, bool) { - prevIP, previouslyTracked := n.peerIPs[nodeID] - shouldUpdateOurIP := previouslyTracked && prevIP.Timestamp < ip.Timestamp - shouldDial := !previouslyTracked && n.WantsConnection(nodeID) - return prevIP, previouslyTracked, shouldUpdateOurIP, shouldDial -} - // dial will spin up a new goroutine and attempt to establish a connection with // [nodeID] at [ip]. // @@ -1094,13 +850,12 @@ func (n *network) dial(nodeID ids.NodeID, ip *trackedIP) { // trackedIPs and this goroutine. This prevents a memory leak when // the tracked nodeID leaves the validator set and is never able to // be connected to. - if !n.WantsConnection(nodeID) { + if !n.ipTracker.WantsConnection(nodeID) { // Typically [n.trackedIPs[nodeID]] will already equal [ip], but // the reference to [ip] is refreshed to avoid any potential // race conditions before removing the entry. if ip, exists := n.trackedIPs[nodeID]; exists { ip.stopTracking() - delete(n.peerIPs, nodeID) delete(n.trackedIPs, nodeID) } n.peersLock.Unlock() @@ -1277,13 +1032,6 @@ func (n *network) upgrade(conn net.Conn, upgrader peer.Upgrader) error { zap.Stringer("nodeID", nodeID), ) - if !n.gossipTracker.StartTrackingPeer(nodeID) { - n.peerConfig.Log.Error( - "started duplicate peer tracker", - zap.Stringer("nodeID", nodeID), - ) - } - // peer.Start requires there is only ever one peer instance running with the // same [peerConfig.InboundMsgThrottler]. This is guaranteed by the above // de-duplications for [connectingPeers] and [connectedPeers]. @@ -1332,7 +1080,6 @@ func (n *network) StartClose() { for nodeID, tracked := range n.trackedIPs { tracked.stopTracking() - delete(n.peerIPs, nodeID) delete(n.trackedIPs, nodeID) } @@ -1404,10 +1151,13 @@ func (n *network) NodeUptime(subnetID ids.ID) (UptimeResult, error) { } func (n *network) runTimers() { - gossipPeerlists := time.NewTicker(n.config.PeerListGossipFreq) + pushGossipPeerlists := time.NewTicker(n.config.PeerListGossipFreq) + pullGossipPeerlists := time.NewTicker(n.config.PeerListPullGossipFreq) + resetPeerListBloom := time.NewTicker(n.config.PeerListBloomResetFreq) updateUptimes := time.NewTicker(n.config.UptimeMetricFreq) defer func() { - gossipPeerlists.Stop() + pushGossipPeerlists.Stop() + resetPeerListBloom.Stop() updateUptimes.Stop() }() @@ -1415,8 +1165,18 @@ func (n *network) runTimers() { select { case <-n.onCloseCtx.Done(): return - case <-gossipPeerlists.C: - n.gossipPeerLists() + case <-pushGossipPeerlists.C: + n.pushGossipPeerLists() + case <-pullGossipPeerlists.C: + n.pullGossipPeerLists() + case <-resetPeerListBloom.C: + if err := n.ipTracker.ResetBloom(); err != nil { + n.peerConfig.Log.Error("failed to reset ip tracker bloom filter", + zap.Error(err), + ) + } else { + n.peerConfig.Log.Debug("reset ip tracker bloom filter") + } case <-updateUptimes.C: primaryUptime, err := n.NodeUptime(constants.PrimaryNetworkID) if err != nil { @@ -1443,8 +1203,8 @@ func (n *network) runTimers() { } } -// gossipPeerLists gossips validators to peers in the network -func (n *network) gossipPeerLists() { +// pushGossipPeerLists gossips validators to peers in the network +func (n *network) pushGossipPeerLists() { peers := n.samplePeers( constants.PrimaryNetworkID, int(n.config.PeerListValidatorGossipSize), @@ -1458,6 +1218,21 @@ func (n *network) gossipPeerLists() { } } +// pullGossipPeerLists requests validators from peers in the network +func (n *network) pullGossipPeerLists() { + peers := n.samplePeers( + constants.PrimaryNetworkID, + 1, // numValidatorsToSample + 0, // numNonValidatorsToSample + 0, // numPeersToSample + subnets.NoOpAllower, + ) + + for _, p := range peers { + p.StartSendGetPeerList() + } +} + func (n *network) getLastReceived() (time.Time, bool) { lastReceived := atomic.LoadInt64(&n.peerConfig.LastReceived) if lastReceived == 0 { diff --git a/network/network_test.go b/network/network_test.go index 916b527da82b..90c063a99bb7 100644 --- a/network/network_test.go +++ b/network/network_test.go @@ -54,6 +54,8 @@ var ( PeerListNonValidatorGossipSize: 100, PeerListPeersGossipSize: 100, PeerListGossipFreq: time.Second, + PeerListPullGossipFreq: time.Second, + PeerListBloomResetFreq: constants.DefaultNetworkPeerListBloomResetFreq, } defaultTimeoutConfig = TimeoutConfig{ PingPongTimeout: 30 * time.Second, @@ -215,27 +217,16 @@ func newFullyConnectedTestNetwork(t *testing.T, handlers []router.InboundHandler msgCreator := newMessageCreator(t) registry := prometheus.NewRegistry() - g, err := peer.NewGossipTracker(registry, "foobar") - require.NoError(err) - - log := logging.NoLog{} - gossipTrackerCallback := peer.GossipTrackerCallback{ - Log: log, - GossipTracker: g, - } - beacons := validators.NewManager() require.NoError(beacons.AddStaker(constants.PrimaryNetworkID, nodeIDs[0], nil, ids.GenerateTestID(), 1)) vdrs := validators.NewManager() - vdrs.RegisterCallbackListener(constants.PrimaryNetworkID, &gossipTrackerCallback) for _, nodeID := range nodeIDs { require.NoError(vdrs.AddStaker(constants.PrimaryNetworkID, nodeID, nil, ids.GenerateTestID(), 1)) } config := config - config.GossipTracker = g config.Beacons = beacons config.Validators = vdrs @@ -244,7 +235,7 @@ func newFullyConnectedTestNetwork(t *testing.T, handlers []router.InboundHandler config, msgCreator, registry, - log, + logging.NoLog{}, listeners[i], dialer, &testHandler{ @@ -405,15 +396,17 @@ func TestTrackVerifiesSignatures(t *testing.T) { nodeID, tlsCert, _ := getTLS(t, 1) require.NoError(network.config.Validators.AddStaker(constants.PrimaryNetworkID, nodeID, nil, ids.Empty, 1)) - _, err := network.Track(ids.EmptyNodeID, []*ips.ClaimedIPPort{{ - Cert: staking.CertificateFromX509(tlsCert.Leaf), - IPPort: ips.IPPort{ - IP: net.IPv4(123, 132, 123, 123), - Port: 10000, - }, - Timestamp: 1000, - Signature: nil, - }}) + err := network.Track([]*ips.ClaimedIPPort{ + ips.NewClaimedIPPort( + staking.CertificateFromX509(tlsCert.Leaf), + ips.IPPort{ + IP: net.IPv4(123, 132, 123, 123), + Port: 10000, + }, + 1000, // timestamp + nil, // signature + ), + }) // The signature is wrong so this peer tracking info isn't useful. require.ErrorIs(err, rsa.ErrVerification) @@ -437,27 +430,16 @@ func TestTrackDoesNotDialPrivateIPs(t *testing.T) { msgCreator := newMessageCreator(t) registry := prometheus.NewRegistry() - g, err := peer.NewGossipTracker(registry, "foobar") - require.NoError(err) - - log := logging.NoLog{} - gossipTrackerCallback := peer.GossipTrackerCallback{ - Log: log, - GossipTracker: g, - } - beacons := validators.NewManager() require.NoError(beacons.AddStaker(constants.PrimaryNetworkID, nodeIDs[0], nil, ids.GenerateTestID(), 1)) vdrs := validators.NewManager() - vdrs.RegisterCallbackListener(constants.PrimaryNetworkID, &gossipTrackerCallback) for _, nodeID := range nodeIDs { require.NoError(vdrs.AddStaker(constants.PrimaryNetworkID, nodeID, nil, ids.GenerateTestID(), 1)) } config := config - config.GossipTracker = g config.Beacons = beacons config.Validators = vdrs config.AllowPrivateIPs = false @@ -466,7 +448,7 @@ func TestTrackDoesNotDialPrivateIPs(t *testing.T) { config, msgCreator, registry, - log, + logging.NoLog{}, listeners[i], dialer, &testHandler{ @@ -532,23 +514,11 @@ func TestDialDeletesNonValidators(t *testing.T) { msgCreator := newMessageCreator(t) registry := prometheus.NewRegistry() - g, err := peer.NewGossipTracker(registry, "foobar") - require.NoError(err) - - log := logging.NoLog{} - gossipTrackerCallback := peer.GossipTrackerCallback{ - Log: log, - GossipTracker: g, - } - beacons := validators.NewManager() require.NoError(beacons.AddStaker(constants.PrimaryNetworkID, nodeIDs[0], nil, ids.GenerateTestID(), 1)) - vdrs.RegisterCallbackListener(constants.PrimaryNetworkID, &gossipTrackerCallback) - config := config - config.GossipTracker = g config.Beacons = beacons config.Validators = vdrs config.AllowPrivateIPs = false @@ -557,7 +527,7 @@ func TestDialDeletesNonValidators(t *testing.T) { config, msgCreator, registry, - log, + logging.NoLog{}, listeners[i], dialer, &testHandler{ @@ -581,16 +551,15 @@ func TestDialDeletesNonValidators(t *testing.T) { wg.Add(len(networks)) for i, net := range networks { if i != 0 { - peerAcks, err := net.Track(config.MyNodeID, []*ips.ClaimedIPPort{{ - Cert: staking.CertificateFromX509(config.TLSConfig.Certificates[0].Leaf), - IPPort: ip.IPPort, - Timestamp: ip.Timestamp, - Signature: ip.Signature, - }}) + err := net.Track([]*ips.ClaimedIPPort{ + ips.NewClaimedIPPort( + staking.CertificateFromX509(config.TLSConfig.Certificates[0].Leaf), + ip.IPPort, + ip.Timestamp, + ip.Signature, + ), + }) require.NoError(err) - // peerAcks is empty because we aren't actually connected to - // MyNodeID yet - require.Empty(peerAcks) } go func(net Network) { @@ -694,25 +663,14 @@ func TestAllowConnectionAsAValidator(t *testing.T) { msgCreator := newMessageCreator(t) registry := prometheus.NewRegistry() - g, err := peer.NewGossipTracker(registry, "foobar") - require.NoError(err) - - log := logging.NoLog{} - gossipTrackerCallback := peer.GossipTrackerCallback{ - Log: log, - GossipTracker: g, - } - beacons := validators.NewManager() require.NoError(beacons.AddStaker(constants.PrimaryNetworkID, nodeIDs[0], nil, ids.GenerateTestID(), 1)) vdrs := validators.NewManager() - vdrs.RegisterCallbackListener(constants.PrimaryNetworkID, &gossipTrackerCallback) require.NoError(vdrs.AddStaker(constants.PrimaryNetworkID, nodeIDs[0], nil, ids.GenerateTestID(), 1)) config := config - config.GossipTracker = g config.Beacons = beacons config.Validators = vdrs config.RequireValidatorToConnect = true @@ -721,7 +679,7 @@ func TestAllowConnectionAsAValidator(t *testing.T) { config, msgCreator, registry, - log, + logging.NoLog{}, listeners[i], dialer, &testHandler{ diff --git a/network/peer/gossip_tracker.go b/network/peer/gossip_tracker.go deleted file mode 100644 index 105d3e5ce64d..000000000000 --- a/network/peer/gossip_tracker.go +++ /dev/null @@ -1,323 +0,0 @@ -// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. -// See the file LICENSE for licensing terms. - -package peer - -import ( - "fmt" - "sync" - - "github.com/prometheus/client_golang/prometheus" - - "github.com/ava-labs/avalanchego/ids" - "github.com/ava-labs/avalanchego/utils/set" -) - -// GossipTracker tracks the validators that we're currently aware of, as well as -// the validators we've told each peers about. This data is stored in a bitset -// to optimize space, where only N (num validators) bits will be used per peer. -// -// This is done by recording some state information of both what validators this -// node is aware of, and what validators we've told each peer about. -// As an example, say we track three peers and three validators (MSB first): -// -// trackedPeers: { -// p1: [1, 1, 1] // we have already told [p1] about all validators -// p2: [0, 1, 1] // [p2] doesn't know about [v3] -// p3: [0, 0, 1] // [p3] knows only about [v3] -// } -// -// GetUnknown computes the validators we haven't sent to a given peer. Ex: -// -// GetUnknown(p1) - [0, 0, 0] -// GetUnknown(p2) - [1, 0, 0] -// GetUnknown(p3) - [1, 1, 0] -// -// Using the gossipTracker, we can quickly compute the validators each peer -// doesn't know about using GetUnknown so that in subsequent PeerList gossip -// messages we only send information that this peer (most likely) doesn't -// already know about. The only case where we'll send a redundant set of -// bytes is if another remote peer gossips to the same peer we're trying to -// gossip to first. -type GossipTracker interface { - // Tracked returns if a peer is being tracked - // Returns: - // bool: False if [peerID] is not tracked. True otherwise. - Tracked(peerID ids.NodeID) bool - - // StartTrackingPeer starts tracking a peer - // Returns: - // bool: False if [peerID] was already tracked. True otherwise. - StartTrackingPeer(peerID ids.NodeID) bool - // StopTrackingPeer stops tracking a given peer - // Returns: - // bool: False if [peerID] was not tracked. True otherwise. - StopTrackingPeer(peerID ids.NodeID) bool - - // AddValidator adds a validator that can be gossiped about - // bool: False if a validator with the same node ID or txID as [validator] - // is present. True otherwise. - AddValidator(validator ValidatorID) bool - // GetNodeID maps a txID into a nodeIDs - // nodeID: The nodeID that was registered by [txID] - // bool: False if [validator] was not present. True otherwise. - GetNodeID(txID ids.ID) (ids.NodeID, bool) - // RemoveValidator removes a validator that can be gossiped about - // bool: False if [validator] was already not present. True otherwise. - RemoveValidator(validatorID ids.NodeID) bool - // ResetValidator resets known gossip status of [validatorID] to unknown - // for all peers - // bool: False if [validator] was not present. True otherwise. - ResetValidator(validatorID ids.NodeID) bool - - // AddKnown adds [knownTxIDs] to the txIDs known by [peerID] and filters - // [txIDs] for non-validators. - // Returns: - // txIDs: The txIDs in [txIDs] that are currently validators. - // bool: False if [peerID] is not tracked. True otherwise. - AddKnown( - peerID ids.NodeID, - knownTxIDs []ids.ID, - txIDs []ids.ID, - ) ([]ids.ID, bool) - // GetUnknown gets the peers that we haven't sent to this peer - // Returns: - // []ValidatorID: a slice of ValidatorIDs that [peerID] doesn't know about. - // bool: False if [peerID] is not tracked. True otherwise. - GetUnknown(peerID ids.NodeID) ([]ValidatorID, bool) -} - -type gossipTracker struct { - lock sync.RWMutex - // a mapping of txIDs => the validator added to the validiator set by that - // tx. - txIDsToNodeIDs map[ids.ID]ids.NodeID - // a mapping of validators => the index they occupy in the bitsets - nodeIDsToIndices map[ids.NodeID]int - // each validator in the index it occupies in the bitset - validatorIDs []ValidatorID - // a mapping of each peer => the validators they know about - trackedPeers map[ids.NodeID]set.Bits - - metrics gossipTrackerMetrics -} - -// NewGossipTracker returns an instance of gossipTracker -func NewGossipTracker( - registerer prometheus.Registerer, - namespace string, -) (GossipTracker, error) { - m, err := newGossipTrackerMetrics(registerer, fmt.Sprintf("%s_gossip_tracker", namespace)) - if err != nil { - return nil, err - } - - return &gossipTracker{ - txIDsToNodeIDs: make(map[ids.ID]ids.NodeID), - nodeIDsToIndices: make(map[ids.NodeID]int), - trackedPeers: make(map[ids.NodeID]set.Bits), - metrics: m, - }, nil -} - -func (g *gossipTracker) Tracked(peerID ids.NodeID) bool { - g.lock.RLock() - defer g.lock.RUnlock() - - _, ok := g.trackedPeers[peerID] - return ok -} - -func (g *gossipTracker) StartTrackingPeer(peerID ids.NodeID) bool { - g.lock.Lock() - defer g.lock.Unlock() - - // don't track the peer if it's already being tracked - if _, ok := g.trackedPeers[peerID]; ok { - return false - } - - // start tracking the peer. Initialize their bitset to zero since we - // haven't sent them anything yet. - g.trackedPeers[peerID] = set.NewBits() - - // emit metrics - g.metrics.trackedPeersSize.Set(float64(len(g.trackedPeers))) - - return true -} - -func (g *gossipTracker) StopTrackingPeer(peerID ids.NodeID) bool { - g.lock.Lock() - defer g.lock.Unlock() - - // only stop tracking peers that are actually being tracked - if _, ok := g.trackedPeers[peerID]; !ok { - return false - } - - // stop tracking the peer by removing them - delete(g.trackedPeers, peerID) - g.metrics.trackedPeersSize.Set(float64(len(g.trackedPeers))) - - return true -} - -func (g *gossipTracker) AddValidator(validator ValidatorID) bool { - g.lock.Lock() - defer g.lock.Unlock() - - // only add validators that are not already present - if _, ok := g.txIDsToNodeIDs[validator.TxID]; ok { - return false - } - if _, ok := g.nodeIDsToIndices[validator.NodeID]; ok { - return false - } - - // add the validator to the MSB of the bitset. - msb := len(g.validatorIDs) - g.txIDsToNodeIDs[validator.TxID] = validator.NodeID - g.nodeIDsToIndices[validator.NodeID] = msb - g.validatorIDs = append(g.validatorIDs, validator) - - // emit metrics - g.metrics.validatorsSize.Set(float64(len(g.validatorIDs))) - - return true -} - -func (g *gossipTracker) GetNodeID(txID ids.ID) (ids.NodeID, bool) { - g.lock.RLock() - defer g.lock.RUnlock() - - nodeID, ok := g.txIDsToNodeIDs[txID] - return nodeID, ok -} - -func (g *gossipTracker) RemoveValidator(validatorID ids.NodeID) bool { - g.lock.Lock() - defer g.lock.Unlock() - - // only remove validators that are already present - indexToRemove, ok := g.nodeIDsToIndices[validatorID] - if !ok { - return false - } - validatorToRemove := g.validatorIDs[indexToRemove] - - // swap the validator-to-be-removed with the validator in the last index - // if the element we're swapping with is ourselves, we can skip this swap - // since we only need to delete instead - lastIndex := len(g.validatorIDs) - 1 - if indexToRemove != lastIndex { - lastValidator := g.validatorIDs[lastIndex] - - g.nodeIDsToIndices[lastValidator.NodeID] = indexToRemove - g.validatorIDs[indexToRemove] = lastValidator - } - - delete(g.txIDsToNodeIDs, validatorToRemove.TxID) - delete(g.nodeIDsToIndices, validatorID) - g.validatorIDs = g.validatorIDs[:lastIndex] - - // Invariant: We must remove the validator from everyone else's validator - // bitsets to make sure that each validator occupies the same position in - // each bitset. - for _, knownPeers := range g.trackedPeers { - // swap the element to be removed with the msb - if indexToRemove != lastIndex { - if knownPeers.Contains(lastIndex) { - knownPeers.Add(indexToRemove) - } else { - knownPeers.Remove(indexToRemove) - } - } - knownPeers.Remove(lastIndex) - } - - // emit metrics - g.metrics.validatorsSize.Set(float64(len(g.validatorIDs))) - - return true -} - -func (g *gossipTracker) ResetValidator(validatorID ids.NodeID) bool { - g.lock.Lock() - defer g.lock.Unlock() - - // only reset validators that exist - indexToReset, ok := g.nodeIDsToIndices[validatorID] - if !ok { - return false - } - - for _, knownPeers := range g.trackedPeers { - knownPeers.Remove(indexToReset) - } - - return true -} - -// AddKnown invariants: -// -// 1. [peerID] SHOULD only be a nodeID that has been tracked with -// StartTrackingPeer(). -func (g *gossipTracker) AddKnown( - peerID ids.NodeID, - knownTxIDs []ids.ID, - txIDs []ids.ID, -) ([]ids.ID, bool) { - g.lock.Lock() - defer g.lock.Unlock() - - knownPeers, ok := g.trackedPeers[peerID] - if !ok { - return nil, false - } - for _, txID := range knownTxIDs { - nodeID, ok := g.txIDsToNodeIDs[txID] - if !ok { - // We don't know about this txID, this can happen due to differences - // between our current validator set and the peer's current - // validator set. - continue - } - - // Because we fetched the nodeID from [g.txIDsToNodeIDs], we are - // guaranteed that the index is populated. - index := g.nodeIDsToIndices[nodeID] - knownPeers.Add(index) - } - - validatorTxIDs := make([]ids.ID, 0, len(txIDs)) - for _, txID := range txIDs { - if _, ok := g.txIDsToNodeIDs[txID]; ok { - validatorTxIDs = append(validatorTxIDs, txID) - } - } - return validatorTxIDs, true -} - -func (g *gossipTracker) GetUnknown(peerID ids.NodeID) ([]ValidatorID, bool) { - g.lock.RLock() - defer g.lock.RUnlock() - - // return false if this peer isn't tracked - knownPeers, ok := g.trackedPeers[peerID] - if !ok { - return nil, false - } - - // Calculate the unknown information we need to send to this peer. We do - // this by computing the difference between the validators we know about - // and the validators we know we've sent to [peerID]. - result := make([]ValidatorID, 0, len(g.validatorIDs)) - for i, validatorID := range g.validatorIDs { - if !knownPeers.Contains(i) { - result = append(result, validatorID) - } - } - - return result, true -} diff --git a/network/peer/gossip_tracker_callback.go b/network/peer/gossip_tracker_callback.go deleted file mode 100644 index 5863b236e069..000000000000 --- a/network/peer/gossip_tracker_callback.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. -// See the file LICENSE for licensing terms. - -package peer - -import ( - "go.uber.org/zap" - - "github.com/ava-labs/avalanchego/ids" - "github.com/ava-labs/avalanchego/snow/validators" - "github.com/ava-labs/avalanchego/utils/crypto/bls" - "github.com/ava-labs/avalanchego/utils/logging" -) - -var _ validators.SetCallbackListener = (*GossipTrackerCallback)(nil) - -// GossipTrackerCallback synchronizes GossipTracker's validator state with the -// validator set it's registered to. -type GossipTrackerCallback struct { - Log logging.Logger - GossipTracker GossipTracker -} - -// OnValidatorAdded adds [validatorID] to the set of validators that can be -// gossiped about -func (g *GossipTrackerCallback) OnValidatorAdded( - nodeID ids.NodeID, - _ *bls.PublicKey, - txID ids.ID, - _ uint64, -) { - vdr := ValidatorID{ - NodeID: nodeID, - TxID: txID, - } - if !g.GossipTracker.AddValidator(vdr) { - g.Log.Error("failed to add a validator", - zap.Stringer("nodeID", nodeID), - zap.Stringer("txID", txID), - ) - } -} - -// OnValidatorRemoved removes [validatorID] from the set of validators that can -// be gossiped about. -func (g *GossipTrackerCallback) OnValidatorRemoved(nodeID ids.NodeID, _ uint64) { - if !g.GossipTracker.RemoveValidator(nodeID) { - g.Log.Error("failed to remove a validator", - zap.Stringer("nodeID", nodeID), - ) - } -} - -// OnValidatorWeightChanged does nothing because PeerList gossip doesn't care -// about validator weights. -func (*GossipTrackerCallback) OnValidatorWeightChanged(ids.NodeID, uint64, uint64) {} diff --git a/network/peer/gossip_tracker_metrics.go b/network/peer/gossip_tracker_metrics.go deleted file mode 100644 index 080f37fde5c1..000000000000 --- a/network/peer/gossip_tracker_metrics.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. -// See the file LICENSE for licensing terms. - -package peer - -import ( - "github.com/prometheus/client_golang/prometheus" - - "github.com/ava-labs/avalanchego/utils" -) - -type gossipTrackerMetrics struct { - trackedPeersSize prometheus.Gauge - validatorsSize prometheus.Gauge -} - -func newGossipTrackerMetrics(registerer prometheus.Registerer, namespace string) (gossipTrackerMetrics, error) { - m := gossipTrackerMetrics{ - trackedPeersSize: prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: namespace, - Name: "tracked_peers_size", - Help: "amount of peers that are being tracked", - }, - ), - validatorsSize: prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: namespace, - Name: "validators_size", - Help: "number of validators this node is tracking", - }, - ), - } - - err := utils.Err( - registerer.Register(m.trackedPeersSize), - registerer.Register(m.validatorsSize), - ) - return m, err -} diff --git a/network/peer/gossip_tracker_test.go b/network/peer/gossip_tracker_test.go deleted file mode 100644 index c9ab1ef8e026..000000000000 --- a/network/peer/gossip_tracker_test.go +++ /dev/null @@ -1,620 +0,0 @@ -// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. -// See the file LICENSE for licensing terms. - -package peer - -import ( - "testing" - - "github.com/prometheus/client_golang/prometheus" - - "github.com/stretchr/testify/require" - - "github.com/ava-labs/avalanchego/ids" -) - -var ( - // peers - p1 = ids.GenerateTestNodeID() - p2 = ids.GenerateTestNodeID() - p3 = ids.GenerateTestNodeID() - - // validators - v1 = ValidatorID{ - NodeID: ids.GenerateTestNodeID(), - TxID: ids.GenerateTestID(), - } - v2 = ValidatorID{ - NodeID: ids.GenerateTestNodeID(), - TxID: ids.GenerateTestID(), - } - v3 = ValidatorID{ - NodeID: ids.GenerateTestNodeID(), - TxID: ids.GenerateTestID(), - } -) - -func TestGossipTracker_Contains(t *testing.T) { - tests := []struct { - name string - track []ids.NodeID - contains ids.NodeID - expected bool - }{ - { - name: "empty", - track: []ids.NodeID{}, - contains: p1, - expected: false, - }, - { - name: "populated - does not contain", - track: []ids.NodeID{p1, p2}, - contains: p3, - expected: false, - }, - { - name: "populated - contains", - track: []ids.NodeID{p1, p2, p3}, - contains: p3, - expected: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - require := require.New(t) - - g, err := NewGossipTracker(prometheus.NewRegistry(), "foobar") - require.NoError(err) - - for _, add := range test.track { - require.True(g.StartTrackingPeer(add)) - } - - require.Equal(test.expected, g.Tracked(test.contains)) - }) - } -} - -func TestGossipTracker_StartTrackingPeer(t *testing.T) { - tests := []struct { - name string - toStartTracking []ids.NodeID - expected []bool - }{ - { - // Tracking new peers always works - name: "unique adds", - toStartTracking: []ids.NodeID{p1, p2, p3}, - expected: []bool{true, true, true}, - }, - { - // We shouldn't be able to track a peer more than once - name: "duplicate adds", - toStartTracking: []ids.NodeID{p1, p1, p1}, - expected: []bool{true, false, false}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - require := require.New(t) - - g, err := NewGossipTracker(prometheus.NewRegistry(), "foobar") - require.NoError(err) - - for i, p := range test.toStartTracking { - require.Equal(test.expected[i], g.StartTrackingPeer(p)) - require.True(g.Tracked(p)) - } - }) - } -} - -func TestGossipTracker_StopTrackingPeer(t *testing.T) { - tests := []struct { - name string - toStartTracking []ids.NodeID - expectedStartTracking []bool - toStopTracking []ids.NodeID - expectedStopTracking []bool - }{ - { - // We should be able to stop tracking that we are tracking - name: "stop tracking tracked peers", - toStartTracking: []ids.NodeID{p1, p2, p3}, - toStopTracking: []ids.NodeID{p1, p2, p3}, - expectedStopTracking: []bool{true, true, true}, - }, - { - // We shouldn't be able to stop tracking peers we've stopped tracking - name: "stop tracking twice", - toStartTracking: []ids.NodeID{p1}, - toStopTracking: []ids.NodeID{p1, p1}, - expectedStopTracking: []bool{true, false}, - }, - { - // We shouldn't be able to stop tracking peers we were never tracking - name: "remove non-existent elements", - toStartTracking: []ids.NodeID{}, - expectedStartTracking: []bool{}, - toStopTracking: []ids.NodeID{p1, p2, p3}, - expectedStopTracking: []bool{false, false, false}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - require := require.New(t) - - g, err := NewGossipTracker(prometheus.NewRegistry(), "foobar") - require.NoError(err) - - for _, add := range test.toStartTracking { - require.True(g.StartTrackingPeer(add)) - require.True(g.Tracked(add)) - } - - for i, p := range test.toStopTracking { - require.Equal(test.expectedStopTracking[i], g.StopTrackingPeer(p)) - } - }) - } -} - -func TestGossipTracker_AddValidator(t *testing.T) { - type args struct { - validator ValidatorID - } - - tests := []struct { - name string - validators []ValidatorID - args args - expected bool - }{ - { - name: "not present", - validators: []ValidatorID{}, - args: args{validator: v1}, - expected: true, - }, - { - name: "already present txID but with different nodeID", - validators: []ValidatorID{v1}, - args: args{validator: ValidatorID{ - NodeID: ids.GenerateTestNodeID(), - TxID: v1.TxID, - }}, - expected: false, - }, - { - name: "already present nodeID but with different txID", - validators: []ValidatorID{v1}, - args: args{validator: ValidatorID{ - NodeID: v1.NodeID, - TxID: ids.GenerateTestID(), - }}, - expected: false, - }, - { - name: "already present validatorID", - validators: []ValidatorID{v1}, - args: args{validator: v1}, - expected: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - require := require.New(t) - - g, err := NewGossipTracker(prometheus.NewRegistry(), "foobar") - require.NoError(err) - - for _, v := range test.validators { - require.True(g.AddValidator(v)) - } - - require.Equal(test.expected, g.AddValidator(test.args.validator)) - }) - } -} - -func TestGossipTracker_RemoveValidator(t *testing.T) { - type args struct { - id ids.NodeID - } - - tests := []struct { - name string - validators []ValidatorID - args args - expected bool - }{ - { - name: "not already present", - validators: []ValidatorID{}, - args: args{id: v1.NodeID}, - expected: false, - }, - { - name: "already present", - validators: []ValidatorID{v1}, - args: args{id: v1.NodeID}, - expected: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - require := require.New(t) - - g, err := NewGossipTracker(prometheus.NewRegistry(), "foobar") - require.NoError(err) - - for _, v := range test.validators { - require.True(g.AddValidator(v)) - } - - require.Equal(test.expected, g.RemoveValidator(test.args.id)) - }) - } -} - -func TestGossipTracker_ResetValidator(t *testing.T) { - type args struct { - id ids.NodeID - } - - tests := []struct { - name string - validators []ValidatorID - args args - expected bool - }{ - { - name: "non-existent validator", - validators: []ValidatorID{}, - args: args{id: v1.NodeID}, - expected: false, - }, - { - name: "existing validator", - validators: []ValidatorID{v1}, - args: args{id: v1.NodeID}, - expected: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - require := require.New(t) - - g, err := NewGossipTracker(prometheus.NewRegistry(), "foobar") - require.NoError(err) - - require.True(g.StartTrackingPeer(p1)) - - for _, v := range test.validators { - require.True(g.AddValidator(v)) - g.AddKnown(p1, []ids.ID{v.TxID}, nil) - - unknown, ok := g.GetUnknown(p1) - require.True(ok) - require.NotContains(unknown, v) - } - - require.Equal(test.expected, g.ResetValidator(test.args.id)) - - for _, v := range test.validators { - unknown, ok := g.GetUnknown(p1) - require.True(ok) - require.Contains(unknown, v) - } - }) - } -} - -func TestGossipTracker_AddKnown(t *testing.T) { - type args struct { - peerID ids.NodeID - txIDs []ids.ID - } - - tests := []struct { - name string - trackedPeers []ids.NodeID - validators []ValidatorID - args args - expectedTxIDs []ids.ID - expectedOk bool - }{ - { - // We should not be able to update an untracked peer - name: "untracked peer - empty", - trackedPeers: []ids.NodeID{}, - validators: []ValidatorID{}, - args: args{peerID: p1, txIDs: []ids.ID{}}, - expectedTxIDs: nil, - expectedOk: false, - }, - { - // We should not be able to update an untracked peer - name: "untracked peer - populated", - trackedPeers: []ids.NodeID{p2, p3}, - validators: []ValidatorID{}, - args: args{peerID: p1, txIDs: []ids.ID{}}, - expectedTxIDs: nil, - expectedOk: false, - }, - { - // We shouldn't be able to look up a peer that isn't tracked - name: "untracked peer - unknown validator", - trackedPeers: []ids.NodeID{}, - validators: []ValidatorID{}, - args: args{peerID: p1, txIDs: []ids.ID{v1.TxID}}, - expectedTxIDs: nil, - expectedOk: false, - }, - { - // We shouldn't fail on a validator that's not registered - name: "tracked peer - unknown validator", - trackedPeers: []ids.NodeID{p1}, - validators: []ValidatorID{}, - args: args{peerID: p1, txIDs: []ids.ID{v1.TxID}}, - expectedTxIDs: []ids.ID{}, - expectedOk: true, - }, - { - // We should be able to update a tracked validator - name: "update tracked validator", - trackedPeers: []ids.NodeID{p1, p2, p3}, - validators: []ValidatorID{v1}, - args: args{peerID: p1, txIDs: []ids.ID{v1.TxID}}, - expectedTxIDs: []ids.ID{v1.TxID}, - expectedOk: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - require := require.New(t) - - g, err := NewGossipTracker(prometheus.NewRegistry(), "foobar") - require.NoError(err) - - for _, p := range test.trackedPeers { - require.True(g.StartTrackingPeer(p)) - require.True(g.Tracked(p)) - } - - for _, v := range test.validators { - require.True(g.AddValidator(v)) - } - - txIDs, ok := g.AddKnown(test.args.peerID, test.args.txIDs, test.args.txIDs) - require.Equal(test.expectedOk, ok) - require.Equal(test.expectedTxIDs, txIDs) - }) - } -} - -func TestGossipTracker_GetUnknown(t *testing.T) { - tests := []struct { - name string - peerID ids.NodeID - peersToTrack []ids.NodeID - validators []ValidatorID - expectedUnknown []ValidatorID - expectedOk bool - }{ - { - name: "non tracked peer", - peerID: p1, - validators: []ValidatorID{v2}, - peersToTrack: []ids.NodeID{}, - expectedUnknown: nil, - expectedOk: false, - }, - { - name: "only validators", - peerID: p1, - peersToTrack: []ids.NodeID{p1}, - validators: []ValidatorID{v2}, - expectedUnknown: []ValidatorID{v2}, - expectedOk: true, - }, - { - name: "only non-validators", - peerID: p1, - peersToTrack: []ids.NodeID{p1, p2}, - validators: []ValidatorID{}, - expectedUnknown: []ValidatorID{}, - expectedOk: true, - }, - { - name: "validators and non-validators", - peerID: p1, - peersToTrack: []ids.NodeID{p1, p3}, - validators: []ValidatorID{v2}, - expectedUnknown: []ValidatorID{v2}, - expectedOk: true, - }, - { - name: "same as limit", - peerID: p1, - peersToTrack: []ids.NodeID{p1}, - validators: []ValidatorID{v2, v3}, - expectedUnknown: []ValidatorID{v2, v3}, - expectedOk: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - require := require.New(t) - - g, err := NewGossipTracker(prometheus.NewRegistry(), "foobar") - require.NoError(err) - - // add our validators - for _, validator := range test.validators { - require.True(g.AddValidator(validator)) - } - - // start tracking our peers - for _, nonValidator := range test.peersToTrack { - require.True(g.StartTrackingPeer(nonValidator)) - require.True(g.Tracked(nonValidator)) - } - - // get the unknown peers for this peer - result, ok := g.GetUnknown(test.peerID) - require.Equal(test.expectedOk, ok) - require.Len(result, len(test.expectedUnknown)) - for _, v := range test.expectedUnknown { - require.Contains(result, v) - } - }) - } -} - -func TestGossipTracker_E2E(t *testing.T) { - require := require.New(t) - - g, err := NewGossipTracker(prometheus.NewRegistry(), "foobar") - require.NoError(err) - - // [v1, v2, v3] are validators - require.True(g.AddValidator(v1)) - require.True(g.AddValidator(v2)) - - // we should get an empty unknown since we're not tracking anything - unknown, ok := g.GetUnknown(p1) - require.False(ok) - require.Nil(unknown) - - // we should get a unknown of [v1, v2] since v1 and v2 are registered - require.True(g.StartTrackingPeer(p1)) - require.True(g.Tracked(p1)) - - // check p1's unknown - unknown, ok = g.GetUnknown(p1) - require.True(ok) - require.Contains(unknown, v1) - require.Contains(unknown, v2) - require.Len(unknown, 2) - - // Check p2's unknown. We should get nothing since we're not tracking it - // yet. - unknown, ok = g.GetUnknown(p2) - require.False(ok) - require.Nil(unknown) - - // Start tracking p2 - require.True(g.StartTrackingPeer(p2)) - - // check p2's unknown - unknown, ok = g.GetUnknown(p2) - require.True(ok) - require.Contains(unknown, v1) - require.Contains(unknown, v2) - require.Len(unknown, 2) - - // p1 now knows about v1, but not v2, so it should see [v2] in its unknown - // p2 still knows nothing, so it should see both - txIDs, ok := g.AddKnown(p1, []ids.ID{v1.TxID}, []ids.ID{v1.TxID}) - require.True(ok) - require.Equal([]ids.ID{v1.TxID}, txIDs) - - // p1 should have an unknown of [v2], since it knows v1 - unknown, ok = g.GetUnknown(p1) - require.True(ok) - require.Contains(unknown, v2) - require.Len(unknown, 1) - - // p2 should have a unknown of [v1, v2], since it knows nothing - unknown, ok = g.GetUnknown(p2) - require.True(ok) - require.Contains(unknown, v1) - require.Contains(unknown, v2) - require.Len(unknown, 2) - - // Add v3 - require.True(g.AddValidator(v3)) - - // track p3, who knows of v1, v2, and v3 - // p1 and p2 still don't know of v3 - require.True(g.StartTrackingPeer(p3)) - - txIDs, ok = g.AddKnown(p3, []ids.ID{v1.TxID, v2.TxID, v3.TxID}, []ids.ID{v1.TxID, v2.TxID, v3.TxID}) - require.True(ok) - require.Equal([]ids.ID{v1.TxID, v2.TxID, v3.TxID}, txIDs) - - // p1 doesn't know about [v2, v3] - unknown, ok = g.GetUnknown(p1) - require.True(ok) - require.Contains(unknown, v2) - require.Contains(unknown, v3) - require.Len(unknown, 2) - - // p2 doesn't know about [v1, v2, v3] - unknown, ok = g.GetUnknown(p2) - require.True(ok) - require.Contains(unknown, v1) - require.Contains(unknown, v2) - require.Contains(unknown, v3) - require.Len(unknown, 3) - - // p3 knows about everyone - unknown, ok = g.GetUnknown(p3) - require.True(ok) - require.Empty(unknown) - - // stop tracking p2 - require.True(g.StopTrackingPeer(p2)) - unknown, ok = g.GetUnknown(p2) - require.False(ok) - require.Nil(unknown) - - // p1 doesn't know about [v2, v3] because v2 is still registered as - // a validator - unknown, ok = g.GetUnknown(p1) - require.True(ok) - require.Contains(unknown, v2) - require.Contains(unknown, v3) - require.Len(unknown, 2) - - // Remove p2 from the validator set - require.True(g.RemoveValidator(v2.NodeID)) - - // p1 doesn't know about [v3] since v2 left the validator set - unknown, ok = g.GetUnknown(p1) - require.True(ok) - require.Contains(unknown, v3) - require.Len(unknown, 1) - - // p3 knows about everyone since it learned about v1 and v3 earlier. - unknown, ok = g.GetUnknown(p3) - require.Empty(unknown) - require.True(ok) -} - -func TestGossipTracker_Regression_IncorrectTxIDDeletion(t *testing.T) { - require := require.New(t) - - g, err := NewGossipTracker(prometheus.NewRegistry(), "foobar") - require.NoError(err) - - require.True(g.AddValidator(v1)) - require.True(g.AddValidator(v2)) - - require.True(g.RemoveValidator(v1.NodeID)) - - require.False(g.AddValidator(ValidatorID{ - NodeID: ids.GenerateTestNodeID(), - TxID: v2.TxID, - })) -} diff --git a/network/peer/network.go b/network/peer/network.go index 8c18ef0ac899..b8fb01814546 100644 --- a/network/peer/network.go +++ b/network/peer/network.go @@ -5,7 +5,7 @@ package peer import ( "github.com/ava-labs/avalanchego/ids" - "github.com/ava-labs/avalanchego/proto/pb/p2p" + "github.com/ava-labs/avalanchego/utils/bloom" "github.com/ava-labs/avalanchego/utils/ips" ) @@ -19,15 +19,9 @@ type Network interface { // connection is no longer desired and should be terminated. AllowConnection(peerID ids.NodeID) bool - // Track allows the peer to notify the network of a potential new peer to - // connect to, given the [ips] of the peers it sent us during the peer - // handshake. - // - // Returns which IPs should not be gossipped to this node again. - Track(peerID ids.NodeID, ips []*ips.ClaimedIPPort) ([]*p2p.PeerAck, error) - - // MarkTracked stops sending gossip about [ips] to [peerID]. - MarkTracked(peerID ids.NodeID, ips []*p2p.PeerAck) error + // Track allows the peer to notify the network of potential new peers to + // connect to. + Track(ips []*ips.ClaimedIPPort) error // Disconnected is called when the peer finishes shutting down. It is not // guaranteed that [Connected] was called for the provided peer. However, it @@ -35,6 +29,13 @@ type Network interface { // for a given [Peer] object. Disconnected(peerID ids.NodeID) - // Peers returns peers that [peerID] might not know about. - Peers(peerID ids.NodeID) ([]ips.ClaimedIPPort, error) + // KnownPeers returns the bloom filter of the known peers. + KnownPeers() (bloomFilter []byte, salt []byte) + + // Peers returns peers that are not known. + Peers( + peerID ids.NodeID, + knownPeers *bloom.ReadFilter, + peerSalt []byte, + ) []*ips.ClaimedIPPort } diff --git a/network/peer/peer.go b/network/peer/peer.go index 16b5d3ab3090..db2cb5485b04 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -21,6 +21,7 @@ import ( "github.com/ava-labs/avalanchego/proto/pb/p2p" "github.com/ava-labs/avalanchego/staking" "github.com/ava-labs/avalanchego/utils" + "github.com/ava-labs/avalanchego/utils/bloom" "github.com/ava-labs/avalanchego/utils/constants" "github.com/ava-labs/avalanchego/utils/ips" "github.com/ava-labs/avalanchego/utils/json" @@ -29,6 +30,10 @@ import ( "github.com/ava-labs/avalanchego/version" ) +// maxBloomSaltLen restricts the allowed size of the bloom salt to prevent +// excessively expensive bloom filter contains checks. +const maxBloomSaltLen = 32 + var ( errClosed = errors.New("closed") @@ -91,6 +96,11 @@ type Peer interface { // sent. StartSendPeerList() + // StartSendGetPeerList attempts to send a GetPeerList message to this peer + // on this peer's gossip routine. It is not guaranteed that a GetPeerList + // will be sent. + StartSendGetPeerList() + // StartClose will begin shutting down the peer. It will not block. StartClose() @@ -170,6 +180,10 @@ type peer struct { // peerListChan signals that we should attempt to send a PeerList to this // peer peerListChan chan struct{} + + // getPeerListChan signals that we should attempt to send a GetPeerList to + // this peer + getPeerListChan chan struct{} } // Start a new peer instance. @@ -197,6 +211,7 @@ func Start( onClosed: make(chan struct{}), observedUptimes: make(map[ids.ID]uint32), peerListChan: make(chan struct{}, 1), + getPeerListChan: make(chan struct{}, 1), } go p.readMessages() @@ -310,6 +325,13 @@ func (p *peer) StartSendPeerList() { } } +func (p *peer) StartSendGetPeerList() { + select { + case p.getPeerListChan <- struct{}{}: + default: + } +} + func (p *peer) StartClose() { p.startClosingOnce.Do(func() { if err := p.conn.Close(); err != nil { @@ -516,6 +538,8 @@ func (p *peer) writeMessages() { Patch: myVersion.Patch, } + knownPeersFilter, knownPeersSalt := p.Network.KnownPeers() + msg, err := p.MessageCreator.Handshake( p.NetworkID, p.Clock.Unix(), @@ -530,6 +554,8 @@ func (p *peer) writeMessages() { p.MySubnets.List(), p.SupportedACPs, p.ObjectedACPs, + knownPeersFilter, + knownPeersSalt, ) if err != nil { p.Log.Error("failed to create message", @@ -621,15 +647,7 @@ func (p *peer) sendNetworkMessages() { for { select { case <-p.peerListChan: - peerIPs, err := p.Config.Network.Peers(p.id) - if err != nil { - p.Log.Error("failed to get peers to gossip", - zap.Stringer("nodeID", p.id), - zap.Error(err), - ) - return - } - + peerIPs := p.Config.Network.Peers(p.id, bloom.EmptyFilter, nil) if len(peerIPs) == 0 { p.Log.Verbo( "skipping peer gossip as there are no unknown peers", @@ -654,6 +672,22 @@ func (p *peer) sendNetworkMessages() { zap.Stringer("nodeID", p.id), ) } + case <-p.getPeerListChan: + knownPeersFilter, knownPeersSalt := p.Config.Network.KnownPeers() + msg, err := p.Config.MessageCreator.GetPeerList(knownPeersFilter, knownPeersSalt) + if err != nil { + p.Log.Error("failed to create get peer list message", + zap.Stringer("nodeID", p.id), + zap.Error(err), + ) + continue + } + + if !p.Send(p.onClosingCtx, msg) { + p.Log.Debug("failed to send get peer list", + zap.Stringer("nodeID", p.id), + ) + } case <-sendPingsTicker.C: if !p.Network.AllowConnection(p.id) { p.Log.Debug("disconnecting from peer", @@ -707,12 +741,12 @@ func (p *peer) handle(msg message.InboundMessage) { p.handleHandshake(m) msg.OnFinishedHandling() return - case *p2p.PeerList: - p.handlePeerList(m) + case *p2p.GetPeerList: + p.handleGetPeerList(m) msg.OnFinishedHandling() return - case *p2p.PeerListAck: - p.handlePeerListAck(m) + case *p2p.PeerList: + p.handlePeerList(m) msg.OnFinishedHandling() return } @@ -992,6 +1026,37 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { return } + var ( + knownPeers = bloom.EmptyFilter + salt []byte + ) + if msg.KnownPeers != nil { + var err error + knownPeers, err = bloom.Parse(msg.KnownPeers.Filter) + if err != nil { + p.Log.Debug("message with invalid field", + zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.HandshakeOp), + zap.String("field", "KnownPeers.Filter"), + zap.Error(err), + ) + p.StartClose() + return + } + + salt = msg.KnownPeers.Salt + if saltLen := len(salt); saltLen > maxBloomSaltLen { + p.Log.Debug("message with invalid field", + zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.HandshakeOp), + zap.String("field", "KnownPeers.Salt"), + zap.Int("saltLen", saltLen), + ) + p.StartClose() + return + } + } + // "net.IP" type in Golang is 16-byte if ipLen := len(msg.IpAddr); ipLen != net.IPv6len { p.Log.Debug("message with invalid field", @@ -1035,14 +1100,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { p.gotHandshake.Set(true) - peerIPs, err := p.Network.Peers(p.id) - if err != nil { - p.Log.Error("failed to get peers to gossip for handshake", - zap.Stringer("nodeID", p.id), - zap.Error(err), - ) - return - } + peerIPs := p.Network.Peers(p.id, knownPeers, salt) // We bypass throttling here to ensure that the peerlist message is // acknowledged timely. @@ -1066,6 +1124,65 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { } } +func (p *peer) handleGetPeerList(msg *p2p.GetPeerList) { + if !p.finishedHandshake.Get() { + p.Log.Verbo("dropping get peer list message", + zap.Stringer("nodeID", p.id), + ) + return + } + + knownPeersMsg := msg.GetKnownPeers() + filter, err := bloom.Parse(knownPeersMsg.GetFilter()) + if err != nil { + p.Log.Debug("message with invalid field", + zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.GetPeerListOp), + zap.String("field", "KnownPeers.Filter"), + zap.Error(err), + ) + p.StartClose() + return + } + + salt := knownPeersMsg.GetSalt() + if saltLen := len(salt); saltLen > maxBloomSaltLen { + p.Log.Debug("message with invalid field", + zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.GetPeerListOp), + zap.String("field", "KnownPeers.Salt"), + zap.Int("saltLen", saltLen), + ) + p.StartClose() + return + } + + peerIPs := p.Network.Peers(p.id, filter, salt) + if len(peerIPs) == 0 { + p.Log.Debug("skipping sending of empty peer list", + zap.Stringer("nodeID", p.id), + ) + return + } + + // Bypass throttling is disabled here to follow the non-handshake message + // sending pattern. + peerListMsg, err := p.Config.MessageCreator.PeerList(peerIPs, false /*=bypassThrottling*/) + if err != nil { + p.Log.Error("failed to create peer list message", + zap.Stringer("nodeID", p.id), + zap.Error(err), + ) + return + } + + if !p.Send(p.onClosingCtx, peerListMsg) { + p.Log.Debug("failed to send peer list", + zap.Stringer("nodeID", p.id), + ) + } +} + func (p *peer) handlePeerList(msg *p2p.PeerList) { if !p.finishedHandshake.Get() { if !p.gotHandshake.Get() { @@ -1114,32 +1231,18 @@ func (p *peer) handlePeerList(msg *p2p.PeerList) { continue } - txID, err := ids.ToID(claimedIPPort.TxId) - if err != nil { - p.Log.Debug("message with invalid field", - zap.Stringer("nodeID", p.id), - zap.Stringer("messageOp", message.PeerListOp), - zap.String("field", "txID"), - zap.Error(err), - ) - p.StartClose() - return - } - - discoveredIPs[i] = &ips.ClaimedIPPort{ - Cert: tlsCert, - IPPort: ips.IPPort{ + discoveredIPs[i] = ips.NewClaimedIPPort( + tlsCert, + ips.IPPort{ IP: claimedIPPort.IpAddr, Port: uint16(claimedIPPort.IpPort), }, - Timestamp: claimedIPPort.Timestamp, - Signature: claimedIPPort.Signature, - TxID: txID, - } + claimedIPPort.Timestamp, + claimedIPPort.Signature, + ) } - trackedPeers, err := p.Network.Track(p.id, discoveredIPs) - if err != nil { + if err := p.Network.Track(discoveredIPs); err != nil { p.Log.Debug("message with invalid field", zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PeerListOp), @@ -1147,42 +1250,6 @@ func (p *peer) handlePeerList(msg *p2p.PeerList) { zap.Error(err), ) p.StartClose() - return - } - if len(trackedPeers) == 0 { - p.Log.Debug("skipping peerlist ack as there were no tracked peers", - zap.Stringer("nodeID", p.id), - ) - return - } - - peerListAckMsg, err := p.Config.MessageCreator.PeerListAck(trackedPeers) - if err != nil { - p.Log.Error("failed to create message", - zap.Stringer("messageOp", message.PeerListAckOp), - zap.Stringer("nodeID", p.id), - zap.Error(err), - ) - return - } - - if !p.Send(p.onClosingCtx, peerListAckMsg) { - p.Log.Debug("failed to send peer list ack", - zap.Stringer("nodeID", p.id), - ) - } -} - -func (p *peer) handlePeerListAck(msg *p2p.PeerListAck) { - err := p.Network.MarkTracked(p.id, msg.PeerAcks) - if err != nil { - p.Log.Debug("message with invalid field", - zap.Stringer("nodeID", p.id), - zap.Stringer("messageOp", message.PeerListAckOp), - zap.String("field", "txID"), - zap.Error(err), - ) - p.StartClose() } } diff --git a/network/peer/test_network.go b/network/peer/test_network.go index 1ba047e1d423..01a341ae9abc 100644 --- a/network/peer/test_network.go +++ b/network/peer/test_network.go @@ -5,7 +5,7 @@ package peer import ( "github.com/ava-labs/avalanchego/ids" - "github.com/ava-labs/avalanchego/proto/pb/p2p" + "github.com/ava-labs/avalanchego/utils/bloom" "github.com/ava-labs/avalanchego/utils/ips" ) @@ -19,16 +19,16 @@ func (testNetwork) AllowConnection(ids.NodeID) bool { return true } -func (testNetwork) Track(ids.NodeID, []*ips.ClaimedIPPort) ([]*p2p.PeerAck, error) { - return nil, nil -} - -func (testNetwork) MarkTracked(ids.NodeID, []*p2p.PeerAck) error { +func (testNetwork) Track([]*ips.ClaimedIPPort) error { return nil } func (testNetwork) Disconnected(ids.NodeID) {} -func (testNetwork) Peers(ids.NodeID) ([]ips.ClaimedIPPort, error) { - return nil, nil +func (testNetwork) KnownPeers() ([]byte, []byte) { + return bloom.EmptyFilter.Marshal(), nil +} + +func (testNetwork) Peers(ids.NodeID, *bloom.ReadFilter, []byte) []*ips.ClaimedIPPort { + return nil } diff --git a/network/test_network.go b/network/test_network.go index 0811875b4c43..8079e76240c1 100644 --- a/network/test_network.go +++ b/network/test_network.go @@ -156,6 +156,8 @@ func NewTestNetwork( PeerListNonValidatorGossipSize: constants.DefaultNetworkPeerListNonValidatorGossipSize, PeerListPeersGossipSize: constants.DefaultNetworkPeerListPeersGossipSize, PeerListGossipFreq: constants.DefaultNetworkPeerListGossipFreq, + PeerListPullGossipFreq: constants.DefaultNetworkPeerListPullGossipFreq, + PeerListBloomResetFreq: constants.DefaultNetworkPeerListBloomResetFreq, }, DelayConfig: DelayConfig{ @@ -186,9 +188,8 @@ func NewTestNetwork( networkConfig.TLSConfig = tlsConfig networkConfig.TLSKey = tlsCert.PrivateKey.(crypto.Signer) - beacons := validators.NewManager() networkConfig.Validators = currentValidators - networkConfig.Beacons = beacons + networkConfig.Beacons = validators.NewManager() // This never actually does anything because we never initialize the P-chain networkConfig.UptimeCalculator = uptime.NoOpCalculator @@ -227,11 +228,6 @@ func NewTestNetwork( networkConfig.MyIPPort = ips.NewDynamicIPPort(net.IPv4zero, 1) - networkConfig.GossipTracker, err = peer.NewGossipTracker(metrics, "") - if err != nil { - return nil, err - } - return NewNetwork( &networkConfig, msgCreator, diff --git a/node/node.go b/node/node.go index 8fce5df75550..24f5c1711767 100644 --- a/node/node.go +++ b/node/node.go @@ -583,18 +583,6 @@ func (n *Node) initNetworking() error { }() } - // initialize gossip tracker - gossipTracker, err := peer.NewGossipTracker(n.MetricsRegisterer, n.networkNamespace) - if err != nil { - return err - } - - // keep gossip tracker synchronized with the validator set - n.vdrs.RegisterCallbackListener(constants.PrimaryNetworkID, &peer.GossipTrackerCallback{ - Log: n.Log, - GossipTracker: gossipTracker, - }) - // add node configs to network config n.Config.NetworkConfig.Namespace = n.networkNamespace n.Config.NetworkConfig.MyNodeID = n.ID @@ -610,7 +598,6 @@ func (n *Node) initNetworking() error { n.Config.NetworkConfig.ResourceTracker = n.resourceTracker n.Config.NetworkConfig.CPUTargeter = n.cpuTargeter n.Config.NetworkConfig.DiskTargeter = n.diskTargeter - n.Config.NetworkConfig.GossipTracker = gossipTracker n.Net, err = network.NewNetwork( &n.Config.NetworkConfig, diff --git a/proto/p2p/p2p.proto b/proto/p2p/p2p.proto index 53a4d3d2126a..6b5deacc7e01 100644 --- a/proto/p2p/p2p.proto +++ b/proto/p2p/p2p.proto @@ -8,6 +8,8 @@ option go_package = "github.com/ava-labs/avalanchego/proto/pb/p2p"; // Represents peer-to-peer messages. // Only one type can be non-null. message Message { + reserved 33; // Until after durango activation. + reserved 36; // Next unused field number. // NOTES // Use "oneof" for each message type and set rest to null if not used. // That is because when the compression is enabled, we don't want to include uncompressed fields. @@ -29,6 +31,7 @@ message Message { Ping ping = 11; Pong pong = 12; Handshake handshake = 13; + GetPeerList get_peer_list = 35; PeerList peer_list = 14; // State-sync messages: @@ -56,8 +59,6 @@ message Message { AppRequest app_request = 30; AppResponse app_response = 31; AppGossip app_gossip = 32; - - PeerListAck peer_list_ack = 33; AppError app_error = 34; } } @@ -118,6 +119,7 @@ message Handshake { Client client = 9; repeated uint32 supported_acps = 10; repeated uint32 objected_acps = 11; + BloomFilter known_peers = 12; } // Metadata about a peer's P2P client used to determine compatibility @@ -130,6 +132,12 @@ message Client { uint32 patch = 4; } +// BloomFilter with a random salt to prevent consistent hash collisions +message BloomFilter { + bytes filter = 1; + bytes salt = 2; +} + // ClaimedIpPort contains metadata needed to connect to a peer message ClaimedIpPort { // X509 certificate of the peer @@ -146,38 +154,29 @@ message ClaimedIpPort { bytes tx_id = 6; } +// GetPeerList contains a bloom filter of the currently known validator IPs. +// +// GetPeerList must not be responded to until finishing the handshake. After the +// handshake is completed, GetPeerlist messages should be responded to with a +// Peerlist message containing validators that are not present in the bloom +// filter. +message GetPeerList { + BloomFilter known_peers = 1; +} + // PeerList contains network-level metadata for a set of validators. // // PeerList must be sent in response to an inbound Handshake message from a // remote peer a peer wants to connect to. Once a PeerList is received after // a Handshake message, the p2p handshake is complete and the connection is // established. - -// Peers should periodically send PeerList messages to allow peers to -// discover each other. // -// PeerListAck should be sent in response to a PeerList. +// PeerList should be sent in response to a GetPeerlist message if the handshake +// has been completed. message PeerList { repeated ClaimedIpPort claimed_ip_ports = 1; } -// PeerAck acknowledges that a gossiped peer in a PeerList message will be -// tracked by the remote peer. -message PeerAck { - // P-Chain transaction that added the acknowledged peer to the validator - // set - bytes tx_id = 1; - // Timestamp of the signed ip of the peer - uint64 timestamp = 2; -} - -// PeerListAck is sent in response to PeerList to acknowledge the subset of -// peers that the peer will attempt to connect to. -message PeerListAck { - reserved 1; // deprecated; used to be tx_ids - repeated PeerAck peer_acks = 2; -} - // GetStateSummaryFrontier requests a peer's most recently accepted state // summary message GetStateSummaryFrontier { diff --git a/proto/pb/p2p/p2p.pb.go b/proto/pb/p2p/p2p.pb.go index 166a8d11f493..0732bf1a13c8 100644 --- a/proto/pb/p2p/p2p.pb.go +++ b/proto/pb/p2p/p2p.pb.go @@ -89,7 +89,8 @@ type Message struct { // *Message_Ping // *Message_Pong // *Message_Handshake - // *Message_PeerList + // *Message_GetPeerList + // *Message_PeerList_ // *Message_GetStateSummaryFrontier // *Message_StateSummaryFrontier_ // *Message_GetAcceptedStateSummary @@ -108,7 +109,6 @@ type Message struct { // *Message_AppRequest // *Message_AppResponse // *Message_AppGossip - // *Message_PeerListAck // *Message_AppError Message isMessage_Message `protobuf_oneof:"message"` } @@ -187,9 +187,16 @@ func (x *Message) GetHandshake() *Handshake { return nil } -func (x *Message) GetPeerList() *PeerList { - if x, ok := x.GetMessage().(*Message_PeerList); ok { - return x.PeerList +func (x *Message) GetGetPeerList() *GetPeerList { + if x, ok := x.GetMessage().(*Message_GetPeerList); ok { + return x.GetPeerList + } + return nil +} + +func (x *Message) GetPeerList_() *PeerList { + if x, ok := x.GetMessage().(*Message_PeerList_); ok { + return x.PeerList_ } return nil } @@ -320,13 +327,6 @@ func (x *Message) GetAppGossip() *AppGossip { return nil } -func (x *Message) GetPeerListAck() *PeerListAck { - if x, ok := x.GetMessage().(*Message_PeerListAck); ok { - return x.PeerListAck - } - return nil -} - func (x *Message) GetAppError() *AppError { if x, ok := x.GetMessage().(*Message_AppError); ok { return x.AppError @@ -365,8 +365,12 @@ type Message_Handshake struct { Handshake *Handshake `protobuf:"bytes,13,opt,name=handshake,proto3,oneof"` } -type Message_PeerList struct { - PeerList *PeerList `protobuf:"bytes,14,opt,name=peer_list,json=peerList,proto3,oneof"` +type Message_GetPeerList struct { + GetPeerList *GetPeerList `protobuf:"bytes,35,opt,name=get_peer_list,json=getPeerList,proto3,oneof"` +} + +type Message_PeerList_ struct { + PeerList_ *PeerList `protobuf:"bytes,14,opt,name=peer_list,json=peerList,proto3,oneof"` } type Message_GetStateSummaryFrontier struct { @@ -445,10 +449,6 @@ type Message_AppGossip struct { AppGossip *AppGossip `protobuf:"bytes,32,opt,name=app_gossip,json=appGossip,proto3,oneof"` } -type Message_PeerListAck struct { - PeerListAck *PeerListAck `protobuf:"bytes,33,opt,name=peer_list_ack,json=peerListAck,proto3,oneof"` -} - type Message_AppError struct { AppError *AppError `protobuf:"bytes,34,opt,name=app_error,json=appError,proto3,oneof"` } @@ -463,7 +463,9 @@ func (*Message_Pong) isMessage_Message() {} func (*Message_Handshake) isMessage_Message() {} -func (*Message_PeerList) isMessage_Message() {} +func (*Message_GetPeerList) isMessage_Message() {} + +func (*Message_PeerList_) isMessage_Message() {} func (*Message_GetStateSummaryFrontier) isMessage_Message() {} @@ -501,8 +503,6 @@ func (*Message_AppResponse) isMessage_Message() {} func (*Message_AppGossip) isMessage_Message() {} -func (*Message_PeerListAck) isMessage_Message() {} - func (*Message_AppError) isMessage_Message() {} // Ping reports a peer's perceived uptime percentage. @@ -711,10 +711,11 @@ type Handshake struct { // Signature of the peer IP port pair at a provided timestamp Sig []byte `protobuf:"bytes,7,opt,name=sig,proto3" json:"sig,omitempty"` // Subnets the peer is tracking - TrackedSubnets [][]byte `protobuf:"bytes,8,rep,name=tracked_subnets,json=trackedSubnets,proto3" json:"tracked_subnets,omitempty"` - Client *Client `protobuf:"bytes,9,opt,name=client,proto3" json:"client,omitempty"` - SupportedAcps []uint32 `protobuf:"varint,10,rep,packed,name=supported_acps,json=supportedAcps,proto3" json:"supported_acps,omitempty"` - ObjectedAcps []uint32 `protobuf:"varint,11,rep,packed,name=objected_acps,json=objectedAcps,proto3" json:"objected_acps,omitempty"` + TrackedSubnets [][]byte `protobuf:"bytes,8,rep,name=tracked_subnets,json=trackedSubnets,proto3" json:"tracked_subnets,omitempty"` + Client *Client `protobuf:"bytes,9,opt,name=client,proto3" json:"client,omitempty"` + SupportedAcps []uint32 `protobuf:"varint,10,rep,packed,name=supported_acps,json=supportedAcps,proto3" json:"supported_acps,omitempty"` + ObjectedAcps []uint32 `protobuf:"varint,11,rep,packed,name=objected_acps,json=objectedAcps,proto3" json:"objected_acps,omitempty"` + KnownPeers *BloomFilter `protobuf:"bytes,12,opt,name=known_peers,json=knownPeers,proto3" json:"known_peers,omitempty"` } func (x *Handshake) Reset() { @@ -826,6 +827,13 @@ func (x *Handshake) GetObjectedAcps() []uint32 { return nil } +func (x *Handshake) GetKnownPeers() *BloomFilter { + if x != nil { + return x.KnownPeers + } + return nil +} + // Metadata about a peer's P2P client used to determine compatibility type Client struct { state protoimpl.MessageState @@ -900,6 +908,62 @@ func (x *Client) GetPatch() uint32 { return 0 } +// BloomFilter with a random salt to prevent consistent hash collisions +type BloomFilter struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Filter []byte `protobuf:"bytes,1,opt,name=filter,proto3" json:"filter,omitempty"` + Salt []byte `protobuf:"bytes,2,opt,name=salt,proto3" json:"salt,omitempty"` +} + +func (x *BloomFilter) Reset() { + *x = BloomFilter{} + if protoimpl.UnsafeEnabled { + mi := &file_p2p_p2p_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BloomFilter) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BloomFilter) ProtoMessage() {} + +func (x *BloomFilter) ProtoReflect() protoreflect.Message { + mi := &file_p2p_p2p_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BloomFilter.ProtoReflect.Descriptor instead. +func (*BloomFilter) Descriptor() ([]byte, []int) { + return file_p2p_p2p_proto_rawDescGZIP(), []int{6} +} + +func (x *BloomFilter) GetFilter() []byte { + if x != nil { + return x.Filter + } + return nil +} + +func (x *BloomFilter) GetSalt() []byte { + if x != nil { + return x.Salt + } + return nil +} + // ClaimedIpPort contains metadata needed to connect to a peer type ClaimedIpPort struct { state protoimpl.MessageState @@ -923,7 +987,7 @@ type ClaimedIpPort struct { func (x *ClaimedIpPort) Reset() { *x = ClaimedIpPort{} if protoimpl.UnsafeEnabled { - mi := &file_p2p_p2p_proto_msgTypes[6] + mi := &file_p2p_p2p_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -936,7 +1000,7 @@ func (x *ClaimedIpPort) String() string { func (*ClaimedIpPort) ProtoMessage() {} func (x *ClaimedIpPort) ProtoReflect() protoreflect.Message { - mi := &file_p2p_p2p_proto_msgTypes[6] + mi := &file_p2p_p2p_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -949,7 +1013,7 @@ func (x *ClaimedIpPort) ProtoReflect() protoreflect.Message { // Deprecated: Use ClaimedIpPort.ProtoReflect.Descriptor instead. func (*ClaimedIpPort) Descriptor() ([]byte, []int) { - return file_p2p_p2p_proto_rawDescGZIP(), []int{6} + return file_p2p_p2p_proto_rawDescGZIP(), []int{7} } func (x *ClaimedIpPort) GetX509Certificate() []byte { @@ -994,73 +1058,22 @@ func (x *ClaimedIpPort) GetTxId() []byte { return nil } -// Peers should periodically send PeerList messages to allow peers to -// discover each other. +// GetPeerList contains a bloom filter of the currently known validator IPs. // -// PeerListAck should be sent in response to a PeerList. -type PeerList struct { +// GetPeerList must not be responded to until finishing the handshake. After the +// handshake is completed, GetPeerlist messages should be responded to with a +// Peerlist message containing validators that are not present in the bloom +// filter. +type GetPeerList struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ClaimedIpPorts []*ClaimedIpPort `protobuf:"bytes,1,rep,name=claimed_ip_ports,json=claimedIpPorts,proto3" json:"claimed_ip_ports,omitempty"` + KnownPeers *BloomFilter `protobuf:"bytes,1,opt,name=known_peers,json=knownPeers,proto3" json:"known_peers,omitempty"` } -func (x *PeerList) Reset() { - *x = PeerList{} - if protoimpl.UnsafeEnabled { - mi := &file_p2p_p2p_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *PeerList) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*PeerList) ProtoMessage() {} - -func (x *PeerList) ProtoReflect() protoreflect.Message { - mi := &file_p2p_p2p_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use PeerList.ProtoReflect.Descriptor instead. -func (*PeerList) Descriptor() ([]byte, []int) { - return file_p2p_p2p_proto_rawDescGZIP(), []int{7} -} - -func (x *PeerList) GetClaimedIpPorts() []*ClaimedIpPort { - if x != nil { - return x.ClaimedIpPorts - } - return nil -} - -// PeerAck acknowledges that a gossiped peer in a PeerList message will be -// tracked by the remote peer. -type PeerAck struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // P-Chain transaction that added the acknowledged peer to the validator - // set - TxId []byte `protobuf:"bytes,1,opt,name=tx_id,json=txId,proto3" json:"tx_id,omitempty"` - // Timestamp of the signed ip of the peer - Timestamp uint64 `protobuf:"varint,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"` -} - -func (x *PeerAck) Reset() { - *x = PeerAck{} +func (x *GetPeerList) Reset() { + *x = GetPeerList{} if protoimpl.UnsafeEnabled { mi := &file_p2p_p2p_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -1068,13 +1081,13 @@ func (x *PeerAck) Reset() { } } -func (x *PeerAck) String() string { +func (x *GetPeerList) String() string { return protoimpl.X.MessageStringOf(x) } -func (*PeerAck) ProtoMessage() {} +func (*GetPeerList) ProtoMessage() {} -func (x *PeerAck) ProtoReflect() protoreflect.Message { +func (x *GetPeerList) ProtoReflect() protoreflect.Message { mi := &file_p2p_p2p_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -1086,37 +1099,37 @@ func (x *PeerAck) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use PeerAck.ProtoReflect.Descriptor instead. -func (*PeerAck) Descriptor() ([]byte, []int) { +// Deprecated: Use GetPeerList.ProtoReflect.Descriptor instead. +func (*GetPeerList) Descriptor() ([]byte, []int) { return file_p2p_p2p_proto_rawDescGZIP(), []int{8} } -func (x *PeerAck) GetTxId() []byte { +func (x *GetPeerList) GetKnownPeers() *BloomFilter { if x != nil { - return x.TxId + return x.KnownPeers } return nil } -func (x *PeerAck) GetTimestamp() uint64 { - if x != nil { - return x.Timestamp - } - return 0 -} - -// PeerListAck is sent in response to PeerList to acknowledge the subset of -// peers that the peer will attempt to connect to. -type PeerListAck struct { +// PeerList contains network-level metadata for a set of validators. +// +// PeerList must be sent in response to an inbound Handshake message from a +// remote peer a peer wants to connect to. Once a PeerList is received after +// a Handshake message, the p2p handshake is complete and the connection is +// established. +// +// PeerList should be sent in response to a GetPeerlist message if the handshake +// has been completed. +type PeerList struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - PeerAcks []*PeerAck `protobuf:"bytes,2,rep,name=peer_acks,json=peerAcks,proto3" json:"peer_acks,omitempty"` + ClaimedIpPorts []*ClaimedIpPort `protobuf:"bytes,1,rep,name=claimed_ip_ports,json=claimedIpPorts,proto3" json:"claimed_ip_ports,omitempty"` } -func (x *PeerListAck) Reset() { - *x = PeerListAck{} +func (x *PeerList) Reset() { + *x = PeerList{} if protoimpl.UnsafeEnabled { mi := &file_p2p_p2p_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -1124,13 +1137,13 @@ func (x *PeerListAck) Reset() { } } -func (x *PeerListAck) String() string { +func (x *PeerList) String() string { return protoimpl.X.MessageStringOf(x) } -func (*PeerListAck) ProtoMessage() {} +func (*PeerList) ProtoMessage() {} -func (x *PeerListAck) ProtoReflect() protoreflect.Message { +func (x *PeerList) ProtoReflect() protoreflect.Message { mi := &file_p2p_p2p_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -1142,14 +1155,14 @@ func (x *PeerListAck) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use PeerListAck.ProtoReflect.Descriptor instead. -func (*PeerListAck) Descriptor() ([]byte, []int) { +// Deprecated: Use PeerList.ProtoReflect.Descriptor instead. +func (*PeerList) Descriptor() ([]byte, []int) { return file_p2p_p2p_proto_rawDescGZIP(), []int{9} } -func (x *PeerListAck) GetPeerAcks() []*PeerAck { +func (x *PeerList) GetClaimedIpPorts() []*ClaimedIpPort { if x != nil { - return x.PeerAcks + return x.ClaimedIpPorts } return nil } @@ -2620,7 +2633,7 @@ var File_p2p_p2p_proto protoreflect.FileDescriptor var file_p2p_p2p_proto_rawDesc = []byte{ 0x0a, 0x0d, 0x70, 0x32, 0x70, 0x2f, 0x70, 0x32, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, - 0x03, 0x70, 0x32, 0x70, 0x22, 0x92, 0x0b, 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x03, 0x70, 0x32, 0x70, 0x22, 0x9e, 0x0b, 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x29, 0x0a, 0x0f, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x5f, 0x67, 0x7a, 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x0e, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x47, 0x7a, 0x69, 0x70, 0x12, 0x29, 0x0a, 0x0f, 0x63, @@ -2633,334 +2646,337 @@ var file_p2p_p2p_proto_rawDesc = []byte{ 0x48, 0x00, 0x52, 0x04, 0x70, 0x6f, 0x6e, 0x67, 0x12, 0x2e, 0x0a, 0x09, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x48, 0x00, 0x52, 0x09, 0x68, - 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x2c, 0x0a, 0x09, 0x70, 0x65, 0x65, 0x72, - 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x70, 0x32, - 0x70, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x00, 0x52, 0x08, 0x70, 0x65, - 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x5b, 0x0a, 0x1a, 0x67, 0x65, 0x74, 0x5f, 0x73, 0x74, - 0x61, 0x74, 0x65, 0x5f, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x66, 0x72, 0x6f, 0x6e, - 0x74, 0x69, 0x65, 0x72, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, 0x32, 0x70, - 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, - 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x48, 0x00, 0x52, 0x17, 0x67, 0x65, 0x74, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x46, 0x72, 0x6f, 0x6e, 0x74, - 0x69, 0x65, 0x72, 0x12, 0x51, 0x0a, 0x16, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x73, 0x75, 0x6d, - 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x66, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x18, 0x10, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, - 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x48, 0x00, - 0x52, 0x14, 0x73, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x46, 0x72, - 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x5b, 0x0a, 0x1a, 0x67, 0x65, 0x74, 0x5f, 0x61, 0x63, - 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x73, 0x75, 0x6d, - 0x6d, 0x61, 0x72, 0x79, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, 0x32, 0x70, - 0x2e, 0x47, 0x65, 0x74, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x48, 0x00, 0x52, 0x17, 0x67, 0x65, 0x74, 0x41, - 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, - 0x61, 0x72, 0x79, 0x12, 0x51, 0x0a, 0x16, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x5f, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x12, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, - 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x48, 0x00, - 0x52, 0x14, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, - 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x4e, 0x0a, 0x15, 0x67, 0x65, 0x74, 0x5f, 0x61, 0x63, - 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x5f, 0x66, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x18, - 0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x47, 0x65, 0x74, 0x41, - 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x48, - 0x00, 0x52, 0x13, 0x67, 0x65, 0x74, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x46, 0x72, - 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x44, 0x0a, 0x11, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, - 0x65, 0x64, 0x5f, 0x66, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x18, 0x14, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, - 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x48, 0x00, 0x52, 0x10, 0x61, 0x63, 0x63, 0x65, - 0x70, 0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x35, 0x0a, 0x0c, - 0x67, 0x65, 0x74, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x18, 0x15, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x63, 0x63, 0x65, - 0x70, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0b, 0x67, 0x65, 0x74, 0x41, 0x63, 0x63, 0x65, 0x70, - 0x74, 0x65, 0x64, 0x12, 0x2b, 0x0a, 0x08, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x18, - 0x16, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x41, 0x63, 0x63, 0x65, - 0x70, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x08, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, - 0x12, 0x38, 0x0a, 0x0d, 0x67, 0x65, 0x74, 0x5f, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, - 0x73, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x47, 0x65, - 0x74, 0x41, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x73, 0x48, 0x00, 0x52, 0x0c, 0x67, 0x65, - 0x74, 0x41, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x2e, 0x0a, 0x09, 0x61, 0x6e, - 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x73, 0x18, 0x18, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, - 0x70, 0x32, 0x70, 0x2e, 0x41, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x73, 0x48, 0x00, 0x52, - 0x09, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x1c, 0x0a, 0x03, 0x67, 0x65, - 0x74, 0x18, 0x19, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x47, 0x65, - 0x74, 0x48, 0x00, 0x52, 0x03, 0x67, 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x03, 0x70, 0x75, 0x74, 0x18, - 0x1a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x50, 0x75, 0x74, 0x48, - 0x00, 0x52, 0x03, 0x70, 0x75, 0x74, 0x12, 0x2f, 0x0a, 0x0a, 0x70, 0x75, 0x73, 0x68, 0x5f, 0x71, - 0x75, 0x65, 0x72, 0x79, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x32, 0x70, - 0x2e, 0x50, 0x75, 0x73, 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, 0x48, 0x00, 0x52, 0x09, 0x70, 0x75, - 0x73, 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x2f, 0x0a, 0x0a, 0x70, 0x75, 0x6c, 0x6c, 0x5f, - 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x1c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x32, - 0x70, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x48, 0x00, 0x52, 0x09, 0x70, - 0x75, 0x6c, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x22, 0x0a, 0x05, 0x63, 0x68, 0x69, 0x74, - 0x73, 0x18, 0x1d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x43, 0x68, - 0x69, 0x74, 0x73, 0x48, 0x00, 0x52, 0x05, 0x63, 0x68, 0x69, 0x74, 0x73, 0x12, 0x32, 0x0a, 0x0b, - 0x61, 0x70, 0x70, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x1e, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x0f, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x41, 0x70, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x48, 0x00, 0x52, 0x0a, 0x61, 0x70, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x35, 0x0a, 0x0c, 0x61, 0x70, 0x70, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x18, 0x1f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x41, 0x70, 0x70, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x70, 0x70, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x0a, 0x61, 0x70, 0x70, 0x5f, 0x67, - 0x6f, 0x73, 0x73, 0x69, 0x70, 0x18, 0x20, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x32, - 0x70, 0x2e, 0x41, 0x70, 0x70, 0x47, 0x6f, 0x73, 0x73, 0x69, 0x70, 0x48, 0x00, 0x52, 0x09, 0x61, - 0x70, 0x70, 0x47, 0x6f, 0x73, 0x73, 0x69, 0x70, 0x12, 0x36, 0x0a, 0x0d, 0x70, 0x65, 0x65, 0x72, - 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x61, 0x63, 0x6b, 0x18, 0x21, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x10, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x63, - 0x6b, 0x48, 0x00, 0x52, 0x0b, 0x70, 0x65, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x63, 0x6b, + 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x36, 0x0a, 0x0d, 0x67, 0x65, 0x74, 0x5f, + 0x70, 0x65, 0x65, 0x72, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x23, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x10, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x65, 0x65, 0x72, 0x4c, 0x69, 0x73, + 0x74, 0x48, 0x00, 0x52, 0x0b, 0x67, 0x65, 0x74, 0x50, 0x65, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, + 0x12, 0x2c, 0x0a, 0x09, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x0e, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x4c, 0x69, + 0x73, 0x74, 0x48, 0x00, 0x52, 0x08, 0x70, 0x65, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x5b, + 0x0a, 0x1a, 0x67, 0x65, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x73, 0x75, 0x6d, 0x6d, + 0x61, 0x72, 0x79, 0x5f, 0x66, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x18, 0x0f, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, + 0x48, 0x00, 0x52, 0x17, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, + 0x61, 0x72, 0x79, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x51, 0x0a, 0x16, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x5f, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x66, 0x72, 0x6f, + 0x6e, 0x74, 0x69, 0x65, 0x72, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x70, 0x32, + 0x70, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x46, 0x72, + 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x48, 0x00, 0x52, 0x14, 0x73, 0x74, 0x61, 0x74, 0x65, 0x53, + 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x5b, + 0x0a, 0x1a, 0x67, 0x65, 0x74, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x5f, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x5f, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x11, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x63, 0x63, 0x65, + 0x70, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, + 0x48, 0x00, 0x52, 0x17, 0x67, 0x65, 0x74, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x51, 0x0a, 0x16, 0x61, + 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x73, 0x75, + 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x70, 0x32, + 0x70, 0x2e, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, + 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x48, 0x00, 0x52, 0x14, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, + 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x4e, + 0x0a, 0x15, 0x67, 0x65, 0x74, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x5f, 0x66, + 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x18, 0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, + 0x70, 0x32, 0x70, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x46, + 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x48, 0x00, 0x52, 0x13, 0x67, 0x65, 0x74, 0x41, 0x63, + 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x44, + 0x0a, 0x11, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x5f, 0x66, 0x72, 0x6f, 0x6e, 0x74, + 0x69, 0x65, 0x72, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x32, 0x70, 0x2e, + 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, + 0x48, 0x00, 0x52, 0x10, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6e, + 0x74, 0x69, 0x65, 0x72, 0x12, 0x35, 0x0a, 0x0c, 0x67, 0x65, 0x74, 0x5f, 0x61, 0x63, 0x63, 0x65, + 0x70, 0x74, 0x65, 0x64, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x32, 0x70, + 0x2e, 0x47, 0x65, 0x74, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0b, + 0x67, 0x65, 0x74, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x12, 0x2b, 0x0a, 0x08, 0x61, + 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x18, 0x16, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, + 0x70, 0x32, 0x70, 0x2e, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x08, + 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x12, 0x38, 0x0a, 0x0d, 0x67, 0x65, 0x74, 0x5f, + 0x61, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x73, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x11, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x6f, + 0x72, 0x73, 0x48, 0x00, 0x52, 0x0c, 0x67, 0x65, 0x74, 0x41, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x6f, + 0x72, 0x73, 0x12, 0x2e, 0x0a, 0x09, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x73, 0x18, + 0x18, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x41, 0x6e, 0x63, 0x65, + 0x73, 0x74, 0x6f, 0x72, 0x73, 0x48, 0x00, 0x52, 0x09, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x6f, + 0x72, 0x73, 0x12, 0x1c, 0x0a, 0x03, 0x67, 0x65, 0x74, 0x18, 0x19, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x08, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x00, 0x52, 0x03, 0x67, 0x65, 0x74, + 0x12, 0x1c, 0x0a, 0x03, 0x70, 0x75, 0x74, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x08, 0x2e, + 0x70, 0x32, 0x70, 0x2e, 0x50, 0x75, 0x74, 0x48, 0x00, 0x52, 0x03, 0x70, 0x75, 0x74, 0x12, 0x2f, + 0x0a, 0x0a, 0x70, 0x75, 0x73, 0x68, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x1b, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x51, 0x75, 0x65, + 0x72, 0x79, 0x48, 0x00, 0x52, 0x09, 0x70, 0x75, 0x73, 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, + 0x2f, 0x0a, 0x0a, 0x70, 0x75, 0x6c, 0x6c, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x1c, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x51, 0x75, + 0x65, 0x72, 0x79, 0x48, 0x00, 0x52, 0x09, 0x70, 0x75, 0x6c, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, + 0x12, 0x22, 0x0a, 0x05, 0x63, 0x68, 0x69, 0x74, 0x73, 0x18, 0x1d, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x0a, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x43, 0x68, 0x69, 0x74, 0x73, 0x48, 0x00, 0x52, 0x05, 0x63, + 0x68, 0x69, 0x74, 0x73, 0x12, 0x32, 0x0a, 0x0b, 0x61, 0x70, 0x70, 0x5f, 0x72, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x70, 0x32, 0x70, 0x2e, + 0x41, 0x70, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0a, 0x61, 0x70, + 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x0c, 0x61, 0x70, 0x70, 0x5f, + 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, + 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x41, 0x70, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x48, 0x00, 0x52, 0x0b, 0x61, 0x70, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x2f, 0x0a, 0x0a, 0x61, 0x70, 0x70, 0x5f, 0x67, 0x6f, 0x73, 0x73, 0x69, 0x70, 0x18, 0x20, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x41, 0x70, 0x70, 0x47, 0x6f, 0x73, + 0x73, 0x69, 0x70, 0x48, 0x00, 0x52, 0x09, 0x61, 0x70, 0x70, 0x47, 0x6f, 0x73, 0x73, 0x69, 0x70, 0x12, 0x2c, 0x0a, 0x09, 0x61, 0x70, 0x70, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x22, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x41, 0x70, 0x70, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x48, 0x00, 0x52, 0x08, 0x61, 0x70, 0x70, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x42, 0x09, - 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x58, 0x0a, 0x04, 0x50, 0x69, 0x6e, - 0x67, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x0e, 0x73, 0x75, 0x62, - 0x6e, 0x65, 0x74, 0x5f, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x11, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x55, 0x70, - 0x74, 0x69, 0x6d, 0x65, 0x52, 0x0d, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x55, 0x70, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x22, 0x43, 0x0a, 0x0c, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x55, 0x70, 0x74, - 0x69, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x49, 0x64, - 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x22, 0x58, 0x0a, 0x04, 0x50, 0x6f, 0x6e, 0x67, - 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x0e, 0x73, 0x75, 0x62, 0x6e, - 0x65, 0x74, 0x5f, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x11, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x55, 0x70, 0x74, - 0x69, 0x6d, 0x65, 0x52, 0x0d, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x55, 0x70, 0x74, 0x69, 0x6d, - 0x65, 0x73, 0x22, 0xe8, 0x02, 0x0a, 0x09, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, - 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x49, 0x64, 0x12, - 0x17, 0x0a, 0x07, 0x6d, 0x79, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x06, 0x6d, 0x79, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x70, 0x5f, 0x61, - 0x64, 0x64, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x69, 0x70, 0x41, 0x64, 0x64, - 0x72, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x06, 0x69, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x79, - 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x6d, 0x79, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x26, 0x0a, 0x0f, 0x69, 0x70, 0x5f, - 0x73, 0x69, 0x67, 0x6e, 0x69, 0x6e, 0x67, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x0d, 0x69, 0x70, 0x53, 0x69, 0x67, 0x6e, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, - 0x65, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x69, 0x67, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, - 0x73, 0x69, 0x67, 0x12, 0x27, 0x0a, 0x0f, 0x74, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x73, - 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0e, 0x74, 0x72, - 0x61, 0x63, 0x6b, 0x65, 0x64, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x06, - 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x70, - 0x32, 0x70, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, - 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, - 0x63, 0x70, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x0d, 0x73, 0x75, 0x70, 0x70, 0x6f, - 0x72, 0x74, 0x65, 0x64, 0x41, 0x63, 0x70, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x6f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x63, 0x70, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0d, 0x52, - 0x0c, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x65, 0x64, 0x41, 0x63, 0x70, 0x73, 0x22, 0x5e, 0x0a, - 0x06, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6d, - 0x61, 0x6a, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6d, 0x61, 0x6a, 0x6f, - 0x72, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x69, 0x6e, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x05, 0x6d, 0x69, 0x6e, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x63, 0x68, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x70, 0x61, 0x74, 0x63, 0x68, 0x22, 0xbd, 0x01, - 0x0a, 0x0d, 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x65, 0x64, 0x49, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, - 0x29, 0x0a, 0x10, 0x78, 0x35, 0x30, 0x39, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, - 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0f, 0x78, 0x35, 0x30, 0x39, 0x43, - 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x70, - 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x69, 0x70, 0x41, - 0x64, 0x64, 0x72, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x69, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1c, 0x0a, 0x09, - 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x13, 0x0a, 0x05, 0x74, 0x78, 0x5f, 0x69, - 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x74, 0x78, 0x49, 0x64, 0x22, 0x48, 0x0a, - 0x08, 0x50, 0x65, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x10, 0x63, 0x6c, 0x61, - 0x69, 0x6d, 0x65, 0x64, 0x5f, 0x69, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x65, - 0x64, 0x49, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x52, 0x0e, 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x65, 0x64, - 0x49, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x73, 0x22, 0x3c, 0x0a, 0x07, 0x50, 0x65, 0x65, 0x72, 0x41, - 0x63, 0x6b, 0x12, 0x13, 0x0a, 0x05, 0x74, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x04, 0x74, 0x78, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x3e, 0x0a, 0x0b, 0x50, 0x65, 0x65, 0x72, 0x4c, 0x69, 0x73, - 0x74, 0x41, 0x63, 0x6b, 0x12, 0x29, 0x0a, 0x09, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x61, 0x63, 0x6b, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x50, 0x65, - 0x65, 0x72, 0x41, 0x63, 0x6b, 0x52, 0x08, 0x70, 0x65, 0x65, 0x72, 0x41, 0x63, 0x6b, 0x73, 0x4a, - 0x04, 0x08, 0x01, 0x10, 0x02, 0x22, 0x6f, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, - 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, - 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, - 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, - 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x6a, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, + 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x4a, 0x04, 0x08, 0x21, 0x10, 0x22, 0x4a, + 0x04, 0x08, 0x24, 0x10, 0x25, 0x22, 0x58, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x16, 0x0a, + 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x75, + 0x70, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x0e, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x5f, + 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, + 0x70, 0x32, 0x70, 0x2e, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65, + 0x52, 0x0d, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x22, + 0x43, 0x0a, 0x0c, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x12, + 0x1b, 0x0a, 0x09, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x08, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, + 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x75, 0x70, + 0x74, 0x69, 0x6d, 0x65, 0x22, 0x58, 0x0a, 0x04, 0x50, 0x6f, 0x6e, 0x67, 0x12, 0x16, 0x0a, 0x06, + 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x75, 0x70, + 0x74, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x0e, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x5f, 0x75, + 0x70, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x70, + 0x32, 0x70, 0x2e, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x52, + 0x0d, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x22, 0x9b, + 0x03, 0x0a, 0x09, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x1d, 0x0a, 0x0a, + 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x09, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x6d, + 0x79, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6d, 0x79, + 0x54, 0x69, 0x6d, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x70, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x12, 0x17, 0x0a, + 0x07, 0x69, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, + 0x69, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x79, 0x5f, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x79, 0x56, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x26, 0x0a, 0x0f, 0x69, 0x70, 0x5f, 0x73, 0x69, 0x67, 0x6e, + 0x69, 0x6e, 0x67, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, + 0x69, 0x70, 0x53, 0x69, 0x67, 0x6e, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x10, 0x0a, + 0x03, 0x73, 0x69, 0x67, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x73, 0x69, 0x67, 0x12, + 0x27, 0x0a, 0x0f, 0x74, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, + 0x74, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0e, 0x74, 0x72, 0x61, 0x63, 0x6b, 0x65, + 0x64, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x06, 0x63, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x43, + 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x25, 0x0a, + 0x0e, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x63, 0x70, 0x73, 0x18, + 0x0a, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x0d, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, + 0x41, 0x63, 0x70, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x65, 0x64, + 0x5f, 0x61, 0x63, 0x70, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x0c, 0x6f, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x41, 0x63, 0x70, 0x73, 0x12, 0x31, 0x0a, 0x0b, 0x6b, 0x6e, 0x6f, + 0x77, 0x6e, 0x5f, 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, + 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, + 0x52, 0x0a, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x50, 0x65, 0x65, 0x72, 0x73, 0x22, 0x5e, 0x0a, 0x06, + 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x61, + 0x6a, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6d, 0x61, 0x6a, 0x6f, 0x72, + 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x69, 0x6e, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x05, 0x6d, 0x69, 0x6e, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x63, 0x68, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x70, 0x61, 0x74, 0x63, 0x68, 0x22, 0x39, 0x0a, 0x0b, + 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x66, + 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x66, 0x69, 0x6c, + 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x61, 0x6c, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x04, 0x73, 0x61, 0x6c, 0x74, 0x22, 0xbd, 0x01, 0x0a, 0x0d, 0x43, 0x6c, 0x61, 0x69, + 0x6d, 0x65, 0x64, 0x49, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x78, 0x35, 0x30, + 0x39, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x0f, 0x78, 0x35, 0x30, 0x39, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, + 0x63, 0x61, 0x74, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x70, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x12, 0x17, 0x0a, + 0x07, 0x69, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, + 0x69, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x12, 0x13, 0x0a, 0x05, 0x74, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x04, 0x74, 0x78, 0x49, 0x64, 0x22, 0x40, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x50, 0x65, + 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x31, 0x0a, 0x0b, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x5f, + 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x32, + 0x70, 0x2e, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x0a, 0x6b, + 0x6e, 0x6f, 0x77, 0x6e, 0x50, 0x65, 0x65, 0x72, 0x73, 0x22, 0x48, 0x0a, 0x08, 0x50, 0x65, 0x65, + 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x10, 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x65, 0x64, + 0x5f, 0x69, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x12, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x65, 0x64, 0x49, 0x70, 0x50, + 0x6f, 0x72, 0x74, 0x52, 0x0e, 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x65, 0x64, 0x49, 0x70, 0x50, 0x6f, + 0x72, 0x74, 0x73, 0x22, 0x6f, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x6d, 0x6d, - 0x61, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, - 0x72, 0x79, 0x22, 0x89, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, - 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x19, - 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, - 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x04, 0x52, 0x07, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x73, 0x22, 0x71, - 0x0a, 0x14, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, - 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, + 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x6a, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, + 0x6d, 0x61, 0x72, 0x79, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x19, 0x0a, 0x08, + 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, + 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, + 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, + 0x22, 0x89, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, + 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, + 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, + 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, + 0x6e, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x73, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x04, 0x52, 0x07, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x73, 0x22, 0x71, 0x0a, 0x14, + 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, + 0x6d, 0x61, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, + 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1f, + 0x0a, 0x0b, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, + 0x03, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x49, 0x64, 0x73, 0x22, + 0x9d, 0x01, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x46, + 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, + 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, + 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x30, 0x0a, + 0x0b, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, + 0x79, 0x70, 0x65, 0x52, 0x0a, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x22, + 0x75, 0x0a, 0x10, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6e, 0x74, + 0x69, 0x65, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, + 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, + 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, + 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0xba, 0x01, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x41, 0x63, + 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, - 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x69, 0x64, 0x73, 0x18, - 0x03, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x49, 0x64, - 0x73, 0x22, 0x9d, 0x01, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, - 0x64, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, - 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, - 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, - 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, - 0x30, 0x0a, 0x0b, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, - 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, - 0x65, 0x22, 0x75, 0x0a, 0x10, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, - 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, + 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x23, 0x0a, 0x0d, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x0c, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, + 0x73, 0x12, 0x30, 0x0a, 0x0b, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x45, 0x6e, 0x67, + 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, + 0x79, 0x70, 0x65, 0x22, 0x6f, 0x0a, 0x08, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x12, + 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0c, + 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x73, 0x4a, 0x04, + 0x08, 0x04, 0x10, 0x05, 0x22, 0xb9, 0x01, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x41, 0x6e, 0x63, 0x65, + 0x73, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, - 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x49, 0x64, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0xba, 0x01, 0x0a, 0x0b, 0x47, 0x65, 0x74, - 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, + 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x30, + 0x0a, 0x0b, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, + 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, + 0x22, 0x6b, 0x0a, 0x09, 0x41, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x19, 0x0a, + 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0a, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0xb0, 0x01, + 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, + 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, + 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x30, + 0x0a, 0x0b, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, + 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, + 0x22, 0x8f, 0x01, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x23, - 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x49, 0x64, 0x73, 0x12, 0x30, 0x0a, 0x0b, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x5f, 0x74, 0x79, - 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x45, - 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x65, 0x6e, 0x67, 0x69, 0x6e, - 0x65, 0x54, 0x79, 0x70, 0x65, 0x22, 0x6f, 0x0a, 0x08, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, - 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, - 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x0c, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x73, - 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0xb9, 0x01, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x41, 0x6e, - 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, - 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x21, 0x0a, - 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, + 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x30, 0x0a, 0x0b, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x45, 0x6e, 0x67, 0x69, + 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, - 0x70, 0x65, 0x22, 0x6b, 0x0a, 0x09, 0x41, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x73, 0x12, + 0x70, 0x65, 0x22, 0xdc, 0x01, 0x0a, 0x09, 0x50, 0x75, 0x73, 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, + 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, + 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, + 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x12, 0x30, 0x0a, 0x0b, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x5f, 0x74, + 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x70, 0x32, 0x70, 0x2e, + 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x65, 0x6e, 0x67, 0x69, + 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x65, 0x64, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x0f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x48, 0x65, 0x69, 0x67, 0x68, + 0x74, 0x22, 0xe1, 0x01, 0x0a, 0x09, 0x50, 0x75, 0x6c, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, - 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0a, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, - 0xb0, 0x01, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, - 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x21, 0x0a, - 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, - 0x12, 0x30, 0x0a, 0x0b, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x45, 0x6e, 0x67, 0x69, - 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, - 0x70, 0x65, 0x22, 0x8f, 0x01, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, - 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, - 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, - 0x65, 0x72, 0x12, 0x30, 0x0a, 0x0b, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x5f, 0x74, 0x79, 0x70, - 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x45, 0x6e, - 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, - 0x54, 0x79, 0x70, 0x65, 0x22, 0xdc, 0x01, 0x0a, 0x09, 0x50, 0x75, 0x73, 0x68, 0x51, 0x75, 0x65, - 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, - 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, - 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, - 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, - 0x61, 0x69, 0x6e, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x30, 0x0a, 0x0b, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, - 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x70, 0x32, - 0x70, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x65, 0x6e, - 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x0f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x48, 0x65, 0x69, - 0x67, 0x68, 0x74, 0x22, 0xe1, 0x01, 0x0a, 0x09, 0x50, 0x75, 0x6c, 0x6c, 0x51, 0x75, 0x65, 0x72, - 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, - 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, - 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, - 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, - 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x0b, 0x65, 0x6e, - 0x67, 0x69, 0x6e, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x0f, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, - 0x52, 0x0a, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x29, 0x0a, 0x10, - 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, - 0x64, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0xba, 0x01, 0x0a, 0x05, 0x43, 0x68, 0x69, 0x74, - 0x73, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, - 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x70, - 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x0b, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x49, 0x64, 0x12, 0x1f, - 0x0a, 0x0b, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x49, 0x64, 0x12, - 0x33, 0x0a, 0x16, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x5f, 0x69, 0x64, 0x5f, - 0x61, 0x74, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x13, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x49, 0x64, 0x41, 0x74, 0x48, 0x65, - 0x69, 0x67, 0x68, 0x74, 0x22, 0x7f, 0x0a, 0x0a, 0x41, 0x70, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, + 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, + 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x0b, 0x65, 0x6e, 0x67, 0x69, + 0x6e, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, + 0x70, 0x32, 0x70, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, + 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x48, + 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0xba, 0x01, 0x0a, 0x05, 0x43, 0x68, 0x69, 0x74, 0x73, 0x12, + 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, 0x65, + 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x0b, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, + 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x49, 0x64, 0x12, 0x33, 0x0a, + 0x16, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x5f, 0x69, 0x64, 0x5f, 0x61, 0x74, + 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x13, 0x70, + 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x49, 0x64, 0x41, 0x74, 0x48, 0x65, 0x69, 0x67, + 0x68, 0x74, 0x22, 0x7f, 0x0a, 0x0a, 0x41, 0x70, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, + 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, + 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x61, 0x70, 0x70, 0x5f, 0x62, 0x79, + 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x61, 0x70, 0x70, 0x42, 0x79, + 0x74, 0x65, 0x73, 0x22, 0x64, 0x0a, 0x0b, 0x41, 0x70, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, - 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, - 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x61, 0x70, 0x70, 0x5f, - 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x61, 0x70, 0x70, - 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x64, 0x0a, 0x0b, 0x41, 0x70, 0x70, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, - 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1b, - 0x0a, 0x09, 0x61, 0x70, 0x70, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x08, 0x61, 0x70, 0x70, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x88, 0x01, 0x0a, 0x08, - 0x41, 0x70, 0x70, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, - 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, - 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, - 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x64, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x11, 0x52, 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, - 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x43, 0x0a, 0x09, 0x41, 0x70, 0x70, 0x47, 0x6f, 0x73, - 0x73, 0x69, 0x70, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1b, - 0x0a, 0x09, 0x61, 0x70, 0x70, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x08, 0x61, 0x70, 0x70, 0x42, 0x79, 0x74, 0x65, 0x73, 0x2a, 0x5d, 0x0a, 0x0a, 0x45, - 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1b, 0x0a, 0x17, 0x45, 0x4e, 0x47, - 0x49, 0x4e, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, - 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x45, 0x4e, 0x47, 0x49, 0x4e, 0x45, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x56, 0x41, 0x4c, 0x41, 0x4e, 0x43, 0x48, 0x45, 0x10, - 0x01, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x4e, 0x47, 0x49, 0x4e, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x53, 0x4e, 0x4f, 0x57, 0x4d, 0x41, 0x4e, 0x10, 0x02, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x76, 0x61, 0x2d, 0x6c, 0x61, 0x62, - 0x73, 0x2f, 0x61, 0x76, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x67, 0x6f, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x2f, 0x70, 0x32, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, + 0x61, 0x70, 0x70, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x08, 0x61, 0x70, 0x70, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x88, 0x01, 0x0a, 0x08, 0x41, 0x70, + 0x70, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, + 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, + 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x11, 0x52, 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, + 0x23, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x22, 0x43, 0x0a, 0x09, 0x41, 0x70, 0x70, 0x47, 0x6f, 0x73, 0x73, 0x69, + 0x70, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, + 0x61, 0x70, 0x70, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x08, 0x61, 0x70, 0x70, 0x42, 0x79, 0x74, 0x65, 0x73, 0x2a, 0x5d, 0x0a, 0x0a, 0x45, 0x6e, 0x67, + 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1b, 0x0a, 0x17, 0x45, 0x4e, 0x47, 0x49, 0x4e, + 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x45, 0x4e, 0x47, 0x49, 0x4e, 0x45, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x41, 0x56, 0x41, 0x4c, 0x41, 0x4e, 0x43, 0x48, 0x45, 0x10, 0x01, 0x12, + 0x17, 0x0a, 0x13, 0x45, 0x4e, 0x47, 0x49, 0x4e, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, + 0x4e, 0x4f, 0x57, 0x4d, 0x41, 0x4e, 0x10, 0x02, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x76, 0x61, 0x2d, 0x6c, 0x61, 0x62, 0x73, 0x2f, + 0x61, 0x76, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2f, 0x70, 0x62, 0x2f, 0x70, 0x32, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -2985,10 +3001,10 @@ var file_p2p_p2p_proto_goTypes = []interface{}{ (*Pong)(nil), // 4: p2p.Pong (*Handshake)(nil), // 5: p2p.Handshake (*Client)(nil), // 6: p2p.Client - (*ClaimedIpPort)(nil), // 7: p2p.ClaimedIpPort - (*PeerList)(nil), // 8: p2p.PeerList - (*PeerAck)(nil), // 9: p2p.PeerAck - (*PeerListAck)(nil), // 10: p2p.PeerListAck + (*BloomFilter)(nil), // 7: p2p.BloomFilter + (*ClaimedIpPort)(nil), // 8: p2p.ClaimedIpPort + (*GetPeerList)(nil), // 9: p2p.GetPeerList + (*PeerList)(nil), // 10: p2p.PeerList (*GetStateSummaryFrontier)(nil), // 11: p2p.GetStateSummaryFrontier (*StateSummaryFrontier)(nil), // 12: p2p.StateSummaryFrontier (*GetAcceptedStateSummary)(nil), // 13: p2p.GetAcceptedStateSummary @@ -3013,44 +3029,45 @@ var file_p2p_p2p_proto_depIdxs = []int32{ 2, // 0: p2p.Message.ping:type_name -> p2p.Ping 4, // 1: p2p.Message.pong:type_name -> p2p.Pong 5, // 2: p2p.Message.handshake:type_name -> p2p.Handshake - 8, // 3: p2p.Message.peer_list:type_name -> p2p.PeerList - 11, // 4: p2p.Message.get_state_summary_frontier:type_name -> p2p.GetStateSummaryFrontier - 12, // 5: p2p.Message.state_summary_frontier:type_name -> p2p.StateSummaryFrontier - 13, // 6: p2p.Message.get_accepted_state_summary:type_name -> p2p.GetAcceptedStateSummary - 14, // 7: p2p.Message.accepted_state_summary:type_name -> p2p.AcceptedStateSummary - 15, // 8: p2p.Message.get_accepted_frontier:type_name -> p2p.GetAcceptedFrontier - 16, // 9: p2p.Message.accepted_frontier:type_name -> p2p.AcceptedFrontier - 17, // 10: p2p.Message.get_accepted:type_name -> p2p.GetAccepted - 18, // 11: p2p.Message.accepted:type_name -> p2p.Accepted - 19, // 12: p2p.Message.get_ancestors:type_name -> p2p.GetAncestors - 20, // 13: p2p.Message.ancestors:type_name -> p2p.Ancestors - 21, // 14: p2p.Message.get:type_name -> p2p.Get - 22, // 15: p2p.Message.put:type_name -> p2p.Put - 23, // 16: p2p.Message.push_query:type_name -> p2p.PushQuery - 24, // 17: p2p.Message.pull_query:type_name -> p2p.PullQuery - 25, // 18: p2p.Message.chits:type_name -> p2p.Chits - 26, // 19: p2p.Message.app_request:type_name -> p2p.AppRequest - 27, // 20: p2p.Message.app_response:type_name -> p2p.AppResponse - 29, // 21: p2p.Message.app_gossip:type_name -> p2p.AppGossip - 10, // 22: p2p.Message.peer_list_ack:type_name -> p2p.PeerListAck + 9, // 3: p2p.Message.get_peer_list:type_name -> p2p.GetPeerList + 10, // 4: p2p.Message.peer_list:type_name -> p2p.PeerList + 11, // 5: p2p.Message.get_state_summary_frontier:type_name -> p2p.GetStateSummaryFrontier + 12, // 6: p2p.Message.state_summary_frontier:type_name -> p2p.StateSummaryFrontier + 13, // 7: p2p.Message.get_accepted_state_summary:type_name -> p2p.GetAcceptedStateSummary + 14, // 8: p2p.Message.accepted_state_summary:type_name -> p2p.AcceptedStateSummary + 15, // 9: p2p.Message.get_accepted_frontier:type_name -> p2p.GetAcceptedFrontier + 16, // 10: p2p.Message.accepted_frontier:type_name -> p2p.AcceptedFrontier + 17, // 11: p2p.Message.get_accepted:type_name -> p2p.GetAccepted + 18, // 12: p2p.Message.accepted:type_name -> p2p.Accepted + 19, // 13: p2p.Message.get_ancestors:type_name -> p2p.GetAncestors + 20, // 14: p2p.Message.ancestors:type_name -> p2p.Ancestors + 21, // 15: p2p.Message.get:type_name -> p2p.Get + 22, // 16: p2p.Message.put:type_name -> p2p.Put + 23, // 17: p2p.Message.push_query:type_name -> p2p.PushQuery + 24, // 18: p2p.Message.pull_query:type_name -> p2p.PullQuery + 25, // 19: p2p.Message.chits:type_name -> p2p.Chits + 26, // 20: p2p.Message.app_request:type_name -> p2p.AppRequest + 27, // 21: p2p.Message.app_response:type_name -> p2p.AppResponse + 29, // 22: p2p.Message.app_gossip:type_name -> p2p.AppGossip 28, // 23: p2p.Message.app_error:type_name -> p2p.AppError 3, // 24: p2p.Ping.subnet_uptimes:type_name -> p2p.SubnetUptime 3, // 25: p2p.Pong.subnet_uptimes:type_name -> p2p.SubnetUptime 6, // 26: p2p.Handshake.client:type_name -> p2p.Client - 7, // 27: p2p.PeerList.claimed_ip_ports:type_name -> p2p.ClaimedIpPort - 9, // 28: p2p.PeerListAck.peer_acks:type_name -> p2p.PeerAck - 0, // 29: p2p.GetAcceptedFrontier.engine_type:type_name -> p2p.EngineType - 0, // 30: p2p.GetAccepted.engine_type:type_name -> p2p.EngineType - 0, // 31: p2p.GetAncestors.engine_type:type_name -> p2p.EngineType - 0, // 32: p2p.Get.engine_type:type_name -> p2p.EngineType - 0, // 33: p2p.Put.engine_type:type_name -> p2p.EngineType - 0, // 34: p2p.PushQuery.engine_type:type_name -> p2p.EngineType - 0, // 35: p2p.PullQuery.engine_type:type_name -> p2p.EngineType - 36, // [36:36] is the sub-list for method output_type - 36, // [36:36] is the sub-list for method input_type - 36, // [36:36] is the sub-list for extension type_name - 36, // [36:36] is the sub-list for extension extendee - 0, // [0:36] is the sub-list for field type_name + 7, // 27: p2p.Handshake.known_peers:type_name -> p2p.BloomFilter + 7, // 28: p2p.GetPeerList.known_peers:type_name -> p2p.BloomFilter + 8, // 29: p2p.PeerList.claimed_ip_ports:type_name -> p2p.ClaimedIpPort + 0, // 30: p2p.GetAcceptedFrontier.engine_type:type_name -> p2p.EngineType + 0, // 31: p2p.GetAccepted.engine_type:type_name -> p2p.EngineType + 0, // 32: p2p.GetAncestors.engine_type:type_name -> p2p.EngineType + 0, // 33: p2p.Get.engine_type:type_name -> p2p.EngineType + 0, // 34: p2p.Put.engine_type:type_name -> p2p.EngineType + 0, // 35: p2p.PushQuery.engine_type:type_name -> p2p.EngineType + 0, // 36: p2p.PullQuery.engine_type:type_name -> p2p.EngineType + 37, // [37:37] is the sub-list for method output_type + 37, // [37:37] is the sub-list for method input_type + 37, // [37:37] is the sub-list for extension type_name + 37, // [37:37] is the sub-list for extension extendee + 0, // [0:37] is the sub-list for field type_name } func init() { file_p2p_p2p_proto_init() } @@ -3132,7 +3149,7 @@ func file_p2p_p2p_proto_init() { } } file_p2p_p2p_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ClaimedIpPort); i { + switch v := v.(*BloomFilter); i { case 0: return &v.state case 1: @@ -3144,7 +3161,7 @@ func file_p2p_p2p_proto_init() { } } file_p2p_p2p_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PeerList); i { + switch v := v.(*ClaimedIpPort); i { case 0: return &v.state case 1: @@ -3156,7 +3173,7 @@ func file_p2p_p2p_proto_init() { } } file_p2p_p2p_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PeerAck); i { + switch v := v.(*GetPeerList); i { case 0: return &v.state case 1: @@ -3168,7 +3185,7 @@ func file_p2p_p2p_proto_init() { } } file_p2p_p2p_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PeerListAck); i { + switch v := v.(*PeerList); i { case 0: return &v.state case 1: @@ -3414,7 +3431,8 @@ func file_p2p_p2p_proto_init() { (*Message_Ping)(nil), (*Message_Pong)(nil), (*Message_Handshake)(nil), - (*Message_PeerList)(nil), + (*Message_GetPeerList)(nil), + (*Message_PeerList_)(nil), (*Message_GetStateSummaryFrontier)(nil), (*Message_StateSummaryFrontier_)(nil), (*Message_GetAcceptedStateSummary)(nil), @@ -3433,7 +3451,6 @@ func file_p2p_p2p_proto_init() { (*Message_AppRequest)(nil), (*Message_AppResponse)(nil), (*Message_AppGossip)(nil), - (*Message_PeerListAck)(nil), (*Message_AppError)(nil), } type x struct{} diff --git a/utils/bloom/read_filter.go b/utils/bloom/read_filter.go index 8c32e143b1c4..075d77ed7a38 100644 --- a/utils/bloom/read_filter.go +++ b/utils/bloom/read_filter.go @@ -8,6 +8,23 @@ import ( "fmt" ) +var ( + EmptyFilter = &ReadFilter{ + hashSeeds: make([]uint64, minHashes), + entries: make([]byte, minEntries), + } + FullFilter = &ReadFilter{ + hashSeeds: make([]uint64, minHashes), + entries: make([]byte, minEntries), + } +) + +func init() { + for i := range FullFilter.entries { + FullFilter.entries[i] = 0xFF + } +} + type ReadFilter struct { hashSeeds []uint64 entries []byte diff --git a/utils/constants/networking.go b/utils/constants/networking.go index 8d60d27af58a..7a4ea89b8241 100644 --- a/utils/constants/networking.go +++ b/utils/constants/networking.go @@ -32,6 +32,8 @@ const ( DefaultNetworkPeerListNonValidatorGossipSize = 0 DefaultNetworkPeerListPeersGossipSize = 10 DefaultNetworkPeerListGossipFreq = time.Minute + DefaultNetworkPeerListPullGossipFreq = 2 * time.Second + DefaultNetworkPeerListBloomResetFreq = time.Minute // Inbound Connection Throttling DefaultInboundConnUpgradeThrottlerCooldown = 10 * time.Second diff --git a/utils/ips/claimed_ip_port.go b/utils/ips/claimed_ip_port.go index 76920f31559d..2ef6c0a71087 100644 --- a/utils/ips/claimed_ip_port.go +++ b/utils/ips/claimed_ip_port.go @@ -6,16 +6,14 @@ package ips import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/staking" + "github.com/ava-labs/avalanchego/utils/hashing" + "github.com/ava-labs/avalanchego/utils/wrappers" ) -// Can't import these from wrappers package due to circular import. const ( - intLen = 4 - longLen = 8 - ipLen = 18 - idLen = 32 // Certificate length, signature length, IP, timestamp, tx ID - baseIPCertDescLen = 2*intLen + ipLen + longLen + idLen + baseIPCertDescLen = 2*wrappers.IntLen + IPPortLen + wrappers.LongLen + ids.IDLen + preimageLen = ids.IDLen + wrappers.LongLen ) // A self contained proof that a peer is claiming ownership of an IPPort at a @@ -32,12 +30,36 @@ type ClaimedIPPort struct { // actually claimed by the peer in question, and not by a malicious peer // trying to get us to dial bogus IPPorts. Signature []byte - // The txID that added this peer into the validator set - TxID ids.ID + // NodeID derived from the peer certificate. + NodeID ids.NodeID + // GossipID derived from the nodeID and timestamp. + GossipID ids.ID } -// Returns the length of the byte representation of this ClaimedIPPort. -func (i *ClaimedIPPort) BytesLen() int { - // See wrappers.PackPeerTrackInfo. +func NewClaimedIPPort( + cert *staking.Certificate, + ipPort IPPort, + timestamp uint64, + signature []byte, +) *ClaimedIPPort { + ip := &ClaimedIPPort{ + Cert: cert, + IPPort: ipPort, + Timestamp: timestamp, + Signature: signature, + NodeID: ids.NodeIDFromCert(cert), + } + + packer := wrappers.Packer{ + Bytes: make([]byte, preimageLen), + } + packer.PackFixedBytes(ip.NodeID[:]) + packer.PackLong(timestamp) + ip.GossipID = hashing.ComputeHash256Array(packer.Bytes) + return ip +} + +// Returns the approximate size of the binary representation of this ClaimedIPPort. +func (i *ClaimedIPPort) Size() int { return baseIPCertDescLen + len(i.Cert.Raw) + len(i.Signature) } diff --git a/utils/ips/ip_port.go b/utils/ips/ip_port.go index 661747cb9cbe..a60e6300ed49 100644 --- a/utils/ips/ip_port.go +++ b/utils/ips/ip_port.go @@ -12,7 +12,10 @@ import ( "github.com/ava-labs/avalanchego/utils/wrappers" ) -const nullStr = "null" +const ( + IPPortLen = 16 + wrappers.ShortLen + nullStr = "null" +) var ( errMissingQuotes = errors.New("first and last characters should be quotes")