From 8dbb088ca3df54bb9299b788fcdd2223dab60e83 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 17 Jun 2024 18:33:52 +0300 Subject: [PATCH 01/45] Make approval-distribution logic runnable on a separate thread Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 356 ++++++++++++------ 1 file changed, 234 insertions(+), 122 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index d48fb08a311c..26cedf1a2a13 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -320,7 +320,7 @@ enum Resend { /// It tracks metadata about our view of the unfinalized chain, /// which assignments and approvals we have seen, and our peers' views. #[derive(Default)] -struct State { +pub struct State { /// These two fields are used in conjunction to construct a view over the unfinalized chain. blocks_by_number: BTreeMap>, blocks: HashMap, @@ -662,9 +662,13 @@ enum PendingMessage { #[overseer::contextbounds(ApprovalDistribution, prefix = self::overseer)] impl State { - async fn handle_network_msg( + async fn handle_network_msg< + N: overseer::SubsystemSender, + A: overseer::SubsystemSender, + >( &mut self, - ctx: &mut Context, + approval_voting_sender: &mut A, + network_sender: &mut N, metrics: &Metrics, event: NetworkBridgeEvent, rng: &mut (impl CryptoRng + Rng), @@ -689,7 +693,7 @@ impl State { }, NetworkBridgeEvent::NewGossipTopology(topology) => { self.handle_new_session_topology( - ctx, + network_sender, topology.session, topology.topology, topology.local_index, @@ -697,7 +701,7 @@ impl State { .await; }, NetworkBridgeEvent::PeerViewChange(peer_id, view) => { - self.handle_peer_view_change(ctx, metrics, peer_id, view, rng).await; + self.handle_peer_view_change(network_sender, metrics, peer_id, view, rng).await; }, NetworkBridgeEvent::OurViewChange(view) => { gum::trace!(target: LOG_TARGET, ?view, "Own view change"); @@ -720,7 +724,15 @@ impl State { }); }, NetworkBridgeEvent::PeerMessage(peer_id, message) => { - self.process_incoming_peer_message(ctx, metrics, peer_id, message, rng).await; + self.process_incoming_peer_message( + approval_voting_sender, + network_sender, + metrics, + peer_id, + message, + rng, + ) + .await; }, NetworkBridgeEvent::UpdatedAuthorityIds(peer_id, authority_ids) => { gum::debug!(target: LOG_TARGET, ?peer_id, ?authority_ids, "Update Authority Ids"); @@ -743,7 +755,7 @@ impl State { let view_intersection = View::new(intersection.cloned(), view.finalized_number); Self::unify_with_peer( - ctx.sender(), + network_sender, metrics, &mut self.blocks, &self.topologies, @@ -761,9 +773,13 @@ impl State { } } - async fn handle_new_blocks( + async fn handle_new_blocks< + N: overseer::SubsystemSender, + A: overseer::SubsystemSender, + >( &mut self, - ctx: &mut Context, + approval_voting_sender: &mut A, + network_sender: &mut N, metrics: &Metrics, metas: Vec, rng: &mut (impl CryptoRng + Rng), @@ -814,12 +830,11 @@ impl State { ); { - let sender = ctx.sender(); for (peer_id, PeerEntry { view, version }) in self.peer_views.iter() { let intersection = view.iter().filter(|h| new_hashes.contains(h)); let view_intersection = View::new(intersection.cloned(), view.finalized_number); Self::unify_with_peer( - sender, + network_sender, metrics, &mut self.blocks, &self.topologies, @@ -866,7 +881,8 @@ impl State { match message { PendingMessage::Assignment(assignment, claimed_indices) => { self.import_and_circulate_assignment( - ctx, + approval_voting_sender, + network_sender, metrics, MessageSource::Peer(peer_id), assignment, @@ -877,7 +893,8 @@ impl State { }, PendingMessage::Approval(approval_vote) => { self.import_and_circulate_approval( - ctx, + approval_voting_sender, + network_sender, metrics, MessageSource::Peer(peer_id), approval_vote, @@ -889,12 +906,12 @@ impl State { } } - self.enable_aggression(ctx, Resend::Yes, metrics).await; + self.enable_aggression(network_sender, Resend::Yes, metrics).await; } - async fn handle_new_session_topology( + async fn handle_new_session_topology>( &mut self, - ctx: &mut Context, + network_sender: &mut N, session: SessionIndex, topology: SessionGridTopology, local_index: Option, @@ -908,7 +925,7 @@ impl State { let topology = self.topologies.get_topology(session).expect("just inserted above; qed"); adjust_required_routing_and_propagate( - ctx, + network_sender, &mut self.blocks, &self.topologies, |block_entry| block_entry.session == session, @@ -926,9 +943,14 @@ impl State { .await; } - async fn process_incoming_assignments( + async fn process_incoming_assignments< + N: overseer::SubsystemSender, + A: overseer::SubsystemSender, + R, + >( &mut self, - ctx: &mut Context, + approval_voting_sender: &mut A, + network_sender: &mut N, metrics: &Metrics, peer_id: PeerId, assignments: Vec<(IndirectAssignmentCertV2, CandidateBitfield)>, @@ -956,7 +978,8 @@ impl State { } self.import_and_circulate_assignment( - ctx, + approval_voting_sender, + network_sender, metrics, MessageSource::Peer(peer_id), assignment, @@ -968,9 +991,13 @@ impl State { } // Entry point for processing an approval coming from a peer. - async fn process_incoming_approvals( + async fn process_incoming_approvals< + N: overseer::SubsystemSender, + A: overseer::SubsystemSender, + >( &mut self, - ctx: &mut Context, + approval_voting_sender: &mut A, + network_sender: &mut N, metrics: &Metrics, peer_id: PeerId, approvals: Vec, @@ -1001,7 +1028,8 @@ impl State { } self.import_and_circulate_approval( - ctx, + approval_voting_sender, + network_sender, metrics, MessageSource::Peer(peer_id), approval_vote, @@ -1010,9 +1038,14 @@ impl State { } } - async fn process_incoming_peer_message( + async fn process_incoming_peer_message< + N: overseer::SubsystemSender, + A: overseer::SubsystemSender, + R, + >( &mut self, - ctx: &mut Context, + approval_voting_sender: &mut A, + network_sender: &mut N, metrics: &Metrics, peer_id: PeerId, msg: Versioned< @@ -1033,10 +1066,11 @@ impl State { "Processing assignments from a peer", ); let sanitized_assignments = - self.sanitize_v2_assignments(peer_id, ctx.sender(), assignments).await; + self.sanitize_v2_assignments(peer_id, network_sender, assignments).await; self.process_incoming_assignments( - ctx, + approval_voting_sender, + network_sender, metrics, peer_id, sanitized_assignments, @@ -1054,10 +1088,11 @@ impl State { ); let sanitized_assignments = - self.sanitize_v1_assignments(peer_id, ctx.sender(), assignments).await; + self.sanitize_v1_assignments(peer_id, network_sender, assignments).await; self.process_incoming_assignments( - ctx, + approval_voting_sender, + network_sender, metrics, peer_id, sanitized_assignments, @@ -1067,25 +1102,37 @@ impl State { }, Versioned::V3(protocol_v3::ApprovalDistributionMessage::Approvals(approvals)) => { let sanitized_approvals = - self.sanitize_v2_approvals(peer_id, ctx.sender(), approvals).await; - self.process_incoming_approvals(ctx, metrics, peer_id, sanitized_approvals) - .await; + self.sanitize_v2_approvals(peer_id, network_sender, approvals).await; + self.process_incoming_approvals( + approval_voting_sender, + network_sender, + metrics, + peer_id, + sanitized_approvals, + ) + .await; }, Versioned::V1(protocol_v1::ApprovalDistributionMessage::Approvals(approvals)) | Versioned::V2(protocol_v2::ApprovalDistributionMessage::Approvals(approvals)) => { let sanitized_approvals = - self.sanitize_v1_approvals(peer_id, ctx.sender(), approvals).await; - self.process_incoming_approvals(ctx, metrics, peer_id, sanitized_approvals) - .await; + self.sanitize_v1_approvals(peer_id, network_sender, approvals).await; + self.process_incoming_approvals( + approval_voting_sender, + network_sender, + metrics, + peer_id, + sanitized_approvals, + ) + .await; }, } } // handle a peer view change: requires that the peer is already connected // and has an entry in the `PeerData` struct. - async fn handle_peer_view_change( + async fn handle_peer_view_change, R>( &mut self, - ctx: &mut Context, + network_sender: &mut N, metrics: &Metrics, peer_id: PeerId, view: View, @@ -1132,7 +1179,7 @@ impl State { } Self::unify_with_peer( - ctx.sender(), + network_sender, metrics, &mut self.blocks, &self.topologies, @@ -1146,9 +1193,9 @@ impl State { .await; } - async fn handle_block_finalized( + async fn handle_block_finalized>( &mut self, - ctx: &mut Context, + network_sender: &mut N, metrics: &Metrics, finalized_number: BlockNumber, ) { @@ -1172,12 +1219,17 @@ impl State { // If a block was finalized, this means we may need to move our aggression // forward to the now oldest block(s). - self.enable_aggression(ctx, Resend::No, metrics).await; + self.enable_aggression(network_sender, Resend::No, metrics).await; } - async fn import_and_circulate_assignment( + async fn import_and_circulate_assignment< + N: overseer::SubsystemSender, + A: overseer::SubsystemSender, + R, + >( &mut self, - ctx: &mut Context, + approval_voting_sender: &mut A, + network_sender: &mut N, metrics: &Metrics, source: MessageSource, assignment: IndirectAssignmentCertV2, @@ -1218,7 +1270,7 @@ impl State { if !self.recent_outdated_blocks.is_recent_outdated(&block_hash) { modify_reputation( &mut self.reputation, - ctx.sender(), + network_sender, peer_id, COST_UNEXPECTED_MESSAGE, ) @@ -1255,7 +1307,7 @@ impl State { modify_reputation( &mut self.reputation, - ctx.sender(), + network_sender, peer_id, COST_DUPLICATE_MESSAGE, ) @@ -1283,7 +1335,7 @@ impl State { ); modify_reputation( &mut self.reputation, - ctx.sender(), + network_sender, peer_id, COST_UNEXPECTED_MESSAGE, ) @@ -1296,7 +1348,7 @@ impl State { if entry.knowledge.contains(&message_subject, message_kind) { modify_reputation( &mut self.reputation, - ctx.sender(), + network_sender, peer_id, BENEFIT_VALID_MESSAGE, ) @@ -1311,12 +1363,13 @@ impl State { let (tx, rx) = oneshot::channel(); - ctx.send_message(ApprovalVotingMessage::CheckAndImportAssignment( - assignment.clone(), - claimed_candidate_indices.clone(), - tx, - )) - .await; + approval_voting_sender + .send_message(ApprovalVotingMessage::CheckAndImportAssignment( + assignment.clone(), + claimed_candidate_indices.clone(), + tx, + )) + .await; let timer = metrics.time_awaiting_approval_voting(); let result = match rx.await { @@ -1339,7 +1392,7 @@ impl State { AssignmentCheckResult::Accepted => { modify_reputation( &mut self.reputation, - ctx.sender(), + network_sender, peer_id, BENEFIT_VALID_MESSAGE_FIRST, ) @@ -1375,7 +1428,7 @@ impl State { ); modify_reputation( &mut self.reputation, - ctx.sender(), + network_sender, peer_id, COST_ASSIGNMENT_TOO_FAR_IN_THE_FUTURE, ) @@ -1394,7 +1447,7 @@ impl State { ); modify_reputation( &mut self.reputation, - ctx.sender(), + network_sender, peer_id, COST_INVALID_MESSAGE, ) @@ -1526,14 +1579,16 @@ impl State { }) .collect::>(); - send_assignments_batched(ctx.sender(), assignments, &peers).await; + send_assignments_batched(network_sender, assignments, &peers).await; } } // Checks if an approval can be processed. // Returns true if we can continue with processing the approval and false otherwise. - async fn check_approval_can_be_processed( - ctx: &mut Context, + async fn check_approval_can_be_processed< + N: overseer::SubsystemSender, + >( + network_sender: &mut N, assignments_knowledge_key: &Vec<(MessageSubject, MessageKind)>, approval_knowledge_key: &(MessageSubject, MessageKind), entry: &mut BlockEntry, @@ -1549,7 +1604,8 @@ impl State { ?message_subject, "Unknown approval assignment", ); - modify_reputation(reputation, ctx.sender(), peer_id, COST_UNEXPECTED_MESSAGE).await; + modify_reputation(reputation, network_sender, peer_id, COST_UNEXPECTED_MESSAGE) + .await; metrics.on_approval_unknown_assignment(); return false } @@ -1573,7 +1629,7 @@ impl State { modify_reputation( reputation, - ctx.sender(), + network_sender, peer_id, COST_DUPLICATE_MESSAGE, ) @@ -1590,7 +1646,8 @@ impl State { ?approval_knowledge_key, "Approval from a peer is out of view", ); - modify_reputation(reputation, ctx.sender(), peer_id, COST_UNEXPECTED_MESSAGE).await; + modify_reputation(reputation, network_sender, peer_id, COST_UNEXPECTED_MESSAGE) + .await; metrics.on_approval_out_of_view(); }, } @@ -1605,16 +1662,20 @@ impl State { // We already processed this approval no need to continue. gum::trace!(target: LOG_TARGET, ?peer_id, ?approval_knowledge_key, "Known approval"); metrics.on_approval_good_known(); - modify_reputation(reputation, ctx.sender(), peer_id, BENEFIT_VALID_MESSAGE).await; + modify_reputation(reputation, network_sender, peer_id, BENEFIT_VALID_MESSAGE).await; false } else { true } } - async fn import_and_circulate_approval( + async fn import_and_circulate_approval< + N: overseer::SubsystemSender, + A: overseer::SubsystemSender, + >( &mut self, - ctx: &mut Context, + approval_voting_sender: &mut A, + network_sender: &mut N, metrics: &Metrics, source: MessageSource, vote: IndirectSignedApprovalVoteV2, @@ -1652,7 +1713,7 @@ impl State { ); modify_reputation( &mut self.reputation, - ctx.sender(), + network_sender, peer_id, COST_UNEXPECTED_MESSAGE, ) @@ -1672,7 +1733,7 @@ impl State { if let Some(peer_id) = source.peer_id() { if !Self::check_approval_can_be_processed( - ctx, + network_sender, &assignments_knowledge_keys, &approval_knwowledge_key, entry, @@ -1687,7 +1748,8 @@ impl State { let (tx, rx) = oneshot::channel(); - ctx.send_message(ApprovalVotingMessage::CheckAndImportApproval(vote.clone(), tx)) + approval_voting_sender + .send_message(ApprovalVotingMessage::CheckAndImportApproval(vote.clone(), tx)) .await; let timer = metrics.time_awaiting_approval_voting(); @@ -1711,7 +1773,7 @@ impl State { ApprovalCheckResult::Accepted => { modify_reputation( &mut self.reputation, - ctx.sender(), + network_sender, peer_id, BENEFIT_VALID_MESSAGE_FIRST, ) @@ -1729,7 +1791,7 @@ impl State { ApprovalCheckResult::Bad(error) => { modify_reputation( &mut self.reputation, - ctx.sender(), + network_sender, peer_id, COST_INVALID_MESSAGE, ) @@ -1831,7 +1893,7 @@ impl State { num_peers = peers.len(), "Sending an approval to peers", ); - send_approvals_batched(ctx.sender(), approvals, &peers).await; + send_approvals_batched(network_sender, approvals, &peers).await; } } @@ -1882,7 +1944,7 @@ impl State { } async fn unify_with_peer( - sender: &mut impl overseer::ApprovalDistributionSenderTrait, + sender: &mut impl overseer::SubsystemSender, metrics: &Metrics, entries: &mut HashMap, topologies: &SessionGridTopologies, @@ -2027,9 +2089,9 @@ impl State { // // In order to switch to using approval lag as a trigger we need a request/response protocol // to fetch votes from validators rather than use gossip. - async fn enable_aggression( + async fn enable_aggression>( &mut self, - ctx: &mut Context, + network_sender: &mut N, resend: Resend, metrics: &Metrics, ) { @@ -2058,7 +2120,7 @@ impl State { gum::debug!(target: LOG_TARGET, min_age, max_age, "Aggression enabled",); adjust_required_routing_and_propagate( - ctx, + network_sender, &mut self.blocks, &self.topologies, |block_entry| { @@ -2086,7 +2148,7 @@ impl State { .await; adjust_required_routing_and_propagate( - ctx, + network_sender, &mut self.blocks, &self.topologies, |block_entry| { @@ -2137,7 +2199,7 @@ impl State { async fn sanitize_v1_assignments( &mut self, peer_id: PeerId, - sender: &mut impl overseer::ApprovalDistributionSenderTrait, + sender: &mut impl overseer::SubsystemSender, assignments: Vec<(IndirectAssignmentCert, CandidateIndex)>, ) -> Vec<(IndirectAssignmentCertV2, CandidateBitfield)> { let mut sanitized_assignments = Vec::new(); @@ -2172,7 +2234,7 @@ impl State { async fn sanitize_v2_assignments( &mut self, peer_id: PeerId, - sender: &mut impl overseer::ApprovalDistributionSenderTrait, + sender: &mut impl overseer::SubsystemSender, assignments: Vec<(IndirectAssignmentCertV2, CandidateBitfield)>, ) -> Vec<(IndirectAssignmentCertV2, CandidateBitfield)> { let mut sanitized_assignments = Vec::new(); @@ -2216,7 +2278,7 @@ impl State { async fn sanitize_v1_approvals( &mut self, peer_id: PeerId, - sender: &mut impl overseer::ApprovalDistributionSenderTrait, + sender: &mut impl overseer::SubsystemSender, approval: Vec, ) -> Vec { let mut sanitized_approvals = Vec::new(); @@ -2243,7 +2305,7 @@ impl State { async fn sanitize_v2_approvals( &mut self, peer_id: PeerId, - sender: &mut impl overseer::ApprovalDistributionSenderTrait, + sender: &mut impl overseer::SubsystemSender, approval: Vec, ) -> Vec { let mut sanitized_approvals = Vec::new(); @@ -2280,8 +2342,12 @@ impl State { // Note that the required routing of a message can be modified even if the // topology is unknown yet. #[overseer::contextbounds(ApprovalDistribution, prefix = self::overseer)] -async fn adjust_required_routing_and_propagate( - ctx: &mut Context, +async fn adjust_required_routing_and_propagate< + N: overseer::SubsystemSender, + BlockFilter, + RoutingModifier, +>( + network_sender: &mut N, blocks: &mut HashMap, topologies: &SessionGridTopologies, block_filter: BlockFilter, @@ -2363,7 +2429,7 @@ async fn adjust_required_routing_and_propagate, peer_id: PeerId, rep: Rep, ) { @@ -2414,7 +2480,6 @@ impl ApprovalDistribution { async fn run(self, ctx: Context) { let mut state = State::default(); - // According to the docs of `rand`, this is a ChaCha12 RNG in practice // and will always be chosen for strong performance and security properties. let mut rng = rand::rngs::StdRng::from_entropy(); @@ -2431,7 +2496,8 @@ impl ApprovalDistribution { ) { let new_reputation_delay = || futures_timer::Delay::new(reputation_interval).fuse(); let mut reputation_delay = new_reputation_delay(); - + let mut approval_voting_sender = ctx.sender().clone(); + let mut network_sender = ctx.sender().clone(); loop { select! { _ = reputation_delay => { @@ -2446,35 +2512,65 @@ impl ApprovalDistribution { return }, }; - match message { - FromOrchestra::Communication { msg } => - Self::handle_incoming(&mut ctx, state, msg, &self.metrics, rng).await, - FromOrchestra::Signal(OverseerSignal::ActiveLeaves(update)) => { - gum::trace!(target: LOG_TARGET, "active leaves signal (ignored)"); - // the relay chain blocks relevant to the approval subsystems - // are those that are available, but not finalized yet - // activated and deactivated heads hence are irrelevant to this subsystem, other than - // for tracing purposes. - if let Some(activated) = update.activated { - let head = activated.hash; - let approval_distribution_span = - jaeger::PerLeafSpan::new(activated.span, "approval-distribution"); - state.spans.insert(head, approval_distribution_span); - } - }, - FromOrchestra::Signal(OverseerSignal::BlockFinalized(_hash, number)) => { - gum::trace!(target: LOG_TARGET, number = %number, "finalized signal"); - state.handle_block_finalized(&mut ctx, &self.metrics, number).await; - }, - FromOrchestra::Signal(OverseerSignal::Conclude) => return, - } + + + self.handle_from_orchestra(message, &mut approval_voting_sender, &mut network_sender, state, rng).await; + }, } } } - async fn handle_incoming( - ctx: &mut Context, + /// Handles a from orchestra message received by approval distribution subystem. + pub async fn handle_from_orchestra< + N: overseer::SubsystemSender, + A: overseer::SubsystemSender, + >( + &self, + message: FromOrchestra, + approval_voting_sender: &mut A, + network_sender: &mut N, + state: &mut State, + rng: &mut (impl CryptoRng + Rng), + ) { + match message { + FromOrchestra::Communication { msg } => + Self::handle_incoming( + approval_voting_sender, + network_sender, + state, + msg, + &self.metrics, + rng, + ) + .await, + FromOrchestra::Signal(OverseerSignal::ActiveLeaves(update)) => { + gum::trace!(target: LOG_TARGET, "active leaves signal (ignored)"); + // the relay chain blocks relevant to the approval subsystems + // are those that are available, but not finalized yet + // activated and deactivated heads hence are irrelevant to this subsystem, other + // than for tracing purposes. + if let Some(activated) = update.activated { + let head = activated.hash; + let approval_distribution_span = + jaeger::PerLeafSpan::new(activated.span, "approval-distribution"); + state.spans.insert(head, approval_distribution_span); + } + }, + FromOrchestra::Signal(OverseerSignal::BlockFinalized(_hash, number)) => { + gum::trace!(target: LOG_TARGET, number = %number, "finalized signal"); + state.handle_block_finalized(network_sender, &self.metrics, number).await; + }, + FromOrchestra::Signal(OverseerSignal::Conclude) => return, + } + } + + async fn handle_incoming< + N: overseer::SubsystemSender, + A: overseer::SubsystemSender, + >( + approval_voting_sender: &mut A, + network_sender: &mut N, state: &mut State, msg: ApprovalDistributionMessage, metrics: &Metrics, @@ -2482,10 +2578,14 @@ impl ApprovalDistribution { ) { match msg { ApprovalDistributionMessage::NetworkBridgeUpdate(event) => { - state.handle_network_msg(ctx, metrics, event, rng).await; + state + .handle_network_msg(approval_voting_sender, network_sender, metrics, event, rng) + .await; }, ApprovalDistributionMessage::NewBlocks(metas) => { - state.handle_new_blocks(ctx, metrics, metas, rng).await; + state + .handle_new_blocks(approval_voting_sender, network_sender, metrics, metas, rng) + .await; }, ApprovalDistributionMessage::DistributeAssignment(cert, candidate_indices) => { let _span = state @@ -2506,7 +2606,8 @@ impl ApprovalDistribution { state .import_and_circulate_assignment( - ctx, + approval_voting_sender, + network_sender, &metrics, MessageSource::Local, cert, @@ -2524,7 +2625,13 @@ impl ApprovalDistribution { ); state - .import_and_circulate_approval(ctx, metrics, MessageSource::Local, vote) + .import_and_circulate_approval( + approval_voting_sender, + network_sender, + metrics, + MessageSource::Local, + vote, + ) .await; }, ApprovalDistributionMessage::GetApprovalSignatures(indices, tx) => { @@ -2579,7 +2686,7 @@ pub const MAX_APPROVAL_BATCH_SIZE: usize = ensure_size_not_zero( // Low level helper for sending assignments. async fn send_assignments_batched_inner( - sender: &mut impl overseer::ApprovalDistributionSenderTrait, + sender: &mut impl overseer::SubsystemSender, batch: impl IntoIterator, peers: Vec, peer_version: ValidationVersion, @@ -2634,7 +2741,7 @@ async fn send_assignments_batched_inner( /// destination, such that the subsystem doesn't get stuck for long processing a batch /// of assignments and can `select!` other tasks. pub(crate) async fn send_assignments_batched( - sender: &mut impl overseer::ApprovalDistributionSenderTrait, + network_sender: &mut impl overseer::SubsystemSender, v2_assignments: impl IntoIterator + Clone, peers: &[(PeerId, ProtocolVersion)], ) { @@ -2658,7 +2765,7 @@ pub(crate) async fn send_assignments_batched( let batch: Vec<_> = v1_batches.by_ref().take(MAX_ASSIGNMENT_BATCH_SIZE).collect(); if !v1_peers.is_empty() { send_assignments_batched_inner( - sender, + network_sender, batch.clone(), v1_peers.clone(), ValidationVersion::V1, @@ -2668,7 +2775,7 @@ pub(crate) async fn send_assignments_batched( if !v2_peers.is_empty() { send_assignments_batched_inner( - sender, + network_sender, batch, v2_peers.clone(), ValidationVersion::V2, @@ -2683,15 +2790,20 @@ pub(crate) async fn send_assignments_batched( while v3.peek().is_some() { let batch = v3.by_ref().take(MAX_ASSIGNMENT_BATCH_SIZE).collect::>(); - send_assignments_batched_inner(sender, batch, v3_peers.clone(), ValidationVersion::V3) - .await; + send_assignments_batched_inner( + network_sender, + batch, + v3_peers.clone(), + ValidationVersion::V3, + ) + .await; } } } /// Send approvals while honoring the `max_notification_size` of the protocol and peer version. pub(crate) async fn send_approvals_batched( - sender: &mut impl overseer::ApprovalDistributionSenderTrait, + sender: &mut impl overseer::SubsystemSender, approvals: impl IntoIterator + Clone, peers: &[(PeerId, ProtocolVersion)], ) { From 14727c5a1fb27365667e48506055be9da64967df Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 19 Jun 2024 12:18:44 +0300 Subject: [PATCH 02/45] Move crypto checks in the approval-distribution Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 3 + .../node/core/approval-voting/src/criteria.rs | 8 +- .../node/core/approval-voting/src/import.rs | 6 +- polkadot/node/core/approval-voting/src/lib.rs | 182 +- .../node/core/approval-voting/src/tests.rs | 168 +- .../network/approval-distribution/Cargo.toml | 3 + .../network/approval-distribution/src/lib.rs | 464 +- .../approval-distribution/src/metrics.rs | 16 - .../approval-distribution/src/tests.rs | 5883 +++++++++-------- polkadot/node/overseer/src/lib.rs | 1 + polkadot/node/primitives/src/approval.rs | 8 +- polkadot/node/service/src/overseer.rs | 5 +- .../subsystem-bench/src/lib/approval/mod.rs | 7 +- polkadot/node/subsystem-types/src/messages.rs | 10 +- 14 files changed, 3733 insertions(+), 3031 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9972285780f3..e91bd6400042 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12755,6 +12755,7 @@ dependencies = [ "futures-timer", "itertools 0.11.0", "log", + "polkadot-node-core-approval-voting", "polkadot-node-jaeger", "polkadot-node-metrics", "polkadot-node-network-protocol", @@ -12767,7 +12768,9 @@ dependencies = [ "rand", "rand_chacha", "rand_core", + "sc-keystore", "schnorrkel 0.11.4", + "sp-application-crypto", "sp-authority-discovery", "sp-core", "tracing-gum", diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index fb9d281e43bc..4a1e87868c2d 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -254,7 +254,7 @@ impl<'a> From<&'a SessionInfo> for Config { } /// A trait for producing and checking assignments. Used to mock. -pub(crate) trait AssignmentCriteria { +pub trait AssignmentCriteria { fn compute_assignments( &self, keystore: &LocalKeystore, @@ -276,7 +276,7 @@ pub(crate) trait AssignmentCriteria { ) -> Result; } -pub(crate) struct RealAssignmentCriteria; +pub struct RealAssignmentCriteria; impl AssignmentCriteria for RealAssignmentCriteria { fn compute_assignments( @@ -614,7 +614,7 @@ fn compute_relay_vrf_delay_assignments( /// Assignment invalid. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct InvalidAssignment(pub(crate) InvalidAssignmentReason); +pub struct InvalidAssignment(pub InvalidAssignmentReason); impl std::fmt::Display for InvalidAssignment { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { @@ -626,7 +626,7 @@ impl std::error::Error for InvalidAssignment {} /// Failure conditions when checking an assignment cert. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum InvalidAssignmentReason { +pub enum InvalidAssignmentReason { ValidatorIndexOutOfBounds, SampleOutOfBounds, CoreIndexOutOfBounds, diff --git a/polkadot/node/core/approval-voting/src/import.rs b/polkadot/node/core/approval-voting/src/import.rs index 3ddef1e01c45..a138af1188d2 100644 --- a/polkadot/node/core/approval-voting/src/import.rs +++ b/polkadot/node/core/approval-voting/src/import.rs @@ -574,9 +574,13 @@ pub(crate) async fn handle_new_head( hash: block_hash, number: block_header.number, parent_hash: block_header.parent_hash, - candidates: included_candidates.iter().map(|(hash, _, _, _)| *hash).collect(), + candidates: included_candidates + .iter() + .map(|(hash, _, core_index, group_index)| (*hash, *core_index, *group_index)) + .collect(), slot, session: session_index, + vrf_story: relay_vrf_story, }); imported_candidates.push(BlockImportedCandidates { diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index d4b6855a44d0..5d588a53f3d1 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -55,9 +55,8 @@ use polkadot_node_subsystem_util::{ }; use polkadot_primitives::{ ApprovalVoteMultipleCandidates, ApprovalVotingParams, BlockNumber, CandidateHash, - CandidateIndex, CandidateReceipt, CoreIndex, DisputeStatement, ExecutorParams, GroupIndex, - Hash, PvfExecKind, SessionIndex, SessionInfo, ValidDisputeStatementKind, ValidatorId, - ValidatorIndex, ValidatorPair, ValidatorSignature, + CandidateIndex, CandidateReceipt, CoreIndex, ExecutorParams, GroupIndex, Hash, PvfExecKind, + SessionIndex, SessionInfo, ValidatorId, ValidatorIndex, ValidatorPair, ValidatorSignature, }; use sc_keystore::LocalKeystore; use sp_application_crypto::Pair; @@ -91,7 +90,7 @@ use schnellru::{ByLength, LruMap}; use approval_checking::RequiredTranches; use bitvec::{order::Lsb0, vec::BitVec}; -use criteria::{AssignmentCriteria, RealAssignmentCriteria}; +pub use criteria::{AssignmentCriteria, Config as AssignmentConfig, RealAssignmentCriteria}; use persisted_entries::{ApprovalEntry, BlockEntry, CandidateEntry}; use time::{slot_number_to_tick, Clock, ClockExt, DelayedApprovalTimer, SystemClock, Tick}; @@ -123,7 +122,7 @@ const APPROVAL_CHECKING_TIMEOUT: Duration = Duration::from_secs(120); const WAIT_FOR_SIGS_TIMEOUT: Duration = Duration::from_millis(500); const APPROVAL_CACHE_SIZE: u32 = 1024; -const TICK_TOO_FAR_IN_FUTURE: Tick = 20; // 10 seconds. +pub const TICK_TOO_FAR_IN_FUTURE: Tick = 20; // 10 seconds. const APPROVAL_DELAY: Tick = 2; pub(crate) const LOG_TARGET: &str = "parachain::approval-voting"; @@ -1607,9 +1606,21 @@ async fn distribution_messages_for_activation( hash: block_hash, number: block_entry.block_number(), parent_hash: block_entry.parent_hash(), - candidates: block_entry.candidates().iter().map(|(_, c_hash)| *c_hash).collect(), + candidates: block_entry + .candidates() + .iter() + .flat_map(|(core_index, c_hash)| { + let candidate = db.load_candidate_entry(c_hash).ok().flatten(); + candidate + .and_then(|entry| { + entry.approval_entry(&block_hash).map(|entry| entry.backing_group()) + }) + .and_then(|group_index| Some((*c_hash, *core_index, group_index))) + }) + .collect(), slot: block_entry.slot(), session: block_entry.session(), + vrf_story: block_entry.relay_vrf_story(), }); let mut signatures_queued = HashSet::new(); for (core_index, candidate_hash) in block_entry.candidates() { @@ -1872,7 +1883,7 @@ async fn handle_from_overseer( vec![Action::Conclude] }, FromOrchestra::Communication { msg } => match msg { - ApprovalVotingMessage::CheckAndImportAssignment(a, claimed_cores, res) => { + ApprovalVotingMessage::CheckAndImportAssignment(a, claimed_cores, tranche, tx) => { let (check_outcome, actions) = check_and_import_assignment( ctx.sender(), state, @@ -1880,27 +1891,39 @@ async fn handle_from_overseer( session_info_provider, a, claimed_cores, + tranche, ) .await?; - let _ = res.send(check_outcome); - + // approval-distribution already sanity checks the assignment is valid and expected, + // so this import should never fail if it does it means there is a bug in the + // code. + if let AssignmentCheckResult::Bad(ref err) = check_outcome { + gum::error!(target: LOG_TARGET, ?err, "Unexpected fail to check and import assignment"); + } + let _ = tx.map(|tx| tx.send(check_outcome)); actions }, - ApprovalVotingMessage::CheckAndImportApproval(a, res) => - check_and_import_approval( + ApprovalVotingMessage::CheckAndImportApproval(a, tx) => { + let result = check_and_import_approval( ctx.sender(), state, db, session_info_provider, metrics, a, - |r| { - let _ = res.send(r); - }, &wakeups, ) - .await? - .0, + .await?; + // approval-distribution already sanity checks the vote is valid and expected, + // so this import should never fail if it does it means there is a bug in the + // code. + if let ApprovalCheckResult::Bad(ref err) = result.1 { + gum::error!(target: LOG_TARGET, ?err, "Unexpected fail to check and import approval"); + } + let _ = tx.map(|tx| tx.send(result.1)); + + result.0 + }, ApprovalVotingMessage::ApprovedAncestor(target, lower_bound, res) => { let mut approved_ancestor_span = state .spans @@ -2384,7 +2407,12 @@ fn schedule_wakeup_action( last_assignment_tick.map(|l| l + APPROVAL_DELAY).filter(|t| t > &tick_now), next_no_show, ) - .map(|tick| Action::ScheduleWakeup { block_hash, block_number, candidate_hash, tick }) + .map(|tick| Action::ScheduleWakeup { + block_hash, + block_number, + candidate_hash, + tick, + }) }, RequiredTranches::Pending { considered, next_no_show, clock_drift, .. } => { // select the minimum of `next_no_show`, or the tick of the next non-empty tranche @@ -2446,6 +2474,7 @@ async fn check_and_import_assignment( session_info_provider: &mut RuntimeInfo, assignment: IndirectAssignmentCertV2, candidate_indices: CandidateBitfield, + tranche: DelayTranche, ) -> SubsystemResult<(AssignmentCheckResult, Vec)> where Sender: SubsystemSender, @@ -2514,8 +2543,6 @@ where )) } - // The Compact VRF modulo assignment cert has multiple core assignments. - let mut backing_groups = Vec::new(); let mut claimed_core_indices = Vec::new(); let mut assigned_candidate_hashes = Vec::new(); @@ -2551,7 +2578,7 @@ where format!("{:?}", jaeger::hash_to_trace_identifier(assigned_candidate_hash.0)), ); - let approval_entry = match candidate_entry.approval_entry_mut(&assignment.block_hash) { + let _approval_entry = match candidate_entry.approval_entry_mut(&assignment.block_hash) { Some(a) => a, None => return Ok(( @@ -2563,7 +2590,6 @@ where )), }; - backing_groups.push(approval_entry.backing_group()); claimed_core_indices.push(claimed_core_index); assigned_candidate_hashes.push(assigned_candidate_hash); } @@ -2579,42 +2605,6 @@ where )) } - // Check the assignment certificate. - let res = state.assignment_criteria.check_assignment_cert( - claimed_core_indices - .clone() - .try_into() - .expect("Checked for null assignment above; qed"), - assignment.validator, - &criteria::Config::from(session_info), - block_entry.relay_vrf_story(), - &assignment.cert, - backing_groups, - ); - - let tranche = match res { - Err(crate::criteria::InvalidAssignment(reason)) => - return Ok(( - AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCert( - assignment.validator, - format!("{:?}", reason), - )), - Vec::new(), - )), - Ok(tranche) => { - let current_tranche = - state.clock.tranche_now(state.slot_duration_millis, block_entry.slot()); - - let too_far_in_future = current_tranche + TICK_TOO_FAR_IN_FUTURE as DelayTranche; - - if tranche >= too_far_in_future { - return Ok((AssignmentCheckResult::TooFarInFuture, Vec::new())) - } - - tranche - }, - }; - let mut actions = Vec::new(); let res = { let mut is_duplicate = true; @@ -2704,23 +2694,21 @@ where Ok((res, actions)) } -async fn check_and_import_approval( +async fn check_and_import_approval( sender: &mut Sender, state: &mut State, db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, metrics: &Metrics, approval: IndirectSignedApprovalVoteV2, - with_response: impl FnOnce(ApprovalCheckResult) -> T, wakeups: &Wakeups, -) -> SubsystemResult<(Vec, T)> +) -> SubsystemResult<(Vec, ApprovalCheckResult)> where Sender: SubsystemSender, { macro_rules! respond_early { ($e: expr) => {{ - let t = with_response($e); - return Ok((Vec::new(), t)) + return Ok((Vec::new(), $e)) }}; } let mut span = state @@ -2774,67 +2762,11 @@ where ), ); - { - let session_info = match get_session_info( - session_info_provider, - sender, - approval.block_hash, - block_entry.session(), - ) - .await - { - Some(s) => s, - None => { - respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::UnknownSessionIndex( - block_entry.session() - ),)) - }, - }; - - let pubkey = match session_info.validators.get(approval.validator) { - Some(k) => k, - None => respond_early!(ApprovalCheckResult::Bad( - ApprovalCheckError::InvalidValidatorIndex(approval.validator), - )), - }; - - gum::trace!( - target: LOG_TARGET, - "Received approval for num_candidates {:}", - approval.candidate_indices.count_ones() - ); - - let candidate_hashes: Vec = - approved_candidates_info.iter().map(|candidate| candidate.1).collect(); - // Signature check: - match DisputeStatement::Valid( - ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(candidate_hashes.clone()), - ) - .check_signature( - &pubkey, - if let Some(candidate_hash) = candidate_hashes.first() { - *candidate_hash - } else { - respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::InvalidValidatorIndex( - approval.validator - ),)) - }, - block_entry.session(), - &approval.signature, - ) { - Err(_) => { - gum::error!( - target: LOG_TARGET, - "Error while checking signature {:}", - approval.candidate_indices.count_ones() - ); - respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::InvalidSignature( - approval.validator - ),)) - }, - Ok(()) => {}, - }; - } + gum::trace!( + target: LOG_TARGET, + "Received approval for num_candidates {:}", + approval.candidate_indices.count_ones() + ); let mut actions = Vec::new(); for (approval_candidate_index, approved_candidate_hash) in approved_candidates_info { @@ -2898,9 +2830,7 @@ where } // importing the approval can be heavy as it may trigger acceptance for a series of blocks. - let t = with_response(ApprovalCheckResult::Accepted); - - Ok((actions, t)) + Ok((actions, ApprovalCheckResult::Accepted)) } #[derive(Debug)] diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 64ae86bc013a..8aab353f9b84 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -41,8 +41,9 @@ use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::TimeoutExt; use polkadot_overseer::HeadSupportsParachains; use polkadot_primitives::{ - ApprovalVote, CandidateCommitments, CandidateEvent, CoreIndex, GroupIndex, Header, - Id as ParaId, IndexedVec, NodeFeatures, ValidationCode, ValidatorSignature, + ApprovalVote, CandidateCommitments, CandidateEvent, CoreIndex, DisputeStatement, GroupIndex, + Header, Id as ParaId, IndexedVec, NodeFeatures, ValidDisputeStatementKind, ValidationCode, + ValidatorSignature, }; use std::{cmp::max, time::Duration}; @@ -262,7 +263,8 @@ where _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, _assignment: &polkadot_node_primitives::approval::v2::AssignmentCertV2, _backing_groups: Vec, - ) -> Result { + ) -> Result + { self.1(validator_index) } } @@ -677,7 +679,7 @@ async fn check_and_import_approval( validator, signature, }, - tx, + Some(tx), ), }, ) @@ -698,6 +700,7 @@ async fn check_and_import_assignment( block_hash: Hash, candidate_index: CandidateIndex, validator: ValidatorIndex, + tranche: DelayTranche, ) -> oneshot::Receiver { let (tx, rx) = oneshot::channel(); overseer_send( @@ -711,7 +714,8 @@ async fn check_and_import_assignment( .into(), }, candidate_index.into(), - tx, + tranche, + Some(tx), ), }, ) @@ -744,7 +748,8 @@ async fn check_and_import_assignment_v2( }), }, core_indices.try_into().unwrap(), - tx, + 0, + Some(tx), ), }, ) @@ -1130,6 +1135,7 @@ fn subsystem_rejects_bad_assignment_ok_criteria() { block_hash, candidate_index, validator, + 0, ) .await; @@ -1143,6 +1149,7 @@ fn subsystem_rejects_bad_assignment_ok_criteria() { unknown_hash, candidate_index, validator, + 0, ) .await; @@ -1155,59 +1162,6 @@ fn subsystem_rejects_bad_assignment_ok_criteria() { }); } -#[test] -fn subsystem_rejects_bad_assignment_err_criteria() { - let assignment_criteria = Box::new(MockAssignmentCriteria::check_only(move |_| { - Err(criteria::InvalidAssignment( - criteria::InvalidAssignmentReason::ValidatorIndexOutOfBounds, - )) - })); - let config = HarnessConfigBuilder::default().assignment_criteria(assignment_criteria).build(); - test_harness(config, |test_harness| async move { - let TestHarness { mut virtual_overseer, sync_oracle_handle: _sync_oracle_handle, .. } = - test_harness; - assert_matches!( - overseer_recv(&mut virtual_overseer).await, - AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { - rx.send(Ok(0)).unwrap(); - } - ); - - let block_hash = Hash::repeat_byte(0x01); - let candidate_index = 0; - let validator = ValidatorIndex(0); - - let head: Hash = ChainBuilder::GENESIS_HASH; - let mut builder = ChainBuilder::new(); - let slot = Slot::from(1 as u64); - builder.add_block( - block_hash, - head, - 1, - BlockConfig { slot, candidates: None, session_info: None, end_syncing: false }, - ); - builder.build(&mut virtual_overseer).await; - - let rx = check_and_import_assignment( - &mut virtual_overseer, - block_hash, - candidate_index, - validator, - ) - .await; - - assert_eq!( - rx.await, - Ok(AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCert( - ValidatorIndex(0), - "ValidatorIndexOutOfBounds".to_string(), - ))), - ); - - virtual_overseer - }); -} - #[test] fn blank_subsystem_act_on_bad_block() { test_harness(HarnessConfig::default(), |test_harness| async move { @@ -1236,7 +1190,8 @@ fn blank_subsystem_act_on_bad_block() { .into(), }, 0u32.into(), - tx, + 0, + Some(tx), ), }, ) @@ -1428,68 +1383,6 @@ fn subsystem_rejects_approval_before_assignment() { }); } -#[test] -fn subsystem_rejects_assignment_in_future() { - let assignment_criteria = - Box::new(MockAssignmentCriteria::check_only(|_| Ok(TICK_TOO_FAR_IN_FUTURE as _))); - let config = HarnessConfigBuilder::default().assignment_criteria(assignment_criteria).build(); - test_harness(config, |test_harness| async move { - let TestHarness { mut virtual_overseer, clock, sync_oracle_handle: _sync_oracle_handle } = - test_harness; - assert_matches!( - overseer_recv(&mut virtual_overseer).await, - AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { - rx.send(Ok(0)).unwrap(); - } - ); - - let block_hash = Hash::repeat_byte(0x01); - let candidate_index = 0; - let validator = ValidatorIndex(0); - - // Add block hash 00. - ChainBuilder::new() - .add_block( - block_hash, - ChainBuilder::GENESIS_HASH, - 1, - BlockConfig { - slot: Slot::from(0), - candidates: None, - session_info: None, - end_syncing: false, - }, - ) - .build(&mut virtual_overseer) - .await; - - let rx = check_and_import_assignment( - &mut virtual_overseer, - block_hash, - candidate_index, - validator, - ) - .await; - - assert_eq!(rx.await, Ok(AssignmentCheckResult::TooFarInFuture)); - - // Advance clock to make assignment reasonably near. - clock.inner.lock().set_tick(9); - - let rx = check_and_import_assignment( - &mut virtual_overseer, - block_hash, - candidate_index, - validator, - ) - .await; - - assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted)); - - virtual_overseer - }); -} - #[test] fn subsystem_accepts_duplicate_assignment() { test_harness(HarnessConfig::default(), |test_harness| async move { @@ -1555,6 +1448,7 @@ fn subsystem_accepts_duplicate_assignment() { block_hash, candidate_index, validator, + 0, ) .await; @@ -1613,6 +1507,7 @@ fn subsystem_rejects_assignment_with_unknown_candidate() { block_hash, candidate_index, validator, + 0, ) .await; @@ -1663,6 +1558,7 @@ fn subsystem_rejects_oversized_bitfields() { block_hash, candidate_index, validator, + 0, ) .await; @@ -1736,6 +1632,7 @@ fn subsystem_accepts_and_imports_approval_after_assignment() { block_hash, candidate_index, validator, + 0, ) .await; @@ -1828,6 +1725,7 @@ fn subsystem_second_approval_import_only_schedules_wakeups() { block_hash, candidate_index, validator, + 0, ) .await; @@ -1916,6 +1814,7 @@ fn subsystem_assignment_import_updates_candidate_entry_and_schedules_wakeup() { block_hash, candidate_index, validator, + 0, ) .await; @@ -2038,6 +1937,7 @@ fn test_approvals_on_fork_are_always_considered_after_no_show( block_hash, candidate_index, validator, + 0, ) .await; assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted)); @@ -2047,6 +1947,7 @@ fn test_approvals_on_fork_are_always_considered_after_no_show( block_hash_fork, candidate_index, validator, + 0, ) .await; @@ -2151,6 +2052,7 @@ fn subsystem_process_wakeup_schedules_wakeup() { block_hash, candidate_index, validator, + 0, ) .await; @@ -2215,7 +2117,8 @@ fn linear_import_act_on_leaf() { .into(), }, 0u32.into(), - tx, + 0, + Some(tx), ), }, ) @@ -2286,7 +2189,8 @@ fn forkful_import_at_same_height_act_on_leaf() { .into(), }, 0u32.into(), - tx, + 0, + Some(tx), ), }, ) @@ -2448,6 +2352,7 @@ fn import_checked_approval_updates_entries_and_schedules() { block_hash, candidate_index, validator_index_a, + 0, ) .await; @@ -2458,6 +2363,7 @@ fn import_checked_approval_updates_entries_and_schedules() { block_hash, candidate_index, validator_index_b, + 0, ) .await; @@ -2611,6 +2517,7 @@ fn subsystem_import_checked_approval_sets_one_block_bit_at_a_time() { block_hash, candidate_index, validator, + 0, ) .await; assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted)); @@ -2896,6 +2803,7 @@ fn approved_ancestor_test( *block_hash, candidate_index, validator, + 0, ) .await; assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted)); @@ -3359,7 +3267,8 @@ where F1: 'static + Fn(ValidatorIndex) -> Result + Send - + Sync, + + Sync + + Clone, F2: Fn(Tick) -> bool, { let TriggersAssignmentConfig { @@ -3388,7 +3297,7 @@ where ); assignments }, - assign_validator_tranche, + assign_validator_tranche.clone(), )); let config = HarnessConfigBuilder::default().assignment_criteria(assignment_criteria).build(); let store = config.backend(); @@ -3454,6 +3363,7 @@ where block_hash, candidate_index, ValidatorIndex(validator), + assign_validator_tranche(ValidatorIndex(validator)).unwrap(), ) .await; assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted)); @@ -3772,6 +3682,7 @@ fn pre_covers_dont_stall_approval() { block_hash, candidate_index, validator_index_a, + 0, ) .await; @@ -3782,6 +3693,7 @@ fn pre_covers_dont_stall_approval() { block_hash, candidate_index, validator_index_b, + 0, ) .await; @@ -3792,6 +3704,7 @@ fn pre_covers_dont_stall_approval() { block_hash, candidate_index, validator_index_c, + 1, ) .await; @@ -3950,6 +3863,7 @@ fn waits_until_approving_assignments_are_old_enough() { block_hash, candidate_index, validator_index_a, + 0, ) .await; @@ -3960,6 +3874,7 @@ fn waits_until_approving_assignments_are_old_enough() { block_hash, candidate_index, validator_index_b, + 0, ) .await; @@ -4996,7 +4911,6 @@ fn subsystem_sends_pending_approvals_on_approval_restart() { })); } ); - assert_matches!( overseer_recv(&mut virtual_overseer).await, AllMessages::RuntimeApi( diff --git a/polkadot/node/network/approval-distribution/Cargo.toml b/polkadot/node/network/approval-distribution/Cargo.toml index a85cde303b61..d19ded99c7e6 100644 --- a/polkadot/node/network/approval-distribution/Cargo.toml +++ b/polkadot/node/network/approval-distribution/Cargo.toml @@ -10,6 +10,7 @@ license.workspace = true workspace = true [dependencies] +polkadot-node-core-approval-voting = { workspace = true, default-features = true } polkadot-node-metrics = { workspace = true, default-features = true } polkadot-node-network-protocol = { workspace = true, default-features = true } polkadot-node-primitives = { workspace = true, default-features = true } @@ -26,6 +27,8 @@ gum = { workspace = true, default-features = true } bitvec = { features = ["alloc"], workspace = true } [dev-dependencies] +sc-keystore = { workspace = true } +sp-application-crypto = { workspace = true, default-features = true } sp-authority-discovery = { workspace = true, default-features = true } sp-core = { features = ["std"], workspace = true, default-features = true } diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 26cedf1a2a13..33136c1b864d 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -24,9 +24,14 @@ #![warn(missing_docs)] use self::metrics::Metrics; -use futures::{channel::oneshot, select, FutureExt as _}; +use futures::{select, FutureExt as _}; use itertools::Itertools; use net_protocol::peer_set::{ProtocolVersion, ValidationVersion}; +use polkadot_node_core_approval_voting::{ + criteria::InvalidAssignment, + time::{Clock, ClockExt, SystemClock}, + AssignmentCriteria, RealAssignmentCriteria, TICK_TOO_FAR_IN_FUTURE, +}; use polkadot_node_jaeger as jaeger; use polkadot_node_network_protocol::{ self as net_protocol, filter_by_peer_version, @@ -35,25 +40,33 @@ use polkadot_node_network_protocol::{ v1 as protocol_v1, v2 as protocol_v2, v3 as protocol_v3, PeerId, UnifiedReputationChange as Rep, Versioned, View, }; -use polkadot_node_primitives::approval::{ - v1::{ - AssignmentCertKind, BlockApprovalMeta, IndirectAssignmentCert, IndirectSignedApprovalVote, - }, - v2::{ - AsBitIndex, AssignmentCertKindV2, CandidateBitfield, IndirectAssignmentCertV2, - IndirectSignedApprovalVoteV2, +use polkadot_node_primitives::{ + approval::{ + v1::{ + AssignmentCertKind, BlockApprovalMeta, DelayTranche, IndirectAssignmentCert, + IndirectSignedApprovalVote, RelayVRFStory, + }, + v2::{ + AsBitIndex, AssignmentCertKindV2, CandidateBitfield, IndirectAssignmentCertV2, + IndirectSignedApprovalVoteV2, + }, }, + DISPUTE_WINDOW, }; use polkadot_node_subsystem::{ messages::{ - ApprovalCheckResult, ApprovalDistributionMessage, ApprovalVotingMessage, - AssignmentCheckResult, NetworkBridgeEvent, NetworkBridgeTxMessage, + ApprovalDistributionMessage, ApprovalVotingMessage, NetworkBridgeEvent, + NetworkBridgeTxMessage, RuntimeApiMessage, }, overseer, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, }; -use polkadot_node_subsystem_util::reputation::{ReputationAggregator, REPUTATION_CHANGE_INTERVAL}; +use polkadot_node_subsystem_util::{ + reputation::{ReputationAggregator, REPUTATION_CHANGE_INTERVAL}, + runtime::{Config as RuntimeInfoConfig, ExtendedSessionInfo, RuntimeInfo}, +}; use polkadot_primitives::{ - BlockNumber, CandidateIndex, Hash, SessionIndex, ValidatorIndex, ValidatorSignature, + BlockNumber, CandidateHash, CandidateIndex, CoreIndex, DisputeStatement, GroupIndex, Hash, + SessionIndex, Slot, ValidDisputeStatementKind, ValidatorIndex, ValidatorSignature, }; use rand::{CryptoRng, Rng, SeedableRng}; use std::{ @@ -86,6 +99,8 @@ const MAX_BITFIELD_SIZE: usize = 500; /// The Approval Distribution subsystem. pub struct ApprovalDistribution { metrics: Metrics, + slot_duration_millis: u64, + clock: Box, } /// Contains recently finalized @@ -354,6 +369,8 @@ pub struct State { /// Aggregated reputation change reputation: ReputationAggregator, + /// Slot duration in millis + slot_duration_millis: u64, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -488,11 +505,17 @@ struct BlockEntry { knowledge: Knowledge, /// A votes entry for each candidate indexed by [`CandidateIndex`]. candidates: Vec, + /// Information about candidate metadata. + candidates_metadata: Vec<(CandidateHash, CoreIndex, GroupIndex)>, /// The session index of this block. session: SessionIndex, /// Approval entries for whole block. These also contain all approvals in the case of multiple /// candidates being claimed by assignments. approval_entries: HashMap<(ValidatorIndex, CandidateBitfield), ApprovalEntry>, + /// The block vrf story. + pub vrf_story: RelayVRFStory, + /// The block slot. + slot: Slot, } impl BlockEntry { @@ -646,6 +669,33 @@ enum MessageSource { Local, } +// Encountered error while validating an assignment. +#[derive(Debug)] +enum InvalidAssignmentError { + // The the vrf check for the assignment failed. + #[allow(dead_code)] + CryptoCheckFailed(InvalidAssignment), + // The assignment did not claim any valid candidate. + NoClaimedCandidates, + // Session Info was not found for the block hash in the assignment. + #[allow(dead_code)] + SessionInfoNotFound(polkadot_node_subsystem_util::runtime::Error), +} + +// Encountered error while validating an approval. +#[derive(Debug)] +enum InvalidVoteError { + // The candidate index was out of bounds. + CandidateIndexOutOfBounds, + // The validator index was out of bounds. + ValidatorIndexOutOfBounds, + // The signature of the vote was invalid. + InvalidSignature, + // Session Info was not found for the block hash in the approval. + #[allow(dead_code)] + SessionInfoNotFound(polkadot_node_subsystem_util::runtime::Error), +} + impl MessageSource { fn peer_id(&self) -> Option { match self { @@ -662,16 +712,26 @@ enum PendingMessage { #[overseer::contextbounds(ApprovalDistribution, prefix = self::overseer)] impl State { + /// Build State with specified slot duration. + pub fn with_config(slot_duration_millis: u64) -> Self { + Self { slot_duration_millis, ..Default::default() } + } + async fn handle_network_msg< N: overseer::SubsystemSender, A: overseer::SubsystemSender, + RA: overseer::SubsystemSender, >( &mut self, approval_voting_sender: &mut A, network_sender: &mut N, + runtime_api_sender: &mut RA, metrics: &Metrics, event: NetworkBridgeEvent, rng: &mut (impl CryptoRng + Rng), + assignment_criteria: &impl AssignmentCriteria, + clock: &(impl Clock + ?Sized), + session_info_provider: &mut RuntimeInfo, ) { match event { NetworkBridgeEvent::PeerConnected(peer_id, role, version, authority_ids) => { @@ -727,10 +787,14 @@ impl State { self.process_incoming_peer_message( approval_voting_sender, network_sender, + runtime_api_sender, metrics, peer_id, message, rng, + assignment_criteria, + clock, + session_info_provider, ) .await; }, @@ -776,16 +840,28 @@ impl State { async fn handle_new_blocks< N: overseer::SubsystemSender, A: overseer::SubsystemSender, + RA: overseer::SubsystemSender, >( &mut self, approval_voting_sender: &mut A, network_sender: &mut N, + runtime_api_sender: &mut RA, metrics: &Metrics, metas: Vec, rng: &mut (impl CryptoRng + Rng), + assignment_criteria: &impl AssignmentCriteria, + clock: &(impl Clock + ?Sized), + session_info_provider: &mut RuntimeInfo, ) { let mut new_hashes = HashSet::new(); - for meta in &metas { + + gum::debug!( + target: LOG_TARGET, + "Got new blocks {:?}", + metas.iter().map(|m| (m.hash, m.number)).collect::>(), + ); + + for meta in metas { let mut span = self .spans .get(&meta.hash) @@ -809,6 +885,9 @@ impl State { candidates, session: meta.session, approval_entries: HashMap::new(), + candidates_metadata: meta.candidates, + vrf_story: meta.vrf_story, + slot: meta.slot, }); self.topologies.inc_session_refs(meta.session); @@ -823,12 +902,6 @@ impl State { } } - gum::debug!( - target: LOG_TARGET, - "Got new blocks {:?}", - metas.iter().map(|m| (m.hash, m.number)).collect::>(), - ); - { for (peer_id, PeerEntry { view, version }) in self.peer_views.iter() { let intersection = view.iter().filter(|h| new_hashes.contains(h)); @@ -883,11 +956,15 @@ impl State { self.import_and_circulate_assignment( approval_voting_sender, network_sender, + runtime_api_sender, metrics, MessageSource::Peer(peer_id), assignment, claimed_indices, rng, + assignment_criteria, + clock, + session_info_provider, ) .await; }, @@ -895,9 +972,11 @@ impl State { self.import_and_circulate_approval( approval_voting_sender, network_sender, + runtime_api_sender, metrics, MessageSource::Peer(peer_id), approval_vote, + session_info_provider, ) .await; }, @@ -946,15 +1025,20 @@ impl State { async fn process_incoming_assignments< N: overseer::SubsystemSender, A: overseer::SubsystemSender, + RA: overseer::SubsystemSender, R, >( &mut self, approval_voting_sender: &mut A, network_sender: &mut N, + runtime_api_sender: &mut RA, metrics: &Metrics, peer_id: PeerId, assignments: Vec<(IndirectAssignmentCertV2, CandidateBitfield)>, rng: &mut R, + assignment_criteria: &impl AssignmentCriteria, + clock: &(impl Clock + ?Sized), + session_info_provider: &mut RuntimeInfo, ) where R: CryptoRng + Rng, { @@ -980,11 +1064,15 @@ impl State { self.import_and_circulate_assignment( approval_voting_sender, network_sender, + runtime_api_sender, metrics, MessageSource::Peer(peer_id), assignment, claimed_indices, rng, + assignment_criteria, + clock, + session_info_provider, ) .await; } @@ -994,13 +1082,16 @@ impl State { async fn process_incoming_approvals< N: overseer::SubsystemSender, A: overseer::SubsystemSender, + RA: overseer::SubsystemSender, >( &mut self, approval_voting_sender: &mut A, network_sender: &mut N, + runtime_api_sender: &mut RA, metrics: &Metrics, peer_id: PeerId, approvals: Vec, + session_info_provider: &mut RuntimeInfo, ) { gum::trace!( target: LOG_TARGET, @@ -1030,9 +1121,11 @@ impl State { self.import_and_circulate_approval( approval_voting_sender, network_sender, + runtime_api_sender, metrics, MessageSource::Peer(peer_id), approval_vote, + session_info_provider, ) .await; } @@ -1041,11 +1134,13 @@ impl State { async fn process_incoming_peer_message< N: overseer::SubsystemSender, A: overseer::SubsystemSender, + RA: overseer::SubsystemSender, R, >( &mut self, approval_voting_sender: &mut A, network_sender: &mut N, + runtime_api_sender: &mut RA, metrics: &Metrics, peer_id: PeerId, msg: Versioned< @@ -1054,6 +1149,9 @@ impl State { protocol_v3::ApprovalDistributionMessage, >, rng: &mut R, + assignment_criteria: &impl AssignmentCriteria, + clock: &(impl Clock + ?Sized), + session_info_provider: &mut RuntimeInfo, ) where R: CryptoRng + Rng, { @@ -1071,10 +1169,14 @@ impl State { self.process_incoming_assignments( approval_voting_sender, network_sender, + runtime_api_sender, metrics, peer_id, sanitized_assignments, rng, + assignment_criteria, + clock, + session_info_provider, ) .await; }, @@ -1093,10 +1195,14 @@ impl State { self.process_incoming_assignments( approval_voting_sender, network_sender, + runtime_api_sender, metrics, peer_id, sanitized_assignments, rng, + assignment_criteria, + clock, + session_info_provider, ) .await; }, @@ -1106,9 +1212,11 @@ impl State { self.process_incoming_approvals( approval_voting_sender, network_sender, + runtime_api_sender, metrics, peer_id, sanitized_approvals, + session_info_provider, ) .await; }, @@ -1119,9 +1227,11 @@ impl State { self.process_incoming_approvals( approval_voting_sender, network_sender, + runtime_api_sender, metrics, peer_id, sanitized_approvals, + session_info_provider, ) .await; }, @@ -1225,16 +1335,21 @@ impl State { async fn import_and_circulate_assignment< N: overseer::SubsystemSender, A: overseer::SubsystemSender, + RA: overseer::SubsystemSender, R, >( &mut self, approval_voting_sender: &mut A, network_sender: &mut N, + runtime_api_sender: &mut RA, metrics: &Metrics, source: MessageSource, assignment: IndirectAssignmentCertV2, claimed_candidate_indices: CandidateBitfield, rng: &mut R, + assignment_criteria: &impl AssignmentCriteria, + clock: &(impl Clock + ?Sized), + session_info_provider: &mut RuntimeInfo, ) where R: CryptoRng + Rng, { @@ -1361,35 +1476,49 @@ impl State { return } - let (tx, rx) = oneshot::channel(); + let result = Self::check_assignment_valid( + assignment_criteria, + &entry, + &assignment, + &claimed_candidate_indices, + session_info_provider, + runtime_api_sender, + ) + .await; - approval_voting_sender - .send_message(ApprovalVotingMessage::CheckAndImportAssignment( - assignment.clone(), - claimed_candidate_indices.clone(), - tx, - )) - .await; + match result { + Ok(tranche) => { + let current_tranche = clock.tranche_now(self.slot_duration_millis, entry.slot); + let too_far_in_future = + current_tranche + TICK_TOO_FAR_IN_FUTURE as DelayTranche; - let timer = metrics.time_awaiting_approval_voting(); - let result = match rx.await { - Ok(result) => result, - Err(_) => { - gum::debug!(target: LOG_TARGET, "The approval voting subsystem is down"); - return - }, - }; - drop(timer); + if tranche >= too_far_in_future { + gum::debug!( + target: LOG_TARGET, + hash = ?block_hash, + ?peer_id, + "Got an assignment too far in the future", + ); + modify_reputation( + &mut self.reputation, + network_sender, + peer_id, + COST_ASSIGNMENT_TOO_FAR_IN_THE_FUTURE, + ) + .await; + metrics.on_assignment_far(); - gum::trace!( - target: LOG_TARGET, - ?source, - ?message_subject, - ?result, - "Checked assignment", - ); - match result { - AssignmentCheckResult::Accepted => { + return + } + + approval_voting_sender + .send_message(ApprovalVotingMessage::CheckAndImportAssignment( + assignment.clone(), + claimed_candidate_indices.clone(), + tranche, + None, + )) + .await; modify_reputation( &mut self.reputation, network_sender, @@ -1402,47 +1531,12 @@ impl State { peer_knowledge.received.insert(message_subject.clone(), message_kind); } }, - AssignmentCheckResult::AcceptedDuplicate => { - // "duplicate" assignments aren't necessarily equal. - // There is more than one way each validator can be assigned to each core. - // cf. https://github.com/paritytech/polkadot/pull/2160#discussion_r557628699 - if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { - peer_knowledge.received.insert(message_subject.clone(), message_kind); - } - gum::debug!( - target: LOG_TARGET, - hash = ?block_hash, - ?peer_id, - "Got an `AcceptedDuplicate` assignment", - ); - metrics.on_assignment_duplicatevoting(); - - return - }, - AssignmentCheckResult::TooFarInFuture => { - gum::debug!( - target: LOG_TARGET, - hash = ?block_hash, - ?peer_id, - "Got an assignment too far in the future", - ); - modify_reputation( - &mut self.reputation, - network_sender, - peer_id, - COST_ASSIGNMENT_TOO_FAR_IN_THE_FUTURE, - ) - .await; - metrics.on_assignment_far(); - - return - }, - AssignmentCheckResult::Bad(error) => { + Err(error) => { gum::info!( target: LOG_TARGET, hash = ?block_hash, ?peer_id, - %error, + ?error, "Got a bad assignment from peer", ); modify_reputation( @@ -1583,6 +1677,46 @@ impl State { } } + async fn check_assignment_valid>( + assignment_criteria: &impl AssignmentCriteria, + entry: &BlockEntry, + assignment: &IndirectAssignmentCertV2, + claimed_candidate_indices: &CandidateBitfield, + runtime_info: &mut RuntimeInfo, + runtime_api_sender: &mut RA, + ) -> Result { + let claimed_cores: Vec = claimed_candidate_indices + .iter_ones() + .flat_map(|candidate_index| { + entry.candidates_metadata.get(candidate_index).map(|(_, core, _)| *core) + }) + .collect::>(); + let backing_groups = claimed_candidate_indices + .iter_ones() + .flat_map(|candidate_index| { + entry.candidates_metadata.get(candidate_index).map(|(_, _, group)| *group) + }) + .collect::>(); + if claimed_cores.is_empty() { + return Err(InvalidAssignmentError::NoClaimedCandidates) + } + + let ExtendedSessionInfo { ref session_info, .. } = runtime_info + .get_session_info_by_index(runtime_api_sender, assignment.block_hash, entry.session) + .await + .map_err(|err| InvalidAssignmentError::SessionInfoNotFound(err))?; + + assignment_criteria + .check_assignment_cert( + claimed_cores.try_into().expect("Non-empty vec; qed"), + assignment.validator, + &polkadot_node_core_approval_voting::AssignmentConfig::from(session_info), + entry.vrf_story.clone(), + &assignment.cert, + backing_groups, + ) + .map_err(|err| InvalidAssignmentError::CryptoCheckFailed(err)) + } // Checks if an approval can be processed. // Returns true if we can continue with processing the approval and false otherwise. async fn check_approval_can_be_processed< @@ -1672,13 +1806,16 @@ impl State { async fn import_and_circulate_approval< N: overseer::SubsystemSender, A: overseer::SubsystemSender, + RA: overseer::SubsystemSender, >( &mut self, approval_voting_sender: &mut A, network_sender: &mut N, + runtime_api_sender: &mut RA, metrics: &Metrics, source: MessageSource, vote: IndirectSignedApprovalVoteV2, + session_info_provider: &mut RuntimeInfo, ) { let _span = self .spans @@ -1746,31 +1883,19 @@ impl State { return } - let (tx, rx) = oneshot::channel(); - - approval_voting_sender - .send_message(ApprovalVotingMessage::CheckAndImportApproval(vote.clone(), tx)) - .await; - - let timer = metrics.time_awaiting_approval_voting(); - let result = match rx.await { - Ok(result) => result, - Err(_) => { - gum::debug!(target: LOG_TARGET, "The approval voting subsystem is down"); - return - }, - }; - drop(timer); + let result = + Self::check_vote_valid(&vote, &entry, session_info_provider, runtime_api_sender) + .await; - gum::trace!( - target: LOG_TARGET, - ?peer_id, - ?result, - ?vote, - "Checked approval", - ); match result { - ApprovalCheckResult::Accepted => { + Ok(_) => { + approval_voting_sender + .send_message(ApprovalVotingMessage::CheckAndImportApproval( + vote.clone(), + None, + )) + .await; + modify_reputation( &mut self.reputation, network_sender, @@ -1788,7 +1913,7 @@ impl State { .insert(approval_knwowledge_key.0.clone(), approval_knwowledge_key.1); } }, - ApprovalCheckResult::Bad(error) => { + Err(err) => { modify_reputation( &mut self.reputation, network_sender, @@ -1796,10 +1921,11 @@ impl State { COST_INVALID_MESSAGE, ) .await; + gum::info!( target: LOG_TARGET, ?peer_id, - %error, + ?err, "Got a bad approval from peer", ); metrics.on_approval_bad(); @@ -1897,6 +2023,49 @@ impl State { } } + // Checks if the approval vote is valid. + async fn check_vote_valid>( + vote: &IndirectSignedApprovalVoteV2, + entry: &BlockEntry, + runtime_info: &mut RuntimeInfo, + runtime_api_sender: &mut RA, + ) -> Result<(), InvalidVoteError> { + if vote.candidate_indices.len() > entry.candidates_metadata.len() { + return Err(InvalidVoteError::CandidateIndexOutOfBounds) + } + + let candidate_hashes = vote + .candidate_indices + .iter_ones() + .flat_map(|candidate_index| { + entry + .candidates_metadata + .get(candidate_index) + .map(|(candidate_hash, _, _)| *candidate_hash) + }) + .collect::>(); + + let ExtendedSessionInfo { ref session_info, .. } = runtime_info + .get_session_info_by_index(runtime_api_sender, vote.block_hash, entry.session) + .await + .map_err(|err| InvalidVoteError::SessionInfoNotFound(err))?; + + let pubkey = session_info + .validators + .get(vote.validator) + .ok_or(InvalidVoteError::ValidatorIndexOutOfBounds)?; + DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates( + candidate_hashes.clone(), + )) + .check_signature( + &pubkey, + *candidate_hashes.first().unwrap(), + entry.session, + &vote.signature, + ) + .map_err(|_| InvalidVoteError::InvalidSignature) + } + /// Retrieve approval signatures from state for the given relay block/indices: fn get_approval_signatures( &mut self, @@ -2474,16 +2643,40 @@ async fn modify_reputation( #[overseer::contextbounds(ApprovalDistribution, prefix = self::overseer)] impl ApprovalDistribution { /// Create a new instance of the [`ApprovalDistribution`] subsystem. - pub fn new(metrics: Metrics) -> Self { - Self { metrics } + pub fn new(metrics: Metrics, slot_duration_millis: u64) -> Self { + Self::new_with_clock(metrics, slot_duration_millis, Box::new(SystemClock)) + } + + /// Create a new instance of the [`ApprovalDistribution`] subsystem, with a custom clock. + pub fn new_with_clock( + metrics: Metrics, + slot_duration_millis: u64, + clock: Box, + ) -> Self { + Self { metrics, slot_duration_millis, clock } } async fn run(self, ctx: Context) { - let mut state = State::default(); + let mut state = + State { slot_duration_millis: self.slot_duration_millis, ..Default::default() }; // According to the docs of `rand`, this is a ChaCha12 RNG in practice // and will always be chosen for strong performance and security properties. let mut rng = rand::rngs::StdRng::from_entropy(); - self.run_inner(ctx, &mut state, REPUTATION_CHANGE_INTERVAL, &mut rng).await + let assignment_criteria = RealAssignmentCriteria {}; + let mut session_info_provider = RuntimeInfo::new_with_config(RuntimeInfoConfig { + keystore: None, + session_cache_lru_size: DISPUTE_WINDOW.get(), + }); + + self.run_inner( + ctx, + &mut state, + REPUTATION_CHANGE_INTERVAL, + &mut rng, + &assignment_criteria, + &mut session_info_provider, + ) + .await } /// Used for testing. @@ -2493,11 +2686,15 @@ impl ApprovalDistribution { state: &mut State, reputation_interval: Duration, rng: &mut (impl CryptoRng + Rng), + assignment_criteria: &impl AssignmentCriteria, + session_info_provider: &mut RuntimeInfo, ) { let new_reputation_delay = || futures_timer::Delay::new(reputation_interval).fuse(); let mut reputation_delay = new_reputation_delay(); let mut approval_voting_sender = ctx.sender().clone(); let mut network_sender = ctx.sender().clone(); + let mut runtime_api_sender = ctx.sender().clone(); + loop { select! { _ = reputation_delay => { @@ -2514,7 +2711,7 @@ impl ApprovalDistribution { }; - self.handle_from_orchestra(message, &mut approval_voting_sender, &mut network_sender, state, rng).await; + self.handle_from_orchestra(message, &mut approval_voting_sender, &mut network_sender, &mut runtime_api_sender, state, rng, assignment_criteria, session_info_provider).await; }, } @@ -2525,23 +2722,31 @@ impl ApprovalDistribution { pub async fn handle_from_orchestra< N: overseer::SubsystemSender, A: overseer::SubsystemSender, + RA: overseer::SubsystemSender, >( &self, message: FromOrchestra, approval_voting_sender: &mut A, network_sender: &mut N, + runtime_api_sender: &mut RA, state: &mut State, rng: &mut (impl CryptoRng + Rng), + assignment_criteria: &impl AssignmentCriteria, + session_info_provider: &mut RuntimeInfo, ) { match message { FromOrchestra::Communication { msg } => Self::handle_incoming( approval_voting_sender, network_sender, + runtime_api_sender, state, msg, &self.metrics, rng, + assignment_criteria, + self.clock.as_ref(), + session_info_provider, ) .await, FromOrchestra::Signal(OverseerSignal::ActiveLeaves(update)) => { @@ -2568,23 +2773,48 @@ impl ApprovalDistribution { async fn handle_incoming< N: overseer::SubsystemSender, A: overseer::SubsystemSender, + RA: overseer::SubsystemSender, >( approval_voting_sender: &mut A, network_sender: &mut N, + runtime_api_sender: &mut RA, state: &mut State, msg: ApprovalDistributionMessage, metrics: &Metrics, rng: &mut (impl CryptoRng + Rng), + assignment_criteria: &impl AssignmentCriteria, + clock: &(impl Clock + ?Sized), + session_info_provider: &mut RuntimeInfo, ) { match msg { ApprovalDistributionMessage::NetworkBridgeUpdate(event) => { state - .handle_network_msg(approval_voting_sender, network_sender, metrics, event, rng) + .handle_network_msg( + approval_voting_sender, + network_sender, + runtime_api_sender, + metrics, + event, + rng, + assignment_criteria, + clock, + session_info_provider, + ) .await; }, ApprovalDistributionMessage::NewBlocks(metas) => { state - .handle_new_blocks(approval_voting_sender, network_sender, metrics, metas, rng) + .handle_new_blocks( + approval_voting_sender, + network_sender, + runtime_api_sender, + metrics, + metas, + rng, + assignment_criteria, + clock, + session_info_provider, + ) .await; }, ApprovalDistributionMessage::DistributeAssignment(cert, candidate_indices) => { @@ -2608,11 +2838,15 @@ impl ApprovalDistribution { .import_and_circulate_assignment( approval_voting_sender, network_sender, + runtime_api_sender, &metrics, MessageSource::Local, cert, candidate_indices, rng, + assignment_criteria, + clock, + session_info_provider, ) .await; }, @@ -2628,9 +2862,11 @@ impl ApprovalDistribution { .import_and_circulate_approval( approval_voting_sender, network_sender, + runtime_api_sender, metrics, MessageSource::Local, vote, + session_info_provider, ) .await; }, diff --git a/polkadot/node/network/approval-distribution/src/metrics.rs b/polkadot/node/network/approval-distribution/src/metrics.rs index 60c7f2f6d3b8..10553c352966 100644 --- a/polkadot/node/network/approval-distribution/src/metrics.rs +++ b/polkadot/node/network/approval-distribution/src/metrics.rs @@ -30,7 +30,6 @@ struct MetricsInner { aggression_l2_messages_total: prometheus::Counter, time_unify_with_peer: prometheus::Histogram, time_import_pending_now_known: prometheus::Histogram, - time_awaiting_approval_voting: prometheus::Histogram, assignments_received_result: prometheus::CounterVec, approvals_received_result: prometheus::CounterVec, } @@ -206,14 +205,6 @@ impl Metrics { } } - pub(crate) fn time_awaiting_approval_voting( - &self, - ) -> Option { - self.0 - .as_ref() - .map(|metrics| metrics.time_awaiting_approval_voting.start_timer()) - } - pub(crate) fn on_aggression_l1(&self) { if let Some(metrics) = &self.0 { metrics.aggression_l1_messages_total.inc(); @@ -288,13 +279,6 @@ impl MetricsTrait for Metrics { ).buckets(vec![0.0001, 0.0004, 0.0016, 0.0064, 0.0256, 0.1024, 0.4096, 1.6384, 3.2768, 4.9152, 6.5536,]))?, registry, )?, - time_awaiting_approval_voting: prometheus::register( - prometheus::Histogram::with_opts(prometheus::HistogramOpts::new( - "polkadot_parachain_time_awaiting_approval_voting", - "Time spent awaiting a reply from the Approval Voting Subsystem.", - ).buckets(vec![0.0001, 0.0004, 0.0016, 0.0064, 0.0256, 0.1024, 0.4096, 1.6384, 3.2768, 4.9152, 6.5536,]))?, - registry, - )?, assignments_received_result: prometheus::register( prometheus::CounterVec::new( prometheus::Opts::new( diff --git a/polkadot/node/network/approval-distribution/src/tests.rs b/polkadot/node/network/approval-distribution/src/tests.rs index 2d08807f97b6..3f926746449d 100644 --- a/polkadot/node/network/approval-distribution/src/tests.rs +++ b/polkadot/node/network/approval-distribution/src/tests.rs @@ -16,7 +16,8 @@ use super::*; use assert_matches::assert_matches; -use futures::{executor, future, Future}; +use futures::{channel::oneshot, executor, future, Future}; +use polkadot_node_core_approval_voting::criteria; use polkadot_node_network_protocol::{ grid_topology::{SessionGridTopology, TopologyPeerInfo}, our_view, @@ -34,12 +35,17 @@ use polkadot_node_primitives::approval::{ }, }; use polkadot_node_subsystem::messages::{ - network_bridge_event, AllMessages, ApprovalCheckError, ReportPeerMessage, + network_bridge_event, AllMessages, ReportPeerMessage, RuntimeApiRequest, }; use polkadot_node_subsystem_util::{reputation::add_reputation, TimeoutExt as _}; -use polkadot_primitives::{AuthorityDiscoveryId, BlakeTwo256, CoreIndex, HashT}; +use polkadot_primitives::{ + ApprovalVoteMultipleCandidates, AuthorityDiscoveryId, BlakeTwo256, CoreIndex, ExecutorParams, + HashT, NodeFeatures, SessionInfo, ValidatorId, +}; use polkadot_primitives_test_helpers::dummy_signature; use rand::SeedableRng; +use sc_keystore::{Keystore, LocalKeystore}; +use sp_application_crypto::AppCrypto; use sp_authority_discovery::AuthorityPair as AuthorityDiscoveryPair; use sp_core::crypto::Pair as PairT; use std::time::Duration; @@ -47,6 +53,8 @@ type VirtualOverseer = polkadot_node_subsystem_test_helpers::TestSubsystemContextHandle; fn test_harness>( + assignment_criteria: &impl AssignmentCriteria, + clock: Box, mut state: State, test_fn: impl FnOnce(VirtualOverseer) -> T, ) -> State { @@ -59,12 +67,22 @@ fn test_harness>( let (context, virtual_overseer) = polkadot_node_subsystem_test_helpers::make_subsystem_context(pool.clone()); - let subsystem = ApprovalDistribution::new(Default::default()); + let subsystem = + ApprovalDistribution::new_with_clock(Metrics::default(), Default::default(), clock); { let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(12345); - - let subsystem = - subsystem.run_inner(context, &mut state, REPUTATION_CHANGE_TEST_INTERVAL, &mut rng); + let mut session_info_provider = RuntimeInfo::new_with_config(RuntimeInfoConfig { + keystore: None, + session_cache_lru_size: DISPUTE_WINDOW.get(), + }); + let subsystem = subsystem.run_inner( + context, + &mut state, + REPUTATION_CHANGE_TEST_INTERVAL, + &mut rng, + assignment_criteria, + &mut session_info_provider, + ); let test_fut = test_fn(virtual_overseer); @@ -118,6 +136,41 @@ async fn overseer_recv(overseer: &mut VirtualOverseer) -> AllMessages { msg } +async fn provide_session(virtual_overseer: &mut VirtualOverseer, session_info: SessionInfo) { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request( + _, + RuntimeApiRequest::SessionInfo(_, si_tx), + ) + ) => { + si_tx.send(Ok(Some(session_info.clone()))).unwrap(); + } + ); + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request( + _, + RuntimeApiRequest::SessionExecutorParams(_, si_tx), + ) + ) => { + // Make sure all SessionExecutorParams calls are not made for the leaf (but for its relay parent) + si_tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); + } + ); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(_, RuntimeApiRequest::NodeFeatures(_, si_tx), ) + ) => { + si_tx.send(Ok(NodeFeatures::EMPTY)).unwrap(); + } + ); +} + fn make_peers_and_authority_ids(n: usize) -> Vec<(PeerId, AuthorityDiscoveryId)> { (0..n) .map(|_| { @@ -375,6 +428,86 @@ fn state_with_reputation_delay() -> State { State { reputation: ReputationAggregator::new(|_| false), ..Default::default() } } +fn dummy_session_info_valid( + index: SessionIndex, + keystore: &mut LocalKeystore, + num_validators: usize, +) -> SessionInfo { + let keys = (0..num_validators) + .map(|_| { + keystore + .sr25519_generate_new(ValidatorId::ID, Some("//Node")) + .expect("Insert key into keystore") + }) + .collect_vec(); + + SessionInfo { + validators: keys.clone().into_iter().map(|key| key.into()).collect(), + discovery_keys: keys.clone().into_iter().map(|key| key.into()).collect(), + assignment_keys: keys.clone().into_iter().map(|key| key.into()).collect(), + validator_groups: Default::default(), + n_cores: index as _, + zeroth_delay_tranche_width: index as _, + relay_vrf_modulo_samples: index as _, + n_delay_tranches: index as _, + no_show_slots: index as _, + needed_approvals: index as _, + active_validator_indices: Vec::new(), + dispute_period: 6, + random_seed: [0u8; 32], + } +} + +fn signature_for( + keystore: &LocalKeystore, + session: &SessionInfo, + candidate_hashes: Vec, + validator_index: ValidatorIndex, +) -> ValidatorSignature { + let payload = ApprovalVoteMultipleCandidates(&candidate_hashes).signing_payload(1); + let sign_key = session.validators.get(validator_index).unwrap().clone(); + let signature = keystore + .sr25519_sign(ValidatorId::ID, &sign_key.into(), &payload[..]) + .unwrap() + .unwrap(); + signature.into() +} + +struct MockAssignmentCriteria { + tranche: + Result, +} + +impl AssignmentCriteria for MockAssignmentCriteria { + fn compute_assignments( + &self, + _keystore: &LocalKeystore, + _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, + _config: &criteria::Config, + _leaving_cores: Vec<( + CandidateHash, + polkadot_primitives::CoreIndex, + polkadot_primitives::GroupIndex, + )>, + _enable_assignments_v2: bool, + ) -> HashMap { + HashMap::new() + } + + fn check_assignment_cert( + &self, + _claimed_core_bitfield: polkadot_node_primitives::approval::v2::CoreBitfield, + _validator_index: polkadot_primitives::ValidatorIndex, + _config: &criteria::Config, + _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, + _assignment: &polkadot_node_primitives::approval::v2::AssignmentCertV2, + _backing_groups: Vec, + ) -> Result + { + self.tranche + } +} + /// import an assignment /// connect a new peer /// the new peer sends us the same assignment @@ -388,89 +521,100 @@ fn try_import_the_same_assignment() { let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); - let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // setup peers - setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; - setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V1).await; - setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V1).await; - - // Set up a gossip topology, where a, b, c and d are topology neighbors to the node under - // testing. - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - setup_gossip_topology( - overseer, - make_gossip_topology(1, &peers_with_optional_peer_id, &[0, 1], &[2, 4], 3), - ) - .await; + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state_without_reputation_delay(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // setup peers + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V1).await; + + // Set up a gossip topology, where a, b, c and d are topology neighbors to the node + // under testing. + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + setup_gossip_topology( + overseer, + make_gossip_topology(1, &peers_with_optional_peer_id, &[0, 1], &[2, 4], 3), + ) + .await; - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 2, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - // send the assignment related to `hash` - let validator_index = ValidatorIndex(0); - let cert = fake_assignment_cert(hash, validator_index); - let assignments = vec![(cert.clone(), 0u32)]; - - let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, &peer_a, msg).await; - - expect_reputation_change(overseer, &peer_a, COST_UNEXPECTED_MESSAGE).await; - - // send an `Accept` message from the Approval Voting subsystem - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( - assignment, - claimed_indices, - tx, - )) => { - assert_eq!(claimed_indices, 0u32.into()); - assert_eq!(assignment, cert.into()); - tx.send(AssignmentCheckResult::Accepted).unwrap(); - } - ); + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 2, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // send the assignment related to `hash` + let validator_index = ValidatorIndex(0); + let cert = fake_assignment_cert(hash, validator_index); + let assignments = vec![(cert.clone(), 0u32)]; - expect_reputation_change(overseer, &peer_a, BENEFIT_VALID_MESSAGE_FIRST).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert_eq!(peers.len(), 2); - assert_eq!(assignments.len(), 1); - } - ); + let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer(overseer, &peer_a, msg).await; + + expect_reputation_change(overseer, &peer_a, COST_UNEXPECTED_MESSAGE).await; + provide_session( + overseer, + dummy_session_info_valid(1, &mut LocalKeystore::in_memory(), 1), + ) + .await; + // send an `Accept` message from the Approval Voting subsystem + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + assignment, + claimed_indices, + tranche, + _, + )) => { + assert_eq!(claimed_indices, 0u32.into()); + assert_eq!(assignment, cert.into()); + assert_eq!(tranche, 0); + } + ); - // setup new peer with V2 - setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::V3).await; + expect_reputation_change(overseer, &peer_a, BENEFIT_VALID_MESSAGE_FIRST).await; - // send the same assignment from peer_d - let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments); - send_message_from_peer(overseer, &peer_d, msg).await; + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 2); + assert_eq!(assignments.len(), 1); + } + ); - expect_reputation_change(overseer, &peer_d, COST_UNEXPECTED_MESSAGE).await; - expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE).await; + // setup new peer with V2 + setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::V3).await; - assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); - virtual_overseer - }); + // send the same assignment from peer_d + let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments); + send_message_from_peer(overseer, &peer_d, msg).await; + + expect_reputation_change(overseer, &peer_d, COST_UNEXPECTED_MESSAGE).await; + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE).await; + + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }, + ); } /// Just like `try_import_the_same_assignment` but use `VRFModuloCompact` assignments for multiple @@ -485,97 +629,108 @@ fn try_import_the_same_assignment_v2() { let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); - let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // setup peers - setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V3).await; - setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; - setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V3).await; - - // Set up a gossip topology, where a, b, c and d are topology neighbors to the node under - // testing. - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - setup_gossip_topology( - overseer, - make_gossip_topology(1, &peers_with_optional_peer_id, &[0, 1], &[2, 4], 3), - ) - .await; + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state_without_reputation_delay(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // setup peers + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V3).await; + + // Set up a gossip topology, where a, b, c and d are topology neighbors to the node + // under testing. + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + setup_gossip_topology( + overseer, + make_gossip_topology(1, &peers_with_optional_peer_id, &[0, 1], &[2, 4], 3), + ) + .await; - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 2, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - // send the assignment related to `hash` - let validator_index = ValidatorIndex(0); - let cores = vec![1, 2, 3, 4]; - let core_bitfield: CoreBitfield = cores - .iter() - .map(|index| CoreIndex(*index)) - .collect::>() - .try_into() - .unwrap(); - - let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfield.clone()); - let assignments = vec![(cert.clone(), cores.clone().try_into().unwrap())]; - - let msg = protocol_v3::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer_v3(overseer, &peer_a, msg).await; - - expect_reputation_change(overseer, &peer_a, COST_UNEXPECTED_MESSAGE).await; - - // send an `Accept` message from the Approval Voting subsystem - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( - assignment, - claimed_indices, - tx, - )) => { - assert_eq!(claimed_indices, cores.try_into().unwrap()); - assert_eq!(assignment, cert.into()); - tx.send(AssignmentCheckResult::Accepted).unwrap(); - } - ); + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 2, + candidates: vec![Default::default(); 4], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // send the assignment related to `hash` + let validator_index = ValidatorIndex(0); + let cores = vec![1, 2, 3, 4]; + let core_bitfield: CoreBitfield = cores + .iter() + .map(|index| CoreIndex(*index)) + .collect::>() + .try_into() + .unwrap(); + + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfield.clone()); + let assignments = vec![(cert.clone(), cores.clone().try_into().unwrap())]; + + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer_v3(overseer, &peer_a, msg).await; + + expect_reputation_change(overseer, &peer_a, COST_UNEXPECTED_MESSAGE).await; + provide_session( + overseer, + dummy_session_info_valid(1, &mut LocalKeystore::in_memory(), 1), + ) + .await; + // send an `Accept` message from the Approval Voting subsystem + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + assignment, + claimed_indices, + tranche, + _, + )) => { + assert_eq!(claimed_indices, cores.try_into().unwrap()); + assert_eq!(assignment, cert.into()); + assert_eq!(tranche, 0); + } + ); - expect_reputation_change(overseer, &peer_a, BENEFIT_VALID_MESSAGE_FIRST).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( - protocol_v3::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert_eq!(peers.len(), 2); - assert_eq!(assignments.len(), 1); - } - ); + expect_reputation_change(overseer, &peer_a, BENEFIT_VALID_MESSAGE_FIRST).await; - // setup new peer - setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::V3).await; + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 2); + assert_eq!(assignments.len(), 1); + } + ); - // send the same assignment from peer_d - let msg = protocol_v3::ApprovalDistributionMessage::Assignments(assignments); - send_message_from_peer_v3(overseer, &peer_d, msg).await; + // setup new peer + setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::V3).await; - expect_reputation_change(overseer, &peer_d, COST_UNEXPECTED_MESSAGE).await; - expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE).await; + // send the same assignment from peer_d + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(assignments); + send_message_from_peer_v3(overseer, &peer_d, msg).await; - assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); - virtual_overseer - }); + expect_reputation_change(overseer, &peer_d, COST_UNEXPECTED_MESSAGE).await; + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE).await; + + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }, + ); } /// import an assignment @@ -587,55 +742,67 @@ fn delay_reputation_change() { let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); - let _ = test_harness(state_with_reputation_delay(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state_with_reputation_delay(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + + // Setup peers + setup_peer_with_view(overseer, &peer, view![], ValidationVersion::V1).await; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 2, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // send the assignment related to `hash` + let validator_index = ValidatorIndex(0); + let cert = fake_assignment_cert(hash, validator_index); + let assignments = vec![(cert.clone(), 0u32)]; - // Setup peers - setup_peer_with_view(overseer, &peer, view![], ValidationVersion::V1).await; - - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 2, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - // send the assignment related to `hash` - let validator_index = ValidatorIndex(0); - let cert = fake_assignment_cert(hash, validator_index); - let assignments = vec![(cert.clone(), 0u32)]; - - let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, &peer, msg).await; - - // send an `Accept` message from the Approval Voting subsystem - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( - assignment, - claimed_candidates, - tx, - )) => { - assert_eq!(assignment.cert, cert.cert.into()); - assert_eq!(claimed_candidates, vec![0u32].try_into().unwrap()); - tx.send(AssignmentCheckResult::Accepted).unwrap(); - } - ); - expect_reputation_changes( - overseer, - &peer, - vec![COST_UNEXPECTED_MESSAGE, BENEFIT_VALID_MESSAGE_FIRST], - ) - .await; - assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer(overseer, &peer, msg).await; + provide_session( + overseer, + dummy_session_info_valid(1, &mut LocalKeystore::in_memory(), 1), + ) + .await; - virtual_overseer - }); + // send an `Accept` message from the Approval Voting subsystem + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + assignment, + claimed_candidates, + tranche, + _, + )) => { + assert_eq!(assignment.cert, cert.cert.into()); + assert_eq!(claimed_candidates, vec![0u32].try_into().unwrap()); + assert_eq!(tranche, 0); + } + ); + expect_reputation_changes( + overseer, + &peer, + vec![COST_UNEXPECTED_MESSAGE, BENEFIT_VALID_MESSAGE_FIRST], + ) + .await; + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + + virtual_overseer + }, + ); } /// @@ -650,77 +817,90 @@ fn spam_attack_results_in_negative_reputation_change() { let peer_a = PeerId::random(); let hash_b = Hash::repeat_byte(0xBB); - let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - let peer = &peer_a; - setup_peer_with_view(overseer, peer, view![], ValidationVersion::V1).await; - - // new block `hash_b` with 20 candidates - let candidates_count = 20; - let meta = BlockApprovalMeta { - hash: hash_b, - parent_hash, - number: 2, - candidates: vec![Default::default(); candidates_count], - slot: 1.into(), - session: 1, - }; - - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - // send 20 assignments related to `hash_b` - // to populate our knowledge - let assignments: Vec<_> = (0..candidates_count) - .map(|candidate_index| { - let validator_index = ValidatorIndex(candidate_index as u32); - let cert = fake_assignment_cert(hash_b, validator_index); - (cert, candidate_index as u32) - }) - .collect(); - - let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, peer, msg.clone()).await; - - for i in 0..candidates_count { - expect_reputation_change(overseer, peer, COST_UNEXPECTED_MESSAGE).await; + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state_without_reputation_delay(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + let peer = &peer_a; + setup_peer_with_view(overseer, peer, view![], ValidationVersion::V1).await; + + // new block `hash_b` with 20 candidates + let candidates_count = 20; + let meta = BlockApprovalMeta { + hash: hash_b, + parent_hash, + number: 2, + candidates: vec![Default::default(); candidates_count], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // send 20 assignments related to `hash_b` + // to populate our knowledge + let assignments: Vec<_> = (0..candidates_count) + .map(|candidate_index| { + let validator_index = ValidatorIndex(candidate_index as u32); + let cert = fake_assignment_cert(hash_b, validator_index); + (cert, candidate_index as u32) + }) + .collect(); - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( - assignment, - claimed_candidate_index, - tx, - )) => { - assert_eq!(assignment, assignments[i].0.clone().into()); - assert_eq!(claimed_candidate_index, assignments[i].1.into()); - tx.send(AssignmentCheckResult::Accepted).unwrap(); + let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer(overseer, peer, msg.clone()).await; + + for i in 0..candidates_count { + expect_reputation_change(overseer, peer, COST_UNEXPECTED_MESSAGE).await; + if i == 0 { + provide_session( + overseer, + dummy_session_info_valid(1, &mut LocalKeystore::in_memory(), 1), + ) + .await; } - ); + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + assignment, + claimed_candidate_index, + tranche, + _, + )) => { + assert_eq!(assignment, assignments[i].0.clone().into()); + assert_eq!(claimed_candidate_index, assignments[i].1.into()); + assert_eq!(tranche, 0); + } + ); - expect_reputation_change(overseer, peer, BENEFIT_VALID_MESSAGE_FIRST).await; - } + expect_reputation_change(overseer, peer, BENEFIT_VALID_MESSAGE_FIRST).await; + } - // send a view update that removes block B from peer's view by bumping the finalized_number - overseer_send( - overseer, - ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerViewChange( - *peer, - View::with_finalized(2), - )), - ) - .await; + // send a view update that removes block B from peer's view by bumping the + // finalized_number + overseer_send( + overseer, + ApprovalDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::PeerViewChange(*peer, View::with_finalized(2)), + ), + ) + .await; - // send the assignments again - send_message_from_peer(overseer, peer, msg.clone()).await; + // send the assignments again + send_message_from_peer(overseer, peer, msg.clone()).await; - // each of them will incur `COST_UNEXPECTED_MESSAGE`, not only the first one - for _ in 0..candidates_count { - expect_reputation_change(overseer, peer, COST_UNEXPECTED_MESSAGE).await; - expect_reputation_change(overseer, peer, BENEFIT_VALID_MESSAGE).await; - } - virtual_overseer - }); + // each of them will incur `COST_UNEXPECTED_MESSAGE`, not only the first one + for _ in 0..candidates_count { + expect_reputation_change(overseer, peer, COST_UNEXPECTED_MESSAGE).await; + expect_reputation_change(overseer, peer, BENEFIT_VALID_MESSAGE).await; + } + virtual_overseer + }, + ); } /// Imagine we send a message to peer A and peer B. @@ -736,86 +916,94 @@ fn peer_sending_us_the_same_we_just_sent_them_is_ok() { let peers = make_peers_and_authority_ids(8); let peer_a = peers.first().unwrap().0; - let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - let peer = &peer_a; - setup_peer_with_view(overseer, peer, view![], ValidationVersion::V1).await; - - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - // Setup a topology where peer_a is neighbor to current node. - setup_gossip_topology( - overseer, - make_gossip_topology(1, &peers_with_optional_peer_id, &[0], &[2], 1), - ) - .await; + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state_without_reputation_delay(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + let peer = &peer_a; + setup_peer_with_view(overseer, peer, view![], ValidationVersion::V1).await; + + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + // Setup a topology where peer_a is neighbor to current node. + setup_gossip_topology( + overseer, + make_gossip_topology(1, &peers_with_optional_peer_id, &[0], &[2], 1), + ) + .await; - // new block `hash` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - // import an assignment related to `hash` locally - let validator_index = ValidatorIndex(0); - let candidate_index = 0u32; - let cert = fake_assignment_cert(hash, validator_index); - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment( - cert.clone().into(), - candidate_index.into(), - ), - ) - .await; + // new block `hash` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // import an assignment related to `hash` locally + let validator_index = ValidatorIndex(0); + let candidate_index = 0u32; + let cert = fake_assignment_cert(hash, validator_index); + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), + ) + .await; - // update peer view to include the hash - overseer_send( - overseer, - ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerViewChange( - *peer, - view![hash], - )), - ) - .await; + // update peer view to include the hash + overseer_send( + overseer, + ApprovalDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::PeerViewChange(*peer, view![hash]), + ), + ) + .await; - // we should send them the assignment - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert_eq!(peers.len(), 1); - assert_eq!(assignments.len(), 1); - } - ); + // we should send them the assignment + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 1); + assert_eq!(assignments.len(), 1); + } + ); - // but if someone else is sending it the same assignment - // the peer could send us it as well - let assignments = vec![(cert, candidate_index)]; - let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments); - send_message_from_peer(overseer, peer, msg.clone()).await; + // but if someone else is sending it the same assignment + // the peer could send us it as well + let assignments = vec![(cert, candidate_index)]; + let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments); + send_message_from_peer(overseer, peer, msg.clone()).await; - assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "we should not punish the peer"); + assert!( + overseer.recv().timeout(TIMEOUT).await.is_none(), + "we should not punish the peer" + ); - // send the assignments again - send_message_from_peer(overseer, peer, msg).await; + // send the assignments again + send_message_from_peer(overseer, peer, msg).await; - // now we should - expect_reputation_change(overseer, peer, COST_DUPLICATE_MESSAGE).await; - virtual_overseer - }); + // now we should + expect_reputation_change(overseer, peer, COST_DUPLICATE_MESSAGE).await; + virtual_overseer + }, + ); } #[test] @@ -827,116 +1015,134 @@ fn import_approval_happy_path_v1_v2_peers() { let peer_c = peers.get(2).unwrap().0; let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); + let candidate_hash = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xBB)); + + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state_without_reputation_delay(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // setup peers with V1 and V2 protocol versions + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V1).await; + + let mut keystore = LocalKeystore::in_memory(); + let session = dummy_session_info_valid(1, &mut keystore, 1); + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![(candidate_hash, 0.into(), 0.into()); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + // Set up a gossip topology, where a, b, and c are topology neighbors to the node. + setup_gossip_topology( + overseer, + make_gossip_topology(1, &peers_with_optional_peer_id, &[0, 1], &[2, 4], 3), + ) + .await; - let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // setup peers with V1 and V2 protocol versions - setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; - setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; - setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V1).await; - - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - // Set up a gossip topology, where a, b, and c are topology neighbors to the node. - setup_gossip_topology( - overseer, - make_gossip_topology(1, &peers_with_optional_peer_id, &[0, 1], &[2, 4], 3), - ) - .await; - - // import an assignment related to `hash` locally - let validator_index = ValidatorIndex(0); - let candidate_index = 0u32; - let cert = fake_assignment_cert(hash, validator_index); - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment( - cert.clone().into(), - candidate_index.into(), - ), - ) - .await; + // import an assignment related to `hash` locally + let validator_index = ValidatorIndex(0); + let candidate_index = 0u32; + let cert = fake_assignment_cert(hash, validator_index); + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), + ) + .await; - // 1 peer is v1 - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert_eq!(peers.len(), 1); - assert_eq!(assignments.len(), 1); - } - ); + // 1 peer is v1 + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 1); + assert_eq!(assignments.len(), 1); + } + ); - // 1 peer is v2 - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( - protocol_v3::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert_eq!(peers.len(), 1); - assert_eq!(assignments.len(), 1); - } - ); + // 1 peer is v2 + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 1); + assert_eq!(assignments.len(), 1); + } + ); - // send the an approval from peer_b - let approval = IndirectSignedApprovalVoteV2 { - block_hash: hash, - candidate_indices: candidate_index.into(), - validator: validator_index, - signature: dummy_signature(), - }; - let msg: protocol_v3::ApprovalDistributionMessage = - protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v3(overseer, &peer_b, msg).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( - vote, - tx, - )) => { - assert_eq!(vote, approval); - tx.send(ApprovalCheckResult::Accepted).unwrap(); - } - ); + // send the an approval from peer_b + let approval = IndirectSignedApprovalVoteV2 { + block_hash: hash, + candidate_indices: candidate_index.into(), + validator: validator_index, + signature: signature_for( + &keystore, + &session, + vec![candidate_hash], + validator_index, + ), + }; + let msg: protocol_v3::ApprovalDistributionMessage = + protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_b, msg).await; + provide_session( + overseer, + dummy_session_info_valid(1, &mut LocalKeystore::in_memory(), 1), + ) + .await; - expect_reputation_change(overseer, &peer_b, BENEFIT_VALID_MESSAGE_FIRST).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(approvals) - )) - )) => { - assert_eq!(peers.len(), 1); - assert_eq!(approvals.len(), 1); - } - ); - virtual_overseer - }); + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + vote, _, + )) => { + assert_eq!(vote, approval); + } + ); + + expect_reputation_change(overseer, &peer_b, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Approvals(approvals) + )) + )) => { + assert_eq!(peers.len(), 1); + assert_eq!(approvals.len(), 1); + } + ); + virtual_overseer + }, + ); } // Test a v2 approval that signs multiple candidate is correctly processed. @@ -949,103 +1155,123 @@ fn import_approval_happy_path_v2() { let peer_c = peers.get(2).unwrap().0; let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); + let candidate_hash_first = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xBB)); + let candidate_hash_second = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xCC)); + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state_without_reputation_delay(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // setup peers with V2 protocol versions + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V3).await; + let mut keystore = LocalKeystore::in_memory(); + let session = dummy_session_info_valid(1, &mut keystore, 1); + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![ + (candidate_hash_first, 0.into(), 0.into()), + (candidate_hash_second, 1.into(), 1.into()), + ], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + // Set up a gossip topology, where a, b, and c are topology neighbors to the node. + setup_gossip_topology( + overseer, + make_gossip_topology(1, &peers_with_optional_peer_id, &[0, 1], &[2, 4], 3), + ) + .await; - let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // setup peers with V2 protocol versions - setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V3).await; - setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; - setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V3).await; - - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 2], - slot: 1.into(), - session: 1, - }; - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - // Set up a gossip topology, where a, b, and c are topology neighbors to the node. - setup_gossip_topology( - overseer, - make_gossip_topology(1, &peers_with_optional_peer_id, &[0, 1], &[2, 4], 3), - ) - .await; + // import an assignment related to `hash` locally + let validator_index = ValidatorIndex(0); + let candidate_indices: CandidateBitfield = + vec![0 as CandidateIndex, 1 as CandidateIndex].try_into().unwrap(); + let candidate_bitfields = vec![CoreIndex(0), CoreIndex(1)].try_into().unwrap(); + let cert = fake_assignment_cert_v2(hash, validator_index, candidate_bitfields); + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_indices.clone(), + ), + ) + .await; - // import an assignment related to `hash` locally - let validator_index = ValidatorIndex(0); - let candidate_indices: CandidateBitfield = - vec![0 as CandidateIndex, 1 as CandidateIndex].try_into().unwrap(); - let candidate_bitfields = vec![CoreIndex(0), CoreIndex(1)].try_into().unwrap(); - let cert = fake_assignment_cert_v2(hash, validator_index, candidate_bitfields); - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment( - cert.clone().into(), - candidate_indices.clone(), - ), - ) - .await; + // 1 peer is v2 + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 2); + assert_eq!(assignments.len(), 1); + } + ); - // 1 peer is v2 - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( - protocol_v3::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert_eq!(peers.len(), 2); - assert_eq!(assignments.len(), 1); - } - ); + // send the an approval from peer_b + let approval = IndirectSignedApprovalVoteV2 { + block_hash: hash, + candidate_indices, + validator: validator_index, + signature: signature_for( + &keystore, + &session, + vec![candidate_hash_first, candidate_hash_second], + validator_index, + ), + }; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_b, msg).await; + provide_session( + overseer, + dummy_session_info_valid(1, &mut LocalKeystore::in_memory(), 1), + ) + .await; - // send the an approval from peer_b - let approval = IndirectSignedApprovalVoteV2 { - block_hash: hash, - candidate_indices, - validator: validator_index, - signature: dummy_signature(), - }; - let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v3(overseer, &peer_b, msg).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( - vote, - tx, - )) => { - assert_eq!(vote, approval); - tx.send(ApprovalCheckResult::Accepted).unwrap(); - } - ); + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + vote, _, + )) => { + assert_eq!(vote, approval); + } + ); - expect_reputation_change(overseer, &peer_b, BENEFIT_VALID_MESSAGE_FIRST).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( - protocol_v3::ApprovalDistributionMessage::Approvals(approvals) - )) - )) => { - assert_eq!(peers.len(), 1); - assert_eq!(approvals.len(), 1); - } - ); - virtual_overseer - }); + expect_reputation_change(overseer, &peer_b, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) + )) + )) => { + assert_eq!(peers.len(), 1); + assert_eq!(approvals.len(), 1); + } + ); + virtual_overseer + }, + ); } // Tests that votes that cover multiple assignments candidates are correctly processed on importing @@ -1059,187 +1285,205 @@ fn multiple_assignments_covered_with_one_approval_vote() { let peer_d = peers.get(4).unwrap().0; let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); + let candidate_hash_first = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xBB)); + let candidate_hash_second = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xCC)); + + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state_without_reputation_delay(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // setup peers with V2 protocol versions + setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_d, view![hash], ValidationVersion::V3).await; + + let mut keystore = LocalKeystore::in_memory(); + let session = dummy_session_info_valid(1, &mut keystore, 5); + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![ + (candidate_hash_first, 0.into(), 0.into()), + (candidate_hash_second, 1.into(), 1.into()), + ], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + // Set up a gossip topology, where a, b, and c, d are topology neighbors to the node. + setup_gossip_topology( + overseer, + make_gossip_topology(1, &peers_with_optional_peer_id, &[0, 1], &[2, 4], 3), + ) + .await; - let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // setup peers with V2 protocol versions - setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::V3).await; - setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; - setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V3).await; - setup_peer_with_view(overseer, &peer_d, view![hash], ValidationVersion::V3).await; - - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 2], - slot: 1.into(), - session: 1, - }; - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - // Set up a gossip topology, where a, b, and c, d are topology neighbors to the node. - setup_gossip_topology( - overseer, - make_gossip_topology(1, &peers_with_optional_peer_id, &[0, 1], &[2, 4], 3), - ) - .await; + // import an assignment related to `hash` locally + let validator_index = ValidatorIndex(2); // peer_c is the originator + let candidate_indices: CandidateBitfield = + vec![0 as CandidateIndex, 1 as CandidateIndex].try_into().unwrap(); - // import an assignment related to `hash` locally - let validator_index = ValidatorIndex(2); // peer_c is the originator - let candidate_indices: CandidateBitfield = - vec![0 as CandidateIndex, 1 as CandidateIndex].try_into().unwrap(); - - let core_bitfields = vec![CoreIndex(0)].try_into().unwrap(); - let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfields); - - // send the candidate 0 assignment from peer_b - let assignment = IndirectAssignmentCertV2 { - block_hash: hash, - validator: validator_index, - cert: cert.cert, - }; - let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( - assignment, - (0 as CandidateIndex).into(), - )]); - send_message_from_peer_v3(overseer, &peer_d, msg).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( - _, _, - tx, - )) => { - tx.send(AssignmentCheckResult::Accepted).unwrap(); - } - ); - expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( - protocol_v3::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert!(peers.len() >= 2); - assert!(peers.contains(&peer_a)); - assert!(peers.contains(&peer_b)); - assert_eq!(assignments.len(), 1); - } - ); + let core_bitfields = vec![CoreIndex(0)].try_into().unwrap(); + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfields); - let candidate_bitfields = vec![CoreIndex(1)].try_into().unwrap(); - let cert = fake_assignment_cert_v2(hash, validator_index, candidate_bitfields); - - // send the candidate 1 assignment from peer_c - let assignment = IndirectAssignmentCertV2 { - block_hash: hash, - validator: validator_index, - cert: cert.cert, - }; - let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( - assignment, - (1 as CandidateIndex).into(), - )]); - - send_message_from_peer_v3(overseer, &peer_c, msg).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( - _, _, - tx, - )) => { - tx.send(AssignmentCheckResult::Accepted).unwrap(); - } - ); - expect_reputation_change(overseer, &peer_c, BENEFIT_VALID_MESSAGE_FIRST).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( - protocol_v3::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert!(peers.len() >= 2); - assert!(peers.contains(&peer_b)); - assert!(peers.contains(&peer_a)); - assert_eq!(assignments.len(), 1); - } - ); + // send the candidate 0 assignment from peer_b + let assignment = IndirectAssignmentCertV2 { + block_hash: hash, + validator: validator_index, + cert: cert.cert, + }; + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( + assignment, + (0 as CandidateIndex).into(), + )]); + send_message_from_peer_v3(overseer, &peer_d, msg).await; + provide_session(overseer, session.clone()).await; - // send an approval from peer_b - let approval = IndirectSignedApprovalVoteV2 { - block_hash: hash, - candidate_indices, - validator: validator_index, - signature: dummy_signature(), - }; - let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v3(overseer, &peer_d, msg).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( - vote, - tx, - )) => { - assert_eq!(vote, approval); - tx.send(ApprovalCheckResult::Accepted).unwrap(); - } - ); + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + _, _, + tranche, + _, + )) => { + assert_eq!(tranche, 0); + } + ); + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; - expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( - protocol_v3::ApprovalDistributionMessage::Approvals(approvals) - )) - )) => { - assert!(peers.len() >= 2); - assert!(peers.contains(&peer_b)); - assert!(peers.contains(&peer_a)); - assert_eq!(approvals.len(), 1); - } - ); - for candidate_index in 0..1 { - let (tx_distribution, rx_distribution) = oneshot::channel(); - let mut candidates_requesting_signatures = HashSet::new(); - candidates_requesting_signatures.insert((hash, candidate_index)); - overseer_send( - overseer, - ApprovalDistributionMessage::GetApprovalSignatures( - candidates_requesting_signatures, - tx_distribution, + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert!(peers.len() >= 2); + assert!(peers.contains(&peer_a)); + assert!(peers.contains(&peer_b)); + assert_eq!(assignments.len(), 1); + } + ); + + let candidate_bitfields = vec![CoreIndex(1)].try_into().unwrap(); + let cert = fake_assignment_cert_v2(hash, validator_index, candidate_bitfields); + + // send the candidate 1 assignment from peer_c + let assignment = IndirectAssignmentCertV2 { + block_hash: hash, + validator: validator_index, + cert: cert.cert, + }; + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( + assignment, + (1 as CandidateIndex).into(), + )]); + + send_message_from_peer_v3(overseer, &peer_c, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + _, _, + tranche, _, + )) => { + assert_eq!(tranche, 0); + } + ); + expect_reputation_change(overseer, &peer_c, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert!(peers.len() >= 2); + assert!(peers.contains(&peer_b)); + assert!(peers.contains(&peer_a)); + assert_eq!(assignments.len(), 1); + } + ); + + // send an approval from peer_b + let approval = IndirectSignedApprovalVoteV2 { + block_hash: hash, + candidate_indices, + validator: validator_index, + signature: signature_for( + &keystore, + &session, + vec![candidate_hash_first, candidate_hash_second], + validator_index, ), - ) - .await; - let signatures = rx_distribution.await.unwrap(); - - assert_eq!(signatures.len(), 1); - for (signing_validator, signature) in signatures { - assert_eq!(validator_index, signing_validator); - assert_eq!(signature.0, hash); - assert_eq!(signature.2, approval.signature); - assert_eq!(signature.1, vec![0, 1]); + }; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_d, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + vote, _, + )) => { + assert_eq!(vote, approval); + } + ); + + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) + )) + )) => { + assert!(peers.len() >= 2); + assert!(peers.contains(&peer_b)); + assert!(peers.contains(&peer_a)); + assert_eq!(approvals.len(), 1); + } + ); + for candidate_index in 0..1 { + let (tx_distribution, rx_distribution) = oneshot::channel(); + let mut candidates_requesting_signatures = HashSet::new(); + candidates_requesting_signatures.insert((hash, candidate_index)); + overseer_send( + overseer, + ApprovalDistributionMessage::GetApprovalSignatures( + candidates_requesting_signatures, + tx_distribution, + ), + ) + .await; + let signatures = rx_distribution.await.unwrap(); + + assert_eq!(signatures.len(), 1); + for (signing_validator, signature) in signatures { + assert_eq!(validator_index, signing_validator); + assert_eq!(signature.0, hash); + assert_eq!(signature.2, approval.signature); + assert_eq!(signature.1, vec![0, 1]); + } } - } - virtual_overseer - }); + virtual_overseer + }, + ); } // Tests that votes that cover multiple assignments candidates are correctly processed when unify @@ -1253,182 +1497,198 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { let peer_d = peers.get(4).unwrap().0; let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); + let candidate_hash_first = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xBB)); + let candidate_hash_second = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xCC)); + + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state_without_reputation_delay(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + setup_peer_with_view(overseer, &peer_d, view![hash], ValidationVersion::V3).await; + let mut keystore = LocalKeystore::in_memory(); + let session = dummy_session_info_valid(1, &mut keystore, 5); + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![ + (candidate_hash_first, 0.into(), 0.into()), + (candidate_hash_second, 1.into(), 1.into()), + ], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + // Set up a gossip topology, where a, b, and c, d are topology neighbors to the node. + setup_gossip_topology( + overseer, + make_gossip_topology(1, &peers_with_optional_peer_id, &[0, 1], &[2, 4], 3), + ) + .await; - let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - setup_peer_with_view(overseer, &peer_d, view![hash], ValidationVersion::V3).await; - - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 2], - slot: 1.into(), - session: 1, - }; - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - // Set up a gossip topology, where a, b, and c, d are topology neighbors to the node. - setup_gossip_topology( - overseer, - make_gossip_topology(1, &peers_with_optional_peer_id, &[0, 1], &[2, 4], 3), - ) - .await; + // import an assignment related to `hash` locally + let validator_index = ValidatorIndex(2); // peer_c is the originator + let candidate_indices: CandidateBitfield = + vec![0 as CandidateIndex, 1 as CandidateIndex].try_into().unwrap(); - // import an assignment related to `hash` locally - let validator_index = ValidatorIndex(2); // peer_c is the originator - let candidate_indices: CandidateBitfield = - vec![0 as CandidateIndex, 1 as CandidateIndex].try_into().unwrap(); - - let core_bitfields = vec![CoreIndex(0)].try_into().unwrap(); - let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfields); - - // send the candidate 0 assignment from peer_b - let assignment = IndirectAssignmentCertV2 { - block_hash: hash, - validator: validator_index, - cert: cert.cert, - }; - let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( - assignment, - (0 as CandidateIndex).into(), - )]); - send_message_from_peer_v3(overseer, &peer_d, msg).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( - _, _, - tx, - )) => { - tx.send(AssignmentCheckResult::Accepted).unwrap(); - } - ); - expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; - - let candidate_bitfields = vec![CoreIndex(1)].try_into().unwrap(); - let cert = fake_assignment_cert_v2(hash, validator_index, candidate_bitfields); - - // send the candidate 1 assignment from peer_c - let assignment = IndirectAssignmentCertV2 { - block_hash: hash, - validator: validator_index, - cert: cert.cert, - }; - let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( - assignment, - (1 as CandidateIndex).into(), - )]); - - send_message_from_peer_v3(overseer, &peer_d, msg).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( - _, _, - tx, - )) => { - tx.send(AssignmentCheckResult::Accepted).unwrap(); - } - ); - expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; - - // send an approval from peer_b - let approval = IndirectSignedApprovalVoteV2 { - block_hash: hash, - candidate_indices, - validator: validator_index, - signature: dummy_signature(), - }; - let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v3(overseer, &peer_d, msg).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( - vote, - tx, - )) => { - assert_eq!(vote, approval); - tx.send(ApprovalCheckResult::Accepted).unwrap(); - } - ); + let core_bitfields = vec![CoreIndex(0)].try_into().unwrap(); + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfields); - expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; - - // setup peers with V2 protocol versions - setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::V3).await; - setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; - let mut expected_peers_assignments = vec![peer_a, peer_b]; - let mut expected_peers_approvals = vec![peer_a, peer_b]; - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( - protocol_v3::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert!(peers.len() == 1); - assert!(expected_peers_assignments.contains(peers.first().unwrap())); - expected_peers_assignments.retain(|peer| peer != peers.first().unwrap()); - assert_eq!(assignments.len(), 2); - } - ); + // send the candidate 0 assignment from peer_b + let assignment = IndirectAssignmentCertV2 { + block_hash: hash, + validator: validator_index, + cert: cert.cert, + }; + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( + assignment, + (0 as CandidateIndex).into(), + )]); + send_message_from_peer_v3(overseer, &peer_d, msg).await; + provide_session(overseer, session.clone()).await; - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( - protocol_v3::ApprovalDistributionMessage::Approvals(approvals) - )) - )) => { - assert!(peers.len() == 1); - assert!(expected_peers_approvals.contains(peers.first().unwrap())); - expected_peers_approvals.retain(|peer| peer != peers.first().unwrap()); - assert_eq!(approvals.len(), 1); - } - ); + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + _, _, + tranche, _, + )) => { + assert_eq!(tranche, 0); + } + ); + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( - protocol_v3::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert!(peers.len() == 1); - assert!(expected_peers_assignments.contains(peers.first().unwrap())); - expected_peers_assignments.retain(|peer| peer != peers.first().unwrap()); - assert_eq!(assignments.len(), 2); - } - ); + let candidate_bitfields = vec![CoreIndex(1)].try_into().unwrap(); + let cert = fake_assignment_cert_v2(hash, validator_index, candidate_bitfields); - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( - protocol_v3::ApprovalDistributionMessage::Approvals(approvals) - )) - )) => { - assert!(peers.len() == 1); - assert!(expected_peers_approvals.contains(peers.first().unwrap())); - expected_peers_approvals.retain(|peer| peer != peers.first().unwrap()); - assert_eq!(approvals.len(), 1); - } - ); + // send the candidate 1 assignment from peer_c + let assignment = IndirectAssignmentCertV2 { + block_hash: hash, + validator: validator_index, + cert: cert.cert, + }; + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( + assignment, + (1 as CandidateIndex).into(), + )]); - virtual_overseer - }); + send_message_from_peer_v3(overseer, &peer_d, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + _, _, + tranche, _, + )) => { + assert_eq!(tranche, 0); + } + ); + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; + + // send an approval from peer_b + let approval = IndirectSignedApprovalVoteV2 { + block_hash: hash, + candidate_indices, + validator: validator_index, + signature: signature_for( + &keystore, + &session, + vec![candidate_hash_first, candidate_hash_second], + validator_index, + ), + }; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_d, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + vote, _, + )) => { + assert_eq!(vote, approval); + } + ); + + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; + + // setup peers with V2 protocol versions + setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; + let mut expected_peers_assignments = vec![peer_a, peer_b]; + let mut expected_peers_approvals = vec![peer_a, peer_b]; + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert!(peers.len() == 1); + assert!(expected_peers_assignments.contains(peers.first().unwrap())); + expected_peers_assignments.retain(|peer| peer != peers.first().unwrap()); + assert_eq!(assignments.len(), 2); + } + ); + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) + )) + )) => { + assert!(peers.len() == 1); + assert!(expected_peers_approvals.contains(peers.first().unwrap())); + expected_peers_approvals.retain(|peer| peer != peers.first().unwrap()); + assert_eq!(approvals.len(), 1); + } + ); + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert!(peers.len() == 1); + assert!(expected_peers_assignments.contains(peers.first().unwrap())); + expected_peers_assignments.retain(|peer| peer != peers.first().unwrap()); + assert_eq!(assignments.len(), 2); + } + ); + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) + )) + )) => { + assert!(peers.len() == 1); + assert!(expected_peers_approvals.contains(peers.first().unwrap())); + expected_peers_approvals.retain(|peer| peer != peers.first().unwrap()); + assert_eq!(approvals.len(), 1); + } + ); + + virtual_overseer + }, + ); } #[test] @@ -1437,79 +1697,88 @@ fn import_approval_bad() { let peer_b = PeerId::random(); let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); + let candidate_hash = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xBB)); + + let diff_candidate_hash = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xCC)); + + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state_without_reputation_delay(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // setup peers + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V1).await; + let mut keystore = LocalKeystore::in_memory(); + let session = dummy_session_info_valid(1, &mut keystore, 1); + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![(candidate_hash, 0.into(), 0.into()); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; - let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // setup peers - setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; - setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V1).await; - - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - let validator_index = ValidatorIndex(0); - let candidate_index = 0u32; - let cert = fake_assignment_cert(hash, validator_index); - - // send the an approval from peer_b, we don't have an assignment yet - let approval = IndirectSignedApprovalVoteV2 { - block_hash: hash, - candidate_indices: candidate_index.into(), - validator: validator_index, - signature: dummy_signature(), - }; - let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v3(overseer, &peer_b, msg).await; - - expect_reputation_change(overseer, &peer_b, COST_UNEXPECTED_MESSAGE).await; - - // now import an assignment from peer_b - let assignments = vec![(cert.clone(), candidate_index)]; - let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments); - send_message_from_peer(overseer, &peer_b, msg).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( - assignment, - i, - tx, - )) => { - assert_eq!(assignment, cert.into()); - assert_eq!(i, candidate_index.into()); - tx.send(AssignmentCheckResult::Accepted).unwrap(); - } - ); + let validator_index = ValidatorIndex(0); + let candidate_index = 0u32; + let cert = fake_assignment_cert(hash, validator_index); - expect_reputation_change(overseer, &peer_b, BENEFIT_VALID_MESSAGE_FIRST).await; + // Sign a different candidate hash. + let payload = + ApprovalVoteMultipleCandidates(&vec![diff_candidate_hash]).signing_payload(1); + let sign_key = session.validators.get(ValidatorIndex(0)).unwrap().clone(); + let signature = keystore + .sr25519_sign(ValidatorId::ID, &sign_key.into(), &payload[..]) + .unwrap() + .unwrap(); + + // send the an approval from peer_b, we don't have an assignment yet + let approval = IndirectSignedApprovalVoteV2 { + block_hash: hash, + candidate_indices: candidate_index.into(), + validator: validator_index, + signature: signature.into(), + }; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_b, msg).await; - // and try again - let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v3(overseer, &peer_b, msg).await; + expect_reputation_change(overseer, &peer_b, COST_UNEXPECTED_MESSAGE).await; - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( - vote, - tx, - )) => { - assert_eq!(vote, approval); - tx.send(ApprovalCheckResult::Bad(ApprovalCheckError::UnknownBlock(hash))).unwrap(); - } - ); + // now import an assignment from peer_b + let assignments = vec![(cert.clone(), candidate_index)]; + let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments); + send_message_from_peer(overseer, &peer_b, msg).await; + provide_session(overseer, session.clone()).await; - expect_reputation_change(overseer, &peer_b, COST_INVALID_MESSAGE).await; - virtual_overseer - }); + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + assignment, + i, + tranche, _, + )) => { + assert_eq!(assignment, cert.into()); + assert_eq!(i, candidate_index.into()); + assert_eq!(tranche, 0); + } + ); + + expect_reputation_change(overseer, &peer_b, BENEFIT_VALID_MESSAGE_FIRST).await; + + // and try again + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_b, msg).await; + + expect_reputation_change(overseer, &peer_b, COST_INVALID_MESSAGE).await; + virtual_overseer + }, + ); } /// make sure we clean up the state on block finalized @@ -1520,38 +1789,46 @@ fn update_our_view() { let hash_b = Hash::repeat_byte(0xBB); let hash_c = Hash::repeat_byte(0xCC); - let state = test_harness(State::default(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // new block `hash_a` with 1 candidates - let meta_a = BlockApprovalMeta { - hash: hash_a, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - let meta_b = BlockApprovalMeta { - hash: hash_b, - parent_hash: hash_a, - number: 2, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - let meta_c = BlockApprovalMeta { - hash: hash_c, - parent_hash: hash_b, - number: 3, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta_a, meta_b, meta_c]); - overseer_send(overseer, msg).await; - virtual_overseer - }); + let state = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + State::default(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // new block `hash_a` with 1 candidates + let meta_a = BlockApprovalMeta { + hash: hash_a, + parent_hash, + number: 1, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let meta_b = BlockApprovalMeta { + hash: hash_b, + parent_hash: hash_a, + number: 2, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let meta_c = BlockApprovalMeta { + hash: hash_c, + parent_hash: hash_b, + number: 3, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta_a, meta_b, meta_c]); + overseer_send(overseer, msg).await; + virtual_overseer + }, + ); assert!(state.blocks_by_number.get(&1).is_some()); assert!(state.blocks_by_number.get(&2).is_some()); @@ -1560,12 +1837,17 @@ fn update_our_view() { assert!(state.blocks.get(&hash_b).is_some()); assert!(state.blocks.get(&hash_c).is_some()); - let state = test_harness(state, |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // finalize a block - overseer_signal_block_finalized(overseer, 2).await; - virtual_overseer - }); + let state = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state, + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // finalize a block + overseer_signal_block_finalized(overseer, 2).await; + virtual_overseer + }, + ); assert!(state.blocks_by_number.get(&1).is_none()); assert!(state.blocks_by_number.get(&2).is_none()); @@ -1574,12 +1856,17 @@ fn update_our_view() { assert!(state.blocks.get(&hash_b).is_none()); assert!(state.blocks.get(&hash_c).is_some()); - let state = test_harness(state, |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // finalize a very high block - overseer_signal_block_finalized(overseer, 4_000_000_000).await; - virtual_overseer - }); + let state = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state, + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // finalize a very high block + overseer_signal_block_finalized(overseer, 4_000_000_000).await; + virtual_overseer + }, + ); assert!(state.blocks_by_number.get(&3).is_none()); assert!(state.blocks.get(&hash_c).is_none()); @@ -1597,81 +1884,89 @@ fn update_peer_view() { let peer_a = peers.first().unwrap().0; let peer = &peer_a; - let state = test_harness(State::default(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // new block `hash_a` with 1 candidates - let meta_a = BlockApprovalMeta { - hash: hash_a, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - let meta_b = BlockApprovalMeta { - hash: hash_b, - parent_hash: hash_a, - number: 2, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - let meta_c = BlockApprovalMeta { - hash: hash_c, - parent_hash: hash_b, - number: 3, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta_a, meta_b, meta_c]); - overseer_send(overseer, msg).await; - - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - // Setup a topology where peer_a is neighbor to current node. - setup_gossip_topology( - overseer, - make_gossip_topology(1, &peers_with_optional_peer_id, &[0], &[2], 1), - ) - .await; + let state = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + State::default(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // new block `hash_a` with 1 candidates + let meta_a = BlockApprovalMeta { + hash: hash_a, + parent_hash, + number: 1, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let meta_b = BlockApprovalMeta { + hash: hash_b, + parent_hash: hash_a, + number: 2, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let meta_c = BlockApprovalMeta { + hash: hash_c, + parent_hash: hash_b, + number: 3, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta_a, meta_b, meta_c]); + overseer_send(overseer, msg).await; + + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + // Setup a topology where peer_a is neighbor to current node. + setup_gossip_topology( + overseer, + make_gossip_topology(1, &peers_with_optional_peer_id, &[0], &[2], 1), + ) + .await; - let cert_a = fake_assignment_cert(hash_a, ValidatorIndex(0)); - let cert_b = fake_assignment_cert(hash_b, ValidatorIndex(0)); + let cert_a = fake_assignment_cert(hash_a, ValidatorIndex(0)); + let cert_b = fake_assignment_cert(hash_b, ValidatorIndex(0)); - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment(cert_a.into(), 0.into()), - ) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment(cert_a.into(), 0.into()), + ) + .await; - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment(cert_b.into(), 0.into()), - ) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment(cert_b.into(), 0.into()), + ) + .await; - // connect a peer - setup_peer_with_view(overseer, peer, view![hash_a], ValidationVersion::V1).await; - - // we should send relevant assignments to the peer - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert_eq!(peers.len(), 1); - assert_eq!(assignments.len(), 1); - } - ); - virtual_overseer - }); + // connect a peer + setup_peer_with_view(overseer, peer, view![hash_a], ValidationVersion::V1).await; + + // we should send relevant assignments to the peer + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 1); + assert_eq!(assignments.len(), 1); + } + ); + virtual_overseer + }, + ); assert_eq!(state.peer_views.get(peer).map(|v| v.view.finalized_number), Some(0)); assert_eq!( @@ -1688,42 +1983,49 @@ fn update_peer_view() { 1, ); - let state = test_harness(state, |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // update peer's view - overseer_send( - overseer, - ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerViewChange( - *peer, - View::new(vec![hash_b, hash_c, hash_d], 2), - )), - ) - .await; + let state = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state, + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // update peer's view + overseer_send( + overseer, + ApprovalDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::PeerViewChange( + *peer, + View::new(vec![hash_b, hash_c, hash_d], 2), + ), + ), + ) + .await; - let cert_c = fake_assignment_cert(hash_c, ValidatorIndex(0)); + let cert_c = fake_assignment_cert(hash_c, ValidatorIndex(0)); - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment(cert_c.clone().into(), 0.into()), - ) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment(cert_c.clone().into(), 0.into()), + ) + .await; - // we should send relevant assignments to the peer - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert_eq!(peers.len(), 1); - assert_eq!(assignments.len(), 1); - assert_eq!(assignments[0].0, cert_c); - } - ); - virtual_overseer - }); + // we should send relevant assignments to the peer + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 1); + assert_eq!(assignments.len(), 1); + assert_eq!(assignments[0].0, cert_c); + } + ); + virtual_overseer + }, + ); assert_eq!(state.peer_views.get(peer).map(|v| v.view.finalized_number), Some(2)); assert_eq!( @@ -1741,19 +2043,26 @@ fn update_peer_view() { ); let finalized_number = 4_000_000_000; - let state = test_harness(state, |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // update peer's view - overseer_send( - overseer, - ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerViewChange( - *peer, - View::with_finalized(finalized_number), - )), - ) - .await; - virtual_overseer - }); + let state = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state, + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // update peer's view + overseer_send( + overseer, + ApprovalDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::PeerViewChange( + *peer, + View::with_finalized(finalized_number), + ), + ), + ) + .await; + virtual_overseer + }, + ); assert_eq!(state.peer_views.get(peer).map(|v| v.view.finalized_number), Some(finalized_number)); assert!(state.blocks.get(&hash_c).unwrap().known_by.get(peer).is_none()); @@ -1776,164 +2085,176 @@ fn update_peer_authority_id() { // Y neighbour, we simulate that PeerId is not known in the beginning. let neighbour_y = peers.get(neighbour_y_index).unwrap().0; - let _state = test_harness(State::default(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // new block `hash_a` with 1 candidates - let meta_a = BlockApprovalMeta { - hash: hash_a, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - let meta_b = BlockApprovalMeta { - hash: hash_b, - parent_hash: hash_a, - number: 2, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - let meta_c = BlockApprovalMeta { - hash: hash_c, - parent_hash: hash_b, - number: 3, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta_a, meta_b, meta_c]); - overseer_send(overseer, msg).await; - - let peers_with_optional_peer_id = peers - .iter() - .enumerate() - .map(|(index, (peer_id, authority))| { - (if index == 0 { None } else { Some(*peer_id) }, authority.clone()) - }) - .collect_vec(); - - // Setup a topology where peer_a is neighbor to current node. - setup_gossip_topology( - overseer, - make_gossip_topology( - 1, - &peers_with_optional_peer_id, - &[neighbour_x_index], - &[neighbour_y_index], - local_index, - ), - ) - .await; + let _state = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + State::default(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // new block `hash_a` with 1 candidates + let meta_a = BlockApprovalMeta { + hash: hash_a, + parent_hash, + number: 1, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let meta_b = BlockApprovalMeta { + hash: hash_b, + parent_hash: hash_a, + number: 2, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let meta_c = BlockApprovalMeta { + hash: hash_c, + parent_hash: hash_b, + number: 3, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta_a, meta_b, meta_c]); + overseer_send(overseer, msg).await; + + let peers_with_optional_peer_id = peers + .iter() + .enumerate() + .map(|(index, (peer_id, authority))| { + (if index == 0 { None } else { Some(*peer_id) }, authority.clone()) + }) + .collect_vec(); + + // Setup a topology where peer_a is neighbor to current node. + setup_gossip_topology( + overseer, + make_gossip_topology( + 1, + &peers_with_optional_peer_id, + &[neighbour_x_index], + &[neighbour_y_index], + local_index, + ), + ) + .await; - let cert_a = fake_assignment_cert(hash_a, ValidatorIndex(local_index as u32)); - let cert_b = fake_assignment_cert(hash_b, ValidatorIndex(local_index as u32)); + let cert_a = fake_assignment_cert(hash_a, ValidatorIndex(local_index as u32)); + let cert_b = fake_assignment_cert(hash_b, ValidatorIndex(local_index as u32)); - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment(cert_a.into(), 0.into()), - ) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment(cert_a.into(), 0.into()), + ) + .await; - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment(cert_b.into(), 0.into()), - ) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment(cert_b.into(), 0.into()), + ) + .await; - // connect a peer - setup_peer_with_view(overseer, &neighbour_x, view![hash_a], ValidationVersion::V1).await; - setup_peer_with_view(overseer, &neighbour_y, view![hash_a], ValidationVersion::V1).await; - - setup_peer_with_view(overseer, &neighbour_x, view![hash_b], ValidationVersion::V1).await; - setup_peer_with_view(overseer, &neighbour_y, view![hash_b], ValidationVersion::V1).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert_eq!(peers.len(), 1); - assert_eq!(assignments.len(), 1); - assert_eq!(peers.get(0), Some(&neighbour_y)); - } - ); + // connect a peer + setup_peer_with_view(overseer, &neighbour_x, view![hash_a], ValidationVersion::V1) + .await; + setup_peer_with_view(overseer, &neighbour_y, view![hash_a], ValidationVersion::V1) + .await; - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert_eq!(peers.len(), 1); - assert_eq!(assignments.len(), 1); - assert_eq!(peers.get(0), Some(&neighbour_y)); - } - ); + setup_peer_with_view(overseer, &neighbour_x, view![hash_b], ValidationVersion::V1) + .await; + setup_peer_with_view(overseer, &neighbour_y, view![hash_b], ValidationVersion::V1) + .await; - overseer_send( - overseer, - ApprovalDistributionMessage::NetworkBridgeUpdate( - NetworkBridgeEvent::UpdatedAuthorityIds( - peers[neighbour_x_index].0, - [peers[neighbour_x_index].1.clone()].into_iter().collect(), + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 1); + assert_eq!(assignments.len(), 1); + assert_eq!(peers.get(0), Some(&neighbour_y)); + } + ); + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 1); + assert_eq!(assignments.len(), 1); + assert_eq!(peers.get(0), Some(&neighbour_y)); + } + ); + + overseer_send( + overseer, + ApprovalDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::UpdatedAuthorityIds( + peers[neighbour_x_index].0, + [peers[neighbour_x_index].1.clone()].into_iter().collect(), + ), ), - ), - ) - .await; + ) + .await; - // we should send relevant assignments to the peer, after we found it's peer id. - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - gum::info!(target: LOG_TARGET, ?peers, ?assignments); - assert_eq!(peers.len(), 1); - assert_eq!(assignments.len(), 2); - assert_eq!(assignments.get(0).unwrap().0.block_hash, hash_a); - assert_eq!(assignments.get(1).unwrap().0.block_hash, hash_b); - assert_eq!(peers.get(0), Some(&neighbour_x)); - } - ); + // we should send relevant assignments to the peer, after we found it's peer id. + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + gum::info!(target: LOG_TARGET, ?peers, ?assignments); + assert_eq!(peers.len(), 1); + assert_eq!(assignments.len(), 2); + assert_eq!(assignments.get(0).unwrap().0.block_hash, hash_a); + assert_eq!(assignments.get(1).unwrap().0.block_hash, hash_b); + assert_eq!(peers.get(0), Some(&neighbour_x)); + } + ); - overseer_send( - overseer, - ApprovalDistributionMessage::NetworkBridgeUpdate( - NetworkBridgeEvent::UpdatedAuthorityIds( - peers[neighbour_y_index].0, - [peers[neighbour_y_index].1.clone()].into_iter().collect(), + overseer_send( + overseer, + ApprovalDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::UpdatedAuthorityIds( + peers[neighbour_y_index].0, + [peers[neighbour_y_index].1.clone()].into_iter().collect(), + ), ), - ), - ) - .await; - overseer_send( - overseer, - ApprovalDistributionMessage::NetworkBridgeUpdate( - NetworkBridgeEvent::UpdatedAuthorityIds( - peers[neighbour_x_index].0, - [peers[neighbour_x_index].1.clone()].into_iter().collect(), + ) + .await; + overseer_send( + overseer, + ApprovalDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::UpdatedAuthorityIds( + peers[neighbour_x_index].0, + [peers[neighbour_x_index].1.clone()].into_iter().collect(), + ), ), - ), - ) - .await; - assert!( - overseer.recv().timeout(TIMEOUT).await.is_none(), - "no message should be sent peers are already known" - ); + ) + .await; + assert!( + overseer.recv().timeout(TIMEOUT).await.is_none(), + "no message should be sent peers are already known" + ); - virtual_overseer - }); + virtual_overseer + }, + ); } /// E.g. if someone copies the keys... @@ -1942,89 +2263,106 @@ fn import_remotely_then_locally() { let peer_a = PeerId::random(); let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); + let candidate_hash = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xBB)); let peer = &peer_a; - let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // setup the peer - setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; - - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - // import the assignment remotely first - let validator_index = ValidatorIndex(0); - let candidate_index = 0u32; - let cert = fake_assignment_cert(hash, validator_index); - let assignments = vec![(cert.clone(), candidate_index)]; - let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, peer, msg).await; - - // send an `Accept` message from the Approval Voting subsystem - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( - assignment, - i, - tx, - )) => { - assert_eq!(assignment, cert.clone().into()); - assert_eq!(i, candidate_index.into()); - tx.send(AssignmentCheckResult::Accepted).unwrap(); - } - ); + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state_without_reputation_delay(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // setup the peer + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; + let mut keystore = LocalKeystore::in_memory(); + + let session = dummy_session_info_valid(1, &mut keystore, 1); + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![(candidate_hash, 0.into(), 0.into()); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + + let payload = ApprovalVoteMultipleCandidates(&vec![candidate_hash]).signing_payload(1); + let sign_key = session.validators.get(ValidatorIndex(0)).unwrap().clone(); + let signature = keystore + .sr25519_sign(ValidatorId::ID, &sign_key.into(), &payload[..]) + .unwrap() + .unwrap(); + + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // import the assignment remotely first + let validator_index = ValidatorIndex(0); + let candidate_index = 0u32; + let cert = fake_assignment_cert(hash, validator_index); + let assignments = vec![(cert.clone(), candidate_index)]; + let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer(overseer, peer, msg).await; + provide_session(overseer, session.clone()).await; + + // send an `Accept` message from the Approval Voting subsystem + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + assignment, + i, + tranche, _, + )) => { + assert_eq!(assignment, cert.clone().into()); + assert_eq!(i, candidate_index.into()); + assert_eq!(tranche, 0); + } + ); - expect_reputation_change(overseer, peer, BENEFIT_VALID_MESSAGE_FIRST).await; + expect_reputation_change(overseer, peer, BENEFIT_VALID_MESSAGE_FIRST).await; - // import the same assignment locally - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment( - cert.clone().into(), - candidate_index.into(), - ), - ) - .await; + // import the same assignment locally + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), + ) + .await; - assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); - - // send the approval remotely - let approval = IndirectSignedApprovalVoteV2 { - block_hash: hash, - candidate_indices: candidate_index.into(), - validator: validator_index, - signature: dummy_signature(), - }; - let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v3(overseer, peer, msg).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( - vote, - tx, - )) => { - assert_eq!(vote, approval); - tx.send(ApprovalCheckResult::Accepted).unwrap(); - } - ); - expect_reputation_change(overseer, peer, BENEFIT_VALID_MESSAGE_FIRST).await; + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); - // import the same approval locally - overseer_send(overseer, ApprovalDistributionMessage::DistributeApproval(approval)).await; + // send the approval remotely + let approval = IndirectSignedApprovalVoteV2 { + block_hash: hash, + candidate_indices: candidate_index.into(), + validator: validator_index, + signature: signature.into(), + }; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, peer, msg).await; - assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); - virtual_overseer - }); + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + vote, _, + )) => { + assert_eq!(vote, approval); + } + ); + expect_reputation_change(overseer, peer, BENEFIT_VALID_MESSAGE_FIRST).await; + + // import the same approval locally + overseer_send(overseer, ApprovalDistributionMessage::DistributeApproval(approval)) + .await; + + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }, + ); } #[test] @@ -2035,94 +2373,100 @@ fn sends_assignments_even_when_state_is_approved() { let hash = Hash::repeat_byte(0xAA); let peer = &peer_a; - let _ = test_harness(State::default(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + State::default(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + // Setup a topology where peer_a is neighbor to current node. + setup_gossip_topology( + overseer, + make_gossip_topology(1, &peers_with_optional_peer_id, &[0], &[2], 1), + ) + .await; - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - // Setup a topology where peer_a is neighbor to current node. - setup_gossip_topology( - overseer, - make_gossip_topology(1, &peers_with_optional_peer_id, &[0], &[2], 1), - ) - .await; + let validator_index = ValidatorIndex(0); + let candidate_index = 0u32; - let validator_index = ValidatorIndex(0); - let candidate_index = 0u32; - - // import an assignment and approval locally. - let cert = fake_assignment_cert(hash, validator_index); - let approval = IndirectSignedApprovalVote { - block_hash: hash, - candidate_index, - validator: validator_index, - signature: dummy_signature(), - }; - - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment( - cert.clone().into(), - candidate_index.into(), - ), - ) - .await; + // import an assignment and approval locally. + let cert = fake_assignment_cert(hash, validator_index); + let approval = IndirectSignedApprovalVote { + block_hash: hash, + candidate_index, + validator: validator_index, + signature: dummy_signature(), + }; - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), - ) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), + ) + .await; - // connect the peer. - setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; - - let assignments = vec![(cert.clone(), candidate_index)]; - let approvals = vec![approval.clone()]; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) - )) - )) => { - assert_eq!(peers, vec![*peer]); - assert_eq!(sent_assignments, assignments); - } - ); + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(sent_approvals) - )) - )) => { - assert_eq!(peers, vec![*peer]); - assert_eq!(sent_approvals, approvals); - } - ); + // connect the peer. + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; - assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); - virtual_overseer - }); + let assignments = vec![(cert.clone(), candidate_index)]; + let approvals = vec![approval.clone()]; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) + )) + )) => { + assert_eq!(peers, vec![*peer]); + assert_eq!(sent_assignments, assignments); + } + ); + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Approvals(sent_approvals) + )) + )) => { + assert_eq!(peers, vec![*peer]); + assert_eq!(sent_approvals, approvals); + } + ); + + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }, + ); } /// Same as `sends_assignments_even_when_state_is_approved_v2` but with `VRFModuloCompact` @@ -2135,112 +2479,118 @@ fn sends_assignments_even_when_state_is_approved_v2() { let hash = Hash::repeat_byte(0xAA); let peer = &peer_a; - let _ = test_harness(State::default(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 4], - slot: 1.into(), - session: 1, - }; - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - // Setup a topology where peer_a is neighbor to current node. - setup_gossip_topology( - overseer, - make_gossip_topology(1, &peers_with_optional_peer_id, &[0], &[2], 1), - ) - .await; - - let validator_index = ValidatorIndex(0); - let cores = vec![0, 1, 2, 3]; - let candidate_bitfield: CandidateBitfield = cores.clone().try_into().unwrap(); - - let core_bitfield: CoreBitfield = cores - .iter() - .map(|index| CoreIndex(*index)) - .collect::>() - .try_into() - .unwrap(); - - let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfield.clone()); + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + State::default(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 4], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + // Setup a topology where peer_a is neighbor to current node. + setup_gossip_topology( + overseer, + make_gossip_topology(1, &peers_with_optional_peer_id, &[0], &[2], 1), + ) + .await; - // Assumes candidate index == core index. - let approvals = cores - .iter() - .map(|core| IndirectSignedApprovalVoteV2 { - block_hash: hash, - candidate_indices: (*core).into(), - validator: validator_index, - signature: dummy_signature(), - }) - .collect::>(); - - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment( - cert.clone().into(), - candidate_bitfield.clone(), - ), - ) - .await; + let validator_index = ValidatorIndex(0); + let cores = vec![0, 1, 2, 3]; + let candidate_bitfield: CandidateBitfield = cores.clone().try_into().unwrap(); + + let core_bitfield: CoreBitfield = cores + .iter() + .map(|index| CoreIndex(*index)) + .collect::>() + .try_into() + .unwrap(); + + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfield.clone()); + + // Assumes candidate index == core index. + let approvals = cores + .iter() + .map(|core| IndirectSignedApprovalVoteV2 { + block_hash: hash, + candidate_indices: (*core).into(), + validator: validator_index, + signature: dummy_signature(), + }) + .collect::>(); - for approval in &approvals { overseer_send( overseer, - ApprovalDistributionMessage::DistributeApproval(approval.clone()), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_bitfield.clone(), + ), ) .await; - } - // connect the peer. - setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V3).await; - - let assignments = vec![(cert.clone(), candidate_bitfield.clone())]; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( - protocol_v3::ApprovalDistributionMessage::Assignments(sent_assignments) - )) - )) => { - assert_eq!(peers, vec![*peer]); - assert_eq!(sent_assignments, assignments); + for approval in &approvals { + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone()), + ) + .await; } - ); - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( - protocol_v3::ApprovalDistributionMessage::Approvals(sent_approvals) - )) - )) => { - // Construct a hashmaps of approvals for comparison. Approval distribution reorders messages because they are kept in a - // hashmap as well. - let sent_approvals = sent_approvals.into_iter().map(|approval| (approval.candidate_indices.clone(), approval)).collect::>(); - let approvals = approvals.into_iter().map(|approval| (approval.candidate_indices.clone(), approval)).collect::>(); - - assert_eq!(peers, vec![*peer]); - assert_eq!(sent_approvals, approvals); - } - ); + // connect the peer. + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V3).await; - assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); - virtual_overseer - }); + let assignments = vec![(cert.clone(), candidate_bitfield.clone())]; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(sent_assignments) + )) + )) => { + assert_eq!(peers, vec![*peer]); + assert_eq!(sent_assignments, assignments); + } + ); + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Approvals(sent_approvals) + )) + )) => { + // Construct a hashmaps of approvals for comparison. Approval distribution reorders messages because they are kept in a + // hashmap as well. + let sent_approvals = sent_approvals.into_iter().map(|approval| (approval.candidate_indices.clone(), approval)).collect::>(); + let approvals = approvals.into_iter().map(|approval| (approval.candidate_indices.clone(), approval)).collect::>(); + + assert_eq!(peers, vec![*peer]); + assert_eq!(sent_approvals, approvals); + } + ); + + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }, + ); } /// @@ -2255,71 +2605,83 @@ fn race_condition_in_local_vs_remote_view_update() { let peer_a = PeerId::random(); let hash_b = Hash::repeat_byte(0xBB); - let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - let peer = &peer_a; - - // Test a small number of candidates - let candidates_count = 1; - let meta = BlockApprovalMeta { - hash: hash_b, - parent_hash, - number: 2, - candidates: vec![Default::default(); candidates_count], - slot: 1.into(), - session: 1, - }; - - // This will send a peer view that is ahead of our view - setup_peer_with_view(overseer, peer, view![hash_b], ValidationVersion::V1).await; - - // Send our view update to include a new head - overseer_send( - overseer, - ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view![hash_b], - )), - ) - .await; + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state_without_reputation_delay(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + let peer = &peer_a; + + // Test a small number of candidates + let candidates_count = 1; + let meta = BlockApprovalMeta { + hash: hash_b, + parent_hash, + number: 2, + candidates: vec![Default::default(); candidates_count], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + + // This will send a peer view that is ahead of our view + setup_peer_with_view(overseer, peer, view![hash_b], ValidationVersion::V1).await; + + // Send our view update to include a new head + overseer_send( + overseer, + ApprovalDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::OurViewChange(our_view![hash_b]), + ), + ) + .await; - // send assignments related to `hash_b` but they will come to the MessagesPending - let assignments: Vec<_> = (0..candidates_count) - .map(|candidate_index| { - let validator_index = ValidatorIndex(candidate_index as u32); - let cert = fake_assignment_cert(hash_b, validator_index); - (cert, candidate_index as u32) - }) - .collect(); + // send assignments related to `hash_b` but they will come to the MessagesPending + let assignments: Vec<_> = (0..candidates_count) + .map(|candidate_index| { + let validator_index = ValidatorIndex(candidate_index as u32); + let cert = fake_assignment_cert(hash_b, validator_index); + (cert, candidate_index as u32) + }) + .collect(); - let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, peer, msg.clone()).await; + let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer(overseer, peer, msg.clone()).await; - // This will handle pending messages being processed - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; + // This will handle pending messages being processed + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; - for i in 0..candidates_count { - // Previously, this has caused out-of-view assignments/approvals - //expect_reputation_change(overseer, peer, COST_UNEXPECTED_MESSAGE).await; + provide_session( + overseer, + dummy_session_info_valid(1, &mut LocalKeystore::in_memory(), 1), + ) + .await; - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( - assignment, - claimed_candidate_index, - tx, - )) => { - assert_eq!(assignment, assignments[i].0.clone().into()); - assert_eq!(claimed_candidate_index, assignments[i].1.into()); - tx.send(AssignmentCheckResult::Accepted).unwrap(); - } - ); + for i in 0..candidates_count { + // Previously, this has caused out-of-view assignments/approvals + //expect_reputation_change(overseer, peer, COST_UNEXPECTED_MESSAGE).await; - // Since we have a valid statement pending, this should always occur - expect_reputation_change(overseer, peer, BENEFIT_VALID_MESSAGE_FIRST).await; - } - virtual_overseer - }); + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + assignment, + claimed_candidate_index, + tranche, _, + )) => { + assert_eq!(assignment, assignments[i].0.clone().into()); + assert_eq!(claimed_candidate_index, assignments[i].1.into()); + assert_eq!(tranche, 0); + } + ); + + // Since we have a valid statement pending, this should always occur + expect_reputation_change(overseer, peer, BENEFIT_VALID_MESSAGE_FIRST).await; + } + virtual_overseer + }, + ); } // Tests that local messages propagate to both dimensions. @@ -2330,198 +2692,86 @@ fn propagates_locally_generated_assignment_to_both_dimensions() { let peers = make_peers_and_authority_ids(100); - let _ = test_harness(State::default(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - - // Connect all peers. - for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; - } - - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - - // Set up a gossip topology. - setup_gossip_topology( - overseer, - make_gossip_topology( - 1, - &peers_with_optional_peer_id, - &[0, 10, 20, 30, 40, 60, 70, 80], - &[50, 51, 52, 53, 54, 55, 56, 57], - 1, - ), - ) - .await; - - let expected_indices = [ - // Both dimensions in the gossip topology - 0, 10, 20, 30, 40, 60, 70, 80, 50, 51, 52, 53, 54, 55, 56, 57, - ]; - - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - let validator_index = ValidatorIndex(0); - let candidate_index = 0u32; - - // import an assignment and approval locally. - let cert = fake_assignment_cert(hash, validator_index); - let approval = IndirectSignedApprovalVote { - block_hash: hash, - candidate_index, - validator: validator_index, - signature: dummy_signature(), - }; - - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment( - cert.clone().into(), - candidate_index.into(), - ), - ) - .await; - - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), - ) - .await; - - let assignments = vec![(cert.clone(), candidate_index)]; - let approvals = vec![approval.clone()]; - - let mut assignment_sent_peers = assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - sent_peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) - )) - )) => { - assert_eq!(sent_peers.len(), expected_indices.len() + 4); - for &i in &expected_indices { - assert!( - sent_peers.contains(&peers[i].0), - "Message not sent to expected peer {}", - i, - ); - } - assert_eq!(sent_assignments, assignments); - sent_peers - } - ); + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + State::default(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - mut sent_peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(sent_approvals) - )) - )) => { - // Random sampling is reused from the assignment. - sent_peers.sort(); - assignment_sent_peers.sort(); - assert_eq!(sent_peers, assignment_sent_peers); - assert_eq!(sent_approvals, approvals); + // Connect all peers. + for (peer, _) in &peers { + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } - ); - - assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); - virtual_overseer - }); -} - -// Tests that messages propagate to the unshared dimension. -#[test] -fn propagates_assignments_along_unshared_dimension() { - let parent_hash = Hash::repeat_byte(0xFF); - let hash = Hash::repeat_byte(0xAA); - - let peers = make_peers_and_authority_ids(100); - let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); - // Connect all peers. - for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; - } + // Set up a gossip topology. + setup_gossip_topology( + overseer, + make_gossip_topology( + 1, + &peers_with_optional_peer_id, + &[0, 10, 20, 30, 40, 60, 70, 80], + &[50, 51, 52, 53, 54, 55, 56, 57], + 1, + ), + ) + .await; - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - - // Set up a gossip topology. - setup_gossip_topology( - overseer, - make_gossip_topology( - 1, - &peers_with_optional_peer_id, - &[0, 10, 20, 30], - &[50, 51, 52, 53], - 1, - ), - ) - .await; + let expected_indices = [ + // Both dimensions in the gossip topology + 0, 10, 20, 30, 40, 60, 70, 80, 50, 51, 52, 53, 54, 55, 56, 57, + ]; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - // Test messages from X direction go to Y peers - { let validator_index = ValidatorIndex(0); let candidate_index = 0u32; // import an assignment and approval locally. let cert = fake_assignment_cert(hash, validator_index); - let assignments = vec![(cert.clone(), candidate_index)]; + let approval = IndirectSignedApprovalVote { + block_hash: hash, + candidate_index, + validator: validator_index, + signature: dummy_signature(), + }; - let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), + ) + .await; - // Issuer of the message is important, not the peer we receive from. - // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, msg).await; - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( - _, - _, - tx, - )) => { - tx.send(AssignmentCheckResult::Accepted).unwrap(); - } - ); - expect_reputation_change(overseer, &peers[99].0, BENEFIT_VALID_MESSAGE_FIRST).await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; - let expected_y = [50, 51, 52, 53]; + let assignments = vec![(cert.clone(), candidate_index)]; + let approvals = vec![approval.clone()]; - assert_matches!( + let mut assignment_sent_peers = assert_matches!( overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( sent_peers, @@ -2529,8 +2779,8 @@ fn propagates_assignments_along_unshared_dimension() { protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) )) )) => { - assert_eq!(sent_peers.len(), expected_y.len() + 4); - for &i in &expected_y { + assert_eq!(sent_peers.len(), expected_indices.len() + 4); + for &i in &expected_indices { assert!( sent_peers.contains(&peers[i].0), "Message not sent to expected peer {}", @@ -2538,189 +2788,285 @@ fn propagates_assignments_along_unshared_dimension() { ); } assert_eq!(sent_assignments, assignments); + sent_peers } ); - }; - - // Test messages from X direction go to Y peers - { - let validator_index = ValidatorIndex(50); - let candidate_index = 0u32; - - // import an assignment and approval locally. - let cert = fake_assignment_cert(hash, validator_index); - let assignments = vec![(cert.clone(), candidate_index)]; - - let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - - // Issuer of the message is important, not the peer we receive from. - // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, msg).await; - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( - _, - _, - tx, - )) => { - tx.send(AssignmentCheckResult::Accepted).unwrap(); - } - ); - expect_reputation_change(overseer, &peers[99].0, BENEFIT_VALID_MESSAGE_FIRST).await; - - let expected_x = [0, 10, 20, 30]; assert_matches!( overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - sent_peers, + mut sent_peers, Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) + protocol_v1::ApprovalDistributionMessage::Approvals(sent_approvals) )) )) => { - assert_eq!(sent_peers.len(), expected_x.len() + 4); - for &i in &expected_x { - assert!( - sent_peers.contains(&peers[i].0), - "Message not sent to expected peer {}", - i, - ); - } - assert_eq!(sent_assignments, assignments); + // Random sampling is reused from the assignment. + sent_peers.sort(); + assignment_sent_peers.sort(); + assert_eq!(sent_peers, assignment_sent_peers); + assert_eq!(sent_approvals, approvals); } ); - }; - assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); - virtual_overseer - }); + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }, + ); } -// tests that messages are propagated to necessary peers after they connect +// Tests that messages propagate to the unshared dimension. #[test] -fn propagates_to_required_after_connect() { +fn propagates_assignments_along_unshared_dimension() { let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); let peers = make_peers_and_authority_ids(100); - let _ = test_harness(State::default(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - - let omitted = [0, 10, 50, 51]; + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state_without_reputation_delay(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; - // Connect all peers except omitted. - for (i, (peer, _)) in peers.iter().enumerate() { - if !omitted.contains(&i) { + // Connect all peers. + for (peer, _) in &peers { setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } - } - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - // Set up a gossip topology. - setup_gossip_topology( - overseer, - make_gossip_topology( - 1, - &peers_with_optional_peer_id, - &[0, 10, 20, 30, 40, 60, 70, 80], - &[50, 51, 52, 53, 54, 55, 56, 57], - 1, - ), - ) - .await; - let expected_indices = [ - // Both dimensions in the gossip topology, minus omitted. - 20, 30, 40, 60, 70, 80, 52, 53, 54, 55, 56, 57, - ]; - - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - let validator_index = ValidatorIndex(0); - let candidate_index = 0u32; - - // import an assignment and approval locally. - let cert = fake_assignment_cert(hash, validator_index); - let approval = IndirectSignedApprovalVote { - block_hash: hash, - candidate_index, - validator: validator_index, - signature: dummy_signature(), - }; - - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment( - cert.clone().into(), - candidate_index.into(), - ), - ) - .await; + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), - ) - .await; + // Set up a gossip topology. + setup_gossip_topology( + overseer, + make_gossip_topology( + 1, + &peers_with_optional_peer_id, + &[0, 10, 20, 30], + &[50, 51, 52, 53], + 1, + ), + ) + .await; - let assignments = vec![(cert.clone(), candidate_index)]; - let approvals = vec![approval.clone()]; - - let mut assignment_sent_peers = assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - sent_peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) - )) - )) => { - assert_eq!(sent_peers.len(), expected_indices.len() + 4); - for &i in &expected_indices { - assert!( - sent_peers.contains(&peers[i].0), - "Message not sent to expected peer {}", - i, - ); + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // Test messages from X direction go to Y peers + { + let validator_index = ValidatorIndex(0); + let candidate_index = 0u32; + + // import an assignment and approval locally. + let cert = fake_assignment_cert(hash, validator_index); + let assignments = vec![(cert.clone(), candidate_index)]; + + let msg = + protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); + + // Issuer of the message is important, not the peer we receive from. + // 99 deliberately chosen because it's not in X or Y. + send_message_from_peer(overseer, &peers[99].0, msg).await; + provide_session( + overseer, + dummy_session_info_valid(1, &mut LocalKeystore::in_memory(), 1), + ) + .await; + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + _, + _, + tranche, _, + )) => { + assert_eq!(tranche, 0); + } + ); + expect_reputation_change(overseer, &peers[99].0, BENEFIT_VALID_MESSAGE_FIRST).await; + + let expected_y = [50, 51, 52, 53]; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + sent_peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) + )) + )) => { + assert_eq!(sent_peers.len(), expected_y.len() + 4); + for &i in &expected_y { + assert!( + sent_peers.contains(&peers[i].0), + "Message not sent to expected peer {}", + i, + ); + } + assert_eq!(sent_assignments, assignments); + } + ); + }; + + // Test messages from X direction go to Y peers + { + let validator_index = ValidatorIndex(50); + let candidate_index = 0u32; + + // import an assignment and approval locally. + let cert = fake_assignment_cert(hash, validator_index); + let assignments = vec![(cert.clone(), candidate_index)]; + + let msg = + protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); + + // Issuer of the message is important, not the peer we receive from. + // 99 deliberately chosen because it's not in X or Y. + send_message_from_peer(overseer, &peers[99].0, msg).await; + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + _, + _, + tranche, _, + )) => { + assert_eq!(tranche, 0); + } + ); + expect_reputation_change(overseer, &peers[99].0, BENEFIT_VALID_MESSAGE_FIRST).await; + + let expected_x = [0, 10, 20, 30]; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + sent_peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) + )) + )) => { + assert_eq!(sent_peers.len(), expected_x.len() + 4); + for &i in &expected_x { + assert!( + sent_peers.contains(&peers[i].0), + "Message not sent to expected peer {}", + i, + ); + } + assert_eq!(sent_assignments, assignments); + } + ); + }; + + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }, + ); +} + +// tests that messages are propagated to necessary peers after they connect +#[test] +fn propagates_to_required_after_connect() { + let parent_hash = Hash::repeat_byte(0xFF); + let hash = Hash::repeat_byte(0xAA); + + let peers = make_peers_and_authority_ids(100); + + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + State::default(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + + let omitted = [0, 10, 50, 51]; + + // Connect all peers except omitted. + for (i, (peer, _)) in peers.iter().enumerate() { + if !omitted.contains(&i) { + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } - assert_eq!(sent_assignments, assignments); - sent_peers } - ); + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + // Set up a gossip topology. + setup_gossip_topology( + overseer, + make_gossip_topology( + 1, + &peers_with_optional_peer_id, + &[0, 10, 20, 30, 40, 60, 70, 80], + &[50, 51, 52, 53, 54, 55, 56, 57], + 1, + ), + ) + .await; - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - mut sent_peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(sent_approvals) - )) - )) => { - // Random sampling is reused from the assignment. - sent_peers.sort(); - assignment_sent_peers.sort(); - assert_eq!(sent_peers, assignment_sent_peers); - assert_eq!(sent_approvals, approvals); - } - ); + let expected_indices = [ + // Both dimensions in the gossip topology, minus omitted. + 20, 30, 40, 60, 70, 80, 52, 53, 54, 55, 56, 57, + ]; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; - for i in omitted.iter().copied() { - setup_peer_with_view(overseer, &peers[i].0, view![hash], ValidationVersion::V1).await; + let validator_index = ValidatorIndex(0); + let candidate_index = 0u32; - assert_matches!( + // import an assignment and approval locally. + let cert = fake_assignment_cert(hash, validator_index); + let approval = IndirectSignedApprovalVote { + block_hash: hash, + candidate_index, + validator: validator_index, + signature: dummy_signature(), + }; + + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), + ) + .await; + + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; + + let assignments = vec![(cert.clone(), candidate_index)]; + let approvals = vec![approval.clone()]; + + let mut assignment_sent_peers = assert_matches!( overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( sent_peers, @@ -2728,30 +3074,72 @@ fn propagates_to_required_after_connect() { protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) )) )) => { - assert_eq!(sent_peers.len(), 1); - assert_eq!(&sent_peers[0], &peers[i].0); + assert_eq!(sent_peers.len(), expected_indices.len() + 4); + for &i in &expected_indices { + assert!( + sent_peers.contains(&peers[i].0), + "Message not sent to expected peer {}", + i, + ); + } assert_eq!(sent_assignments, assignments); + sent_peers } ); assert_matches!( overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - sent_peers, + mut sent_peers, Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( protocol_v1::ApprovalDistributionMessage::Approvals(sent_approvals) )) )) => { - assert_eq!(sent_peers.len(), 1); - assert_eq!(&sent_peers[0], &peers[i].0); + // Random sampling is reused from the assignment. + sent_peers.sort(); + assignment_sent_peers.sort(); + assert_eq!(sent_peers, assignment_sent_peers); assert_eq!(sent_approvals, approvals); } ); - } - assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); - virtual_overseer - }); + for i in omitted.iter().copied() { + setup_peer_with_view(overseer, &peers[i].0, view![hash], ValidationVersion::V1) + .await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + sent_peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) + )) + )) => { + assert_eq!(sent_peers.len(), 1); + assert_eq!(&sent_peers[0], &peers[i].0); + assert_eq!(sent_assignments, assignments); + } + ); + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + sent_peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Approvals(sent_approvals) + )) + )) => { + assert_eq!(sent_peers.len(), 1); + assert_eq!(&sent_peers[0], &peers[i].0); + assert_eq!(sent_approvals, approvals); + } + ); + } + + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }, + ); } // test that new gossip topology triggers send of messages. @@ -2762,124 +3150,130 @@ fn sends_to_more_peers_after_getting_topology() { let peers = make_peers_and_authority_ids(100); - let _ = test_harness(State::default(), |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + State::default(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; - // Connect all peers except omitted. - for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; - } + // Connect all peers except omitted. + for (peer, _) in &peers { + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; + } - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - let validator_index = ValidatorIndex(0); - let candidate_index = 0u32; - - // import an assignment and approval locally. - let cert = fake_assignment_cert(hash, validator_index); - let approval = IndirectSignedApprovalVote { - block_hash: hash, - candidate_index, - validator: validator_index, - signature: dummy_signature(), - }; - - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment( - cert.clone().into(), - candidate_index.into(), - ), - ) - .await; + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), - ) - .await; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; - let assignments = vec![(cert.clone(), candidate_index)]; - let approvals = vec![approval.clone()]; - - let expected_indices = vec![0, 10, 20, 30, 50, 51, 52, 53]; - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - // Set up a gossip topology. - setup_gossip_topology( - overseer, - make_gossip_topology( - 1, - &peers_with_optional_peer_id, - &[0, 10, 20, 30], - &[50, 51, 52, 53], - 1, - ), - ) - .await; + let validator_index = ValidatorIndex(0); + let candidate_index = 0u32; + + // import an assignment and approval locally. + let cert = fake_assignment_cert(hash, validator_index); + let approval = IndirectSignedApprovalVote { + block_hash: hash, + candidate_index, + validator: validator_index, + signature: dummy_signature(), + }; - let mut expected_indices_assignments = expected_indices.clone(); - let mut expected_indices_approvals = expected_indices.clone(); + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), + ) + .await; - for _ in 0..expected_indices_assignments.len() { - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - sent_peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) - )) - )) => { - // Sends to all expected peers. - assert_eq!(sent_peers.len(), 1); - assert_eq!(sent_assignments, assignments); + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; + + let assignments = vec![(cert.clone(), candidate_index)]; + let approvals = vec![approval.clone()]; + + let expected_indices = vec![0, 10, 20, 30, 50, 51, 52, 53]; + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + // Set up a gossip topology. + setup_gossip_topology( + overseer, + make_gossip_topology( + 1, + &peers_with_optional_peer_id, + &[0, 10, 20, 30], + &[50, 51, 52, 53], + 1, + ), + ) + .await; + + let mut expected_indices_assignments = expected_indices.clone(); + let mut expected_indices_approvals = expected_indices.clone(); + + for _ in 0..expected_indices_assignments.len() { + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + sent_peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) + )) + )) => { + // Sends to all expected peers. + assert_eq!(sent_peers.len(), 1); + assert_eq!(sent_assignments, assignments); - let pos = expected_indices_assignments.iter() - .position(|i| &peers[*i].0 == &sent_peers[0]) - .unwrap(); - expected_indices_assignments.remove(pos); - } - ); - } + let pos = expected_indices_assignments.iter() + .position(|i| &peers[*i].0 == &sent_peers[0]) + .unwrap(); + expected_indices_assignments.remove(pos); + } + ); + } - for _ in 0..expected_indices_approvals.len() { - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - sent_peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(sent_approvals) - )) - )) => { - // Sends to all expected peers. - assert_eq!(sent_peers.len(), 1); - assert_eq!(sent_approvals, approvals); + for _ in 0..expected_indices_approvals.len() { + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + sent_peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Approvals(sent_approvals) + )) + )) => { + // Sends to all expected peers. + assert_eq!(sent_peers.len(), 1); + assert_eq!(sent_approvals, approvals); - let pos = expected_indices_approvals.iter() - .position(|i| &peers[*i].0 == &sent_peers[0]) - .unwrap(); + let pos = expected_indices_approvals.iter() + .position(|i| &peers[*i].0 == &sent_peers[0]) + .unwrap(); - expected_indices_approvals.remove(pos); - } - ); - } + expected_indices_approvals.remove(pos); + } + ); + } - assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); - virtual_overseer - }); + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }, + ); } // test aggression L1 @@ -2894,284 +3288,178 @@ fn originator_aggression_l1() { state.aggression_config.resend_unfinalized_period = None; let aggression_l1_threshold = state.aggression_config.l1_threshold.unwrap(); - let _ = test_harness(state, |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state, + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; - // Connect all peers except omitted. - for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; - } + // Connect all peers except omitted. + for (peer, _) in &peers { + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; + } - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - let validator_index = ValidatorIndex(0); - let candidate_index = 0u32; - - // import an assignment and approval locally. - let cert = fake_assignment_cert(hash, validator_index); - let approval = IndirectSignedApprovalVote { - block_hash: hash, - candidate_index, - validator: validator_index, - signature: dummy_signature(), - }; - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - // Set up a gossip topology. - setup_gossip_topology( - overseer, - make_gossip_topology( - 1, - &peers_with_optional_peer_id, - &[0, 10, 20, 30], - &[50, 51, 52, 53], - 1, - ), - ) - .await; + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment( - cert.clone().into(), - candidate_index.into(), - ), - ) - .await; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), - ) - .await; + let validator_index = ValidatorIndex(0); + let candidate_index = 0u32; - let assignments = vec![(cert.clone(), candidate_index)]; - let approvals = vec![approval.clone()]; - - let prev_sent_indices = assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - sent_peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(_) - )) - )) => { - sent_peers.into_iter() - .filter_map(|sp| peers.iter().position(|p| &p.0 == &sp)) - .collect::>() - } - ); + // import an assignment and approval locally. + let cert = fake_assignment_cert(hash, validator_index); + let approval = IndirectSignedApprovalVote { + block_hash: hash, + candidate_index, + validator: validator_index, + signature: dummy_signature(), + }; + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + // Set up a gossip topology. + setup_gossip_topology( + overseer, + make_gossip_topology( + 1, + &peers_with_optional_peer_id, + &[0, 10, 20, 30], + &[50, 51, 52, 53], + 1, + ), + ) + .await; - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - _, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(_) - )) - )) => { } - ); + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), + ) + .await; - // Add blocks until aggression L1 is triggered. - { - let mut parent_hash = hash; - for level in 0..aggression_l1_threshold { - let number = 1 + level + 1; // first block had number 1 - let hash = BlakeTwo256::hash_of(&(parent_hash, number)); - let meta = BlockApprovalMeta { - hash, - parent_hash, - number, - candidates: vec![], - slot: (level as u64).into(), - session: 1, - }; - - let msg = ApprovalDistributionMessage::ApprovalCheckingLagUpdate(level + 1); - overseer_send(overseer, msg).await; - - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - parent_hash = hash; - } - } + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; - let unsent_indices = - (0..peers.len()).filter(|i| !prev_sent_indices.contains(&i)).collect::>(); + let assignments = vec![(cert.clone(), candidate_index)]; + let approvals = vec![approval.clone()]; - for _ in 0..unsent_indices.len() { - assert_matches!( + let prev_sent_indices = assert_matches!( overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( sent_peers, Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) + protocol_v1::ApprovalDistributionMessage::Assignments(_) )) )) => { - // Sends to all expected peers. - assert_eq!(sent_peers.len(), 1); - assert_eq!(sent_assignments, assignments); - - assert!(unsent_indices.iter() - .any(|i| &peers[*i].0 == &sent_peers[0])); + sent_peers.into_iter() + .filter_map(|sp| peers.iter().position(|p| &p.0 == &sp)) + .collect::>() } ); - } - for _ in 0..unsent_indices.len() { assert_matches!( overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - sent_peers, + _, Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(sent_approvals) + protocol_v1::ApprovalDistributionMessage::Approvals(_) )) - )) => { - // Sends to all expected peers. - assert_eq!(sent_peers.len(), 1); - assert_eq!(sent_approvals, approvals); - - assert!(unsent_indices.iter() - .any(|i| &peers[*i].0 == &sent_peers[0])); - } + )) => { } ); - } - - assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); - virtual_overseer - }); -} - -// test aggression L1 -#[test] -fn non_originator_aggression_l1() { - let parent_hash = Hash::repeat_byte(0xFF); - let hash = Hash::repeat_byte(0xAA); - let peers = make_peers_and_authority_ids(100); + // Add blocks until aggression L1 is triggered. + { + let mut parent_hash = hash; + for level in 0..aggression_l1_threshold { + let number = 1 + level + 1; // first block had number 1 + let hash = BlakeTwo256::hash_of(&(parent_hash, number)); + let meta = BlockApprovalMeta { + hash, + parent_hash, + number, + candidates: vec![], + slot: (level as u64).into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; - let mut state = state_without_reputation_delay(); - state.aggression_config.resend_unfinalized_period = None; - let aggression_l1_threshold = state.aggression_config.l1_threshold.unwrap(); + let msg = ApprovalDistributionMessage::ApprovalCheckingLagUpdate(level + 1); + overseer_send(overseer, msg).await; - let _ = test_harness(state, |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; - // Connect all peers except omitted. - for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; - } + parent_hash = hash; + } + } - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - let validator_index = ValidatorIndex(0); - let candidate_index = 0u32; - - // import an assignment and approval locally. - let cert = fake_assignment_cert(hash, validator_index); - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - // Set up a gossip topology. - setup_gossip_topology( - overseer, - make_gossip_topology( - 1, - &peers_with_optional_peer_id, - &[0, 10, 20, 30], - &[50, 51, 52, 53], - 1, - ), - ) - .await; + let unsent_indices = + (0..peers.len()).filter(|i| !prev_sent_indices.contains(&i)).collect::>(); - let assignments = vec![(cert.clone().into(), candidate_index)]; - let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); + for _ in 0..unsent_indices.len() { + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + sent_peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) + )) + )) => { + // Sends to all expected peers. + assert_eq!(sent_peers.len(), 1); + assert_eq!(sent_assignments, assignments); - // Issuer of the message is important, not the peer we receive from. - // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, msg).await; - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( - _, - _, - tx, - )) => { - tx.send(AssignmentCheckResult::Accepted).unwrap(); + assert!(unsent_indices.iter() + .any(|i| &peers[*i].0 == &sent_peers[0])); + } + ); } - ); - - expect_reputation_change(overseer, &peers[99].0, BENEFIT_VALID_MESSAGE_FIRST).await; - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - _, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(_) - )) - )) => { } - ); + for _ in 0..unsent_indices.len() { + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + sent_peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Approvals(sent_approvals) + )) + )) => { + // Sends to all expected peers. + assert_eq!(sent_peers.len(), 1); + assert_eq!(sent_approvals, approvals); - // Add blocks until aggression L1 is triggered. - { - let mut parent_hash = hash; - for level in 0..aggression_l1_threshold { - let number = 1 + level + 1; // first block had number 1 - let hash = BlakeTwo256::hash_of(&(parent_hash, number)); - let meta = BlockApprovalMeta { - hash, - parent_hash, - number, - candidates: vec![], - slot: (level as u64).into(), - session: 1, - }; - - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - parent_hash = hash; + assert!(unsent_indices.iter() + .any(|i| &peers[*i].0 == &sent_peers[0])); + } + ); } - } - - // No-op on non-originator - assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); - virtual_overseer - }); + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }, + ); } -// test aggression L2 on non-originator +// test aggression L1 #[test] -fn non_originator_aggression_l2() { +fn non_originator_aggression_l1() { let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); @@ -3179,275 +3467,256 @@ fn non_originator_aggression_l2() { let mut state = state_without_reputation_delay(); state.aggression_config.resend_unfinalized_period = None; - let aggression_l1_threshold = state.aggression_config.l1_threshold.unwrap(); - let aggression_l2_threshold = state.aggression_config.l2_threshold.unwrap(); - let _ = test_harness(state, |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // Connect all peers except omitted. - for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; - } + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state, + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - let validator_index = ValidatorIndex(0); - let candidate_index = 0u32; - - // import an assignment and approval locally. - let cert = fake_assignment_cert(hash, validator_index); - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - // Set up a gossip topology. - setup_gossip_topology( - overseer, - make_gossip_topology( - 1, - &peers_with_optional_peer_id, - &[0, 10, 20, 30], - &[50, 51, 52, 53], - 1, - ), - ) - .await; + // Connect all peers except omitted. + for (peer, _) in &peers { + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; + } - let assignments = vec![(cert.clone(), candidate_index)]; - let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; - // Issuer of the message is important, not the peer we receive from. - // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, msg).await; - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( - _, - _, - tx, - )) => { - tx.send(AssignmentCheckResult::Accepted).unwrap(); - } - ); + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; - expect_reputation_change(overseer, &peers[99].0, BENEFIT_VALID_MESSAGE_FIRST).await; - - let prev_sent_indices = assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - sent_peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(_) - )) - )) => { - sent_peers.into_iter() - .filter_map(|sp| peers.iter().position(|p| &p.0 == &sp)) - .collect::>() - } - ); + let validator_index = ValidatorIndex(0); + let candidate_index = 0u32; - // Add blocks until aggression L1 is triggered. - let chain_head = { - let mut parent_hash = hash; - for level in 0..aggression_l1_threshold { - let number = 1 + level + 1; // first block had number 1 - let hash = BlakeTwo256::hash_of(&(parent_hash, number)); - let meta = BlockApprovalMeta { - hash, - parent_hash, - number, - candidates: vec![], - slot: (level as u64).into(), - session: 1, - }; - - let msg = ApprovalDistributionMessage::ApprovalCheckingLagUpdate(level + 1); - overseer_send(overseer, msg).await; - - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - parent_hash = hash; - } + // import an assignment and approval locally. + let cert = fake_assignment_cert(hash, validator_index); + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + // Set up a gossip topology. + setup_gossip_topology( + overseer, + make_gossip_topology( + 1, + &peers_with_optional_peer_id, + &[0, 10, 20, 30], + &[50, 51, 52, 53], + 1, + ), + ) + .await; - parent_hash - }; - - // No-op on non-originator - - // Add blocks until aggression L2 is triggered. - { - let mut parent_hash = chain_head; - for level in 0..aggression_l2_threshold - aggression_l1_threshold { - let number = aggression_l1_threshold + level + 1 + 1; // first block had number 1 - let hash = BlakeTwo256::hash_of(&(parent_hash, number)); - let meta = BlockApprovalMeta { - hash, - parent_hash, - number, - candidates: vec![], - slot: (level as u64).into(), - session: 1, - }; - - let msg = ApprovalDistributionMessage::ApprovalCheckingLagUpdate( - aggression_l1_threshold + level + 1, - ); - overseer_send(overseer, msg).await; - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; + let assignments = vec![(cert.clone().into(), candidate_index)]; + let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - parent_hash = hash; - } - } + // Issuer of the message is important, not the peer we receive from. + // 99 deliberately chosen because it's not in X or Y. + send_message_from_peer(overseer, &peers[99].0, msg).await; + provide_session( + overseer, + dummy_session_info_valid(1, &mut LocalKeystore::in_memory(), 1), + ) + .await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + _, + _, + tranche, _, + )) => { + assert_eq!(tranche, 0); + } + ); - // XY dimension - previously sent. - let unsent_indices = [0, 10, 20, 30, 50, 51, 52, 53] - .iter() - .cloned() - .filter(|i| !prev_sent_indices.contains(&i)) - .collect::>(); + expect_reputation_change(overseer, &peers[99].0, BENEFIT_VALID_MESSAGE_FIRST).await; - for _ in 0..unsent_indices.len() { assert_matches!( overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - sent_peers, + _, Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) + protocol_v1::ApprovalDistributionMessage::Assignments(_) )) - )) => { - // Sends to all expected peers. - assert_eq!(sent_peers.len(), 1); - assert_eq!(sent_assignments, assignments); + )) => { } + ); + + // Add blocks until aggression L1 is triggered. + { + let mut parent_hash = hash; + for level in 0..aggression_l1_threshold { + let number = 1 + level + 1; // first block had number 1 + let hash = BlakeTwo256::hash_of(&(parent_hash, number)); + let meta = BlockApprovalMeta { + hash, + parent_hash, + number, + candidates: vec![], + slot: (level as u64).into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; - assert!(unsent_indices.iter() - .any(|i| &peers[*i].0 == &sent_peers[0])); + parent_hash = hash; } - ); - } + } - assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); - virtual_overseer - }); + // No-op on non-originator + + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }, + ); } -// Tests that messages propagate to the unshared dimension. +// test aggression L2 on non-originator #[test] -fn resends_messages_periodically() { +fn non_originator_aggression_l2() { let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); let peers = make_peers_and_authority_ids(100); let mut state = state_without_reputation_delay(); - state.aggression_config.l1_threshold = None; - state.aggression_config.l2_threshold = None; - state.aggression_config.resend_unfinalized_period = Some(2); - let _ = test_harness(state, |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - - // Connect all peers. - for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; - } - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - // Set up a gossip topology. - setup_gossip_topology( - overseer, - make_gossip_topology( - 1, - &peers_with_optional_peer_id, - &[0, 10, 20, 30], - &[50, 51, 52, 53], - 1, - ), - ) - .await; + state.aggression_config.resend_unfinalized_period = None; - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; + let aggression_l1_threshold = state.aggression_config.l1_threshold.unwrap(); + let aggression_l2_threshold = state.aggression_config.l2_threshold.unwrap(); + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state, + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + + // Connect all peers except omitted. + for (peer, _) in &peers { + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; + } - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; - let validator_index = ValidatorIndex(0); - let candidate_index = 0u32; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; - // import an assignment and approval locally. - let cert = fake_assignment_cert(hash, validator_index); - let assignments = vec![(cert.clone(), candidate_index)]; + let validator_index = ValidatorIndex(0); + let candidate_index = 0u32; + + // import an assignment and approval locally. + let cert = fake_assignment_cert(hash, validator_index); + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + // Set up a gossip topology. + setup_gossip_topology( + overseer, + make_gossip_topology( + 1, + &peers_with_optional_peer_id, + &[0, 10, 20, 30], + &[50, 51, 52, 53], + 1, + ), + ) + .await; - { + let assignments = vec![(cert.clone(), candidate_index)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); // Issuer of the message is important, not the peer we receive from. // 99 deliberately chosen because it's not in X or Y. send_message_from_peer(overseer, &peers[99].0, msg).await; + provide_session( + overseer, + dummy_session_info_valid(1, &mut LocalKeystore::in_memory(), 1), + ) + .await; assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( _, _, - tx, + tranche, _, )) => { - tx.send(AssignmentCheckResult::Accepted).unwrap(); + assert_eq!(tranche, 0); } ); - expect_reputation_change(overseer, &peers[99].0, BENEFIT_VALID_MESSAGE_FIRST).await; - let expected_y = [50, 51, 52, 53]; + expect_reputation_change(overseer, &peers[99].0, BENEFIT_VALID_MESSAGE_FIRST).await; - assert_matches!( + let prev_sent_indices = assert_matches!( overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( sent_peers, Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) + protocol_v1::ApprovalDistributionMessage::Assignments(_) )) )) => { - assert_eq!(sent_peers.len(), expected_y.len() + 4); - for &i in &expected_y { - assert!( - sent_peers.contains(&peers[i].0), - "Message not sent to expected peer {}", - i, - ); - } - assert_eq!(sent_assignments, assignments); + sent_peers.into_iter() + .filter_map(|sp| peers.iter().position(|p| &p.0 == &sp)) + .collect::>() } ); - }; - let mut number = 1; - for _ in 0..10 { - // Add blocks until resend is done. - { + // Add blocks until aggression L1 is triggered. + let chain_head = { let mut parent_hash = hash; - for level in 0..2 { - number = number + 1; + for level in 0..aggression_l1_threshold { + let number = 1 + level + 1; // first block had number 1 + let hash = BlakeTwo256::hash_of(&(parent_hash, number)); + let meta = BlockApprovalMeta { + hash, + parent_hash, + number, + candidates: vec![], + slot: (level as u64).into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + + let msg = ApprovalDistributionMessage::ApprovalCheckingLagUpdate(level + 1); + overseer_send(overseer, msg).await; + + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + parent_hash = hash; + } + + parent_hash + }; + + // No-op on non-originator + + // Add blocks until aggression L2 is triggered. + { + let mut parent_hash = chain_head; + for level in 0..aggression_l2_threshold - aggression_l1_threshold { + let number = aggression_l1_threshold + level + 1 + 1; // first block had number 1 let hash = BlakeTwo256::hash_of(&(parent_hash, number)); let meta = BlockApprovalMeta { hash, @@ -3456,9 +3725,12 @@ fn resends_messages_periodically() { candidates: vec![], slot: (level as u64).into(), session: 1, + vrf_story: RelayVRFStory(Default::default()), }; - let msg = ApprovalDistributionMessage::ApprovalCheckingLagUpdate(2); + let msg = ApprovalDistributionMessage::ApprovalCheckingLagUpdate( + aggression_l1_threshold + level + 1, + ); overseer_send(overseer, msg).await; let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); overseer_send(overseer, msg).await; @@ -3467,10 +3739,14 @@ fn resends_messages_periodically() { } } - let mut expected_y = vec![50, 51, 52, 53]; + // XY dimension - previously sent. + let unsent_indices = [0, 10, 20, 30, 50, 51, 52, 53] + .iter() + .cloned() + .filter(|i| !prev_sent_indices.contains(&i)) + .collect::>(); - // Expect messages sent only to topology peers, one by one. - for _ in 0..expected_y.len() { + for _ in 0..unsent_indices.len() { assert_matches!( overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( @@ -3479,21 +3755,186 @@ fn resends_messages_periodically() { protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) )) )) => { + // Sends to all expected peers. assert_eq!(sent_peers.len(), 1); - let expected_pos = expected_y.iter() - .position(|&i| &peers[i].0 == &sent_peers[0]) - .unwrap(); + assert_eq!(sent_assignments, assignments); + + assert!(unsent_indices.iter() + .any(|i| &peers[*i].0 == &sent_peers[0])); + } + ); + } + + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }, + ); +} + +// Tests that messages propagate to the unshared dimension. +#[test] +fn resends_messages_periodically() { + let parent_hash = Hash::repeat_byte(0xFF); + let hash = Hash::repeat_byte(0xAA); + + let peers = make_peers_and_authority_ids(100); + + let mut state = state_without_reputation_delay(); + state.aggression_config.l1_threshold = None; + state.aggression_config.l2_threshold = None; + state.aggression_config.resend_unfinalized_period = Some(2); + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state, + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + + // Connect all peers. + for (peer, _) in &peers { + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; + } + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + // Set up a gossip topology. + setup_gossip_topology( + overseer, + make_gossip_topology( + 1, + &peers_with_optional_peer_id, + &[0, 10, 20, 30], + &[50, 51, 52, 53], + 1, + ), + ) + .await; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + let validator_index = ValidatorIndex(0); + let candidate_index = 0u32; + + // import an assignment and approval locally. + let cert = fake_assignment_cert(hash, validator_index); + let assignments = vec![(cert.clone(), candidate_index)]; + + { + let msg = + protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); + + // Issuer of the message is important, not the peer we receive from. + // 99 deliberately chosen because it's not in X or Y. + send_message_from_peer(overseer, &peers[99].0, msg).await; + provide_session( + overseer, + dummy_session_info_valid(1, &mut LocalKeystore::in_memory(), 1), + ) + .await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + _, + _, + tranche, _, + )) => { + assert_eq!(tranche, 0); + } + ); + expect_reputation_change(overseer, &peers[99].0, BENEFIT_VALID_MESSAGE_FIRST).await; + + let expected_y = [50, 51, 52, 53]; - expected_y.remove(expected_pos); + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + sent_peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) + )) + )) => { + assert_eq!(sent_peers.len(), expected_y.len() + 4); + for &i in &expected_y { + assert!( + sent_peers.contains(&peers[i].0), + "Message not sent to expected peer {}", + i, + ); + } assert_eq!(sent_assignments, assignments); } ); + }; + + let mut number = 1; + for _ in 0..10 { + // Add blocks until resend is done. + { + let mut parent_hash = hash; + for level in 0..2 { + number = number + 1; + let hash = BlakeTwo256::hash_of(&(parent_hash, number)); + let meta = BlockApprovalMeta { + hash, + parent_hash, + number, + candidates: vec![], + slot: (level as u64).into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + + let msg = ApprovalDistributionMessage::ApprovalCheckingLagUpdate(2); + overseer_send(overseer, msg).await; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + parent_hash = hash; + } + } + + let mut expected_y = vec![50, 51, 52, 53]; + + // Expect messages sent only to topology peers, one by one. + for _ in 0..expected_y.len() { + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + sent_peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) + )) + )) => { + assert_eq!(sent_peers.len(), 1); + let expected_pos = expected_y.iter() + .position(|&i| &peers[i].0 == &sent_peers[0]) + .unwrap(); + + expected_y.remove(expected_pos); + assert_eq!(sent_assignments, assignments); + } + ); + } } - } - assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); - virtual_overseer - }); + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }, + ); } /// Tests that peers correctly receive versioned messages. @@ -3507,154 +3948,170 @@ fn import_versioned_approval() { let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); let state = state_without_reputation_delay(); - let _ = test_harness(state, |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // All peers are aware of relay parent. - setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::V2).await; - setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V1).await; - setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V2).await; - - // Set up a gossip topology, where a, b, c and d are topology neighbors to the node under - // testing. - let peers_with_optional_peer_id = peers - .iter() - .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) - .collect_vec(); - setup_gossip_topology( - overseer, - make_gossip_topology(1, &peers_with_optional_peer_id, &[0, 1], &[2, 4], 3), - ) - .await; + let candidate_hash = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xBB)); + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(SystemClock {}), + state, + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // All peers are aware of relay parent. + setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::V2).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V2).await; + + // Set up a gossip topology, where a, b, c and d are topology neighbors to the node + // under testing. + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + setup_gossip_topology( + overseer, + make_gossip_topology(1, &peers_with_optional_peer_id, &[0, 1], &[2, 4], 3), + ) + .await; - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - // import an assignment related to `hash` locally - let validator_index = ValidatorIndex(0); - let candidate_index = 0u32; - let cert = fake_assignment_cert(hash, validator_index); - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.into(), candidate_index.into()), - ) - .await; + let mut keystore = LocalKeystore::in_memory(); + let session = dummy_session_info_valid(1, &mut keystore, 1); + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![(candidate_hash, 0.into(), 0.into()); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // import an assignment related to `hash` locally + let validator_index = ValidatorIndex(0); + let candidate_index = 0u32; + let cert = fake_assignment_cert(hash, validator_index); + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment( + cert.into(), + candidate_index.into(), + ), + ) + .await; - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert_eq!(peers, vec![peer_b]); - assert_eq!(assignments.len(), 1); - } - ); + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers, vec![peer_b]); + assert_eq!(assignments.len(), 1); + } + ); - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V2(protocol_v2::ValidationProtocol::ApprovalDistribution( - protocol_v2::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert_eq!(peers.len(), 2); - assert!(peers.contains(&peer_a)); - assert!(peers.contains(&peer_c)); - - assert_eq!(assignments.len(), 1); - } - ); + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V2(protocol_v2::ValidationProtocol::ApprovalDistribution( + protocol_v2::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 2); + assert!(peers.contains(&peer_a)); + assert!(peers.contains(&peer_c)); - // send the an approval from peer_a - let approval = IndirectSignedApprovalVote { - block_hash: hash, - candidate_index, - validator: validator_index, - signature: dummy_signature(), - }; - let msg = protocol_v2::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v2(overseer, &peer_a, msg).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( - vote, - tx, - )) => { - assert_eq!(vote, approval.into()); - tx.send(ApprovalCheckResult::Accepted).unwrap(); - } - ); + assert_eq!(assignments.len(), 1); + } + ); - expect_reputation_change(overseer, &peer_a, BENEFIT_VALID_MESSAGE_FIRST).await; - - // Peers b and c receive versioned approval messages. - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(approvals) - )) - )) => { - assert_eq!(peers, vec![peer_b]); - assert_eq!(approvals.len(), 1); - } - ); + // send the an approval from peer_a + let approval = IndirectSignedApprovalVote { + block_hash: hash, + candidate_index, + validator: validator_index, + signature: signature_for( + &keystore, + &session, + vec![candidate_hash], + validator_index, + ), + }; + let msg = protocol_v2::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, &peer_a, msg).await; + provide_session(overseer, session).await; + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + vote, _, + )) => { + assert_eq!(vote, approval.into()); + } + ); - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V2(protocol_v2::ValidationProtocol::ApprovalDistribution( - protocol_v2::ApprovalDistributionMessage::Approvals(approvals) - )) - )) => { - assert_eq!(peers, vec![peer_c]); - assert_eq!(approvals.len(), 1); - } - ); + expect_reputation_change(overseer, &peer_a, BENEFIT_VALID_MESSAGE_FIRST).await; + + // Peers b and c receive versioned approval messages. + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Approvals(approvals) + )) + )) => { + assert_eq!(peers, vec![peer_b]); + assert_eq!(approvals.len(), 1); + } + ); + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V2(protocol_v2::ValidationProtocol::ApprovalDistribution( + protocol_v2::ApprovalDistributionMessage::Approvals(approvals) + )) + )) => { + assert_eq!(peers, vec![peer_c]); + assert_eq!(approvals.len(), 1); + } + ); + + // send an obviously invalid approval + let approval = IndirectSignedApprovalVote { + block_hash: hash, + // Invalid candidate index, should not pass sanitization. + candidate_index: 16777284, + validator: validator_index, + signature: dummy_signature(), + }; + let msg = protocol_v2::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, &peer_a, msg).await; - // send an obviously invalid approval - let approval = IndirectSignedApprovalVote { - block_hash: hash, - // Invalid candidate index, should not pass sanitization. - candidate_index: 16777284, - validator: validator_index, - signature: dummy_signature(), - }; - let msg = protocol_v2::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v2(overseer, &peer_a, msg).await; - - expect_reputation_change(overseer, &peer_a, COST_OVERSIZED_BITFIELD).await; - - // send an obviously invalid approval - let approval = IndirectSignedApprovalVoteV2 { - block_hash: hash, - // Invalid candidates len, should not pass sanitization. - candidate_indices: 16777284.into(), - validator: validator_index, - signature: dummy_signature(), - }; - let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v3(overseer, &peer_a, msg).await; - - expect_reputation_change(overseer, &peer_a, COST_OVERSIZED_BITFIELD).await; + expect_reputation_change(overseer, &peer_a, COST_OVERSIZED_BITFIELD).await; - virtual_overseer - }); + // send an obviously invalid approval + let approval = IndirectSignedApprovalVoteV2 { + block_hash: hash, + // Invalid candidates len, should not pass sanitization. + candidate_indices: 16777284.into(), + validator: validator_index, + signature: dummy_signature(), + }; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_a, msg).await; + + expect_reputation_change(overseer, &peer_a, COST_OVERSIZED_BITFIELD).await; + + virtual_overseer + }, + ); } fn batch_test_round(message_count: usize) { @@ -3664,11 +4121,26 @@ fn batch_test_round(message_count: usize) { let (mut context, mut virtual_overseer) = polkadot_node_subsystem_test_helpers::make_subsystem_context(pool.clone()); - let subsystem = ApprovalDistribution::new(Default::default()); + let subsystem = ApprovalDistribution::new_with_clock( + Default::default(), + Default::default(), + Box::new(SystemClock {}), + ); let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(12345); let mut sender = context.sender().clone(); - let subsystem = - subsystem.run_inner(context, &mut state, REPUTATION_CHANGE_TEST_INTERVAL, &mut rng); + let mut session_info_provider = RuntimeInfo::new_with_config(RuntimeInfoConfig { + keystore: None, + session_cache_lru_size: DISPUTE_WINDOW.get(), + }); + + let subsystem = subsystem.run_inner( + context, + &mut state, + REPUTATION_CHANGE_TEST_INTERVAL, + &mut rng, + &MockAssignmentCriteria { tranche: Ok(0) }, + &mut session_info_provider, + ); let test_fut = async move { let overseer = &mut virtual_overseer; @@ -3814,3 +4286,146 @@ fn const_ensure_size_not_zero() { crate::ensure_size_not_zero(super::MAX_ASSIGNMENT_BATCH_SIZE); crate::ensure_size_not_zero(super::MAX_APPROVAL_BATCH_SIZE); } + +struct DummyClock; +impl Clock for DummyClock { + fn tick_now(&self) -> polkadot_node_core_approval_voting::time::Tick { + 0 + } + + fn wait( + &self, + _tick: polkadot_node_core_approval_voting::time::Tick, + ) -> std::pin::Pin + Send + 'static>> { + todo!() + } +} + +/// Subsystem rejects assignments too far into the future. +#[test] +fn subsystem_rejects_assignment_in_future() { + let peers = make_peers_and_authority_ids(15); + let peer_a = peers.get(0).unwrap().0; + let parent_hash = Hash::repeat_byte(0xFF); + let hash = Hash::repeat_byte(0xAA); + + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(89) }, + Box::new(DummyClock {}), + state_without_reputation_delay(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // setup peers + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; + + // Set up a gossip topology, where a, b, c and d are topology neighbors to the node + // under testing. + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + setup_gossip_topology( + overseer, + make_gossip_topology(1, &peers_with_optional_peer_id, &[0, 1], &[2, 4], 3), + ) + .await; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 2, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // send the assignment related to `hash` + let validator_index = ValidatorIndex(0); + let cert = fake_assignment_cert(hash, validator_index); + let assignments = vec![(cert.clone(), 0u32)]; + setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::V3).await; + + let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer(overseer, &peer_a, msg).await; + provide_session( + overseer, + dummy_session_info_valid(1, &mut LocalKeystore::in_memory(), 1), + ) + .await; + + expect_reputation_change(overseer, &peer_a, COST_ASSIGNMENT_TOO_FAR_IN_THE_FUTURE) + .await; + + virtual_overseer + }, + ); +} + +/// Subsystem rejects assignments too far into the future. +#[test] +fn subsystem_rejects_bad_assignments() { + let peers = make_peers_and_authority_ids(15); + let peer_a = peers.get(0).unwrap().0; + let parent_hash = Hash::repeat_byte(0xFF); + let hash = Hash::repeat_byte(0xAA); + + let _ = test_harness( + &MockAssignmentCriteria { + tranche: Err(InvalidAssignment(criteria::InvalidAssignmentReason::NullAssignment)), + }, + Box::new(DummyClock {}), + state_without_reputation_delay(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // setup peers + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; + + // Set up a gossip topology, where a, b, c and d are topology neighbors to the node + // under testing. + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + setup_gossip_topology( + overseer, + make_gossip_topology(1, &peers_with_optional_peer_id, &[0, 1], &[2, 4], 3), + ) + .await; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 2, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // send the assignment related to `hash` + let validator_index = ValidatorIndex(0); + let cert = fake_assignment_cert(hash, validator_index); + let assignments = vec![(cert.clone(), 0u32)]; + setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::V3).await; + + let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer(overseer, &peer_a, msg).await; + provide_session( + overseer, + dummy_session_info_valid(1, &mut LocalKeystore::in_memory(), 1), + ) + .await; + + expect_reputation_change(overseer, &peer_a, COST_INVALID_MESSAGE).await; + + virtual_overseer + }, + ); +} diff --git a/polkadot/node/overseer/src/lib.rs b/polkadot/node/overseer/src/lib.rs index 24985a99913d..2c113f81c85f 100644 --- a/polkadot/node/overseer/src/lib.rs +++ b/polkadot/node/overseer/src/lib.rs @@ -580,6 +580,7 @@ pub struct Overseer { #[subsystem(blocking, message_capacity: 64000, ApprovalDistributionMessage, sends: [ NetworkBridgeTxMessage, ApprovalVotingMessage, + RuntimeApiMessage, ])] approval_distribution: ApprovalDistribution, diff --git a/polkadot/node/primitives/src/approval.rs b/polkadot/node/primitives/src/approval.rs index 66883b33367b..ec41647fc3d1 100644 --- a/polkadot/node/primitives/src/approval.rs +++ b/polkadot/node/primitives/src/approval.rs @@ -25,8 +25,8 @@ pub mod v1 { use codec::{Decode, Encode}; use polkadot_primitives::{ - BlockNumber, CandidateHash, CandidateIndex, CoreIndex, Hash, Header, SessionIndex, - ValidatorIndex, ValidatorSignature, + BlockNumber, CandidateHash, CandidateIndex, CoreIndex, GroupIndex, Hash, Header, + SessionIndex, ValidatorIndex, ValidatorSignature, }; use sp_application_crypto::ByteArray; @@ -128,11 +128,13 @@ pub mod v1 { pub parent_hash: Hash, /// The candidates included by the block. /// Note that these are not the same as the candidates that appear within the block body. - pub candidates: Vec, + pub candidates: Vec<(CandidateHash, CoreIndex, GroupIndex)>, /// The consensus slot of the block. pub slot: Slot, /// The session of the block. pub session: SessionIndex, + /// The vrf story. + pub vrf_story: RelayVRFStory, } /// Errors that can occur during the approvals protocol. diff --git a/polkadot/node/service/src/overseer.rs b/polkadot/node/service/src/overseer.rs index 5f4db99b00ef..eedc92572e08 100644 --- a/polkadot/node/service/src/overseer.rs +++ b/polkadot/node/service/src/overseer.rs @@ -308,7 +308,10 @@ where Metrics::register(registry)?, rand::rngs::StdRng::from_entropy(), )) - .approval_distribution(ApprovalDistributionSubsystem::new(Metrics::register(registry)?)) + .approval_distribution(ApprovalDistributionSubsystem::new( + Metrics::register(registry)?, + approval_voting_config.slot_duration_millis, + )) .approval_voting(ApprovalVotingSubsystem::with_config( approval_voting_config, parachains_db.clone(), diff --git a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs index 4ac044ea3459..4d00b803033a 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs @@ -804,8 +804,11 @@ fn build_overseer( Box::new(system_clock.clone()), ); - let approval_distribution = - ApprovalDistribution::new(Metrics::register(Some(&dependencies.registry)).unwrap()); + let approval_distribution = ApprovalDistribution::new_with_clock( + Metrics::register(Some(&dependencies.registry)).unwrap(), + SLOT_DURATION_MILLIS, + Box::new(system_clock.clone()), + ); let mock_chain_api = MockChainApi::new(state.build_chain_api_state()); let mock_chain_selection = MockChainSelection { state: state.clone(), clock: system_clock }; let mock_runtime_api = MockRuntimeApi::new( diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index ee937bca05bf..e0c078de48b7 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -33,7 +33,7 @@ use polkadot_node_network_protocol::{ }; use polkadot_node_primitives::{ approval::{ - v1::BlockApprovalMeta, + v1::{BlockApprovalMeta, DelayTranche}, v2::{CandidateBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2}, }, AvailableData, BabeEpoch, BlockWeight, CandidateVotes, CollationGenerationConfig, @@ -967,13 +967,17 @@ pub enum ApprovalVotingMessage { CheckAndImportAssignment( IndirectAssignmentCertV2, CandidateBitfield, - oneshot::Sender, + DelayTranche, + Option>, ), /// Check if the approval vote is valid and can be accepted by our view of the /// protocol. /// /// Should not be sent unless the block hash within the indirect vote is known. - CheckAndImportApproval(IndirectSignedApprovalVoteV2, oneshot::Sender), + CheckAndImportApproval( + IndirectSignedApprovalVoteV2, + Option>, + ), /// Returns the highest possible ancestor hash of the provided block hash which is /// acceptable to vote on finality for. /// The `BlockNumber` provided is the number of the block's ancestor which is the From 19421396f963a8a8097f6eb98dd77c9aff06f366 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 18 Jun 2024 18:37:47 +0300 Subject: [PATCH 03/45] Make approval-voting runable on a worker thread Signed-off-by: Alexandru Gheorghe --- polkadot/cli/src/command.rs | 8 +- polkadot/node/core/approval-voting/Cargo.toml | 1 + .../node/core/approval-voting/src/import.rs | 109 +++--- polkadot/node/core/approval-voting/src/lib.rs | 370 +++++++++++++----- .../node/core/approval-voting/src/tests.rs | 17 +- polkadot/node/service/src/lib.rs | 13 +- polkadot/node/service/src/overseer.rs | 1 + .../subsystem-bench/src/lib/approval/mod.rs | 3 +- 8 files changed, 354 insertions(+), 168 deletions(-) diff --git a/polkadot/cli/src/command.rs b/polkadot/cli/src/command.rs index 62d99122c301..61f3b1ad55ec 100644 --- a/polkadot/cli/src/command.rs +++ b/polkadot/cli/src/command.rs @@ -373,16 +373,16 @@ pub fn run() -> Result<()> { Ok(runner.async_run(|mut config| { let (client, backend, _, task_manager) = polkadot_service::new_chain_ops(&mut config, None)?; + let task_handle = task_manager.spawn_handle(); let aux_revert = Box::new(|client, backend, blocks| { - polkadot_service::revert_backend(client, backend, blocks, config).map_err( - |err| { + polkadot_service::revert_backend(client, backend, blocks, config, task_handle) + .map_err(|err| { match err { polkadot_service::Error::Blockchain(err) => err.into(), // Generic application-specific error. err => sc_cli::Error::Application(err.into()), } - }, - ) + }) }); Ok(( cmd.run(client, backend, Some(aux_revert)).map_err(Error::SubstrateCli), diff --git a/polkadot/node/core/approval-voting/Cargo.toml b/polkadot/node/core/approval-voting/Cargo.toml index 65985c0a5db9..50708a77ad7f 100644 --- a/polkadot/node/core/approval-voting/Cargo.toml +++ b/polkadot/node/core/approval-voting/Cargo.toml @@ -22,6 +22,7 @@ kvdb = { workspace = true } derive_more = { workspace = true, default-features = true } thiserror = { workspace = true } itertools = { workspace = true } +async-trait = { workspace = true } polkadot-node-subsystem = { workspace = true, default-features = true } polkadot-node-subsystem-util = { workspace = true, default-features = true } diff --git a/polkadot/node/core/approval-voting/src/import.rs b/polkadot/node/core/approval-voting/src/import.rs index a138af1188d2..9ab16ba1cd6b 100644 --- a/polkadot/node/core/approval-voting/src/import.rs +++ b/polkadot/node/core/approval-voting/src/import.rs @@ -44,6 +44,7 @@ use polkadot_node_subsystem::{ overseer, RuntimeApiError, SubsystemError, SubsystemResult, }; use polkadot_node_subsystem_util::{determine_new_blocks, runtime::RuntimeInfo}; +use polkadot_overseer::SubsystemSender; use polkadot_primitives::{ node_features, BlockNumber, CandidateEvent, CandidateHash, CandidateReceipt, ConsensusLog, CoreIndex, GroupIndex, Hash, Header, SessionIndex, @@ -110,8 +111,8 @@ enum ImportedBlockInfoError { /// Computes information about the imported block. Returns an error if the info couldn't be /// extracted. #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] -async fn imported_block_info( - ctx: &mut Context, +async fn imported_block_info>( + sender: &mut Sender, env: ImportedBlockInfoEnv<'_>, block_hash: Hash, block_header: &Header, @@ -123,11 +124,12 @@ async fn imported_block_info( // fetch candidates let included_candidates: Vec<_> = { let (c_tx, c_rx) = oneshot::channel(); - ctx.send_message(RuntimeApiMessage::Request( - block_hash, - RuntimeApiRequest::CandidateEvents(c_tx), - )) - .await; + sender + .send_message(RuntimeApiMessage::Request( + block_hash, + RuntimeApiRequest::CandidateEvents(c_tx), + )) + .await; let events: Vec = match c_rx.await { Ok(Ok(events)) => events, @@ -150,11 +152,12 @@ async fn imported_block_info( // short, that shouldn't happen. let session_index = { let (s_tx, s_rx) = oneshot::channel(); - ctx.send_message(RuntimeApiMessage::Request( - block_header.parent_hash, - RuntimeApiRequest::SessionIndexForChild(s_tx), - )) - .await; + sender + .send_message(RuntimeApiMessage::Request( + block_header.parent_hash, + RuntimeApiRequest::SessionIndexForChild(s_tx), + )) + .await; let session_index = match s_rx.await { Ok(Ok(s)) => s, @@ -200,11 +203,12 @@ async fn imported_block_info( // by one block. This gives us the opposite invariant for sessions - the parent block's // post-state gives us the canonical information about the session index for any of its // children, regardless of which slot number they might be produced at. - ctx.send_message(RuntimeApiMessage::Request( - block_hash, - RuntimeApiRequest::CurrentBabeEpoch(s_tx), - )) - .await; + sender + .send_message(RuntimeApiMessage::Request( + block_hash, + RuntimeApiRequest::CurrentBabeEpoch(s_tx), + )) + .await; match s_rx.await { Ok(Ok(s)) => s, @@ -215,7 +219,7 @@ async fn imported_block_info( }; let extended_session_info = - get_extended_session_info(env.runtime_info, ctx.sender(), block_hash, session_index).await; + get_extended_session_info(env.runtime_info, sender, block_hash, session_index).await; let enable_v2_assignments = extended_session_info.map_or(false, |extended_session_info| { *extended_session_info .node_features @@ -224,7 +228,7 @@ async fn imported_block_info( .unwrap_or(&false) }); - let session_info = get_session_info(env.runtime_info, ctx.sender(), block_hash, session_index) + let session_info = get_session_info(env.runtime_info, sender, block_hash, session_index) .await .ok_or(ImportedBlockInfoError::SessionInfoUnavailable)?; @@ -328,9 +332,15 @@ pub struct BlockImportedCandidates { /// * and return information about all candidates imported under each block. /// /// It is the responsibility of the caller to schedule wakeups for each block. -#[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] -pub(crate) async fn handle_new_head( - ctx: &mut Context, +pub(crate) async fn handle_new_head< + Sender: SubsystemSender + + SubsystemSender + + SubsystemSender, + AVSender: SubsystemSender, + B: Backend, +>( + sender: &mut Sender, + approval_voting_sender: &mut AVSender, state: &State, db: &mut OverlayedBackend<'_, B>, session_info_provider: &mut RuntimeInfo, @@ -348,7 +358,7 @@ pub(crate) async fn handle_new_head( let header = { let (h_tx, h_rx) = oneshot::channel(); - ctx.send_message(ChainApiMessage::BlockHeader(head, h_tx)).await; + sender.send_message(ChainApiMessage::BlockHeader(head, h_tx)).await; match h_rx.await? { Err(e) => { gum::debug!( @@ -374,7 +384,7 @@ pub(crate) async fn handle_new_head( let lower_bound_number = finalized_number.unwrap_or(lower_bound_number).max(lower_bound_number); let new_blocks = determine_new_blocks( - ctx.sender(), + sender, |h| db.load_block_entry(h).map(|e| e.is_some()), head, &header, @@ -400,12 +410,15 @@ pub(crate) async fn handle_new_head( keystore: &state.keystore, }; - match imported_block_info(ctx, env, block_hash, &block_header, finalized_number).await { + match imported_block_info(sender, env, block_hash, &block_header, finalized_number) + .await + { Ok(i) => imported_blocks_and_info.push((block_hash, block_header, i)), Err(error) => { // It's possible that we've lost a race with finality. let (tx, rx) = oneshot::channel(); - ctx.send_message(ChainApiMessage::FinalizedBlockHash(block_header.number, tx)) + sender + .send_message(ChainApiMessage::FinalizedBlockHash(block_header.number, tx)) .await; let lost_to_finality = match rx.await { @@ -449,17 +462,11 @@ pub(crate) async fn handle_new_head( force_approve, } = imported_block_info; - let session_info = match get_session_info( - session_info_provider, - ctx.sender(), - head, - session_index, - ) - .await - { - Some(session_info) => session_info, - None => return Ok(Vec::new()), - }; + let session_info = + match get_session_info(session_info_provider, sender, head, session_index).await { + Some(session_info) => session_info, + None => return Ok(Vec::new()), + }; let (block_tick, no_show_duration) = { let block_tick = slot_number_to_tick(state.slot_duration_millis, slot); @@ -509,7 +516,7 @@ pub(crate) async fn handle_new_head( }; // If all bits are already set, then send an approve message. if approved_bitfield.count_ones() == approved_bitfield.len() { - ctx.send_message(ChainSelectionMessage::Approved(block_hash)).await; + sender.send_message(ChainSelectionMessage::Approved(block_hash)).await; } let block_entry = v3::BlockEntry { @@ -566,7 +573,7 @@ pub(crate) async fn handle_new_head( // Notify chain-selection of all approved hashes. for hash in approved_hashes { - ctx.send_message(ChainSelectionMessage::Approved(hash)).await; + sender.send_message(ChainSelectionMessage::Approved(hash)).await; } } @@ -602,7 +609,8 @@ pub(crate) async fn handle_new_head( "Informing distribution of newly imported chain", ); - ctx.send_unbounded_message(ApprovalDistributionMessage::NewBlocks(approval_meta)); + approval_voting_sender + .send_unbounded_message(ApprovalDistributionMessage::NewBlocks(approval_meta)); Ok(imported_candidates) } @@ -660,7 +668,7 @@ pub(crate) mod tests { State { keystore: Arc::new(LocalKeystore::in_memory()), slot_duration_millis: 6_000, - clock: Box::new(MockClock::default()), + clock: Arc::new(MockClock::default()), assignment_criteria: Box::new(MockAssignmentCriteria::default()), spans: HashMap::new(), per_block_assignments_gathering_times: LruMap::new(ByLength::new( @@ -804,8 +812,9 @@ pub(crate) mod tests { keystore: &LocalKeystore::in_memory(), }; - let info = - imported_block_info(&mut ctx, env, hash, &header, &Some(4)).await.unwrap(); + let info = imported_block_info(ctx.sender(), env, hash, &header, &Some(4)) + .await + .unwrap(); assert_eq!(info.included_candidates, included_candidates); assert_eq!(info.session_index, session); @@ -951,7 +960,7 @@ pub(crate) mod tests { keystore: &LocalKeystore::in_memory(), }; - let info = imported_block_info(&mut ctx, env, hash, &header, &Some(4)).await; + let info = imported_block_info(ctx.sender(), env, hash, &header, &Some(4)).await; assert_matches!(info, Err(ImportedBlockInfoError::VrfInfoUnavailable)); }) @@ -1090,7 +1099,7 @@ pub(crate) mod tests { keystore: &LocalKeystore::in_memory(), }; - let info = imported_block_info(&mut ctx, env, hash, &header, &Some(6)).await; + let info = imported_block_info(ctx.sender(), env, hash, &header, &Some(6)).await; assert_matches!(info, Err(ImportedBlockInfoError::BlockAlreadyFinalized)); }) @@ -1126,7 +1135,8 @@ pub(crate) mod tests { #[test] fn imported_block_info_extracts_force_approve() { let pool = TaskExecutor::new(); - let (mut ctx, mut handle) = make_subsystem_context(pool.clone()); + let (mut ctx, mut handle) = + make_subsystem_context::(pool.clone()); let session = 5; let session_info = dummy_session_info(session); @@ -1189,7 +1199,7 @@ pub(crate) mod tests { }; let info = - imported_block_info(&mut ctx, env, hash, &header, &Some(4)).await.unwrap(); + imported_block_info(ctx.sender(), env, hash, &header, &Some(4)).await.unwrap(); assert_eq!(info.included_candidates, included_candidates); assert_eq!(info.session_index, session); @@ -1382,8 +1392,11 @@ pub(crate) mod tests { let test_fut = { Box::pin(async move { let mut overlay_db = OverlayedBackend::new(&db); + + let mut approval_voting_sender = ctx.sender().clone(); let result = handle_new_head( - &mut ctx, + ctx.sender(), + &mut approval_voting_sender, &state, &mut overlay_db, &mut session_info_provider, diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 5d588a53f3d1..4195a1fe490d 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -164,7 +164,8 @@ pub struct ApprovalVotingSubsystem { db: Arc, mode: Mode, metrics: Metrics, - clock: Box, + clock: Arc, + spawner: Arc, } #[derive(Clone)] @@ -483,6 +484,7 @@ impl ApprovalVotingSubsystem { keystore: Arc, sync_oracle: Box, metrics: Metrics, + spawner: Arc, ) -> Self { ApprovalVotingSubsystem::with_config_and_clock( config, @@ -490,7 +492,8 @@ impl ApprovalVotingSubsystem { keystore, sync_oracle, metrics, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), + spawner, ) } @@ -501,7 +504,8 @@ impl ApprovalVotingSubsystem { keystore: Arc, sync_oracle: Box, metrics: Metrics, - clock: Box, + clock: Arc, + spawner: Arc, ) -> Self { ApprovalVotingSubsystem { keystore, @@ -511,6 +515,7 @@ impl ApprovalVotingSubsystem { mode: Mode::Syncing(sync_oracle), metrics, clock, + spawner, } } @@ -550,12 +555,21 @@ fn db_sanity_check(db: Arc, config: DatabaseConfig) -> SubsystemRe #[overseer::subsystem(ApprovalVoting, error = SubsystemError, prefix = self::overseer)] impl ApprovalVotingSubsystem { - fn start(self, ctx: Context) -> SpawnedSubsystem { + fn start(self, mut ctx: Context) -> SpawnedSubsystem { let backend = DbBackend::new(self.db.clone(), self.db_config); - let future = - run::(ctx, self, Box::new(RealAssignmentCriteria), backend) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) - .boxed(); + let to_other_subsystems = ctx.sender().clone(); + let to_approval_distr = ctx.sender().clone(); + + let future = run::( + ctx, + to_other_subsystems, + to_approval_distr, + self, + Box::new(RealAssignmentCriteria), + backend, + ) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) + .boxed(); SpawnedSubsystem { name: "approval-voting-subsystem", future } } @@ -824,7 +838,7 @@ where struct State { keystore: Arc, slot_duration_millis: u64, - clock: Box, + clock: Arc, assignment_criteria: Box, spans: HashMap, // Per block, candidate records about how long we take until we gather enough @@ -960,20 +974,20 @@ impl State { } // Returns the approval voting params from the RuntimeApi. - #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] - async fn get_approval_voting_params_or_default( + async fn get_approval_voting_params_or_default>( &self, - ctx: &mut Context, + sender: &mut Sender, session_index: SessionIndex, block_hash: Hash, ) -> Option { let (s_tx, s_rx) = oneshot::channel(); - ctx.send_message(RuntimeApiMessage::Request( - block_hash, - RuntimeApiRequest::ApprovalVotingParams(session_index, s_tx), - )) - .await; + sender + .send_message(RuntimeApiMessage::Request( + block_hash, + RuntimeApiRequest::ApprovalVotingParams(session_index, s_tx), + )) + .await; match s_rx.await { Ok(Ok(params)) => { @@ -1142,9 +1156,36 @@ enum Action { Conclude, } +/// Trait for providing approval voting subsystem with work. +#[async_trait::async_trait] +pub trait ApprovalVotingWorkProvider { + async fn recv(&mut self) -> SubsystemResult>; +} + +#[async_trait::async_trait] #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] -async fn run( - mut ctx: Context, +impl ApprovalVotingWorkProvider for Context { + async fn recv(&mut self) -> SubsystemResult> { + self.recv().await + } +} + +#[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] +async fn run< + B, + WorkProvider: ApprovalVotingWorkProvider, + Sender: SubsystemSender + + SubsystemSender + + SubsystemSender + + SubsystemSender + + SubsystemSender + + SubsystemSender + + Clone, + ADSender: SubsystemSender, +>( + mut work_provider: WorkProvider, + mut to_other_subsystems: Sender, + mut to_approval_distr: ADSender, mut subsystem: ApprovalVotingSubsystem, assignment_criteria: Box, mut backend: B, @@ -1168,19 +1209,11 @@ where no_show_stats: NoShowStats::default(), }; - // `None` on start-up. Gets initialized/updated on leaf update - let mut session_info_provider = RuntimeInfo::new_with_config(RuntimeInfoConfig { - keystore: None, - session_cache_lru_size: DISPUTE_WINDOW.get(), - }); - let mut wakeups = Wakeups::default(); - let mut currently_checking_set = CurrentlyCheckingSet::default(); - let mut delayed_approvals_timers = DelayedApprovalTimer::default(); - let mut approvals_cache = LruMap::new(ByLength::new(APPROVAL_CACHE_SIZE)); - let mut last_finalized_height: Option = { let (tx, rx) = oneshot::channel(); - ctx.send_message(ChainApiMessage::FinalizedBlockNumber(tx)).await; + to_other_subsystems + .send_message(ChainApiMessage::FinalizedBlockNumber(tx)) + .await; match rx.await? { Ok(number) => Some(number), Err(err) => { @@ -1190,13 +1223,24 @@ where } }; + // `None` on start-up. Gets initialized/updated on leaf update + let mut session_info_provider = RuntimeInfo::new_with_config(RuntimeInfoConfig { + keystore: None, + session_cache_lru_size: DISPUTE_WINDOW.get(), + }); + + let mut wakeups = Wakeups::default(); + let mut currently_checking_set = CurrentlyCheckingSet::default(); + let mut delayed_approvals_timers = DelayedApprovalTimer::default(); + let mut approvals_cache = LruMap::new(ByLength::new(APPROVAL_CACHE_SIZE)); + loop { let mut overlayed_db = OverlayedBackend::new(&backend); let actions = futures::select! { (_tick, woken_block, woken_candidate) = wakeups.next(&*state.clock).fuse() => { subsystem.metrics.on_wakeup(); process_wakeup( - &mut ctx, + &mut to_other_subsystems, &mut state, &mut overlayed_db, &mut session_info_provider, @@ -1206,9 +1250,11 @@ where &wakeups, ).await? } - next_msg = ctx.recv().fuse() => { + next_msg = work_provider.recv().fuse() => { let mut actions = handle_from_overseer( - &mut ctx, + &mut to_other_subsystems, + &mut to_approval_distr, + &subsystem.spawner, &mut state, &mut overlayed_db, &mut session_info_provider, @@ -1268,7 +1314,8 @@ where &mut overlayed_db, &mut session_info_provider, &state, - &mut ctx, + &mut to_other_subsystems, + &mut to_approval_distr, block_hash, validator_index, &subsystem.metrics, @@ -1290,7 +1337,9 @@ where }; if handle_actions( - &mut ctx, + &mut to_other_subsystems, + &mut to_approval_distr, + &subsystem.spawner, &mut state, &mut overlayed_db, &mut session_info_provider, @@ -1317,6 +1366,61 @@ where Ok(()) } +// Starts a worker thread that runs the approval voting subsystem. +pub async fn start_approval_worker< + WorkProvider: ApprovalVotingWorkProvider + Send + 'static, + Sender: SubsystemSender + + SubsystemSender + + SubsystemSender + + SubsystemSender + + SubsystemSender + + SubsystemSender + + Clone, + ADSender: SubsystemSender, +>( + work_provider: WorkProvider, + to_other_subsystems: Sender, + to_approval_distr: ADSender, + config: Config, + db: Arc, + keystore: Arc, + sync_oracle: Box, + metrics: Metrics, + spawner: Arc, + clock: Arc, +) -> SubsystemResult<()> { + let approval_voting = ApprovalVotingSubsystem::with_config_and_clock( + config, + db.clone(), + keystore, + sync_oracle, + metrics, + clock, + spawner, + ); + let backend = DbBackend::new(db.clone(), approval_voting.db_config); + let spawner = approval_voting.spawner.clone(); + spawner.spawn_blocking( + "approval-voting-rewrite-db", + Some("approval-voting-rewrite-subsystem"), + Box::pin(async move { + if let Err(err) = run( + work_provider, + to_other_subsystems, + to_approval_distr, + approval_voting, + Box::new(RealAssignmentCriteria), + backend, + ) + .await + { + gum::error!(target: LOG_TARGET, ?err, "Approval voting worker stoped processing messages"); + }; + }), + ); + Ok(()) +} + // Handle actions is a function that accepts a set of instructions // and subsequently updates the underlying approvals_db in accordance // with the linear set of instructions passed in. Therefore, actions @@ -1337,8 +1441,19 @@ where // // returns `true` if any of the actions was a `Conclude` command. #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] -async fn handle_actions( - ctx: &mut Context, +async fn handle_actions< + Sender: SubsystemSender + + SubsystemSender + + SubsystemSender + + SubsystemSender + + SubsystemSender + + SubsystemSender + + Clone, + ADSender: SubsystemSender, +>( + sender: &mut Sender, + approval_voting_sender: &mut ADSender, + spawn_handle: &Arc, state: &mut State, overlayed_db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, @@ -1370,7 +1485,8 @@ async fn handle_actions( // Note that chaining these iterators is O(n) as we must consume // the prior iterator. let next_actions: Vec = issue_approval( - ctx, + sender, + approval_voting_sender, state, overlayed_db, session_info_provider, @@ -1421,10 +1537,12 @@ async fn handle_actions( let validator_index = indirect_cert.validator; if distribute_assignment { - ctx.send_unbounded_message(ApprovalDistributionMessage::DistributeAssignment( - indirect_cert, - claimed_candidate_indices, - )); + approval_voting_sender.send_unbounded_message( + ApprovalDistributionMessage::DistributeAssignment( + indirect_cert, + claimed_candidate_indices, + ), + ); } match approvals_cache.get(&candidate_hash) { @@ -1439,7 +1557,8 @@ async fn handle_actions( actions_iter = new_actions.into_iter(); }, None => { - let ctx = &mut *ctx; + let sender = sender.clone(); + let spawn_handle = spawn_handle.clone(); currently_checking_set .insert_relay_block_hash( @@ -1448,7 +1567,8 @@ async fn handle_actions( relay_block_hash, async move { launch_approval( - ctx, + sender, + spawn_handle, metrics.clone(), session, candidate, @@ -1477,13 +1597,13 @@ async fn handle_actions( }) .with_string_tag("block-hash", format!("{:?}", block_hash)) .with_stage(jaeger::Stage::ApprovalChecking); - ctx.send_message(ChainSelectionMessage::Approved(block_hash)).await; + sender.send_message(ChainSelectionMessage::Approved(block_hash)).await; }, Action::BecomeActive => { *mode = Mode::Active; let (messages, next_actions) = distribution_messages_for_activation( - ctx, + sender, overlayed_db, state, delayed_approvals_timers, @@ -1491,7 +1611,7 @@ async fn handle_actions( ) .await?; - ctx.send_messages(messages.into_iter()).await; + approval_voting_sender.send_messages(messages.into_iter()).await; let next_actions: Vec = next_actions.into_iter().map(|v| v.clone()).chain(actions_iter).collect(); @@ -1565,8 +1685,8 @@ fn get_assignment_core_indices( } #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] -async fn distribution_messages_for_activation( - ctx: &mut Context, +async fn distribution_messages_for_activation>( + sender: &mut Sender, db: &OverlayedBackend<'_, impl Backend>, state: &State, delayed_approvals_timers: &mut DelayedApprovalTimer, @@ -1683,7 +1803,7 @@ async fn distribution_messages_for_activation( let ExtendedSessionInfo { ref executor_params, .. } = match get_extended_session_info( session_info_provider, - ctx.sender(), + sender, block_entry.block_hash(), block_entry.session(), ) @@ -1781,9 +1901,16 @@ async fn distribution_messages_for_activation( } // Handle an incoming signal from the overseer. Returns true if execution should conclude. -#[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] -async fn handle_from_overseer( - ctx: &mut Context, +async fn handle_from_overseer< + Sender: SubsystemSender + + SubsystemSender + + SubsystemSender + + Clone, + ADSender: SubsystemSender, +>( + sender: &mut Sender, + approval_voting_sender: &mut ADSender, + spawn_handle: &Arc, state: &mut State, db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, @@ -1801,7 +1928,8 @@ async fn handle_from_overseer( jaeger::PerLeafSpan::new(activated.span, "approval-voting"); state.spans.insert(head, approval_voting_span); match import::handle_new_head( - ctx, + sender, + approval_voting_sender, state, db, session_info_provider, @@ -1885,7 +2013,7 @@ async fn handle_from_overseer( FromOrchestra::Communication { msg } => match msg { ApprovalVotingMessage::CheckAndImportAssignment(a, claimed_cores, tranche, tx) => { let (check_outcome, actions) = check_and_import_assignment( - ctx.sender(), + sender, state, db, session_info_provider, @@ -1905,7 +2033,7 @@ async fn handle_from_overseer( }, ApprovalVotingMessage::CheckAndImportApproval(a, tx) => { let result = check_and_import_approval( - ctx.sender(), + sender, state, db, session_info_provider, @@ -1933,7 +2061,7 @@ async fn handle_from_overseer( .with_stage(jaeger::Stage::ApprovalChecking) .with_string_tag("leaf", format!("{:?}", target)); match handle_approved_ancestor( - ctx, + sender, db, target, lower_bound, @@ -1956,7 +2084,14 @@ async fn handle_from_overseer( }, ApprovalVotingMessage::GetApprovalSignaturesForCandidate(candidate_hash, tx) => { metrics.on_candidate_signatures_request(); - get_approval_signatures_for_candidate(ctx, db, candidate_hash, tx).await?; + get_approval_signatures_for_candidate( + approval_voting_sender.clone(), + spawn_handle, + db, + candidate_hash, + tx, + ) + .await?; Vec::new() }, }, @@ -1970,8 +2105,11 @@ async fn handle_from_overseer( /// This involves an unbounded message send to approval-distribution, the caller has to ensure that /// calls to this function are infrequent and bounded. #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] -async fn get_approval_signatures_for_candidate( - ctx: &mut Context, +async fn get_approval_signatures_for_candidate< + Sender: SubsystemSender, +>( + mut sender: Sender, + spawn_handle: &Arc, db: &OverlayedBackend<'_, impl Backend>, candidate_hash: CandidateHash, tx: oneshot::Sender, ValidatorSignature)>>, @@ -2030,7 +2168,6 @@ async fn get_approval_signatures_for_candidate( } } - let mut sender = ctx.sender().clone(); let get_approvals = async move { let (tx_distribution, rx_distribution) = oneshot::channel(); sender.send_unbounded_message(ApprovalDistributionMessage::GetApprovalSignatures( @@ -2110,12 +2247,17 @@ async fn get_approval_signatures_for_candidate( ?candidate_hash, "Spawning task for fetching signatures from approval-distribution" ); - ctx.spawn("get-approval-signatures", Box::pin(get_approvals)) + spawn_handle.spawn( + "get-approval-signatures", + Some("approval-voting-subsystem"), + Box::pin(get_approvals), + ); + Ok(()) } #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] -async fn handle_approved_ancestor( - ctx: &mut Context, +async fn handle_approved_ancestor>( + sender: &mut Sender, db: &OverlayedBackend<'_, impl Backend>, target: Hash, lower_bound: BlockNumber, @@ -2135,7 +2277,7 @@ async fn handle_approved_ancestor( let target_number = { let (tx, rx) = oneshot::channel(); - ctx.send_message(ChainApiMessage::BlockNumber(target, tx)).await; + sender.send_message(ChainApiMessage::BlockNumber(target, tx)).await; match rx.await { Ok(Ok(Some(n))) => n, @@ -2156,12 +2298,13 @@ async fn handle_approved_ancestor( let ancestry = if target_number > lower_bound + 1 { let (tx, rx) = oneshot::channel(); - ctx.send_message(ChainApiMessage::Ancestors { - hash: target, - k: (target_number - (lower_bound + 1)) as usize, - response_channel: tx, - }) - .await; + sender + .send_message(ChainApiMessage::Ancestors { + hash: target, + k: (target_number - (lower_bound + 1)) as usize, + response_channel: tx, + }) + .await; match rx.await { Ok(Ok(a)) => a, @@ -3106,9 +3249,8 @@ fn should_trigger_assignment( } } -#[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] -async fn process_wakeup( - ctx: &mut Context, +async fn process_wakeup>( + sender: &mut Sender, state: &mut State, db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, @@ -3139,7 +3281,7 @@ async fn process_wakeup( let ExtendedSessionInfo { ref session_info, ref executor_params, .. } = match get_extended_session_info( session_info_provider, - ctx.sender(), + sender, block_entry.block_hash(), block_entry.session(), ) @@ -3281,7 +3423,7 @@ async fn process_wakeup( // Note that this function also schedules a wakeup as necessary. actions.extend( advance_approval_state( - ctx.sender(), + sender, state, db, session_info_provider, @@ -3302,8 +3444,14 @@ async fn process_wakeup( // spawned. When the background work is no longer needed, the `AbortHandle` should be dropped // to cancel the background work and any requests it has spawned. #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] -async fn launch_approval( - ctx: &mut Context, +async fn launch_approval< + Sender: SubsystemSender + + SubsystemSender + + SubsystemSender + + SubsystemSender, +>( + mut sender: Sender, + spawn_handle: Arc, metrics: Metrics, session_index: SessionIndex, candidate: CandidateReceipt, @@ -3354,14 +3502,15 @@ async fn launch_approval( .with_stage(jaeger::Stage::ApprovalChecking); let timer = metrics.time_recover_and_approve(); - ctx.send_message(AvailabilityRecoveryMessage::RecoverAvailableData( - candidate.clone(), - session_index, - Some(backing_group), - core_index, - a_tx, - )) - .await; + sender + .send_message(AvailabilityRecoveryMessage::RecoverAvailableData( + candidate.clone(), + session_index, + Some(backing_group), + core_index, + a_tx, + )) + .await; let request_validation_result_span = span .child("request-validation-result") @@ -3370,15 +3519,18 @@ async fn launch_approval( .with_string_tag("block-hash", format!("{:?}", block_hash)) .with_stage(jaeger::Stage::ApprovalChecking); - ctx.send_message(RuntimeApiMessage::Request( - block_hash, - RuntimeApiRequest::ValidationCodeByHash(candidate.descriptor.validation_code_hash, code_tx), - )) - .await; + sender + .send_message(RuntimeApiMessage::Request( + block_hash, + RuntimeApiRequest::ValidationCodeByHash( + candidate.descriptor.validation_code_hash, + code_tx, + ), + )) + .await; let candidate = candidate.clone(); let metrics_guard = StaleGuard(Some(metrics)); - let mut sender = ctx.sender().clone(); let background = async move { // Force the move of the timer into the background task. let _timer = timer; @@ -3508,14 +3660,19 @@ async fn launch_approval( } }; let (background, remote_handle) = background.remote_handle(); - ctx.spawn("approval-checks", Box::pin(background)).map(move |()| remote_handle) + spawn_handle.spawn("approval-checks", Some("approval-voting-subsystem"), Box::pin(background)); + Ok(remote_handle) } // Issue and import a local approval vote. Should only be invoked after approval checks // have been done. #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] -async fn issue_approval( - ctx: &mut Context, +async fn issue_approval< + Sender: SubsystemSender, + ADSender: SubsystemSender, +>( + sender: &mut Sender, + approval_voting_sender: &mut ADSender, state: &mut State, db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, @@ -3594,7 +3751,7 @@ async fn issue_approval( let session_info = match get_session_info( session_info_provider, - ctx.sender(), + sender, block_entry.parent_hash(), block_entry.session(), ) @@ -3636,7 +3793,7 @@ async fn issue_approval( ); let actions = advance_approval_state( - ctx.sender(), + sender, state, db, session_info_provider, @@ -3653,7 +3810,8 @@ async fn issue_approval( db, session_info_provider, state, - ctx, + sender, + approval_voting_sender, block_hash, validator_index, metrics, @@ -3672,11 +3830,15 @@ async fn issue_approval( // Create signature for the approved candidates pending signatures #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] -async fn maybe_create_signature( +async fn maybe_create_signature< + Sender: SubsystemSender, + ADSender: SubsystemSender, +>( db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, state: &State, - ctx: &mut Context, + sender: &mut Sender, + approval_voting_sender: &mut ADSender, block_hash: Hash, validator_index: ValidatorIndex, metrics: &Metrics, @@ -3695,7 +3857,7 @@ async fn maybe_create_signature( }; let approval_params = state - .get_approval_voting_params_or_default(ctx, block_entry.session(), block_hash) + .get_approval_voting_params_or_default(sender, block_entry.session(), block_hash) .await .unwrap_or_default(); @@ -3715,7 +3877,7 @@ async fn maybe_create_signature( let session_info = match get_session_info( session_info_provider, - ctx.sender(), + sender, block_entry.parent_hash(), block_entry.session(), ) @@ -3796,7 +3958,7 @@ async fn maybe_create_signature( metrics.on_approval_produced(); - ctx.send_unbounded_message(ApprovalDistributionMessage::DistributeApproval( + approval_voting_sender.send_unbounded_message(ApprovalDistributionMessage::DistributeApproval( IndirectSignedApprovalVoteV2 { block_hash: block_entry.block_hash(), candidate_indices: candidates_indices, @@ -3837,7 +3999,7 @@ fn issue_local_invalid_statement( candidate_hash: CandidateHash, candidate: CandidateReceipt, ) where - Sender: overseer::ApprovalVotingSenderTrait, + Sender: SubsystemSender, { // We need to send an unbounded message here to break a cycle: // DisputeCoordinatorMessage::IssueLocalStatement -> diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 8aab353f9b84..8565e5d14a1a 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -39,7 +39,7 @@ use polkadot_node_subsystem::{ }; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::TimeoutExt; -use polkadot_overseer::HeadSupportsParachains; +use polkadot_overseer::{HeadSupportsParachains, SpawnGlue}; use polkadot_primitives::{ ApprovalVote, CandidateCommitments, CandidateEvent, CoreIndex, DisputeStatement, GroupIndex, Header, Id as ParaId, IndexedVec, NodeFeatures, ValidDisputeStatementKind, ValidationCode, @@ -537,7 +537,7 @@ impl Default for HarnessConfig { struct TestHarness { virtual_overseer: VirtualOverseer, - clock: Box, + clock: Arc, sync_oracle_handle: TestSyncOracleHandle, } @@ -556,7 +556,7 @@ fn test_harness>( let pool = sp_core::testing::TaskExecutor::new(); let (context, virtual_overseer) = - polkadot_node_subsystem_test_helpers::make_subsystem_context(pool); + polkadot_node_subsystem_test_helpers::make_subsystem_context(pool.clone()); let keystore = LocalKeystore::in_memory(); let _ = keystore.sr25519_generate_new( @@ -564,7 +564,7 @@ fn test_harness>( Some(&Sr25519Keyring::Alice.to_seed()), ); - let clock = Box::new(clock); + let clock = Arc::new(clock); let db = kvdb_memorydb::create(test_constants::NUM_COLUMNS); let db = polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[]); @@ -580,6 +580,7 @@ fn test_harness>( sync_oracle, Metrics::default(), clock.clone(), + Arc::new(SpawnGlue(pool)), ), assignment_criteria, backend, @@ -4164,7 +4165,7 @@ async fn handle_approval_on_max_coalesce_count( async fn handle_approval_on_max_wait_time( virtual_overseer: &mut VirtualOverseer, candidate_indices: Vec, - clock: Box, + clock: Arc, ) { const TICK_NOW_BEGIN: u64 = 1; const MAX_COALESCE_COUNT: u32 = 3; @@ -4462,7 +4463,7 @@ async fn build_chain_with_two_blocks_with_one_candidate_each( async fn setup_overseer_with_two_blocks_each_with_one_assignment_triggered( virtual_overseer: &mut VirtualOverseer, store: TestStore, - clock: &Box, + clock: &Arc, sync_oracle_handle: TestSyncOracleHandle, ) { assert_matches!( @@ -4976,7 +4977,7 @@ fn test_gathering_assignments_statements() { let mut state = State { keystore: Arc::new(LocalKeystore::in_memory()), slot_duration_millis: 6_000, - clock: Box::new(MockClock::default()), + clock: Arc::new(MockClock::default()), assignment_criteria: Box::new(MockAssignmentCriteria::check_only(|_| Ok(0))), spans: HashMap::new(), per_block_assignments_gathering_times: LruMap::new(ByLength::new( @@ -5071,7 +5072,7 @@ fn test_observe_assignment_gathering_status() { let mut state = State { keystore: Arc::new(LocalKeystore::in_memory()), slot_duration_millis: 6_000, - clock: Box::new(MockClock::default()), + clock: Arc::new(MockClock::default()), assignment_criteria: Box::new(MockAssignmentCriteria::check_only(|_| Ok(0))), spans: HashMap::new(), per_block_assignments_gathering_times: LruMap::new(ByLength::new( diff --git a/polkadot/node/service/src/lib.rs b/polkadot/node/service/src/lib.rs index b4f63bd2aa06..b76f40dd3102 100644 --- a/polkadot/node/service/src/lib.rs +++ b/polkadot/node/service/src/lib.rs @@ -63,6 +63,7 @@ use { }; use polkadot_node_subsystem_util::database::Database; +use polkadot_overseer::SpawnGlue; #[cfg(feature = "full-node")] pub use { @@ -83,7 +84,7 @@ use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Duration}; use prometheus_endpoint::Registry; #[cfg(feature = "full-node")] use sc_service::KeystoreContainer; -use sc_service::RpcHandlers; +use sc_service::{RpcHandlers, SpawnTaskHandle}; use sc_telemetry::TelemetryWorker; #[cfg(feature = "full-node")] use sc_telemetry::{Telemetry, TelemetryWorkerHandle}; @@ -1481,6 +1482,7 @@ pub fn revert_backend( backend: Arc, blocks: BlockNumber, config: Configuration, + task_handle: SpawnTaskHandle, ) -> Result<(), Error> { let best_number = client.info().best_number; let finalized = client.info().finalized_number; @@ -1501,7 +1503,7 @@ pub fn revert_backend( let parachains_db = open_database(&config.database) .map_err(|err| sp_blockchain::Error::Backend(err.to_string()))?; - revert_approval_voting(parachains_db.clone(), hash)?; + revert_approval_voting(parachains_db.clone(), hash, task_handle)?; revert_chain_selection(parachains_db, hash)?; // Revert Substrate consensus related components sc_consensus_babe::revert(client.clone(), backend, blocks)?; @@ -1524,7 +1526,11 @@ fn revert_chain_selection(db: Arc, hash: Hash) -> sp_blockchain::R .map_err(|err| sp_blockchain::Error::Backend(err.to_string())) } -fn revert_approval_voting(db: Arc, hash: Hash) -> sp_blockchain::Result<()> { +fn revert_approval_voting( + db: Arc, + hash: Hash, + task_handle: SpawnTaskHandle, +) -> sp_blockchain::Result<()> { let config = approval_voting_subsystem::Config { col_approval_data: parachains_db::REAL_COLUMNS.col_approval_data, slot_duration_millis: Default::default(), @@ -1536,6 +1542,7 @@ fn revert_approval_voting(db: Arc, hash: Hash) -> sp_blockchain::R Arc::new(sc_keystore::LocalKeystore::in_memory()), Box::new(sp_consensus::NoNetwork), approval_voting_subsystem::Metrics::default(), + Arc::new(SpawnGlue(task_handle)), ); approval_voting diff --git a/polkadot/node/service/src/overseer.rs b/polkadot/node/service/src/overseer.rs index eedc92572e08..1a14b3f85cb1 100644 --- a/polkadot/node/service/src/overseer.rs +++ b/polkadot/node/service/src/overseer.rs @@ -318,6 +318,7 @@ where keystore.clone(), Box::new(sync_service.clone()), Metrics::register(registry)?, + Arc::new(spawner.clone()), )) .gossip_support(GossipSupportSubsystem::new( keystore.clone(), diff --git a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs index 4d00b803033a..382360eda664 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs @@ -801,7 +801,8 @@ fn build_overseer( Arc::new(keystore), Box::new(TestSyncOracle {}), state.approval_voting_metrics.clone(), - Box::new(system_clock.clone()), + Arc::new(system_clock.clone()), + Arc::new(SpawnGlue(spawn_task_handle.clone())), ); let approval_distribution = ApprovalDistribution::new_with_clock( From 55520b8c9504a01a7d6b60544aeb5d07d7347244 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 20 Jun 2024 14:34:50 +0300 Subject: [PATCH 04/45] Introduce approval-voting-parallel Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 51 ++ Cargo.toml | 2 + .../src/lib.rs | 1 + polkadot/cli/src/cli.rs | 6 + polkadot/cli/src/command.rs | 1 + .../core/approval-voting-parallel/Cargo.toml | 67 ++ .../core/approval-voting-parallel/src/lib.rs | 747 ++++++++++++++++++ .../node/core/approval-voting/src/import.rs | 2 +- polkadot/node/core/approval-voting/src/lib.rs | 16 +- .../node/core/approval-voting/src/tests.rs | 10 +- .../dispute-coordinator/src/initialized.rs | 33 +- .../node/core/dispute-coordinator/src/lib.rs | 6 +- .../core/dispute-coordinator/src/tests.rs | 1 + .../network/approval-distribution/src/lib.rs | 25 +- .../approval-distribution/src/tests.rs | 71 +- polkadot/node/network/bridge/src/rx/mod.rs | 68 +- polkadot/node/network/bridge/src/rx/tests.rs | 1 + polkadot/node/overseer/src/dummy.rs | 4 + polkadot/node/overseer/src/lib.rs | 30 +- polkadot/node/overseer/src/tests.rs | 4 + polkadot/node/primitives/src/approval.rs | 4 +- polkadot/node/service/Cargo.toml | 2 + polkadot/node/service/src/lib.rs | 6 + polkadot/node/service/src/overseer.rs | 31 +- .../node/service/src/relay_chain_selection.rs | 65 +- polkadot/node/service/src/tests.rs | 1 + polkadot/node/subsystem-bench/Cargo.toml | 1 + .../src/lib/approval/helpers.rs | 9 +- .../subsystem-bench/src/lib/approval/mod.rs | 67 +- .../subsystem-bench/src/lib/environment.rs | 33 + .../subsystem-bench/src/lib/mock/dummy.rs | 1 + .../node/subsystem-bench/src/lib/mock/mod.rs | 1 + .../src/lib/mock/network_bridge.rs | 5 +- polkadot/node/subsystem-types/src/messages.rs | 112 +++ polkadot/node/test/service/src/lib.rs | 2 + .../adder/collator/src/main.rs | 1 + .../undying/collator/src/main.rs | 1 + 37 files changed, 1367 insertions(+), 121 deletions(-) create mode 100644 polkadot/node/core/approval-voting-parallel/Cargo.toml create mode 100644 polkadot/node/core/approval-voting-parallel/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index e91bd6400042..090db1321c3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13122,6 +13122,55 @@ dependencies = [ "tracing-gum", ] +[[package]] +name = "polkadot-node-core-approval-voting-parallel" +version = "7.0.0" +dependencies = [ + "assert_matches", + "async-trait", + "bitvec", + "derive_more", + "env_logger 0.11.3", + "futures", + "futures-timer", + "itertools 0.11.0", + "kvdb", + "kvdb-memorydb", + "log", + "merlin", + "parity-scale-codec", + "parking_lot 0.12.3", + "polkadot-approval-distribution", + "polkadot-node-core-approval-voting", + "polkadot-node-jaeger", + "polkadot-node-metrics", + "polkadot-node-network-protocol", + "polkadot-node-primitives", + "polkadot-node-subsystem", + "polkadot-node-subsystem-test-helpers", + "polkadot-node-subsystem-util", + "polkadot-overseer", + "polkadot-primitives", + "polkadot-primitives-test-helpers", + "polkadot-subsystem-bench", + "rand", + "rand_chacha", + "rand_core", + "sc-keystore", + "schnellru", + "schnorrkel 0.11.4", + "sp-application-crypto", + "sp-consensus", + "sp-consensus-babe", + "sp-consensus-slots", + "sp-core", + "sp-keyring", + "sp-keystore", + "sp-runtime", + "thiserror", + "tracing-gum", +] + [[package]] name = "polkadot-node-core-av-store" version = "7.0.0" @@ -14579,6 +14628,7 @@ dependencies = [ "polkadot-network-bridge", "polkadot-node-collation-generation", "polkadot-node-core-approval-voting", + "polkadot-node-core-approval-voting-parallel", "polkadot-node-core-av-store", "polkadot-node-core-backing", "polkadot-node-core-bitfield-signing", @@ -14741,6 +14791,7 @@ dependencies = [ "polkadot-availability-recovery", "polkadot-erasure-coding", "polkadot-node-core-approval-voting", + "polkadot-node-core-approval-voting-parallel", "polkadot-node-core-av-store", "polkadot-node-core-chain-api", "polkadot-node-metrics", diff --git a/Cargo.toml b/Cargo.toml index 5c2677fffeb2..29b7477abd9e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -152,6 +152,7 @@ members = [ "polkadot/erasure-coding/fuzzer", "polkadot/node/collation-generation", "polkadot/node/core/approval-voting", + "polkadot/node/core/approval-voting-parallel", "polkadot/node/core/av-store", "polkadot/node/core/backing", "polkadot/node/core/bitfield-signing", @@ -1009,6 +1010,7 @@ polkadot-gossip-support = { path = "polkadot/node/network/gossip-support", defau polkadot-network-bridge = { path = "polkadot/node/network/bridge", default-features = false } polkadot-node-collation-generation = { path = "polkadot/node/collation-generation", default-features = false } polkadot-node-core-approval-voting = { path = "polkadot/node/core/approval-voting", default-features = false } +polkadot-node-core-approval-voting-parallel = { path = "polkadot/node/core/approval-voting-parallel", default-features = false } polkadot-node-core-av-store = { path = "polkadot/node/core/av-store", default-features = false } polkadot-node-core-backing = { path = "polkadot/node/core/backing", default-features = false } polkadot-node-core-bitfield-signing = { path = "polkadot/node/core/bitfield-signing", default-features = false } diff --git a/cumulus/client/relay-chain-inprocess-interface/src/lib.rs b/cumulus/client/relay-chain-inprocess-interface/src/lib.rs index 38ba84748c1e..2d5ffb6c39c2 100644 --- a/cumulus/client/relay-chain-inprocess-interface/src/lib.rs +++ b/cumulus/client/relay-chain-inprocess-interface/src/lib.rs @@ -334,6 +334,7 @@ fn build_polkadot_full_node( execute_workers_max_num: None, prepare_workers_hard_max_num: None, prepare_workers_soft_max_num: None, + enable_approval_voting_parallel: false, }, )?; diff --git a/polkadot/cli/src/cli.rs b/polkadot/cli/src/cli.rs index 3e5a6ccdd3c2..722521f11efa 100644 --- a/polkadot/cli/src/cli.rs +++ b/polkadot/cli/src/cli.rs @@ -151,6 +151,12 @@ pub struct RunCmd { /// TESTING ONLY: disable the version check between nodes and workers. #[arg(long, hide = true)] pub disable_worker_version_check: bool, + + /// Enable approval-voting message processing in parallel. + /// + ///**Dangerous!** This is an experimental feature and should not be used in production. + #[arg(long)] + pub enable_approval_voting_parallel: bool, } #[allow(missing_docs)] diff --git a/polkadot/cli/src/command.rs b/polkadot/cli/src/command.rs index 61f3b1ad55ec..a66a8354a4c9 100644 --- a/polkadot/cli/src/command.rs +++ b/polkadot/cli/src/command.rs @@ -256,6 +256,7 @@ where execute_workers_max_num: cli.run.execute_workers_max_num, prepare_workers_hard_max_num: cli.run.prepare_workers_hard_max_num, prepare_workers_soft_max_num: cli.run.prepare_workers_soft_max_num, + enable_approval_voting_parallel: cli.run.enable_approval_voting_parallel, }, ) .map(|full| full.task_manager)?; diff --git a/polkadot/node/core/approval-voting-parallel/Cargo.toml b/polkadot/node/core/approval-voting-parallel/Cargo.toml new file mode 100644 index 000000000000..10653cd21891 --- /dev/null +++ b/polkadot/node/core/approval-voting-parallel/Cargo.toml @@ -0,0 +1,67 @@ +[package] +name = "polkadot-node-core-approval-voting-parallel" +version = "7.0.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +description = "Approval Voting Subsystem running approval work in parallel" + +[lints] +workspace = true + +[dependencies] +futures = "0.3.30" +futures-timer = "3.0.2" +codec = { package = "parity-scale-codec", version = "3.6.12", default-features = false, features = ["bit-vec", "derive"] } +gum = { package = "tracing-gum", path = "../../gum" } +bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } +schnellru = "0.2.1" +merlin = "3.0" +schnorrkel = "0.11.4" +kvdb = "0.13.0" +derive_more = "0.99.17" +thiserror = { workspace = true } +itertools = "0.11" +async-trait = { workspace = true } + +polkadot-node-core-approval-voting = { workspace = true, default-features = true } +polkadot-approval-distribution = { workspace = true, default-features = true } + + +polkadot-node-subsystem = { workspace = true, default-features = true } +polkadot-node-subsystem-util = { workspace = true, default-features = true } +polkadot-overseer = { workspace = true, default-features = true } +polkadot-primitives = { workspace = true, default-features = true } +polkadot-node-primitives = { workspace = true, default-features = true } +polkadot-node-jaeger = { workspace = true, default-features = true } + +sc-keystore = { workspace = true, default-features = false } +sp-consensus = { workspace = true, default-features = false } +sp-consensus-slots = { workspace = true, default-features = false } +sp-application-crypto = { workspace = true, default-features = false, features = ["full_crypto"] } +sp-runtime = { workspace = true, default-features = false } +polkadot-node-network-protocol = { workspace = true, default-features = true } +polkadot-node-metrics = { workspace = true, default-features = true} + +rand = "0.8.5" + +# rand_core should match schnorrkel +rand_core = "0.6.2" +rand_chacha = { version = "0.3.1" } + +[dev-dependencies] +async-trait = "0.1.79" +parking_lot = "0.12.1" +sp-keyring = { workspace = true, default-features = true } +sp-keystore = {workspace = true, default-features = true} +sp-core = { workspace = true, default-features = true} +sp-consensus-babe = { workspace = true, default-features = true } +polkadot-node-subsystem-test-helpers = { workspace = true, default-features = true} +assert_matches = "1.4.0" +kvdb-memorydb = "0.13.0" +polkadot-primitives-test-helpers = { workspace = true, default-features = true } +log = { workspace = true, default-features = true } +env_logger = "0.11" + +polkadot-subsystem-bench = { workspace = true, default-features = true} + diff --git a/polkadot/node/core/approval-voting-parallel/src/lib.rs b/polkadot/node/core/approval-voting-parallel/src/lib.rs new file mode 100644 index 000000000000..3142694e1af6 --- /dev/null +++ b/polkadot/node/core/approval-voting-parallel/src/lib.rs @@ -0,0 +1,747 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! The Approval Voting Parallel Subsystem. +//! +//! This subsystem is responsible for orchestrating the work done by +//! approval-voting and approval-distribution subsystem, so they can +//! do their work in parallel, rather than serially, when they are run +//! as independent subsystems. +use itertools::Itertools; +use polkadot_node_core_approval_voting::{ + time::{Clock, SystemClock}, + Config, RealAssignmentCriteria, +}; +use polkadot_node_metrics::metered::{ + self, channel, unbounded, MeteredSender, UnboundedMeteredSender, +}; + +use polkadot_node_primitives::DISPUTE_WINDOW; +use polkadot_node_subsystem::{ + messages::{ApprovalDistributionMessage, ApprovalVotingMessage, ApprovalVotingParallelMessage}, + overseer, FromOrchestra, SpawnedSubsystem, SubsystemError, SubsystemResult, +}; +use polkadot_node_subsystem_util::{ + self, + database::Database, + metrics::{self, prometheus}, + runtime::{Config as RuntimeInfoConfig, RuntimeInfo}, +}; +use polkadot_overseer::{OverseerSignal, SubsystemSender}; +use polkadot_primitives::ValidatorIndex; +use rand::SeedableRng; + +use sc_keystore::LocalKeystore; +use sp_consensus::SyncOracle; + +use futures::{channel::oneshot, prelude::*, StreamExt}; +use polkadot_node_core_approval_voting::{ + approval_db::common::Config as DatabaseConfig, ApprovalVotingWorkProvider, +}; +use std::{collections::HashMap, fmt::Debug, sync::Arc}; +use stream::{select_with_strategy, PollNext}; + +pub(crate) const LOG_TARGET: &str = "parachain::approval-voting-parallel"; + +/// The approval voting subsystem. +pub struct ApprovalVotingParallelSubsystem { + /// `LocalKeystore` is needed for assignment keys, but not necessarily approval keys. + /// + /// We do a lot of VRF signing and need the keys to have low latency. + keystore: Arc, + db_config: DatabaseConfig, + slot_duration_millis: u64, + db: Arc, + sync_oracle: Box, + metrics: Metrics, + spawner: Arc, + clock: Arc, + subsystem_enabled: bool, +} + +/// Approval Voting metrics. +#[derive(Default, Clone)] +pub struct Metrics( + pub polkadot_approval_distribution::metrics::Metrics, + pub polkadot_node_core_approval_voting::Metrics, +); + +impl metrics::Metrics for Metrics { + fn try_register( + registry: &prometheus::Registry, + ) -> std::result::Result { + Ok(Metrics( + polkadot_approval_distribution::metrics::Metrics::try_register(registry)?, + polkadot_node_core_approval_voting::Metrics::try_register(registry)?, + )) + } +} + +impl ApprovalVotingParallelSubsystem { + /// Create a new approval voting subsystem with the given keystore, config, and database. + pub fn with_config( + config: Config, + db: Arc, + keystore: Arc, + sync_oracle: Box, + metrics: Metrics, + spawner: impl overseer::gen::Spawner + 'static + Clone, + subsystem_enabled: bool, + ) -> Self { + ApprovalVotingParallelSubsystem::with_config_and_clock( + config, + db, + keystore, + sync_oracle, + metrics, + Arc::new(SystemClock {}), + spawner, + subsystem_enabled, + ) + } + + /// Create a new approval voting subsystem with the given keystore, config, and database. + pub fn with_config_and_clock( + config: Config, + db: Arc, + keystore: Arc, + sync_oracle: Box, + metrics: Metrics, + clock: Arc, + spawner: impl overseer::gen::Spawner + 'static, + subsystem_enabled: bool, + ) -> Self { + ApprovalVotingParallelSubsystem { + keystore, + slot_duration_millis: config.slot_duration_millis, + db, + db_config: DatabaseConfig { col_approval_data: config.col_approval_data }, + sync_oracle, + metrics, + spawner: Arc::new(spawner), + clock, + subsystem_enabled, + } + } +} + +#[overseer::subsystem(ApprovalVotingParallel, error = SubsystemError, prefix = self::overseer)] +impl ApprovalVotingParallelSubsystem { + fn start(self, ctx: Context) -> SpawnedSubsystem { + let future = run::(ctx, self) + .map_err(|e| SubsystemError::with_origin("approval-voting-parallel", e)) + .boxed(); + + SpawnedSubsystem { name: "approval-voting-parallel-subsystem", future } + } +} + +/// The number of workers used for running the approval-distribution logic. +pub const APPROVAL_DISTRIBUTION_WORKER_COUNT: usize = 2; + +/// The channel size for the workers. +pub const WORKERS_CHANNEL_SIZE: usize = 64000 / APPROVAL_DISTRIBUTION_WORKER_COUNT; +fn prio_right<'a>(_val: &'a mut ()) -> PollNext { + PollNext::Right +} + +#[overseer::contextbounds(ApprovalVotingParallel, prefix = self::overseer)] +async fn run( + mut ctx: Context, + subsystem: ApprovalVotingParallelSubsystem, +) -> SubsystemResult<()> +where +{ + // Build approval voting handles. + let (tx_approval_voting_work, rx_approval_voting_work) = + channel::>(WORKERS_CHANNEL_SIZE); + let (tx_approval_voting_work_unbounded, rx_approval_voting_work_unbounded) = + unbounded::>(); + + let mut to_approval_voting_worker = + ToWorker(tx_approval_voting_work, tx_approval_voting_work_unbounded); + + let prioritised = select_with_strategy( + rx_approval_voting_work, + rx_approval_voting_work_unbounded, + prio_right, + ); + let approval_voting_work_provider = ApprovalVotingWorkProviderImpl(prioritised); + + gum::info!(target: LOG_TARGET, "Starting approval distribution workers"); + + let mut approval_distribution_channels = Vec::new(); + let slot_duration_millis = subsystem.slot_duration_millis; + + for i in 0..APPROVAL_DISTRIBUTION_WORKER_COUNT { + let approval_distro_orig = + polkadot_approval_distribution::ApprovalDistribution::new_with_clock( + subsystem.metrics.0.clone(), + subsystem.slot_duration_millis, + subsystem.clock.clone(), + false, + ); + + let (tx_approval_distribution_work, rx_approval_distribution_work) = + channel::>(WORKERS_CHANNEL_SIZE); + let (tx_approval_distribution_work_unbounded, rx_approval_distribution_unbounded) = + unbounded::>(); + + let to_approval_distribution_worker = + ToWorker(tx_approval_distribution_work, tx_approval_distribution_work_unbounded); + + let task_name = format!("approval-voting-parallel-{}", i); + + let mut network_sender = ctx.sender().clone(); + let mut runtime_api_sender = ctx.sender().clone(); + let mut approval_distribution_to_approval_voting = to_approval_voting_worker.clone(); + + subsystem.spawner.spawn_blocking( + task_name.leak(), + Some("approval-voting-parallel-subsystem"), + Box::pin(async move { + let mut state = + polkadot_approval_distribution::State::with_config(slot_duration_millis); + let mut rng = rand::rngs::StdRng::from_entropy(); + let assignment_criteria = RealAssignmentCriteria {}; + let mut session_info_provider = RuntimeInfo::new_with_config(RuntimeInfoConfig { + keystore: None, + session_cache_lru_size: DISPUTE_WINDOW.get(), + }); + + let mut work_channels = select_with_strategy( + rx_approval_distribution_work, + rx_approval_distribution_unbounded, + prio_right, + ); + + loop { + let message = match work_channels.next().await { + Some(message) => message, + None => { + gum::info!( + target: LOG_TARGET, + "Approval distribution stream finished, most likely shutting down", + ); + break; + }, + }; + approval_distro_orig + .handle_from_orchestra( + message, + &mut approval_distribution_to_approval_voting, + &mut network_sender, + &mut runtime_api_sender, + &mut state, + &mut rng, + &assignment_criteria, + &mut session_info_provider, + ) + .await; + } + }), + ); + approval_distribution_channels.push(to_approval_distribution_worker); + } + + gum::info!(target: LOG_TARGET, "Starting approval voting workers"); + let sender = ctx.sender().clone(); + let to_approval_distribution = ApprovalVotingToApprovalDistribution(sender.clone()); + + polkadot_node_core_approval_voting::start_approval_worker( + approval_voting_work_provider, + sender.clone(), + to_approval_distribution, + polkadot_node_core_approval_voting::Config { + slot_duration_millis: subsystem.slot_duration_millis, + col_approval_data: subsystem.db_config.col_approval_data, + }, + subsystem.db.clone(), + subsystem.keystore.clone(), + subsystem.sync_oracle, + subsystem.metrics.1.clone(), + subsystem.spawner.clone(), + subsystem.clock.clone(), + ) + .await?; + + gum::info!( + target: LOG_TARGET, + subsystem_disabled = ?subsystem.subsystem_enabled, + "Starting main subsystem loop" + ); + + // Main loop of the subsystem, it shouldn't include any logic just dispatching of messages to + // the workers. + loop { + futures::select! { + next_msg = ctx.recv().fuse() => { + let next_msg = match next_msg { + Ok(msg) => msg, + Err(err) => { + gum::info!(target: LOG_TARGET, ?err, "Approval voting parallel subsystem received an error"); + break; + } + }; + if !subsystem.subsystem_enabled { + gum::trace!(target: LOG_TARGET, ?next_msg, "Parallel processing is not enabled skipping message"); + continue; + } + gum::trace!(target: LOG_TARGET, ?next_msg, "Parallel processing is not enabled skipping message"); + + match next_msg { + FromOrchestra::Signal(msg) => { + for worker in approval_distribution_channels.iter_mut() { + worker + .send_signal(msg.clone()).await?; + } + + to_approval_voting_worker.send_signal(msg).await?; + }, + FromOrchestra::Communication { msg } => match msg { + // The message the approval voting subsystem would've handled. + ApprovalVotingParallelMessage::CheckAndImportAssignment(_,_, _) | + ApprovalVotingParallelMessage::CheckAndImportApproval(_)| + ApprovalVotingParallelMessage::ApprovedAncestor(_, _,_) | + ApprovalVotingParallelMessage::GetApprovalSignaturesForCandidate(_, _) => { + // Safe to unwrap because we know the message is of the right type. + to_approval_voting_worker.send_message(msg.try_into().unwrap()).await; + }, + // Now the message the approval distribution subsystem would've handled and need to + // be forwarded to the workers. + ApprovalVotingParallelMessage::NewBlocks(msg) => { + for worker in approval_distribution_channels.iter_mut() { + worker + .send_message( + ApprovalDistributionMessage::NewBlocks(msg.clone()), + ) + .await; + } + }, + ApprovalVotingParallelMessage::DistributeAssignment(assignment, claimed) => { + let worker_index = assignment.validator.0 as usize % approval_distribution_channels.len(); + let worker = approval_distribution_channels.get_mut(worker_index).expect("Worker index is obtained modulo len; qed"); + worker + .send_message( + ApprovalDistributionMessage::DistributeAssignment(assignment, claimed) + ) + .await; + + }, + ApprovalVotingParallelMessage::DistributeApproval(vote) => { + let worker_index = vote.validator.0 as usize % approval_distribution_channels.len(); + let worker = approval_distribution_channels.get_mut(worker_index).expect("Worker index is obtained modulo len; qed"); + worker + .send_message( + ApprovalDistributionMessage::DistributeApproval(vote) + ).await; + + }, + ApprovalVotingParallelMessage::NetworkBridgeUpdate(msg) => { + if let polkadot_node_subsystem::messages::NetworkBridgeEvent::PeerMessage( + peer_id, + msg, + ) = msg + { + let (all_msgs_from_same_validator, messages_split_by_validator) = validator_index_for_msg(msg); + + for (validator_index, msg) in all_msgs_from_same_validator.into_iter().chain(messages_split_by_validator.into_iter().flatten()) { + let worker_index = validator_index.0 as usize % approval_distribution_channels.len(); + let worker = approval_distribution_channels.get_mut(worker_index).expect("Worker index is obtained modulo len; qed"); + + worker + .send_message( + ApprovalDistributionMessage::NetworkBridgeUpdate( + polkadot_node_subsystem::messages::NetworkBridgeEvent::PeerMessage( + peer_id, msg, + ), + ), + ).await; + } + } else { + for worker in approval_distribution_channels.iter_mut() { + worker + .send_message( + ApprovalDistributionMessage::NetworkBridgeUpdate(msg.clone()), + ).await; + } + } + }, + ApprovalVotingParallelMessage::GetApprovalSignatures(indices, tx) => { + let mut sigs = HashMap::new(); + let mut signatures_channels = Vec::new(); + for worker in approval_distribution_channels.iter_mut() { + let (tx, rx) = oneshot::channel(); + worker + .send_message( + ApprovalDistributionMessage::GetApprovalSignatures(indices.clone(), tx) + ).await; + signatures_channels.push(rx); + } + let results = futures::future::join_all(signatures_channels).await; + + for result in results { + let worker_sigs = match result { + Ok(sigs) => sigs, + Err(_) => { + gum::error!( + target: LOG_TARGET, + "Getting approval signatures failed, oneshot got closed" + ); + continue; + }, + }; + sigs.extend(worker_sigs); + } + + if let Err(_) = tx.send(sigs) { + gum::debug!( + target: LOG_TARGET, + "Sending back approval signatures failed, oneshot got closed" + ); + } + }, + ApprovalVotingParallelMessage::ApprovalCheckingLagUpdate(lag) => { + for worker in approval_distribution_channels.iter_mut() { + worker + .send_message( + ApprovalDistributionMessage::ApprovalCheckingLagUpdate(lag) + ).await; + } + }, + }, + }; + + }, + }; + } + Ok(()) +} + +// Returns the validators that initially created this assignments/votes, the validator index +// is later used to decide which approval-distribution worker should receive the message. +// +// Because this is on the hot path and we don't want to be unnecessarily slow, it contains two logic +// paths. The ultra fast path where all messages have the same validator index and we don't don't do +// any cloning or allocation and the path where we need to split the messages into multiple +// messages, because they have different validator indices, where we do need to clone and allocate. +// In practice most of the message will fall on the ultra fast path. +fn validator_index_for_msg( + msg: polkadot_node_network_protocol::ApprovalDistributionMessage, +) -> ( + Option<(ValidatorIndex, polkadot_node_network_protocol::ApprovalDistributionMessage)>, + Option>, +) { + match msg { + polkadot_node_network_protocol::Versioned::V1(ref message) => match message { + polkadot_node_network_protocol::v1::ApprovalDistributionMessage::Assignments(msgs) => + if let Ok(validator) = msgs.iter().map(|(msg, _)| msg.validator).all_equal_value() { + (Some((validator, msg)), None) + } else { + let split = msgs + .iter() + .map(|(msg, claimed_candidates)| { + ( + msg.validator, + polkadot_node_network_protocol::Versioned::V1( + polkadot_node_network_protocol::v1::ApprovalDistributionMessage::Assignments( + vec![(msg.clone(), *claimed_candidates)] + ), + ), + ) + }) + .collect_vec(); + (None, Some(split)) + }, + polkadot_node_network_protocol::v1::ApprovalDistributionMessage::Approvals(msgs) => + if let Ok(validator) = msgs.iter().map(|msg| msg.validator).all_equal_value() { + (Some((validator, msg)), None) + } else { + let split = msgs + .iter() + .map(|vote| { + ( + vote.validator, + polkadot_node_network_protocol::Versioned::V1( + polkadot_node_network_protocol::v1::ApprovalDistributionMessage::Approvals( + vec![vote.clone()] + ), + ), + ) + }) + .collect_vec(); + (None, Some(split)) + }, + }, + polkadot_node_network_protocol::Versioned::V2(ref message) => match message { + polkadot_node_network_protocol::v2::ApprovalDistributionMessage::Assignments(msgs) => + if let Ok(validator) = msgs.iter().map(|(msg, _)| msg.validator).all_equal_value() { + (Some((validator, msg)), None) + } else { + let split = msgs + .iter() + .map(|(msg, claimed_candidates)| { + ( + msg.validator, + polkadot_node_network_protocol::Versioned::V2( + polkadot_node_network_protocol::v2::ApprovalDistributionMessage::Assignments( + vec![(msg.clone(), *claimed_candidates)] + ), + ), + ) + }) + .collect_vec(); + (None, Some(split)) + }, + + polkadot_node_network_protocol::v2::ApprovalDistributionMessage::Approvals(msgs) => + if let Ok(validator) = msgs.iter().map(|msg| msg.validator).all_equal_value() { + (Some((validator, msg)), None) + } else { + let split = msgs + .iter() + .map(|vote| { + ( + vote.validator, + polkadot_node_network_protocol::Versioned::V2( + polkadot_node_network_protocol::v2::ApprovalDistributionMessage::Approvals( + vec![vote.clone()] + ), + ), + ) + }) + .collect_vec(); + (None, Some(split)) + }, + }, + polkadot_node_network_protocol::Versioned::V3(ref message) => match message { + polkadot_node_network_protocol::v3::ApprovalDistributionMessage::Assignments(msgs) => + if let Ok(validator) = msgs.iter().map(|(msg, _)| msg.validator).all_equal_value() { + (Some((validator, msg)), None) + } else { + let split = msgs + .iter() + .map(|(msg, claimed_candidates)| { + ( + msg.validator, + polkadot_node_network_protocol::Versioned::V3( + polkadot_node_network_protocol::v3::ApprovalDistributionMessage::Assignments( + vec![(msg.clone(), claimed_candidates.clone())] + ), + ), + ) + }) + .collect_vec(); + (None, Some(split)) + }, + polkadot_node_network_protocol::v3::ApprovalDistributionMessage::Approvals(msgs) => + if let Ok(validator) = msgs.iter().map(|msg| msg.validator).all_equal_value() { + (Some((validator, msg)), None) + } else { + let split = msgs + .iter() + .map(|vote| { + ( + vote.validator, + polkadot_node_network_protocol::Versioned::V3( + polkadot_node_network_protocol::v3::ApprovalDistributionMessage::Approvals( + vec![vote.clone()] + ), + ), + ) + }) + .collect_vec(); + (None, Some(split)) + }, + }, + } +} + +/// Just a wrapper over a channel Receiver, that is injected into approval-voting worker for +/// providing the messages to be processed. +pub struct ApprovalVotingWorkProviderImpl(T); + +#[async_trait::async_trait] +impl ApprovalVotingWorkProvider for ApprovalVotingWorkProviderImpl +where + T: Stream> + Unpin + Send, +{ + async fn recv(&mut self) -> SubsystemResult> { + self.0.next().await.ok_or(SubsystemError::Context( + "ApprovalVotingWorkProviderImpl: Channel closed".to_string(), + )) + } +} + +/// Just a wrapper for implementing overseer::SubsystemSender and +/// overseer::SubsystemSender, so that we can inject into the +/// workers, so they can talke directly with each other without intermediating in this subsystem +/// loop. +pub struct ToWorker( + MeteredSender>, + UnboundedMeteredSender>, +); + +impl Clone for ToWorker { + fn clone(&self) -> Self { + Self(self.0.clone(), self.1.clone()) + } +} + +impl ToWorker { + async fn send_signal(&mut self, signal: OverseerSignal) -> Result<(), SubsystemError> { + self.1 + .unbounded_send(FromOrchestra::Signal(signal)) + .map_err(|err| SubsystemError::QueueError(err.into_send_error())) + } +} + +impl overseer::SubsystemSender for ToWorker { + fn send_message<'life0, 'async_trait>( + &'life0 mut self, + msg: T, + ) -> ::core::pin::Pin< + Box + ::core::marker::Send + 'async_trait>, + > + where + 'life0: 'async_trait, + Self: 'async_trait, + { + async { + if let Err(err) = + self.0.send(polkadot_overseer::FromOrchestra::Communication { msg }).await + { + gum::error!( + target: LOG_TARGET, + "Failed to send message to approval voting worker: {:?}, subsystem is probably shutting down.", + err + ); + } + } + .boxed() + } + + fn try_send_message(&mut self, msg: T) -> Result<(), metered::TrySendError> { + self.0 + .try_send(polkadot_overseer::FromOrchestra::Communication { msg }) + .map_err(|result| { + let is_full = result.is_full(); + let msg = match result.into_inner() { + polkadot_overseer::FromOrchestra::Signal(_) => + panic!("Cannot happen variant is never built"), + polkadot_overseer::FromOrchestra::Communication { msg } => msg, + }; + if is_full { + metered::TrySendError::Full(msg) + } else { + metered::TrySendError::Closed(msg) + } + }) + } + + fn send_messages<'life0, 'async_trait, I>( + &'life0 mut self, + msgs: I, + ) -> ::core::pin::Pin< + Box + ::core::marker::Send + 'async_trait>, + > + where + I: IntoIterator + Send, + I::IntoIter: Send, + I: 'async_trait, + 'life0: 'async_trait, + Self: 'async_trait, + { + async { + for msg in msgs { + self.send_message(msg).await; + } + } + .boxed() + } + + fn send_unbounded_message(&mut self, msg: T) { + if let Err(err) = + self.1.unbounded_send(polkadot_overseer::FromOrchestra::Communication { msg }) + { + gum::error!( + target: LOG_TARGET, + "Failed to send unbounded message to approval voting worker: {:?}, subsystem is probably shutting down.", + err + ); + } + } +} + +/// Just a wrapper for implementing overseer::SubsystemSender, so that +/// we can inject into the approval voting subsystem. +#[derive(Clone)] +pub struct ApprovalVotingToApprovalDistribution>( + S, +); + +impl> + overseer::SubsystemSender + for ApprovalVotingToApprovalDistribution +{ + #[allow(clippy::type_complexity, clippy::type_repetition_in_bounds)] + fn send_message<'life0, 'async_trait>( + &'life0 mut self, + msg: ApprovalDistributionMessage, + ) -> ::core::pin::Pin< + Box + ::core::marker::Send + 'async_trait>, + > + where + 'life0: 'async_trait, + Self: 'async_trait, + { + self.0.send_message(msg.into()) + } + + fn try_send_message( + &mut self, + msg: ApprovalDistributionMessage, + ) -> Result<(), metered::TrySendError> { + self.0.try_send_message(msg.into()).map_err(|err| match err { + // Safe to unwrap because it was built from the same type. + metered::TrySendError::Closed(msg) => + metered::TrySendError::Closed(msg.try_into().unwrap()), + metered::TrySendError::Full(msg) => + metered::TrySendError::Full(msg.try_into().unwrap()), + }) + } + + #[allow(clippy::type_complexity, clippy::type_repetition_in_bounds)] + fn send_messages<'life0, 'async_trait, I>( + &'life0 mut self, + msgs: I, + ) -> ::core::pin::Pin< + Box + ::core::marker::Send + 'async_trait>, + > + where + I: IntoIterator + Send, + I::IntoIter: Send, + I: 'async_trait, + 'life0: 'async_trait, + Self: 'async_trait, + { + self.0.send_messages(msgs.into_iter().map(|msg| msg.into())) + } + + fn send_unbounded_message(&mut self, msg: ApprovalDistributionMessage) { + self.0.send_unbounded_message(msg.into()) + } +} diff --git a/polkadot/node/core/approval-voting/src/import.rs b/polkadot/node/core/approval-voting/src/import.rs index 9ab16ba1cd6b..6c8294672826 100644 --- a/polkadot/node/core/approval-voting/src/import.rs +++ b/polkadot/node/core/approval-voting/src/import.rs @@ -41,7 +41,7 @@ use polkadot_node_subsystem::{ ApprovalDistributionMessage, ChainApiMessage, ChainSelectionMessage, RuntimeApiMessage, RuntimeApiRequest, }, - overseer, RuntimeApiError, SubsystemError, SubsystemResult, + overseer, RuntimeApiError, SubsystemContext, SubsystemError, SubsystemResult, }; use polkadot_node_subsystem_util::{determine_new_blocks, runtime::RuntimeInfo}; use polkadot_overseer::SubsystemSender; diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 4195a1fe490d..45ac823fd1a2 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -166,6 +166,7 @@ pub struct ApprovalVotingSubsystem { metrics: Metrics, clock: Arc, spawner: Arc, + subsystem_disabled: bool, } #[derive(Clone)] @@ -485,6 +486,7 @@ impl ApprovalVotingSubsystem { sync_oracle: Box, metrics: Metrics, spawner: Arc, + subsystem_disabled: bool, ) -> Self { ApprovalVotingSubsystem::with_config_and_clock( config, @@ -494,6 +496,7 @@ impl ApprovalVotingSubsystem { metrics, Arc::new(SystemClock {}), spawner, + subsystem_disabled, ) } @@ -506,6 +509,7 @@ impl ApprovalVotingSubsystem { metrics: Metrics, clock: Arc, spawner: Arc, + subsystem_disabled: bool, ) -> Self { ApprovalVotingSubsystem { keystore, @@ -516,6 +520,7 @@ impl ApprovalVotingSubsystem { metrics, clock, spawner, + subsystem_disabled, } } @@ -1251,6 +1256,12 @@ where ).await? } next_msg = work_provider.recv().fuse() => { + if subsystem.subsystem_disabled { + // If we are running in parallel mode, we need to ensure that we are not + // processing messages, but also consume the messages, so that the system + // is not marked as stalled because of the signals it receives. + continue; + } let mut actions = handle_from_overseer( &mut to_other_subsystems, &mut to_approval_distr, @@ -1397,12 +1408,13 @@ pub async fn start_approval_worker< metrics, clock, spawner, + false, ); let backend = DbBackend::new(db.clone(), approval_voting.db_config); let spawner = approval_voting.spawner.clone(); spawner.spawn_blocking( - "approval-voting-rewrite-db", - Some("approval-voting-rewrite-subsystem"), + "approval-voting-parallel-db", + Some("approval-voting-parallel-subsystem"), Box::pin(async move { if let Err(err) = run( work_provider, diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 8565e5d14a1a..3a15da7c3989 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -35,7 +35,7 @@ use polkadot_node_subsystem::{ messages::{ AllMessages, ApprovalVotingMessage, AssignmentCheckResult, AvailabilityRecoveryMessage, }, - ActiveLeavesUpdate, + ActiveLeavesUpdate, SubsystemContext, }; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::TimeoutExt; @@ -555,7 +555,7 @@ fn test_harness>( config; let pool = sp_core::testing::TaskExecutor::new(); - let (context, virtual_overseer) = + let (mut context, virtual_overseer) = polkadot_node_subsystem_test_helpers::make_subsystem_context(pool.clone()); let keystore = LocalKeystore::in_memory(); @@ -567,9 +567,12 @@ fn test_harness>( let clock = Arc::new(clock); let db = kvdb_memorydb::create(test_constants::NUM_COLUMNS); let db = polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[]); - + let sender = context.sender().clone(); + let to_approval_distr_sender = context.sender().clone(); let subsystem = run( context, + sender, + to_approval_distr_sender, ApprovalVotingSubsystem::with_config_and_clock( Config { col_approval_data: test_constants::TEST_CONFIG.col_approval_data, @@ -581,6 +584,7 @@ fn test_harness>( Metrics::default(), clock.clone(), Arc::new(SpawnGlue(pool)), + false, ), assignment_criteria, backend, diff --git a/polkadot/node/core/dispute-coordinator/src/initialized.rs b/polkadot/node/core/dispute-coordinator/src/initialized.rs index 5f86da87f21c..0d9da012712d 100644 --- a/polkadot/node/core/dispute-coordinator/src/initialized.rs +++ b/polkadot/node/core/dispute-coordinator/src/initialized.rs @@ -34,8 +34,9 @@ use polkadot_node_primitives::{ }; use polkadot_node_subsystem::{ messages::{ - ApprovalVotingMessage, BlockDescription, ChainSelectionMessage, DisputeCoordinatorMessage, - DisputeDistributionMessage, ImportStatementsResult, + ApprovalVotingMessage, ApprovalVotingParallelMessage, BlockDescription, + ChainSelectionMessage, DisputeCoordinatorMessage, DisputeDistributionMessage, + ImportStatementsResult, }, overseer, ActivatedLeaf, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, RuntimeApiError, }; @@ -117,6 +118,7 @@ pub(crate) struct Initialized { /// `CHAIN_IMPORT_MAX_BATCH_SIZE` and put the rest here for later processing. chain_import_backlog: VecDeque, metrics: Metrics, + approval_voting_parallel_enabled: bool, } #[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)] @@ -130,7 +132,13 @@ impl Initialized { highest_session_seen: SessionIndex, gaps_in_cache: bool, ) -> Self { - let DisputeCoordinatorSubsystem { config: _, store: _, keystore, metrics } = subsystem; + let DisputeCoordinatorSubsystem { + config: _, + store: _, + keystore, + metrics, + approval_voting_parallel_enabled, + } = subsystem; let (participation_sender, participation_receiver) = mpsc::channel(1); let participation = Participation::new(participation_sender, metrics.clone()); @@ -148,6 +156,7 @@ impl Initialized { participation_receiver, chain_import_backlog: VecDeque::new(), metrics, + approval_voting_parallel_enabled, } } @@ -1059,9 +1068,21 @@ impl Initialized { // 4. We are waiting (and blocking the whole subsystem) on a response right after - // therefore even with all else failing we will never have more than // one message in flight at any given time. - ctx.send_unbounded_message( - ApprovalVotingMessage::GetApprovalSignaturesForCandidate(candidate_hash, tx), - ); + if self.approval_voting_parallel_enabled { + ctx.send_unbounded_message( + ApprovalVotingParallelMessage::GetApprovalSignaturesForCandidate( + candidate_hash, + tx, + ), + ); + } else { + ctx.send_unbounded_message( + ApprovalVotingMessage::GetApprovalSignaturesForCandidate( + candidate_hash, + tx, + ), + ); + } match rx.await { Err(_) => { gum::warn!( diff --git a/polkadot/node/core/dispute-coordinator/src/lib.rs b/polkadot/node/core/dispute-coordinator/src/lib.rs index daa384b36ffb..fcac596f4b8f 100644 --- a/polkadot/node/core/dispute-coordinator/src/lib.rs +++ b/polkadot/node/core/dispute-coordinator/src/lib.rs @@ -34,7 +34,7 @@ use gum::CandidateHash; use sc_keystore::LocalKeystore; use polkadot_node_primitives::{ - CandidateVotes, DisputeMessage, DisputeMessageCheckError, SignedDisputeStatement, + approval, CandidateVotes, DisputeMessage, DisputeMessageCheckError, SignedDisputeStatement, DISPUTE_WINDOW, }; use polkadot_node_subsystem::{ @@ -122,6 +122,7 @@ pub struct DisputeCoordinatorSubsystem { store: Arc, keystore: Arc, metrics: Metrics, + approval_voting_parallel_enabled: bool, } /// Configuration for the dispute coordinator subsystem. @@ -164,8 +165,9 @@ impl DisputeCoordinatorSubsystem { config: Config, keystore: Arc, metrics: Metrics, + approval_voting_parallel_enabled: bool, ) -> Self { - Self { store, config, keystore, metrics } + Self { store, config, keystore, metrics, approval_voting_parallel_enabled } } /// Initialize and afterwards run `Initialized::run`. diff --git a/polkadot/node/core/dispute-coordinator/src/tests.rs b/polkadot/node/core/dispute-coordinator/src/tests.rs index f97a625a9528..b41cdb94b4d2 100644 --- a/polkadot/node/core/dispute-coordinator/src/tests.rs +++ b/polkadot/node/core/dispute-coordinator/src/tests.rs @@ -580,6 +580,7 @@ impl TestState { self.config, self.subsystem_keystore.clone(), Metrics::default(), + false, ); let backend = DbBackend::new(self.db.clone(), self.config.column_config(), Metrics::default()); diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 33136c1b864d..a7bd016887f2 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -71,10 +71,12 @@ use polkadot_primitives::{ use rand::{CryptoRng, Rng, SeedableRng}; use std::{ collections::{hash_map, BTreeMap, HashMap, HashSet, VecDeque}, + sync::Arc, time::Duration, }; -mod metrics; +/// Approval distribution metrics. +pub mod metrics; #[cfg(test)] mod tests; @@ -100,7 +102,8 @@ const MAX_BITFIELD_SIZE: usize = 500; pub struct ApprovalDistribution { metrics: Metrics, slot_duration_millis: u64, - clock: Box, + clock: Arc, + subsystem_disabled: bool, } /// Contains recently finalized @@ -2643,17 +2646,23 @@ async fn modify_reputation( #[overseer::contextbounds(ApprovalDistribution, prefix = self::overseer)] impl ApprovalDistribution { /// Create a new instance of the [`ApprovalDistribution`] subsystem. - pub fn new(metrics: Metrics, slot_duration_millis: u64) -> Self { - Self::new_with_clock(metrics, slot_duration_millis, Box::new(SystemClock)) + pub fn new(metrics: Metrics, slot_duration_millis: u64, subsystem_disabled: bool) -> Self { + Self::new_with_clock( + metrics, + slot_duration_millis, + Arc::new(SystemClock), + subsystem_disabled, + ) } /// Create a new instance of the [`ApprovalDistribution`] subsystem, with a custom clock. pub fn new_with_clock( metrics: Metrics, slot_duration_millis: u64, - clock: Box, + clock: Arc, + subsystem_disabled: bool, ) -> Self { - Self { metrics, slot_duration_millis, clock } + Self { metrics, slot_duration_millis, clock, subsystem_disabled } } async fn run(self, ctx: Context) { @@ -2702,6 +2711,10 @@ impl ApprovalDistribution { reputation_delay = new_reputation_delay(); }, message = ctx.recv().fuse() => { + if self.subsystem_disabled { + gum::trace!(target: LOG_TARGET, "Approval voting parallel is enabled skipping messages"); + continue; + } let message = match message { Ok(message) => message, Err(e) => { diff --git a/polkadot/node/network/approval-distribution/src/tests.rs b/polkadot/node/network/approval-distribution/src/tests.rs index 3f926746449d..5c32e67aa80f 100644 --- a/polkadot/node/network/approval-distribution/src/tests.rs +++ b/polkadot/node/network/approval-distribution/src/tests.rs @@ -54,7 +54,7 @@ type VirtualOverseer = fn test_harness>( assignment_criteria: &impl AssignmentCriteria, - clock: Box, + clock: Arc, mut state: State, test_fn: impl FnOnce(VirtualOverseer) -> T, ) -> State { @@ -68,7 +68,7 @@ fn test_harness>( polkadot_node_subsystem_test_helpers::make_subsystem_context(pool.clone()); let subsystem = - ApprovalDistribution::new_with_clock(Metrics::default(), Default::default(), clock); + ApprovalDistribution::new_with_clock(Metrics::default(), Default::default(), clock, false); { let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(12345); let mut session_info_provider = RuntimeInfo::new_with_config(RuntimeInfoConfig { @@ -523,7 +523,7 @@ fn try_import_the_same_assignment() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -631,7 +631,7 @@ fn try_import_the_same_assignment_v2() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -744,7 +744,7 @@ fn delay_reputation_change() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state_with_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -819,7 +819,7 @@ fn spam_attack_results_in_negative_reputation_change() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -918,7 +918,7 @@ fn peer_sending_us_the_same_we_just_sent_them_is_ok() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -1019,7 +1019,7 @@ fn import_approval_happy_path_v1_v2_peers() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -1159,7 +1159,7 @@ fn import_approval_happy_path_v2() { let candidate_hash_second = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xCC)); let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -1290,7 +1290,7 @@ fn multiple_assignments_covered_with_one_approval_vote() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -1502,7 +1502,7 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -1703,7 +1703,7 @@ fn import_approval_bad() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -1791,7 +1791,7 @@ fn update_our_view() { let state = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), State::default(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -1839,7 +1839,7 @@ fn update_our_view() { let state = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state, |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -1858,7 +1858,7 @@ fn update_our_view() { let state = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state, |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -1886,7 +1886,7 @@ fn update_peer_view() { let state = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), State::default(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -1985,7 +1985,7 @@ fn update_peer_view() { let state = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state, |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -2045,7 +2045,7 @@ fn update_peer_view() { let finalized_number = 4_000_000_000; let state = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state, |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -2087,7 +2087,7 @@ fn update_peer_authority_id() { let _state = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), State::default(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -2268,7 +2268,7 @@ fn import_remotely_then_locally() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -2375,7 +2375,7 @@ fn sends_assignments_even_when_state_is_approved() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), State::default(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -2481,7 +2481,7 @@ fn sends_assignments_even_when_state_is_approved_v2() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), State::default(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -2607,7 +2607,7 @@ fn race_condition_in_local_vs_remote_view_update() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -2694,7 +2694,7 @@ fn propagates_locally_generated_assignment_to_both_dimensions() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), State::default(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -2824,7 +2824,7 @@ fn propagates_assignments_along_unshared_dimension() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -2987,7 +2987,7 @@ fn propagates_to_required_after_connect() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), State::default(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -3152,7 +3152,7 @@ fn sends_to_more_peers_after_getting_topology() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), State::default(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -3290,7 +3290,7 @@ fn originator_aggression_l1() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state, |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -3471,7 +3471,7 @@ fn non_originator_aggression_l1() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state, |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -3598,7 +3598,7 @@ fn non_originator_aggression_l2() { let aggression_l2_threshold = state.aggression_config.l2_threshold.unwrap(); let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state, |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -3785,7 +3785,7 @@ fn resends_messages_periodically() { state.aggression_config.resend_unfinalized_period = Some(2); let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state, |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -3951,7 +3951,7 @@ fn import_versioned_approval() { let candidate_hash = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xBB)); let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(SystemClock {}), + Arc::new(SystemClock {}), state, |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -4124,7 +4124,8 @@ fn batch_test_round(message_count: usize) { let subsystem = ApprovalDistribution::new_with_clock( Default::default(), Default::default(), - Box::new(SystemClock {}), + Arc::new(SystemClock {}), + false, ); let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(12345); let mut sender = context.sender().clone(); @@ -4311,7 +4312,7 @@ fn subsystem_rejects_assignment_in_future() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(89) }, - Box::new(DummyClock {}), + Arc::new(DummyClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -4377,7 +4378,7 @@ fn subsystem_rejects_bad_assignments() { &MockAssignmentCriteria { tranche: Err(InvalidAssignment(criteria::InvalidAssignmentReason::NullAssignment)), }, - Box::new(DummyClock {}), + Arc::new(DummyClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; diff --git a/polkadot/node/network/bridge/src/rx/mod.rs b/polkadot/node/network/bridge/src/rx/mod.rs index 84e935366d0c..8053ad5a8d85 100644 --- a/polkadot/node/network/bridge/src/rx/mod.rs +++ b/polkadot/node/network/bridge/src/rx/mod.rs @@ -45,8 +45,9 @@ use polkadot_node_subsystem::{ errors::SubsystemError, messages::{ network_bridge_event::NewGossipTopology, ApprovalDistributionMessage, - BitfieldDistributionMessage, CollatorProtocolMessage, GossipSupportMessage, - NetworkBridgeEvent, NetworkBridgeRxMessage, StatementDistributionMessage, + ApprovalVotingParallelMessage, BitfieldDistributionMessage, CollatorProtocolMessage, + GossipSupportMessage, NetworkBridgeEvent, NetworkBridgeRxMessage, + StatementDistributionMessage, }, overseer, ActivatedLeaf, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, SpawnedSubsystem, }; @@ -89,6 +90,7 @@ pub struct NetworkBridgeRx { validation_service: Box, collation_service: Box, notification_sinks: Arc>>>, + approval_voting_parallel_enabled: bool, } impl NetworkBridgeRx { @@ -105,6 +107,7 @@ impl NetworkBridgeRx { peerset_protocol_names: PeerSetProtocolNames, mut notification_services: HashMap>, notification_sinks: Arc>>>, + approval_voting_parallel_enabled: bool, ) -> Self { let shared = Shared::default(); @@ -125,6 +128,7 @@ impl NetworkBridgeRx { validation_service, collation_service, notification_sinks, + approval_voting_parallel_enabled, } } } @@ -156,6 +160,7 @@ async fn handle_validation_message( peerset_protocol_names: &PeerSetProtocolNames, notification_service: &mut Box, notification_sinks: &mut Arc>>>, + approval_voting_parallel_enabled: bool, ) where AD: validator_discovery::AuthorityDiscovery + Send, { @@ -276,6 +281,7 @@ async fn handle_validation_message( ], sender, &metrics, + approval_voting_parallel_enabled, ) .await; @@ -329,6 +335,7 @@ async fn handle_validation_message( NetworkBridgeEvent::PeerDisconnected(peer), sender, &metrics, + approval_voting_parallel_enabled, ) .await; } @@ -398,7 +405,13 @@ async fn handle_validation_message( network_service.report_peer(peer, report.into()); } - dispatch_validation_events_to_all(events, sender, &metrics).await; + dispatch_validation_events_to_all( + events, + sender, + &metrics, + approval_voting_parallel_enabled, + ) + .await; }, } } @@ -652,6 +665,7 @@ async fn handle_network_messages( mut validation_service: Box, mut collation_service: Box, mut notification_sinks: Arc>>>, + approval_voting_parallel_enabled: bool, ) -> Result<(), Error> where AD: validator_discovery::AuthorityDiscovery + Send, @@ -669,6 +683,7 @@ where &peerset_protocol_names, &mut validation_service, &mut notification_sinks, + approval_voting_parallel_enabled, ).await, None => return Err(Error::EventStreamConcluded), }, @@ -727,6 +742,7 @@ async fn run_incoming_orchestra_signals( sync_oracle: Box, metrics: Metrics, notification_sinks: Arc>>>, + approval_voting_parallel_enabled: bool, ) -> Result<(), Error> where AD: validator_discovery::AuthorityDiscovery + Clone, @@ -766,6 +782,7 @@ where local_index, }), ctx.sender(), + approval_voting_parallel_enabled, ); }, FromOrchestra::Communication { @@ -787,6 +804,7 @@ where dispatch_validation_event_to_all_unbounded( NetworkBridgeEvent::UpdatedAuthorityIds(peer_id, authority_ids), ctx.sender(), + approval_voting_parallel_enabled, ); }, FromOrchestra::Signal(OverseerSignal::Conclude) => return Ok(()), @@ -826,6 +844,7 @@ where finalized_number, &metrics, ¬ification_sinks, + approval_voting_parallel_enabled, ); note_peers_count(&metrics, &shared); } @@ -875,6 +894,7 @@ where validation_service, collation_service, notification_sinks, + approval_voting_parallel_enabled, } = bridge; let (task, network_event_handler) = handle_network_messages( @@ -887,6 +907,7 @@ where validation_service, collation_service, notification_sinks.clone(), + approval_voting_parallel_enabled, ) .remote_handle(); @@ -900,6 +921,7 @@ where sync_oracle, metrics, notification_sinks, + approval_voting_parallel_enabled, ); futures::pin_mut!(orchestra_signal_handler); @@ -926,6 +948,7 @@ fn update_our_view( finalized_number: BlockNumber, metrics: &Metrics, notification_sinks: &Arc>>>, + approval_voting_parallel_enabled: bool, ) { let new_view = construct_view(live_heads.iter().map(|v| v.hash), finalized_number); @@ -1016,6 +1039,7 @@ fn update_our_view( dispatch_validation_event_to_all_unbounded( NetworkBridgeEvent::OurViewChange(our_view.clone()), ctx.sender(), + approval_voting_parallel_enabled, ); dispatch_collation_event_to_all_unbounded( @@ -1081,8 +1105,15 @@ async fn dispatch_validation_event_to_all( event: NetworkBridgeEvent, ctx: &mut impl overseer::NetworkBridgeRxSenderTrait, metrics: &Metrics, + approval_voting_parallel_enabled: bool, ) { - dispatch_validation_events_to_all(std::iter::once(event), ctx, metrics).await + dispatch_validation_events_to_all( + std::iter::once(event), + ctx, + metrics, + approval_voting_parallel_enabled, + ) + .await } async fn dispatch_collation_event_to_all( @@ -1095,6 +1126,7 @@ async fn dispatch_collation_event_to_all( fn dispatch_validation_event_to_all_unbounded( event: NetworkBridgeEvent, sender: &mut impl overseer::NetworkBridgeRxSenderTrait, + approval_voting_parallel_enabled: bool, ) { event .focus() @@ -1106,11 +1138,20 @@ fn dispatch_validation_event_to_all_unbounded( .ok() .map(BitfieldDistributionMessage::from) .and_then(|msg| Some(sender.send_unbounded_message(msg))); - event - .focus() - .ok() - .map(ApprovalDistributionMessage::from) - .and_then(|msg| Some(sender.send_unbounded_message(msg))); + + if approval_voting_parallel_enabled { + event + .focus() + .ok() + .map(ApprovalVotingParallelMessage::from) + .and_then(|msg| Some(sender.send_unbounded_message(msg))); + } else { + event + .focus() + .ok() + .map(ApprovalDistributionMessage::from) + .and_then(|msg| Some(sender.send_unbounded_message(msg))); + } event .focus() .ok() @@ -1131,6 +1172,7 @@ async fn dispatch_validation_events_to_all( events: I, sender: &mut impl overseer::NetworkBridgeRxSenderTrait, _metrics: &Metrics, + approval_voting_parallel_enabled: bool, ) where I: IntoIterator>, I::IntoIter: Send, @@ -1140,7 +1182,13 @@ async fn dispatch_validation_events_to_all( .send_messages(event.focus().map(StatementDistributionMessage::from)) .await; sender.send_messages(event.focus().map(BitfieldDistributionMessage::from)).await; - sender.send_messages(event.focus().map(ApprovalDistributionMessage::from)).await; + if approval_voting_parallel_enabled { + sender + .send_messages(event.focus().map(ApprovalVotingParallelMessage::from)) + .await; + } else { + sender.send_messages(event.focus().map(ApprovalDistributionMessage::from)).await; + } sender.send_messages(event.focus().map(GossipSupportMessage::from)).await; } } diff --git a/polkadot/node/network/bridge/src/rx/tests.rs b/polkadot/node/network/bridge/src/rx/tests.rs index 6182bf3d883b..e267446c2693 100644 --- a/polkadot/node/network/bridge/src/rx/tests.rs +++ b/polkadot/node/network/bridge/src/rx/tests.rs @@ -521,6 +521,7 @@ fn test_harness>( validation_service, collation_service, notification_sinks, + approval_voting_parallel_enabled: false, }; let network_bridge = run_network_in(bridge, context) diff --git a/polkadot/node/overseer/src/dummy.rs b/polkadot/node/overseer/src/dummy.rs index fc5f0070773b..6f9cd9d00403 100644 --- a/polkadot/node/overseer/src/dummy.rs +++ b/polkadot/node/overseer/src/dummy.rs @@ -88,6 +88,7 @@ pub fn dummy_overseer_builder( DummySubsystem, DummySubsystem, DummySubsystem, + DummySubsystem, >, SubsystemError, > @@ -131,6 +132,7 @@ pub fn one_for_all_overseer_builder( Sub, Sub, Sub, + Sub, >, SubsystemError, > @@ -155,6 +157,7 @@ where + Subsystem, SubsystemError> + Subsystem, SubsystemError> + Subsystem, SubsystemError> + + Subsystem, SubsystemError> + Subsystem, SubsystemError> + Subsystem, SubsystemError> + Subsystem, SubsystemError> @@ -183,6 +186,7 @@ where .statement_distribution(subsystem.clone()) .approval_distribution(subsystem.clone()) .approval_voting(subsystem.clone()) + .approval_voting_parallel(subsystem.clone()) .gossip_support(subsystem.clone()) .dispute_coordinator(subsystem.clone()) .dispute_distribution(subsystem.clone()) diff --git a/polkadot/node/overseer/src/lib.rs b/polkadot/node/overseer/src/lib.rs index 2c113f81c85f..8d08f1a3aeb8 100644 --- a/polkadot/node/overseer/src/lib.rs +++ b/polkadot/node/overseer/src/lib.rs @@ -76,13 +76,13 @@ use sc_client_api::{BlockImportNotification, BlockchainEvents, FinalityNotificat use self::messages::{BitfieldSigningMessage, PvfCheckerMessage}; use polkadot_node_subsystem_types::messages::{ - ApprovalDistributionMessage, ApprovalVotingMessage, AvailabilityDistributionMessage, - AvailabilityRecoveryMessage, AvailabilityStoreMessage, BitfieldDistributionMessage, - CandidateBackingMessage, CandidateValidationMessage, ChainApiMessage, ChainSelectionMessage, - CollationGenerationMessage, CollatorProtocolMessage, DisputeCoordinatorMessage, - DisputeDistributionMessage, GossipSupportMessage, NetworkBridgeRxMessage, - NetworkBridgeTxMessage, ProspectiveParachainsMessage, ProvisionerMessage, RuntimeApiMessage, - StatementDistributionMessage, + ApprovalDistributionMessage, ApprovalVotingMessage, ApprovalVotingParallelMessage, + AvailabilityDistributionMessage, AvailabilityRecoveryMessage, AvailabilityStoreMessage, + BitfieldDistributionMessage, CandidateBackingMessage, CandidateValidationMessage, + ChainApiMessage, ChainSelectionMessage, CollationGenerationMessage, CollatorProtocolMessage, + DisputeCoordinatorMessage, DisputeDistributionMessage, GossipSupportMessage, + NetworkBridgeRxMessage, NetworkBridgeTxMessage, ProspectiveParachainsMessage, + ProvisionerMessage, RuntimeApiMessage, StatementDistributionMessage, }; pub use polkadot_node_subsystem_types::{ @@ -549,6 +549,7 @@ pub struct Overseer { BitfieldDistributionMessage, StatementDistributionMessage, ApprovalDistributionMessage, + ApprovalVotingParallelMessage, GossipSupportMessage, DisputeDistributionMessage, CollationGenerationMessage, @@ -594,7 +595,19 @@ pub struct Overseer { RuntimeApiMessage, ])] approval_voting: ApprovalVoting, - + #[subsystem(blocking, message_capacity: 64000, ApprovalVotingParallelMessage, sends: [ + AvailabilityRecoveryMessage, + CandidateValidationMessage, + ChainApiMessage, + ChainSelectionMessage, + DisputeCoordinatorMessage, + RuntimeApiMessage, + NetworkBridgeTxMessage, + ApprovalVotingMessage, + ApprovalDistributionMessage, + ApprovalVotingParallelMessage, + ])] + approval_voting_parallel: ApprovalVotingParallel, #[subsystem(GossipSupportMessage, sends: [ NetworkBridgeTxMessage, NetworkBridgeRxMessage, // TODO @@ -612,6 +625,7 @@ pub struct Overseer { AvailabilityStoreMessage, AvailabilityRecoveryMessage, ChainSelectionMessage, + ApprovalVotingParallelMessage, ])] dispute_coordinator: DisputeCoordinator, diff --git a/polkadot/node/overseer/src/tests.rs b/polkadot/node/overseer/src/tests.rs index 8e78d8fc8921..8833b93c0d1b 100644 --- a/polkadot/node/overseer/src/tests.rs +++ b/polkadot/node/overseer/src/tests.rs @@ -1101,6 +1101,7 @@ fn context_holds_onto_message_until_enough_signals_received() { let (chain_selection_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); let (pvf_checker_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); let (prospective_parachains_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (approval_voting_parallel_tx, _) = metered::channel(CHANNEL_CAPACITY); let (candidate_validation_unbounded_tx, _) = metered::unbounded(); let (candidate_backing_unbounded_tx, _) = metered::unbounded(); @@ -1125,6 +1126,7 @@ fn context_holds_onto_message_until_enough_signals_received() { let (chain_selection_unbounded_tx, _) = metered::unbounded(); let (pvf_checker_unbounded_tx, _) = metered::unbounded(); let (prospective_parachains_unbounded_tx, _) = metered::unbounded(); + let (approval_voting_parallel_unbounded_tx, _) = metered::unbounded(); let channels_out = ChannelsOut { candidate_validation: candidate_validation_bounded_tx.clone(), @@ -1150,6 +1152,7 @@ fn context_holds_onto_message_until_enough_signals_received() { chain_selection: chain_selection_bounded_tx.clone(), pvf_checker: pvf_checker_bounded_tx.clone(), prospective_parachains: prospective_parachains_bounded_tx.clone(), + approval_voting_parallel: approval_voting_parallel_tx.clone(), candidate_validation_unbounded: candidate_validation_unbounded_tx.clone(), candidate_backing_unbounded: candidate_backing_unbounded_tx.clone(), @@ -1174,6 +1177,7 @@ fn context_holds_onto_message_until_enough_signals_received() { chain_selection_unbounded: chain_selection_unbounded_tx.clone(), pvf_checker_unbounded: pvf_checker_unbounded_tx.clone(), prospective_parachains_unbounded: prospective_parachains_unbounded_tx.clone(), + approval_voting_parallel_unbounded: approval_voting_parallel_unbounded_tx.clone(), }; let (mut signal_tx, signal_rx) = metered::channel(CHANNEL_CAPACITY); diff --git a/polkadot/node/primitives/src/approval.rs b/polkadot/node/primitives/src/approval.rs index ec41647fc3d1..d1874f760364 100644 --- a/polkadot/node/primitives/src/approval.rs +++ b/polkadot/node/primitives/src/approval.rs @@ -18,7 +18,7 @@ /// A list of primitives introduced in v1. pub mod v1 { - use sp_consensus_babe as babe_primitives; + use sp_consensus_babe::{self as babe_primitives, SlotDuration}; pub use sp_consensus_babe::{ Randomness, Slot, VrfPreOutput, VrfProof, VrfSignature, VrfTranscript, }; @@ -118,7 +118,7 @@ pub mod v1 { } /// Metadata about a block which is now live in the approval protocol. - #[derive(Debug)] + #[derive(Debug, Clone)] pub struct BlockApprovalMeta { /// The hash of the block. pub hash: Hash, diff --git a/polkadot/node/service/Cargo.toml b/polkadot/node/service/Cargo.toml index c0ddbf7dcfc3..0f05fc14612d 100644 --- a/polkadot/node/service/Cargo.toml +++ b/polkadot/node/service/Cargo.toml @@ -128,6 +128,7 @@ polkadot-gossip-support = { optional = true, workspace = true, default-features polkadot-network-bridge = { optional = true, workspace = true, default-features = true } polkadot-node-collation-generation = { optional = true, workspace = true, default-features = true } polkadot-node-core-approval-voting = { optional = true, workspace = true, default-features = true } +polkadot-node-core-approval-voting-parallel = { optional = true, workspace = true, default-features = true } polkadot-node-core-av-store = { optional = true, workspace = true, default-features = true } polkadot-node-core-backing = { optional = true, workspace = true, default-features = true } polkadot-node-core-bitfield-signing = { optional = true, workspace = true, default-features = true } @@ -172,6 +173,7 @@ full-node = [ "polkadot-network-bridge", "polkadot-node-collation-generation", "polkadot-node-core-approval-voting", + "polkadot-node-core-approval-voting-parallel", "polkadot-node-core-av-store", "polkadot-node-core-backing", "polkadot-node-core-bitfield-signing", diff --git a/polkadot/node/service/src/lib.rs b/polkadot/node/service/src/lib.rs index b76f40dd3102..1a174e094477 100644 --- a/polkadot/node/service/src/lib.rs +++ b/polkadot/node/service/src/lib.rs @@ -668,6 +668,8 @@ pub struct NewFullParams { #[allow(dead_code)] pub malus_finality_delay: Option, pub hwbench: Option, + /// Enable approval voting processing in parallel. + pub enable_approval_voting_parallel: bool, } #[cfg(feature = "full-node")] @@ -761,6 +763,7 @@ pub fn new_full< execute_workers_max_num, prepare_workers_soft_max_num, prepare_workers_hard_max_num, + enable_approval_voting_parallel, }: NewFullParams, ) -> Result { use polkadot_availability_recovery::FETCH_CHUNKS_THRESHOLD; @@ -816,6 +819,7 @@ pub fn new_full< overseer_handle.clone(), metrics, Some(basics.task_manager.spawn_handle()), + enable_approval_voting_parallel, ) } else { SelectRelayChain::new_longest_chain(basics.backend.clone()) @@ -1024,6 +1028,7 @@ pub fn new_full< dispute_coordinator_config, chain_selection_config, fetch_chunks_threshold, + enable_approval_voting_parallel, }) }; @@ -1543,6 +1548,7 @@ fn revert_approval_voting( Box::new(sp_consensus::NoNetwork), approval_voting_subsystem::Metrics::default(), Arc::new(SpawnGlue(task_handle)), + false, ); approval_voting diff --git a/polkadot/node/service/src/overseer.rs b/polkadot/node/service/src/overseer.rs index 1a14b3f85cb1..5a4a5a3d65ed 100644 --- a/polkadot/node/service/src/overseer.rs +++ b/polkadot/node/service/src/overseer.rs @@ -58,6 +58,9 @@ pub use polkadot_network_bridge::{ }; pub use polkadot_node_collation_generation::CollationGenerationSubsystem; pub use polkadot_node_core_approval_voting::ApprovalVotingSubsystem; +pub use polkadot_node_core_approval_voting_parallel::{ + ApprovalVotingParallelSubsystem, Metrics as ApprovalVotingParallelMetrics, +}; pub use polkadot_node_core_av_store::AvailabilityStoreSubsystem; pub use polkadot_node_core_backing::CandidateBackingSubsystem; pub use polkadot_node_core_bitfield_signing::BitfieldSigningSubsystem; @@ -139,6 +142,7 @@ pub struct ExtendedOverseerGenArgs { /// than the value put in here we always try to recovery availability from backers. /// The presence of this parameter here is needed to have different values per chain. pub fetch_chunks_threshold: Option, + pub enable_approval_voting_parallel: bool, } /// Obtain a prepared validator `Overseer`, that is initialized with all default values. @@ -174,6 +178,7 @@ pub fn validator_overseer_builder( dispute_coordinator_config, chain_selection_config, fetch_chunks_threshold, + enable_approval_voting_parallel, }: ExtendedOverseerGenArgs, ) -> Result< InitializedOverseerBuilder< @@ -203,6 +208,7 @@ pub fn validator_overseer_builder( CollatorProtocolSubsystem, ApprovalDistributionSubsystem, ApprovalVotingSubsystem, + ApprovalVotingParallelSubsystem, GossipSupportSubsystem, DisputeCoordinatorSubsystem, DisputeDistributionSubsystem, @@ -223,7 +229,8 @@ where let spawner = SpawnGlue(spawner); let network_bridge_metrics: NetworkBridgeMetrics = Metrics::register(registry)?; - + let approval_voting_parallel_metrics: ApprovalVotingParallelMetrics = + Metrics::register(registry)?; let builder = Overseer::builder() .network_bridge_tx(NetworkBridgeTxSubsystem::new( network_service.clone(), @@ -241,6 +248,7 @@ where peerset_protocol_names, notification_services, notification_sinks, + enable_approval_voting_parallel, )) .availability_distribution(AvailabilityDistributionSubsystem::new( keystore.clone(), @@ -309,16 +317,27 @@ where rand::rngs::StdRng::from_entropy(), )) .approval_distribution(ApprovalDistributionSubsystem::new( - Metrics::register(registry)?, + approval_voting_parallel_metrics.0.clone(), approval_voting_config.slot_duration_millis, + enable_approval_voting_parallel, )) .approval_voting(ApprovalVotingSubsystem::with_config( - approval_voting_config, + approval_voting_config.clone(), parachains_db.clone(), keystore.clone(), Box::new(sync_service.clone()), - Metrics::register(registry)?, + approval_voting_parallel_metrics.1.clone(), Arc::new(spawner.clone()), + enable_approval_voting_parallel, + )) + .approval_voting_parallel(ApprovalVotingParallelSubsystem::with_config( + approval_voting_config, + parachains_db.clone(), + keystore.clone(), + Box::new(sync_service.clone()), + approval_voting_parallel_metrics, + spawner.clone(), + enable_approval_voting_parallel, )) .gossip_support(GossipSupportSubsystem::new( keystore.clone(), @@ -330,6 +349,7 @@ where dispute_coordinator_config, keystore.clone(), Metrics::register(registry)?, + enable_approval_voting_parallel, )) .dispute_distribution(DisputeDistributionSubsystem::new( keystore.clone(), @@ -405,6 +425,7 @@ pub fn collator_overseer_builder( DummySubsystem, DummySubsystem, DummySubsystem, + DummySubsystem, >, Error, > @@ -437,6 +458,7 @@ where peerset_protocol_names, notification_services, notification_sinks, + false, )) .availability_distribution(DummySubsystem) .availability_recovery(AvailabilityRecoverySubsystem::for_collator( @@ -479,6 +501,7 @@ where .statement_distribution(DummySubsystem) .approval_distribution(DummySubsystem) .approval_voting(DummySubsystem) + .approval_voting_parallel(DummySubsystem) .gossip_support(DummySubsystem) .dispute_coordinator(DummySubsystem) .dispute_distribution(DummySubsystem) diff --git a/polkadot/node/service/src/relay_chain_selection.rs b/polkadot/node/service/src/relay_chain_selection.rs index c0b1ce8b0ebe..e48874f01ca6 100644 --- a/polkadot/node/service/src/relay_chain_selection.rs +++ b/polkadot/node/service/src/relay_chain_selection.rs @@ -39,8 +39,8 @@ use super::{HeaderProvider, HeaderProviderProvider}; use futures::channel::oneshot; use polkadot_node_primitives::MAX_FINALITY_LAG as PRIMITIVES_MAX_FINALITY_LAG; use polkadot_node_subsystem::messages::{ - ApprovalDistributionMessage, ApprovalVotingMessage, ChainSelectionMessage, - DisputeCoordinatorMessage, HighestApprovedAncestorBlock, + ApprovalDistributionMessage, ApprovalVotingMessage, ApprovalVotingParallelMessage, + ChainSelectionMessage, DisputeCoordinatorMessage, HighestApprovedAncestorBlock, }; use polkadot_node_subsystem_util::metrics::{self, prometheus}; use polkadot_overseer::{AllMessages, Handle}; @@ -169,6 +169,7 @@ where overseer: Handle, metrics: Metrics, spawn_handle: Option, + approval_voting_parallel_enabled: bool, ) -> Self { gum::debug!(target: LOG_TARGET, "Using dispute aware relay-chain selection algorithm",); @@ -179,6 +180,7 @@ where overseer, metrics, spawn_handle, + approval_voting_parallel_enabled, )), } } @@ -230,6 +232,7 @@ pub struct SelectRelayChainInner { overseer: OH, metrics: Metrics, spawn_handle: Option, + approval_voting_parallel_enabled: bool, } impl SelectRelayChainInner @@ -244,8 +247,15 @@ where overseer: OH, metrics: Metrics, spawn_handle: Option, + approval_voting_parallel_enabled: bool, ) -> Self { - SelectRelayChainInner { backend, overseer, metrics, spawn_handle } + SelectRelayChainInner { + backend, + overseer, + metrics, + spawn_handle, + approval_voting_parallel_enabled, + } } fn block_header(&self, hash: Hash) -> Result { @@ -284,6 +294,7 @@ where overseer: self.overseer.clone(), metrics: self.metrics.clone(), spawn_handle: self.spawn_handle.clone(), + approval_voting_parallel_enabled: self.approval_voting_parallel_enabled, } } } @@ -448,13 +459,25 @@ where // 2. Constrain according to `ApprovedAncestor`. let (subchain_head, subchain_number, subchain_block_descriptions) = { let (tx, rx) = oneshot::channel(); - overseer - .send_msg( - ApprovalVotingMessage::ApprovedAncestor(subchain_head, target_number, tx), - std::any::type_name::(), - ) - .await; - + if self.approval_voting_parallel_enabled { + overseer + .send_msg( + ApprovalVotingParallelMessage::ApprovedAncestor( + subchain_head, + target_number, + tx, + ), + std::any::type_name::(), + ) + .await; + } else { + overseer + .send_msg( + ApprovalVotingMessage::ApprovedAncestor(subchain_head, target_number, tx), + std::any::type_name::(), + ) + .await; + } match rx .await .map_err(Error::ApprovedAncestorCanceled) @@ -476,13 +499,23 @@ where // task for sending the message to not block here and delay finality. if let Some(spawn_handle) = &self.spawn_handle { let mut overseer_handle = self.overseer.clone(); + let approval_voting_parallel_enabled = self.approval_voting_parallel_enabled; let lag_update_task = async move { - overseer_handle - .send_msg( - ApprovalDistributionMessage::ApprovalCheckingLagUpdate(lag), - std::any::type_name::(), - ) - .await; + if approval_voting_parallel_enabled { + overseer_handle + .send_msg( + ApprovalVotingParallelMessage::ApprovalCheckingLagUpdate(lag), + std::any::type_name::(), + ) + .await; + } else { + overseer_handle + .send_msg( + ApprovalDistributionMessage::ApprovalCheckingLagUpdate(lag), + std::any::type_name::(), + ) + .await; + } }; spawn_handle.spawn( diff --git a/polkadot/node/service/src/tests.rs b/polkadot/node/service/src/tests.rs index bebd05071013..0b438fd6c1d5 100644 --- a/polkadot/node/service/src/tests.rs +++ b/polkadot/node/service/src/tests.rs @@ -86,6 +86,7 @@ fn test_harness>( context.sender().clone(), Default::default(), None, + false, ); let target_hash = case_vars.target_block; diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index 0325613d25f9..e429a6fe982a 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -76,6 +76,7 @@ serde_yaml = { workspace = true } serde_json = { workspace = true } polkadot-node-core-approval-voting = { workspace = true, default-features = true } +polkadot-node-core-approval-voting-parallel = { workspace = true, default-features = true } polkadot-approval-distribution = { workspace = true, default-features = true } sp-consensus-babe = { workspace = true, default-features = true } sp-runtime = { workspace = true } diff --git a/polkadot/node/subsystem-bench/src/lib/approval/helpers.rs b/polkadot/node/subsystem-bench/src/lib/approval/helpers.rs index ca58875c8139..f9eaacfeb148 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/helpers.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/helpers.rs @@ -21,8 +21,9 @@ use polkadot_node_network_protocol::{ grid_topology::{SessionGridTopology, TopologyPeerInfo}, View, }; +use polkadot_node_subsystem::messages::ApprovalVotingParallelMessage; use polkadot_node_subsystem_types::messages::{ - network_bridge_event::NewGossipTopology, ApprovalDistributionMessage, NetworkBridgeEvent, + network_bridge_event::NewGossipTopology, NetworkBridgeEvent, }; use polkadot_overseer::AllMessages; use polkadot_primitives::{ @@ -129,14 +130,16 @@ pub fn generate_new_session_topology( topology, local_index: Some(test_node), }); - vec![AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate(event))] + vec![AllMessages::ApprovalVotingParallel(ApprovalVotingParallelMessage::NetworkBridgeUpdate( + event, + ))] } /// Generates a peer view change for the passed `block_hash` pub fn generate_peer_view_change_for(block_hash: Hash, peer_id: PeerId) -> AllMessages { let network = NetworkBridgeEvent::PeerViewChange(peer_id, View::new([block_hash], 0)); - AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate(network)) + AllMessages::ApprovalVotingParallel(ApprovalVotingParallelMessage::NetworkBridgeUpdate(network)) } /// Helper function to create a a signature for the block header. diff --git a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs index 382360eda664..7bdf2af54f54 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs @@ -13,7 +13,6 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . - use crate::{ approval::{ helpers::{ @@ -49,14 +48,15 @@ use overseer::{metrics::Metrics as OverseerMetrics, MetricsTrait}; use polkadot_approval_distribution::ApprovalDistribution; use polkadot_node_core_approval_voting::{ time::{slot_number_to_tick, tick_to_slot_number, Clock, ClockExt, SystemClock}, - ApprovalVotingSubsystem, Config as ApprovalVotingConfig, Metrics as ApprovalVotingMetrics, + ApprovalVotingSubsystem, Config as ApprovalVotingConfig, }; use polkadot_node_network_protocol::v3 as protocol_v3; use polkadot_node_primitives::approval::{self, v1::RelayVRFStory}; -use polkadot_node_subsystem::{overseer, AllMessages, Overseer, OverseerConnector, SpawnGlue}; +use polkadot_node_subsystem::{ + messages::ApprovalVotingParallelMessage, overseer, AllMessages, Overseer, OverseerConnector, + SpawnGlue, +}; use polkadot_node_subsystem_test_helpers::mock::new_block_import_info; -use polkadot_node_subsystem_types::messages::{ApprovalDistributionMessage, ApprovalVotingMessage}; -use polkadot_node_subsystem_util::metrics::Metrics; use polkadot_overseer::Handle as OverseerHandleReal; use polkadot_primitives::{ BlockNumber, CandidateEvent, CandidateIndex, CandidateReceipt, Hash, Header, Slot, @@ -265,7 +265,7 @@ pub struct ApprovalTestState { /// Total unique sent messages. total_unique_messages: Arc, /// Approval voting metrics. - approval_voting_metrics: ApprovalVotingMetrics, + approval_voting_metrics_rewrite: polkadot_node_core_approval_voting_parallel::Metrics, /// The delta ticks from the tick the messages were generated to the the time we start this /// message. delta_tick_from_generated: Arc, @@ -323,7 +323,10 @@ impl ApprovalTestState { total_sent_messages_from_node: Arc::new(AtomicU64::new(0)), total_unique_messages: Arc::new(AtomicU64::new(0)), options, - approval_voting_metrics: ApprovalVotingMetrics::try_register(&dependencies.registry) + approval_voting_metrics_rewrite: + polkadot_node_core_approval_voting_parallel::Metrics::try_register( + &dependencies.registry, + ) .unwrap(), delta_tick_from_generated: Arc::new(AtomicU64::new(630720000)), configuration: configuration.clone(), @@ -590,9 +593,9 @@ impl PeerMessageProducer { // so when the approval-distribution answered to it, we know it doesn't have anything // else to process. let (tx, rx) = oneshot::channel(); - let msg = ApprovalDistributionMessage::GetApprovalSignatures(HashSet::new(), tx); + let msg = ApprovalVotingParallelMessage::GetApprovalSignatures(HashSet::new(), tx); self.send_overseer_message( - AllMessages::ApprovalDistribution(msg), + AllMessages::ApprovalVotingParallel(msg), ValidatorIndex(0), None, ) @@ -795,21 +798,38 @@ fn build_overseer( let system_clock = PastSystemClock::new(SystemClock {}, state.delta_tick_from_generated.clone()); + let keystore = Arc::new(keystore); + let db = Arc::new(db); let approval_voting = ApprovalVotingSubsystem::with_config_and_clock( TEST_CONFIG, - Arc::new(db), - Arc::new(keystore), + db.clone(), + keystore.clone(), Box::new(TestSyncOracle {}), - state.approval_voting_metrics.clone(), + state.approval_voting_metrics_rewrite.1.clone(), Arc::new(system_clock.clone()), Arc::new(SpawnGlue(spawn_task_handle.clone())), + true, ); let approval_distribution = ApprovalDistribution::new_with_clock( - Metrics::register(Some(&dependencies.registry)).unwrap(), - SLOT_DURATION_MILLIS, - Box::new(system_clock.clone()), + state.approval_voting_metrics_rewrite.0.clone(), + TEST_CONFIG.slot_duration_millis as u64, + Arc::new(system_clock.clone()), + true, ); + + let approval_voting_parallel = + polkadot_node_core_approval_voting_parallel::ApprovalVotingParallelSubsystem::with_config_and_clock( + TEST_CONFIG, + db.clone(), + keystore.clone(), + Box::new(TestSyncOracle {}), + state.approval_voting_metrics_rewrite.clone(), + Arc::new(system_clock.clone()), + SpawnGlue(spawn_task_handle.clone()), + true, + ); + let mock_chain_api = MockChainApi::new(state.build_chain_api_state()); let mock_chain_selection = MockChainSelection { state: state.clone(), clock: system_clock }; let mock_runtime_api = MockRuntimeApi::new( @@ -831,6 +851,7 @@ fn build_overseer( let dummy = dummy_builder!(spawn_task_handle, overseer_metrics) .replace_approval_distribution(|_| approval_distribution) .replace_approval_voting(|_| approval_voting) + .replace_approval_voting_parallel(|_| approval_voting_parallel) .replace_chain_api(|_| mock_chain_api) .replace_chain_selection(|_| mock_chain_selection) .replace_runtime_api(|_| mock_runtime_api) @@ -927,7 +948,7 @@ pub async fn bench_approvals_run( // First create the initialization messages that make sure that then node under // tests receives notifications about the topology used and the connected peers. let mut initialization_messages = env.network().generate_peer_connected(|e| { - AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate(e)) + AllMessages::ApprovalVotingParallel(ApprovalVotingParallelMessage::NetworkBridgeUpdate(e)) }); initialization_messages.extend(generate_new_session_topology( &state.test_authorities, @@ -996,7 +1017,7 @@ pub async fn bench_approvals_run( state.total_sent_messages_to_node.load(std::sync::atomic::Ordering::SeqCst) as usize; env.wait_until_metric( "polkadot_parachain_subsystem_bounded_received", - Some(("subsystem_name", "approval-distribution-subsystem")), + Some(("subsystem_name", "approval-voting-parallel-subsystem")), |value| { gum::debug!(target: LOG_TARGET, ?value, ?at_least_messages, "Waiting metric"); value >= at_least_messages as f64 @@ -1013,11 +1034,11 @@ pub async fn bench_approvals_run( CandidateEvent::CandidateIncluded(receipt_fetch, _head, _, _) => { let (tx, rx) = oneshot::channel(); - let msg = ApprovalVotingMessage::GetApprovalSignaturesForCandidate( + let msg = ApprovalVotingParallelMessage::GetApprovalSignaturesForCandidate( receipt_fetch.hash(), tx, ); - env.send_message(AllMessages::ApprovalVoting(msg)).await; + env.send_message(AllMessages::ApprovalVotingParallel(msg)).await; let result = rx.await.unwrap(); @@ -1041,7 +1062,7 @@ pub async fn bench_approvals_run( state.total_sent_messages_to_node.load(std::sync::atomic::Ordering::SeqCst) as usize; env.wait_until_metric( "polkadot_parachain_subsystem_bounded_received", - Some(("subsystem_name", "approval-distribution-subsystem")), + Some(("subsystem_name", "approval-voting-parallel-subsystem")), |value| { gum::debug!(target: LOG_TARGET, ?value, ?at_least_messages, "Waiting metric"); value >= at_least_messages as f64 @@ -1082,5 +1103,9 @@ pub async fn bench_approvals_run( state.total_unique_messages.load(std::sync::atomic::Ordering::SeqCst) ); - env.collect_resource_usage(&["approval-distribution", "approval-voting"]) + env.collect_resource_usage(&[ + "approval-distribution", + "approval-voting", + "approval-voting-parallel", + ]) } diff --git a/polkadot/node/subsystem-bench/src/lib/environment.rs b/polkadot/node/subsystem-bench/src/lib/environment.rs index a63f90da50b3..7e3c7f42b613 100644 --- a/polkadot/node/subsystem-bench/src/lib/environment.rs +++ b/polkadot/node/subsystem-bench/src/lib/environment.rs @@ -24,6 +24,7 @@ use crate::{ }; use core::time::Duration; use futures::{Future, FutureExt}; +use polkadot_node_core_approval_voting_parallel::APPROVAL_DISTRIBUTION_WORKER_COUNT; use polkadot_node_subsystem::{messages::AllMessages, Overseer, SpawnGlue, TimeoutExt}; use polkadot_node_subsystem_types::Hash; use polkadot_node_subsystem_util::metrics::prometheus::{ @@ -392,6 +393,38 @@ impl TestEnvironment { total: total_cpu, per_block: total_cpu / num_blocks, }); + + if subsystem == &"approval-voting-parallel" { + for i in 0..APPROVAL_DISTRIBUTION_WORKER_COUNT { + let task_name = format!("approval-voting-parallel-{}", i); + + let subsystem_cpu_metrics = + test_metrics.subset_with_label_value("task_name", task_name.as_str()); + + let total_cpu = + subsystem_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); + + usage.push(ResourceUsage { + resource_name: task_name.to_string(), + total: total_cpu, + per_block: total_cpu / num_blocks, + }) + } + + let task_name = format!("approval-voting-parallel-db"); + + let subsystem_cpu_metrics = + test_metrics.subset_with_label_value("task_name", task_name.as_str()); + + let total_cpu = + subsystem_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); + + usage.push(ResourceUsage { + resource_name: task_name.to_string(), + total: total_cpu, + per_block: total_cpu / num_blocks, + }) + } } let test_env_cpu_metrics = diff --git a/polkadot/node/subsystem-bench/src/lib/mock/dummy.rs b/polkadot/node/subsystem-bench/src/lib/mock/dummy.rs index 8783b35f1c04..092a8fc5f4c1 100644 --- a/polkadot/node/subsystem-bench/src/lib/mock/dummy.rs +++ b/polkadot/node/subsystem-bench/src/lib/mock/dummy.rs @@ -96,5 +96,6 @@ mock!(NetworkBridgeTx); mock!(ChainApi); mock!(ChainSelection); mock!(ApprovalVoting); +mock!(ApprovalVotingParallel); mock!(ApprovalDistribution); mock!(RuntimeApi); diff --git a/polkadot/node/subsystem-bench/src/lib/mock/mod.rs b/polkadot/node/subsystem-bench/src/lib/mock/mod.rs index 12766374bfa9..3e33c4dde51a 100644 --- a/polkadot/node/subsystem-bench/src/lib/mock/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/mock/mod.rs @@ -45,6 +45,7 @@ macro_rules! dummy_builder { // All subsystem except approval_voting and approval_distribution are mock subsystems. Overseer::builder() .approval_voting(MockApprovalVoting {}) + .approval_voting_parallel(MockApprovalVotingParallel {}) .approval_distribution(MockApprovalDistribution {}) .availability_recovery(MockAvailabilityRecovery {}) .candidate_validation(MockCandidateValidation {}) diff --git a/polkadot/node/subsystem-bench/src/lib/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/lib/mock/network_bridge.rs index d70953926d13..a32b48705463 100644 --- a/polkadot/node/subsystem-bench/src/lib/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/lib/mock/network_bridge.rs @@ -24,7 +24,8 @@ use crate::{ use futures::{channel::mpsc::UnboundedSender, FutureExt, StreamExt}; use polkadot_node_network_protocol::Versioned; use polkadot_node_subsystem::{ - messages::NetworkBridgeTxMessage, overseer, SpawnedSubsystem, SubsystemError, + messages::{ApprovalVotingParallelMessage, NetworkBridgeTxMessage}, + overseer, SpawnedSubsystem, SubsystemError, }; use polkadot_node_subsystem_types::{ messages::{ @@ -200,7 +201,7 @@ impl MockNetworkBridgeRx { polkadot_node_network_protocol::v3::ValidationProtocol::ApprovalDistribution(msg) ) => { ctx.send_message( - ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage(peer_id, polkadot_node_network_protocol::Versioned::V3(msg))) + ApprovalVotingParallelMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage(peer_id, polkadot_node_network_protocol::Versioned::V3(msg))) ).await; } Versioned::V3( diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index e0c078de48b7..01de24996e78 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -945,6 +945,118 @@ pub struct BlockDescription { pub candidates: Vec, } +/// Message to the Approval Voting subsystem running both approval-distribution and approval-voting +/// logic in parallel. This is a combination of all the messages ApprovalVoting and +/// ApprovalDistribution subsystems can receive. +#[derive(Debug, derive_more::From)] +pub enum ApprovalVotingParallelMessage { + /// Check if the assignment is valid and can be accepted by our view of the protocol. + /// Should not be sent unless the block hash is known. + CheckAndImportAssignment(IndirectAssignmentCertV2, CandidateBitfield, DelayTranche), + /// Check if the approval vote is valid and can be accepted by our view of the + /// protocol. + /// + /// Should not be sent unless the block hash within the indirect vote is known. + CheckAndImportApproval(IndirectSignedApprovalVoteV2), + /// Returns the highest possible ancestor hash of the provided block hash which is + /// acceptable to vote on finality for. + /// The `BlockNumber` provided is the number of the block's ancestor which is the + /// earliest possible vote. + /// + /// It can also return the same block hash, if that is acceptable to vote upon. + /// Return `None` if the input hash is unrecognized. + ApprovedAncestor(Hash, BlockNumber, oneshot::Sender>), + + /// Retrieve all available approval signatures for a candidate from approval-voting. + /// + /// This message involves a linear search for candidates on each relay chain fork and also + /// requires calling into `approval-distribution`: Calls should be infrequent and bounded. + GetApprovalSignaturesForCandidate( + CandidateHash, + oneshot::Sender, ValidatorSignature)>>, + ), + /// Notify the `ApprovalDistribution` subsystem about new blocks + /// and the candidates contained within them. + NewBlocks(Vec), + /// Distribute an assignment cert from the local validator. The cert is assumed + /// to be valid, relevant, and for the given relay-parent and validator index. + DistributeAssignment(IndirectAssignmentCertV2, CandidateBitfield), + /// Distribute an approval vote for the local validator. The approval vote is assumed to be + /// valid, relevant, and the corresponding approval already issued. + /// If not, the subsystem is free to drop the message. + DistributeApproval(IndirectSignedApprovalVoteV2), + /// An update from the network bridge. + #[from] + NetworkBridgeUpdate(NetworkBridgeEvent), + + /// Get all approval signatures for all chains a candidate appeared in. + GetApprovalSignatures( + HashSet<(Hash, CandidateIndex)>, + oneshot::Sender, ValidatorSignature)>>, + ), + /// Approval checking lag update measured in blocks. + ApprovalCheckingLagUpdate(BlockNumber), +} + +impl TryFrom for ApprovalVotingMessage { + type Error = (); + + fn try_from(msg: ApprovalVotingParallelMessage) -> Result { + match msg { + ApprovalVotingParallelMessage::CheckAndImportAssignment(cert, bitfield, tranche) => + Ok(ApprovalVotingMessage::CheckAndImportAssignment(cert, bitfield, tranche, None)), + ApprovalVotingParallelMessage::CheckAndImportApproval(vote) => + Ok(ApprovalVotingMessage::CheckAndImportApproval(vote, None)), + ApprovalVotingParallelMessage::ApprovedAncestor(hash, number, tx) => + Ok(ApprovalVotingMessage::ApprovedAncestor(hash, number, tx)), + ApprovalVotingParallelMessage::GetApprovalSignaturesForCandidate(candidate, tx) => + Ok(ApprovalVotingMessage::GetApprovalSignaturesForCandidate(candidate, tx)), + _ => Err(()), + } + } +} + +impl TryFrom for ApprovalDistributionMessage { + type Error = (); + + fn try_from(msg: ApprovalVotingParallelMessage) -> Result { + match msg { + ApprovalVotingParallelMessage::NewBlocks(blocks) => + Ok(ApprovalDistributionMessage::NewBlocks(blocks)), + ApprovalVotingParallelMessage::DistributeAssignment(assignment, claimed_cores) => + Ok(ApprovalDistributionMessage::DistributeAssignment(assignment, claimed_cores)), + ApprovalVotingParallelMessage::DistributeApproval(vote) => + Ok(ApprovalDistributionMessage::DistributeApproval(vote)), + ApprovalVotingParallelMessage::NetworkBridgeUpdate(msg) => + Ok(ApprovalDistributionMessage::NetworkBridgeUpdate(msg)), + ApprovalVotingParallelMessage::GetApprovalSignatures(candidate_indicies, tx) => + Ok(ApprovalDistributionMessage::GetApprovalSignatures(candidate_indicies, tx)), + ApprovalVotingParallelMessage::ApprovalCheckingLagUpdate(lag) => + Ok(ApprovalDistributionMessage::ApprovalCheckingLagUpdate(lag)), + _ => Err(()), + } + } +} + +impl From for ApprovalVotingParallelMessage { + fn from(msg: ApprovalDistributionMessage) -> Self { + match msg { + ApprovalDistributionMessage::NewBlocks(blocks) => + ApprovalVotingParallelMessage::NewBlocks(blocks), + ApprovalDistributionMessage::DistributeAssignment(cert, bitfield) => + ApprovalVotingParallelMessage::DistributeAssignment(cert, bitfield), + ApprovalDistributionMessage::DistributeApproval(vote) => + ApprovalVotingParallelMessage::DistributeApproval(vote), + ApprovalDistributionMessage::NetworkBridgeUpdate(msg) => + ApprovalVotingParallelMessage::NetworkBridgeUpdate(msg), + ApprovalDistributionMessage::GetApprovalSignatures(candidate_indicies, tx) => + ApprovalVotingParallelMessage::GetApprovalSignatures(candidate_indicies, tx), + ApprovalDistributionMessage::ApprovalCheckingLagUpdate(lag) => + ApprovalVotingParallelMessage::ApprovalCheckingLagUpdate(lag), + } + } +} + /// Response type to `ApprovalVotingMessage::ApprovedAncestor`. #[derive(Clone, Debug)] pub struct HighestApprovedAncestorBlock { diff --git a/polkadot/node/test/service/src/lib.rs b/polkadot/node/test/service/src/lib.rs index 35156a3a9372..ed5338fb27a3 100644 --- a/polkadot/node/test/service/src/lib.rs +++ b/polkadot/node/test/service/src/lib.rs @@ -100,6 +100,7 @@ pub fn new_full( execute_workers_max_num: None, prepare_workers_hard_max_num: None, prepare_workers_soft_max_num: None, + enable_approval_voting_parallel: false, }, ), sc_network::config::NetworkBackendType::Litep2p => @@ -122,6 +123,7 @@ pub fn new_full( execute_workers_max_num: None, prepare_workers_hard_max_num: None, prepare_workers_soft_max_num: None, + enable_approval_voting_parallel: false, }, ), } diff --git a/polkadot/parachain/test-parachains/adder/collator/src/main.rs b/polkadot/parachain/test-parachains/adder/collator/src/main.rs index e8588274df27..4660b4d38f7c 100644 --- a/polkadot/parachain/test-parachains/adder/collator/src/main.rs +++ b/polkadot/parachain/test-parachains/adder/collator/src/main.rs @@ -98,6 +98,7 @@ fn main() -> Result<()> { execute_workers_max_num: None, prepare_workers_hard_max_num: None, prepare_workers_soft_max_num: None, + enable_approval_voting_parallel: false, }, ) .map_err(|e| e.to_string())?; diff --git a/polkadot/parachain/test-parachains/undying/collator/src/main.rs b/polkadot/parachain/test-parachains/undying/collator/src/main.rs index 7198a831a477..3dfa714e6d16 100644 --- a/polkadot/parachain/test-parachains/undying/collator/src/main.rs +++ b/polkadot/parachain/test-parachains/undying/collator/src/main.rs @@ -100,6 +100,7 @@ fn main() -> Result<()> { execute_workers_max_num: None, prepare_workers_hard_max_num: None, prepare_workers_soft_max_num: None, + enable_approval_voting_parallel: false, }, ) .map_err(|e| e.to_string())?; From 717595783bcfba762bf02088c86764b4b266a43a Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 26 Jul 2024 10:10:11 +0300 Subject: [PATCH 05/45] Address review feedback Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 26cedf1a2a13..3462aaef1f69 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -943,11 +943,7 @@ impl State { .await; } - async fn process_incoming_assignments< - N: overseer::SubsystemSender, - A: overseer::SubsystemSender, - R, - >( + async fn process_incoming_assignments( &mut self, approval_voting_sender: &mut A, network_sender: &mut N, @@ -956,6 +952,8 @@ impl State { assignments: Vec<(IndirectAssignmentCertV2, CandidateBitfield)>, rng: &mut R, ) where + A: overseer::SubsystemSender, + N: overseer::SubsystemSender, R: CryptoRng + Rng, { for (assignment, claimed_indices) in assignments { @@ -1038,11 +1036,7 @@ impl State { } } - async fn process_incoming_peer_message< - N: overseer::SubsystemSender, - A: overseer::SubsystemSender, - R, - >( + async fn process_incoming_peer_message( &mut self, approval_voting_sender: &mut A, network_sender: &mut N, @@ -1055,6 +1049,8 @@ impl State { >, rng: &mut R, ) where + A: overseer::SubsystemSender, + N: overseer::SubsystemSender, R: CryptoRng + Rng, { match msg { @@ -1222,11 +1218,7 @@ impl State { self.enable_aggression(network_sender, Resend::No, metrics).await; } - async fn import_and_circulate_assignment< - N: overseer::SubsystemSender, - A: overseer::SubsystemSender, - R, - >( + async fn import_and_circulate_assignment( &mut self, approval_voting_sender: &mut A, network_sender: &mut N, @@ -1236,6 +1228,8 @@ impl State { claimed_candidate_indices: CandidateBitfield, rng: &mut R, ) where + A: overseer::SubsystemSender, + N: overseer::SubsystemSender, R: CryptoRng + Rng, { let _span = self From d9fdd519a75a4f6ceeed8a8259f002254310e566 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 26 Jul 2024 10:54:00 +0300 Subject: [PATCH 06/45] Address review feedback - rename check_and_import in import. - refactor un-needed variable. Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 62 +++--- .../node/core/approval-voting/src/tests.rs | 205 +++++++----------- .../network/approval-distribution/src/lib.rs | 7 +- .../approval-distribution/src/tests.rs | 44 ++-- polkadot/node/subsystem-types/src/messages.rs | 18 +- 5 files changed, 135 insertions(+), 201 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 5d588a53f3d1..2359bb5dfbd0 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -1883,8 +1883,8 @@ async fn handle_from_overseer( vec![Action::Conclude] }, FromOrchestra::Communication { msg } => match msg { - ApprovalVotingMessage::CheckAndImportAssignment(a, claimed_cores, tranche, tx) => { - let (check_outcome, actions) = check_and_import_assignment( + ApprovalVotingMessage::ImportAssignment(a, claimed_cores, tranche, tx) => { + let (check_outcome, actions) = import_assignment( ctx.sender(), state, db, @@ -1894,17 +1894,17 @@ async fn handle_from_overseer( tranche, ) .await?; - // approval-distribution already sanity checks the assignment is valid and expected, - // so this import should never fail if it does it means there is a bug in the - // code. + // approval-distribution makes sure this assignment is valid and expected, + // so this import should never fail, if it does it might mean one of two things, + // there is a bug in the code or the two subsystems got out of sync. if let AssignmentCheckResult::Bad(ref err) = check_outcome { - gum::error!(target: LOG_TARGET, ?err, "Unexpected fail to check and import assignment"); + gum::debug!(target: LOG_TARGET, ?err, "Unexpected fail when importing an assignment"); } let _ = tx.map(|tx| tx.send(check_outcome)); actions }, - ApprovalVotingMessage::CheckAndImportApproval(a, tx) => { - let result = check_and_import_approval( + ApprovalVotingMessage::ImportApproval(a, tx) => { + let result = import_approval( ctx.sender(), state, db, @@ -1914,11 +1914,11 @@ async fn handle_from_overseer( &wakeups, ) .await?; - // approval-distribution already sanity checks the vote is valid and expected, - // so this import should never fail if it does it means there is a bug in the - // code. + // approval-distribution makes sure this vote is valid and expected, + // so this import should never fail, if it does it might mean one of two things, + // there is a bug in the code or the two subsystems got out of sync. if let ApprovalCheckResult::Bad(ref err) = result.1 { - gum::error!(target: LOG_TARGET, ?err, "Unexpected fail to check and import approval"); + gum::debug!(target: LOG_TARGET, ?err, "Unexpected fail when importing an approval"); } let _ = tx.map(|tx| tx.send(result.1)); @@ -2467,7 +2467,7 @@ fn schedule_wakeup_action( maybe_action } -async fn check_and_import_assignment( +async fn import_assignment( sender: &mut Sender, state: &State, db: &mut OverlayedBackend<'_, impl Backend>, @@ -2481,16 +2481,16 @@ where { let tick_now = state.clock.tick_now(); - let mut check_and_import_assignment_span = state + let mut import_assignment_span = state .spans .get(&assignment.block_hash) - .map(|span| span.child("check-and-import-assignment")) - .unwrap_or_else(|| jaeger::Span::new(assignment.block_hash, "check-and-import-assignment")) + .map(|span| span.child("import-assignment")) + .unwrap_or_else(|| jaeger::Span::new(assignment.block_hash, "import-assignment")) .with_relay_parent(assignment.block_hash) .with_stage(jaeger::Stage::ApprovalChecking); for candidate_index in candidate_indices.iter_ones() { - check_and_import_assignment_span.add_uint_tag("candidate-index", candidate_index as u64); + import_assignment_span.add_uint_tag("candidate-index", candidate_index as u64); } let block_entry = match db.load_block_entry(&assignment.block_hash)? { @@ -2571,23 +2571,21 @@ where )), // no candidate at core. }; - check_and_import_assignment_span + import_assignment_span .add_string_tag("candidate-hash", format!("{:?}", assigned_candidate_hash)); - check_and_import_assignment_span.add_string_tag( + import_assignment_span.add_string_tag( "traceID", format!("{:?}", jaeger::hash_to_trace_identifier(assigned_candidate_hash.0)), ); - let _approval_entry = match candidate_entry.approval_entry_mut(&assignment.block_hash) { - Some(a) => a, - None => - return Ok(( - AssignmentCheckResult::Bad(AssignmentCheckError::Internal( - assignment.block_hash, - assigned_candidate_hash, - )), - Vec::new(), + if candidate_entry.approval_entry_mut(&assignment.block_hash).is_none() { + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::Internal( + assignment.block_hash, + assigned_candidate_hash, )), + Vec::new(), + )); }; claimed_core_indices.push(claimed_core_index); @@ -2637,7 +2635,7 @@ where }; is_duplicate &= approval_entry.is_assigned(assignment.validator); approval_entry.import_assignment(tranche, assignment.validator, tick_now); - check_and_import_assignment_span.add_uint_tag("tranche", tranche as u64); + import_assignment_span.add_uint_tag("tranche", tranche as u64); // We've imported a new assignment, so we need to schedule a wake-up for when that might // no-show. @@ -2694,7 +2692,7 @@ where Ok((res, actions)) } -async fn check_and_import_approval( +async fn import_approval( sender: &mut Sender, state: &mut State, db: &mut OverlayedBackend<'_, impl Backend>, @@ -2714,8 +2712,8 @@ where let mut span = state .spans .get(&approval.block_hash) - .map(|span| span.child("check-and-import-approval")) - .unwrap_or_else(|| jaeger::Span::new(approval.block_hash, "check-and-import-approval")) + .map(|span| span.child("import-approval")) + .unwrap_or_else(|| jaeger::Span::new(approval.block_hash, "import-approval")) .with_string_fmt_debug_tag("candidate-index", approval.candidate_indices.clone()) .with_relay_parent(approval.block_hash) .with_stage(jaeger::Stage::ApprovalChecking); diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 8aab353f9b84..4687b35a0024 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -653,7 +653,7 @@ fn make_candidate(para_id: ParaId, hash: &Hash) -> CandidateReceipt { r } -async fn check_and_import_approval( +async fn import_approval( overseer: &mut VirtualOverseer, block_hash: Hash, candidate_index: CandidateIndex, @@ -672,7 +672,7 @@ async fn check_and_import_approval( overseer_send( overseer, FromOrchestra::Communication { - msg: ApprovalVotingMessage::CheckAndImportApproval( + msg: ApprovalVotingMessage::ImportApproval( IndirectSignedApprovalVoteV2 { block_hash, candidate_indices: candidate_index.into(), @@ -695,7 +695,7 @@ async fn check_and_import_approval( rx } -async fn check_and_import_assignment( +async fn import_assignment( overseer: &mut VirtualOverseer, block_hash: Hash, candidate_index: CandidateIndex, @@ -706,7 +706,7 @@ async fn check_and_import_assignment( overseer_send( overseer, FromOrchestra::Communication { - msg: ApprovalVotingMessage::CheckAndImportAssignment( + msg: ApprovalVotingMessage::ImportAssignment( IndirectAssignmentCertV2 { block_hash, validator, @@ -723,7 +723,7 @@ async fn check_and_import_assignment( rx } -async fn check_and_import_assignment_v2( +async fn import_assignment_v2( overseer: &mut VirtualOverseer, block_hash: Hash, core_indices: Vec, @@ -733,7 +733,7 @@ async fn check_and_import_assignment_v2( overseer_send( overseer, FromOrchestra::Communication { - msg: ApprovalVotingMessage::CheckAndImportAssignment( + msg: ApprovalVotingMessage::ImportAssignment( IndirectAssignmentCertV2 { block_hash, validator, @@ -1130,28 +1130,18 @@ fn subsystem_rejects_bad_assignment_ok_criteria() { ); builder.build(&mut virtual_overseer).await; - let rx = check_and_import_assignment( - &mut virtual_overseer, - block_hash, - candidate_index, - validator, - 0, - ) - .await; + let rx = + import_assignment(&mut virtual_overseer, block_hash, candidate_index, validator, 0) + .await; assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted),); // unknown hash let unknown_hash = Hash::repeat_byte(0x02); - let rx = check_and_import_assignment( - &mut virtual_overseer, - unknown_hash, - candidate_index, - validator, - 0, - ) - .await; + let rx = + import_assignment(&mut virtual_overseer, unknown_hash, candidate_index, validator, 0) + .await; assert_eq!( rx.await, @@ -1180,7 +1170,7 @@ fn blank_subsystem_act_on_bad_block() { overseer_send( &mut virtual_overseer, FromOrchestra::Communication { - msg: ApprovalVotingMessage::CheckAndImportAssignment( + msg: ApprovalVotingMessage::ImportAssignment( IndirectAssignmentCertV2 { block_hash: bad_block_hash, validator: 0u32.into(), @@ -1254,7 +1244,7 @@ fn subsystem_rejects_approval_if_no_candidate_entry() { }); let session_index = 1; - let rx = check_and_import_approval( + let rx = import_approval( &mut virtual_overseer, block_hash, candidate_index, @@ -1295,7 +1285,7 @@ fn subsystem_rejects_approval_if_no_block_entry() { let candidate_hash = dummy_candidate_receipt(block_hash).hash(); let session_index = 1; - let rx = check_and_import_approval( + let rx = import_approval( &mut virtual_overseer, block_hash, candidate_index, @@ -1360,7 +1350,7 @@ fn subsystem_rejects_approval_before_assignment() { .build(&mut virtual_overseer) .await; - let rx = check_and_import_approval( + let rx = import_approval( &mut virtual_overseer, block_hash, candidate_index, @@ -1432,7 +1422,7 @@ fn subsystem_accepts_duplicate_assignment() { .await; // Initial assignment. - let rx = check_and_import_assignment_v2( + let rx = import_assignment_v2( &mut virtual_overseer, block_hash, vec![candidate_index1, candidate_index2], @@ -1443,20 +1433,15 @@ fn subsystem_accepts_duplicate_assignment() { assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted)); // Test with single assigned core. - let rx = check_and_import_assignment( - &mut virtual_overseer, - block_hash, - candidate_index, - validator, - 0, - ) - .await; + let rx = + import_assignment(&mut virtual_overseer, block_hash, candidate_index, validator, 0) + .await; assert_eq!(rx.await, Ok(AssignmentCheckResult::AcceptedDuplicate)); // Test with multiple assigned cores. This cannot happen in practice, as tranche0 // assignments are sent first, but we should still ensure correct behavior. - let rx = check_and_import_assignment_v2( + let rx = import_assignment_v2( &mut virtual_overseer, block_hash, vec![candidate_index1, candidate_index2], @@ -1502,14 +1487,9 @@ fn subsystem_rejects_assignment_with_unknown_candidate() { .build(&mut virtual_overseer) .await; - let rx = check_and_import_assignment( - &mut virtual_overseer, - block_hash, - candidate_index, - validator, - 0, - ) - .await; + let rx = + import_assignment(&mut virtual_overseer, block_hash, candidate_index, validator, 0) + .await; assert_eq!( rx.await, @@ -1553,14 +1533,9 @@ fn subsystem_rejects_oversized_bitfields() { .build(&mut virtual_overseer) .await; - let rx = check_and_import_assignment( - &mut virtual_overseer, - block_hash, - candidate_index, - validator, - 0, - ) - .await; + let rx = + import_assignment(&mut virtual_overseer, block_hash, candidate_index, validator, 0) + .await; assert_eq!( rx.await, @@ -1569,13 +1544,9 @@ fn subsystem_rejects_oversized_bitfields() { ))), ); - let rx = check_and_import_assignment_v2( - &mut virtual_overseer, - block_hash, - vec![1, 2, 10, 50], - validator, - ) - .await; + let rx = + import_assignment_v2(&mut virtual_overseer, block_hash, vec![1, 2, 10, 50], validator) + .await; assert_eq!( rx.await, @@ -1627,18 +1598,13 @@ fn subsystem_accepts_and_imports_approval_after_assignment() { .build(&mut virtual_overseer) .await; - let rx = check_and_import_assignment( - &mut virtual_overseer, - block_hash, - candidate_index, - validator, - 0, - ) - .await; + let rx = + import_assignment(&mut virtual_overseer, block_hash, candidate_index, validator, 0) + .await; assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted)); - let rx = check_and_import_approval( + let rx = import_approval( &mut virtual_overseer, block_hash, candidate_index, @@ -1720,20 +1686,15 @@ fn subsystem_second_approval_import_only_schedules_wakeups() { .build(&mut virtual_overseer) .await; - let rx = check_and_import_assignment( - &mut virtual_overseer, - block_hash, - candidate_index, - validator, - 0, - ) - .await; + let rx = + import_assignment(&mut virtual_overseer, block_hash, candidate_index, validator, 0) + .await; assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted)); assert!(clock.inner.lock().current_wakeup_is(APPROVAL_DELAY + 2)); - let rx = check_and_import_approval( + let rx = import_approval( &mut virtual_overseer, block_hash, candidate_index, @@ -1750,7 +1711,7 @@ fn subsystem_second_approval_import_only_schedules_wakeups() { futures_timer::Delay::new(Duration::from_millis(100)).await; assert!(clock.inner.lock().current_wakeup_is(APPROVAL_DELAY + 2)); - let rx = check_and_import_approval( + let rx = import_approval( &mut virtual_overseer, block_hash, candidate_index, @@ -1809,14 +1770,9 @@ fn subsystem_assignment_import_updates_candidate_entry_and_schedules_wakeup() { .build(&mut virtual_overseer) .await; - let rx = check_and_import_assignment( - &mut virtual_overseer, - block_hash, - candidate_index, - validator, - 0, - ) - .await; + let rx = + import_assignment(&mut virtual_overseer, block_hash, candidate_index, validator, 0) + .await; assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted)); @@ -1932,17 +1888,12 @@ fn test_approvals_on_fork_are_always_considered_after_no_show( .await; // Send assignments for the same candidate on both forks - let rx = check_and_import_assignment( - &mut virtual_overseer, - block_hash, - candidate_index, - validator, - 0, - ) - .await; + let rx = + import_assignment(&mut virtual_overseer, block_hash, candidate_index, validator, 0) + .await; assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted)); - let rx = check_and_import_assignment( + let rx = import_assignment( &mut virtual_overseer, block_hash_fork, candidate_index, @@ -1977,7 +1928,7 @@ fn test_approvals_on_fork_are_always_considered_after_no_show( futures_timer::Delay::new(Duration::from_millis(100)).await; // Send the approval for candidate just in the context of 0x01 block. - let rx = check_and_import_approval( + let rx = import_approval( &mut virtual_overseer, block_hash, candidate_index, @@ -2047,14 +1998,9 @@ fn subsystem_process_wakeup_schedules_wakeup() { .build(&mut virtual_overseer) .await; - let rx = check_and_import_assignment( - &mut virtual_overseer, - block_hash, - candidate_index, - validator, - 0, - ) - .await; + let rx = + import_assignment(&mut virtual_overseer, block_hash, candidate_index, validator, 0) + .await; assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted)); @@ -2107,7 +2053,7 @@ fn linear_import_act_on_leaf() { overseer_send( &mut virtual_overseer, FromOrchestra::Communication { - msg: ApprovalVotingMessage::CheckAndImportAssignment( + msg: ApprovalVotingMessage::ImportAssignment( IndirectAssignmentCertV2 { block_hash: head, validator: 0u32.into(), @@ -2179,7 +2125,7 @@ fn forkful_import_at_same_height_act_on_leaf() { overseer_send( &mut virtual_overseer, FromOrchestra::Communication { - msg: ApprovalVotingMessage::CheckAndImportAssignment( + msg: ApprovalVotingMessage::ImportAssignment( IndirectAssignmentCertV2 { block_hash: head, validator: 0u32.into(), @@ -2347,7 +2293,7 @@ fn import_checked_approval_updates_entries_and_schedules() { let candidate_index = 0; - let rx = check_and_import_assignment( + let rx = import_assignment( &mut virtual_overseer, block_hash, candidate_index, @@ -2358,7 +2304,7 @@ fn import_checked_approval_updates_entries_and_schedules() { assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted),); - let rx = check_and_import_assignment( + let rx = import_assignment( &mut virtual_overseer, block_hash, candidate_index, @@ -2372,7 +2318,7 @@ fn import_checked_approval_updates_entries_and_schedules() { let session_index = 1; let sig_a = sign_approval(Sr25519Keyring::Alice, candidate_hash, session_index); - let rx = check_and_import_approval( + let rx = import_approval( &mut virtual_overseer, block_hash, candidate_index, @@ -2399,7 +2345,7 @@ fn import_checked_approval_updates_entries_and_schedules() { clock.inner.lock().wakeup_all(2); let sig_b = sign_approval(Sr25519Keyring::Bob, candidate_hash, session_index); - let rx = check_and_import_approval( + let rx = import_approval( &mut virtual_overseer, block_hash, candidate_index, @@ -2512,14 +2458,9 @@ fn subsystem_import_checked_approval_sets_one_block_bit_at_a_time() { ]; for (candidate_index, validator) in assignments { - let rx = check_and_import_assignment( - &mut virtual_overseer, - block_hash, - candidate_index, - validator, - 0, - ) - .await; + let rx = + import_assignment(&mut virtual_overseer, block_hash, candidate_index, validator, 0) + .await; assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted)); } @@ -2540,7 +2481,7 @@ fn subsystem_import_checked_approval_sets_one_block_bit_at_a_time() { } else { sign_approval(Sr25519Keyring::Bob, *candidate_hash, session_index) }; - let rx = check_and_import_approval( + let rx = import_approval( &mut virtual_overseer, block_hash, *candidate_index, @@ -2798,7 +2739,7 @@ fn approved_ancestor_test( for (i, (block_hash, candidate_hash)) in block_hashes.iter().zip(candidate_hashes).enumerate() { - let rx = check_and_import_assignment( + let rx = import_assignment( &mut virtual_overseer, *block_hash, candidate_index, @@ -2812,7 +2753,7 @@ fn approved_ancestor_test( continue } - let rx = check_and_import_approval( + let rx = import_approval( &mut virtual_overseer, *block_hash, candidate_index, @@ -3358,7 +3299,7 @@ where .await; for validator in assignments_to_import { - let rx = check_and_import_assignment( + let rx = import_assignment( &mut virtual_overseer, block_hash, candidate_index, @@ -3372,7 +3313,7 @@ where let n_validators = validators.len(); for (i, &validator_index) in approvals_to_import.iter().enumerate() { let expect_chain_approved = 3 * (i + 1) > n_validators; - let rx = check_and_import_approval( + let rx = import_approval( &mut virtual_overseer, block_hash, candidate_index, @@ -3677,7 +3618,7 @@ fn pre_covers_dont_stall_approval() { let candidate_index = 0; - let rx = check_and_import_assignment( + let rx = import_assignment( &mut virtual_overseer, block_hash, candidate_index, @@ -3688,7 +3629,7 @@ fn pre_covers_dont_stall_approval() { assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted),); - let rx = check_and_import_assignment( + let rx = import_assignment( &mut virtual_overseer, block_hash, candidate_index, @@ -3699,7 +3640,7 @@ fn pre_covers_dont_stall_approval() { assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted),); - let rx = check_and_import_assignment( + let rx = import_assignment( &mut virtual_overseer, block_hash, candidate_index, @@ -3713,7 +3654,7 @@ fn pre_covers_dont_stall_approval() { let session_index = 1; let sig_b = sign_approval(Sr25519Keyring::Bob, candidate_hash, session_index); - let rx = check_and_import_approval( + let rx = import_approval( &mut virtual_overseer, block_hash, candidate_index, @@ -3728,7 +3669,7 @@ fn pre_covers_dont_stall_approval() { assert_eq!(rx.await, Ok(ApprovalCheckResult::Accepted),); let sig_c = sign_approval(Sr25519Keyring::Charlie, candidate_hash, session_index); - let rx = check_and_import_approval( + let rx = import_approval( &mut virtual_overseer, block_hash, candidate_index, @@ -3858,7 +3799,7 @@ fn waits_until_approving_assignments_are_old_enough() { let candidate_index = 0; - let rx = check_and_import_assignment( + let rx = import_assignment( &mut virtual_overseer, block_hash, candidate_index, @@ -3869,7 +3810,7 @@ fn waits_until_approving_assignments_are_old_enough() { assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted),); - let rx = check_and_import_assignment( + let rx = import_assignment( &mut virtual_overseer, block_hash, candidate_index, @@ -3885,7 +3826,7 @@ fn waits_until_approving_assignments_are_old_enough() { let session_index = 1; let sig_a = sign_approval(Sr25519Keyring::Alice, candidate_hash, session_index); - let rx = check_and_import_approval( + let rx = import_approval( &mut virtual_overseer, block_hash, candidate_index, @@ -3901,7 +3842,7 @@ fn waits_until_approving_assignments_are_old_enough() { let sig_b = sign_approval(Sr25519Keyring::Bob, candidate_hash, session_index); - let rx = check_and_import_approval( + let rx = import_approval( &mut virtual_overseer, block_hash, candidate_index, diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index d863489b35ff..c61a4ee91b6c 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -1506,7 +1506,7 @@ impl State { } approval_voting_sender - .send_message(ApprovalVotingMessage::CheckAndImportAssignment( + .send_message(ApprovalVotingMessage::ImportAssignment( assignment.clone(), claimed_candidate_indices.clone(), tranche, @@ -1884,10 +1884,7 @@ impl State { match result { Ok(_) => { approval_voting_sender - .send_message(ApprovalVotingMessage::CheckAndImportApproval( - vote.clone(), - None, - )) + .send_message(ApprovalVotingMessage::ImportApproval(vote.clone(), None)) .await; modify_reputation( diff --git a/polkadot/node/network/approval-distribution/src/tests.rs b/polkadot/node/network/approval-distribution/src/tests.rs index 3f926746449d..ec68e0e42bf0 100644 --- a/polkadot/node/network/approval-distribution/src/tests.rs +++ b/polkadot/node/network/approval-distribution/src/tests.rs @@ -574,7 +574,7 @@ fn try_import_the_same_assignment() { // send an `Accept` message from the Approval Voting subsystem assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( assignment, claimed_indices, tranche, @@ -690,7 +690,7 @@ fn try_import_the_same_assignment_v2() { // send an `Accept` message from the Approval Voting subsystem assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( assignment, claimed_indices, tranche, @@ -781,7 +781,7 @@ fn delay_reputation_change() { // send an `Accept` message from the Approval Voting subsystem assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( assignment, claimed_candidates, tranche, @@ -865,7 +865,7 @@ fn spam_attack_results_in_negative_reputation_change() { } assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( assignment, claimed_candidate_index, tranche, @@ -1119,7 +1119,7 @@ fn import_approval_happy_path_v1_v2_peers() { assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportApproval( vote, _, )) => { assert_eq!(vote, approval); @@ -1248,7 +1248,7 @@ fn import_approval_happy_path_v2() { assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportApproval( vote, _, )) => { assert_eq!(vote, approval); @@ -1352,7 +1352,7 @@ fn multiple_assignments_covered_with_one_approval_vote() { assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( _, _, tranche, _, @@ -1395,7 +1395,7 @@ fn multiple_assignments_covered_with_one_approval_vote() { assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( _, _, tranche, _, )) => { @@ -1436,7 +1436,7 @@ fn multiple_assignments_covered_with_one_approval_vote() { assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportApproval( vote, _, )) => { assert_eq!(vote, approval); @@ -1559,7 +1559,7 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( _, _, tranche, _, )) => { @@ -1586,7 +1586,7 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( _, _, tranche, _, )) => { @@ -1612,7 +1612,7 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportApproval( vote, _, )) => { assert_eq!(vote, approval); @@ -1758,7 +1758,7 @@ fn import_approval_bad() { assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( assignment, i, tranche, _, @@ -2310,7 +2310,7 @@ fn import_remotely_then_locally() { // send an `Accept` message from the Approval Voting subsystem assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( assignment, i, tranche, _, @@ -2347,7 +2347,7 @@ fn import_remotely_then_locally() { assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportApproval( vote, _, )) => { assert_eq!(vote, approval); @@ -2665,7 +2665,7 @@ fn race_condition_in_local_vs_remote_view_update() { assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( assignment, claimed_candidate_index, tranche, _, @@ -2888,7 +2888,7 @@ fn propagates_assignments_along_unshared_dimension() { .await; assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( _, _, tranche, _, @@ -2938,7 +2938,7 @@ fn propagates_assignments_along_unshared_dimension() { send_message_from_peer(overseer, &peers[99].0, msg).await; assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( _, _, tranche, _, @@ -3531,7 +3531,7 @@ fn non_originator_aggression_l1() { assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( _, _, tranche, _, @@ -3657,7 +3657,7 @@ fn non_originator_aggression_l2() { .await; assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( _, _, tranche, _, @@ -3847,7 +3847,7 @@ fn resends_messages_periodically() { assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( _, _, tranche, _, @@ -4047,7 +4047,7 @@ fn import_versioned_approval() { provide_session(overseer, session).await; assert_matches!( overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportApproval( vote, _, )) => { assert_eq!(vote, approval.into()); diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index e0c078de48b7..f413add1e1a3 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -962,22 +962,20 @@ pub struct HighestApprovedAncestorBlock { /// Message to the Approval Voting subsystem. #[derive(Debug)] pub enum ApprovalVotingMessage { - /// Check if the assignment is valid and can be accepted by our view of the protocol. - /// Should not be sent unless the block hash is known. - CheckAndImportAssignment( + /// Import an assignment into the approval-voting database. + /// + /// Should not be sent unless the block hash is known and the VRF assignment checks out. + ImportAssignment( IndirectAssignmentCertV2, CandidateBitfield, DelayTranche, Option>, ), - /// Check if the approval vote is valid and can be accepted by our view of the - /// protocol. + /// Import an approval vote into approval-voting database /// - /// Should not be sent unless the block hash within the indirect vote is known. - CheckAndImportApproval( - IndirectSignedApprovalVoteV2, - Option>, - ), + /// Should not be sent unless the block hash within the indirect vote is known, vote is + /// correctly signed and we had a previous assignment for the candidate. + ImportApproval(IndirectSignedApprovalVoteV2, Option>), /// Returns the highest possible ancestor hash of the provided block hash which is /// acceptable to vote on finality for. /// The `BlockNumber` provided is the number of the block's ancestor which is the From 2752d6ef6bf70bfd92056f78dec8c44f30a4cb66 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 29 Jul 2024 10:38:45 +0300 Subject: [PATCH 07/45] Fail early on oversized claims Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 40 ++++++++++++++----- .../approval-distribution/src/tests.rs | 4 +- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index c61a4ee91b6c..4c8f0f0467ad 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -677,6 +677,13 @@ enum InvalidAssignmentError { CryptoCheckFailed(InvalidAssignment), // The assignment did not claim any valid candidate. NoClaimedCandidates, + // Claimed invalid candidate. + ClaimedInvalidCandidateIndex { + claimed_index: usize, + max_index: usize, + }, + // The assignment claimes more candidates than the maximum allowed. + OversizedClaimedBitfield, // Session Info was not found for the block hash in the assignment. #[allow(dead_code)] SessionInfoNotFound(polkadot_node_subsystem_util::runtime::Error), @@ -1679,26 +1686,37 @@ impl State { runtime_info: &mut RuntimeInfo, runtime_api_sender: &mut RA, ) -> Result { + let ExtendedSessionInfo { ref session_info, .. } = runtime_info + .get_session_info_by_index(runtime_api_sender, assignment.block_hash, entry.session) + .await + .map_err(|err| InvalidAssignmentError::SessionInfoNotFound(err))?; + + if claimed_candidate_indices.len() > session_info.n_cores as usize { + return Err(InvalidAssignmentError::OversizedClaimedBitfield) + } + let claimed_cores: Vec = claimed_candidate_indices .iter_ones() - .flat_map(|candidate_index| { - entry.candidates_metadata.get(candidate_index).map(|(_, core, _)| *core) + .map(|candidate_index| { + entry.candidates_metadata.get(candidate_index).map(|(_, core, _)| *core).ok_or( + InvalidAssignmentError::ClaimedInvalidCandidateIndex { + claimed_index: candidate_index, + max_index: entry.candidates_metadata.len(), + }, + ) }) - .collect::>(); + .collect::, InvalidAssignmentError>>()?; + + if claimed_cores.is_empty() { + return Err(InvalidAssignmentError::NoClaimedCandidates) + } + let backing_groups = claimed_candidate_indices .iter_ones() .flat_map(|candidate_index| { entry.candidates_metadata.get(candidate_index).map(|(_, _, group)| *group) }) .collect::>(); - if claimed_cores.is_empty() { - return Err(InvalidAssignmentError::NoClaimedCandidates) - } - - let ExtendedSessionInfo { ref session_info, .. } = runtime_info - .get_session_info_by_index(runtime_api_sender, assignment.block_hash, entry.session) - .await - .map_err(|err| InvalidAssignmentError::SessionInfoNotFound(err))?; assignment_criteria .check_assignment_cert( diff --git a/polkadot/node/network/approval-distribution/src/tests.rs b/polkadot/node/network/approval-distribution/src/tests.rs index ec68e0e42bf0..ce3ec93458d5 100644 --- a/polkadot/node/network/approval-distribution/src/tests.rs +++ b/polkadot/node/network/approval-distribution/src/tests.rs @@ -446,7 +446,7 @@ fn dummy_session_info_valid( discovery_keys: keys.clone().into_iter().map(|key| key.into()).collect(), assignment_keys: keys.clone().into_iter().map(|key| key.into()).collect(), validator_groups: Default::default(), - n_cores: index as _, + n_cores: 20, zeroth_delay_tranche_width: index as _, relay_vrf_modulo_samples: index as _, n_delay_tranches: index as _, @@ -657,7 +657,7 @@ fn try_import_the_same_assignment_v2() { hash, parent_hash, number: 2, - candidates: vec![Default::default(); 4], + candidates: vec![Default::default(); 5], slot: 1.into(), session: 1, vrf_story: RelayVRFStory(Default::default()), From 00d22ee3b0013d8614ba49638f67a94fac1d69a6 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 29 Jul 2024 13:08:12 +0300 Subject: [PATCH 08/45] Add more unittests Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 1 + .../approval-distribution/src/tests.rs | 306 +++++++++++++++++- 2 files changed, 306 insertions(+), 1 deletion(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 4c8f0f0467ad..5fdb95bdc91e 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -678,6 +678,7 @@ enum InvalidAssignmentError { // The assignment did not claim any valid candidate. NoClaimedCandidates, // Claimed invalid candidate. + #[allow(dead_code)] ClaimedInvalidCandidateIndex { claimed_index: usize, max_index: usize, diff --git a/polkadot/node/network/approval-distribution/src/tests.rs b/polkadot/node/network/approval-distribution/src/tests.rs index ce3ec93458d5..62cf0ec5317e 100644 --- a/polkadot/node/network/approval-distribution/src/tests.rs +++ b/polkadot/node/network/approval-distribution/src/tests.rs @@ -385,6 +385,30 @@ fn fake_assignment_cert_v2( } } +fn fake_assignment_cert_delay( + block_hash: Hash, + validator: ValidatorIndex, + core_bitfield: CoreBitfield, +) -> IndirectAssignmentCertV2 { + let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); + let msg = b"WhenParachains?"; + let mut prng = rand_core::OsRng; + let keypair = schnorrkel::Keypair::generate_with(&mut prng); + let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); + let preout = inout.to_preout(); + + IndirectAssignmentCertV2 { + block_hash, + validator, + cert: AssignmentCertV2 { + kind: AssignmentCertKindV2::RelayVRFDelay { + core_index: CoreIndex(core_bitfield.iter_ones().next().unwrap() as u32), + }, + vrf: VrfSignature { pre_output: VrfPreOutput(preout), proof: VrfProof(proof) }, + }, + } +} + async fn expect_reputation_change( virtual_overseer: &mut VirtualOverseer, peer_id: &PeerId, @@ -4365,7 +4389,7 @@ fn subsystem_rejects_assignment_in_future() { ); } -/// Subsystem rejects assignments too far into the future. +/// Subsystem rejects bad vrf assignments. #[test] fn subsystem_rejects_bad_assignments() { let peers = make_peers_and_authority_ids(15); @@ -4429,3 +4453,283 @@ fn subsystem_rejects_bad_assignments() { }, ); } + +/// Subsystem rejects assignments that have invalid claimed candidates. +#[test] +fn subsystem_rejects_wrong_claimed_assignments() { + let peers = make_peers_and_authority_ids(15); + let peer_a = peers.get(0).unwrap().0; + let parent_hash = Hash::repeat_byte(0xFF); + let hash = Hash::repeat_byte(0xAA); + + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(DummyClock {}), + state_without_reputation_delay(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // setup peers + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; + + // Set up a gossip topology, where a, b, c and d are topology neighbors to the node + // under testing. + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + setup_gossip_topology( + overseer, + make_gossip_topology(1, &peers_with_optional_peer_id, &[0, 1], &[2, 4], 3), + ) + .await; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 2, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // send the assignment related to `hash` + let validator_index = ValidatorIndex(0); + + // Claimed core 1 which does not have a candidate included on it, so the assignment + // should be rejected. + let cores = vec![1]; + let core_bitfield: CoreBitfield = cores + .iter() + .map(|index| CoreIndex(*index)) + .collect::>() + .try_into() + .unwrap(); + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfield); + let assignments: Vec<(IndirectAssignmentCertV2, CandidateBitfield)> = + vec![(cert.clone(), cores.try_into().unwrap())]; + setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::V3).await; + + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer_v3(overseer, &peer_a, msg).await; + provide_session( + overseer, + dummy_session_info_valid(1, &mut LocalKeystore::in_memory(), 1), + ) + .await; + + expect_reputation_change(overseer, &peer_a, COST_INVALID_MESSAGE).await; + + virtual_overseer + }, + ); +} + +/// Subsystem accepts tranche0 duplicate assignments, sometimes on validator Compact tranche0 +/// assignment and Delay tranche assignments land on the same candidate. The delay tranche0 can be +/// safely ignored and we don't need to gossip it however, the compact tranche0 assignment should be +/// gossiped, because other candidates are included in it, this test makes sure this invariant is +/// upheld, see https://github.com/paritytech/polkadot/pull/2160#discussion_r557628699, for +/// this edge case. +#[test] +fn subsystem_accepts_tranche0_duplicate_assignments() { + let peers = make_peers_and_authority_ids(15); + let peer_a = peers.get(0).unwrap().0; + let peer_b = peers.get(1).unwrap().0; + let parent_hash = Hash::repeat_byte(0xFF); + let hash = Hash::repeat_byte(0xAA); + let candidate_hash_first = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xBB)); + let candidate_hash_second = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xCC)); + let candidate_hash_third = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xBB)); + let candidate_hash_fourth = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xBB)); + + let _ = test_harness( + &MockAssignmentCriteria { tranche: Ok(0) }, + Box::new(DummyClock {}), + state_without_reputation_delay(), + |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // setup peers + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_b, view![], ValidationVersion::V3).await; + + // Set up a gossip topology, where a, b, c and d are topology neighbors to the node + // under testing. + let peers_with_optional_peer_id = peers + .iter() + .map(|(peer_id, authority)| (Some(*peer_id), authority.clone())) + .collect_vec(); + setup_gossip_topology( + overseer, + make_gossip_topology(1, &peers_with_optional_peer_id, &[0, 1], &[2, 4], 3), + ) + .await; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 2, + candidates: vec![ + (candidate_hash_first, 0.into(), 0.into()), + (candidate_hash_second, 1.into(), 1.into()), + (candidate_hash_third, 2.into(), 2.into()), + (candidate_hash_fourth, 3.into(), 3.into()), + ], + slot: 1.into(), + session: 1, + vrf_story: RelayVRFStory(Default::default()), + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // send the assignment related to `hash` + let validator_index = ValidatorIndex(0); + + setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; + + // 1. Compact assignment with multiple candidates, coming after delay assignment which + // covered just one of the candidate is still imported and gossiped. + let candidate_indices: CandidateBitfield = + vec![1 as CandidateIndex].try_into().unwrap(); + let core_bitfield = vec![CoreIndex(1)].try_into().unwrap(); + let cert = fake_assignment_cert_delay(hash, validator_index, core_bitfield); + let assignments: Vec<(IndirectAssignmentCertV2, CandidateBitfield)> = + vec![(cert.clone(), candidate_indices.clone())]; + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer_v3(overseer, &peer_a, msg).await; + provide_session( + overseer, + dummy_session_info_valid(1, &mut LocalKeystore::in_memory(), 1), + ) + .await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( + assignment, + claimed_indices, + tranche, + _, + )) => { + assert_eq!(claimed_indices, candidate_indices); + assert_eq!(assignment, cert.into()); + assert_eq!(tranche, 0); + } + ); + + expect_reputation_change(overseer, &peer_a, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 1); + assert_eq!(assignments.len(), 1); + } + ); + + let candidate_indices: CandidateBitfield = + vec![0 as CandidateIndex, 1 as CandidateIndex].try_into().unwrap(); + let core_bitfield = vec![CoreIndex(0), CoreIndex(1)].try_into().unwrap(); + + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfield); + + let assignments: Vec<(IndirectAssignmentCertV2, CandidateBitfield)> = + vec![(cert.clone(), candidate_indices.clone())]; + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer_v3(overseer, &peer_a, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( + assignment, + claimed_indices, + tranche, + _, + )) => { + assert_eq!(claimed_indices, candidate_indices); + assert_eq!(assignment, cert.into()); + assert_eq!(tranche, 0); + } + ); + + expect_reputation_change(overseer, &peer_a, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 1); + assert_eq!(assignments.len(), 1); + } + ); + + // 2. Delay assignment coming after compact assignment that already covered the + // candidate is not gossiped anymore. + let candidate_indices: CandidateBitfield = + vec![2 as CandidateIndex, 3 as CandidateIndex].try_into().unwrap(); + let core_bitfield = vec![CoreIndex(2), CoreIndex(3)].try_into().unwrap(); + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfield); + let assignments: Vec<(IndirectAssignmentCertV2, CandidateBitfield)> = + vec![(cert.clone(), candidate_indices.clone())]; + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer_v3(overseer, &peer_a, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( + assignment, + claimed_indices, + tranche, + _, + )) => { + assert_eq!(claimed_indices, candidate_indices); + assert_eq!(assignment, cert.into()); + assert_eq!(tranche, 0); + } + ); + + expect_reputation_change(overseer, &peer_a, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 1); + assert_eq!(assignments.len(), 1); + } + ); + + let candidate_indices: CandidateBitfield = vec![3].try_into().unwrap(); + let core_bitfield = vec![CoreIndex(3)].try_into().unwrap(); + + let cert = fake_assignment_cert_delay(hash, validator_index, core_bitfield); + + let assignments: Vec<(IndirectAssignmentCertV2, CandidateBitfield)> = + vec![(cert.clone(), candidate_indices.clone())]; + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer_v3(overseer, &peer_a, msg).await; + + expect_reputation_change(overseer, &peer_a, COST_DUPLICATE_MESSAGE).await; + + virtual_overseer + }, + ); +} From 00dbcb094eec4fbbae545b3932a12aa65f17e2e4 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 30 Jul 2024 09:53:53 +0300 Subject: [PATCH 09/45] Some refactoring to make the code a bit more testable Signed-off-by: Alexandru Gheorghe --- .../core/approval-voting-parallel/src/lib.rs | 82 ++++++++++++++----- 1 file changed, 61 insertions(+), 21 deletions(-) diff --git a/polkadot/node/core/approval-voting-parallel/src/lib.rs b/polkadot/node/core/approval-voting-parallel/src/lib.rs index e6cf04123d25..c02ab58daf0f 100644 --- a/polkadot/node/core/approval-voting-parallel/src/lib.rs +++ b/polkadot/node/core/approval-voting-parallel/src/lib.rs @@ -158,11 +158,15 @@ fn prio_right<'a>(_val: &'a mut ()) -> PollNext { PollNext::Right } +// It starts worker for the approval voting subsystem and the `APPROVAL_DISTRIBUTION_WORKER_COUNT` +// workers for the approval distribution subsystem. +// +// It returns handles that can be used to send messages to the workers. #[overseer::contextbounds(ApprovalVotingParallel, prefix = self::overseer)] -async fn run( - mut ctx: Context, +async fn start_workers( + ctx: &mut Context, subsystem: ApprovalVotingParallelSubsystem, -) -> SubsystemResult<()> +) -> SubsystemResult<(ToWorker, Vec>)> where { // Build approval voting handles. @@ -171,7 +175,7 @@ where let (tx_approval_voting_work_unbounded, rx_approval_voting_work_unbounded) = unbounded::>(); - let mut to_approval_voting_worker = + let to_approval_voting_worker = ToWorker(tx_approval_voting_work, tx_approval_voting_work_unbounded); let prioritised = select_with_strategy( @@ -183,11 +187,11 @@ where gum::info!(target: LOG_TARGET, "Starting approval distribution workers"); - let mut approval_distribution_channels = Vec::new(); + let mut to_approval_distribution_workers = Vec::new(); let slot_duration_millis = subsystem.slot_duration_millis; for i in 0..APPROVAL_DISTRIBUTION_WORKER_COUNT { - let approval_distro_orig = + let approval_distr_instance = polkadot_approval_distribution::ApprovalDistribution::new_with_clock( subsystem.metrics.0.clone(), subsystem.slot_duration_millis, @@ -239,7 +243,7 @@ where break; }, }; - approval_distro_orig + approval_distr_instance .handle_from_orchestra( message, &mut approval_distribution_to_approval_voting, @@ -254,7 +258,7 @@ where } }), ); - approval_distribution_channels.push(to_approval_distribution_worker); + to_approval_distribution_workers.push(to_approval_distribution_worker); } gum::info!(target: LOG_TARGET, "Starting approval voting workers"); @@ -278,14 +282,50 @@ where ) .await?; + Ok((to_approval_voting_worker, to_approval_distribution_workers)) +} + +// The main run function of the approval voting subsystem. +#[overseer::contextbounds(ApprovalVotingParallel, prefix = self::overseer)] +async fn run( + mut ctx: Context, + subsystem: ApprovalVotingParallelSubsystem, +) -> SubsystemResult<()> { + let subsystem_enabled = subsystem.subsystem_enabled; + gum::info!( target: LOG_TARGET, - subsystem_disabled = ?subsystem.subsystem_enabled, + subsystem_enabled, + "Starting workers" + ); + + let (to_approval_voting_worker, to_approval_distribution_workers) = + start_workers(&mut ctx, subsystem).await?; + + gum::info!( + target: LOG_TARGET, + subsystem_enabled, "Starting main subsystem loop" ); // Main loop of the subsystem, it shouldn't include any logic just dispatching of messages to // the workers. + run_main_loop( + ctx, + subsystem_enabled, + to_approval_voting_worker, + to_approval_distribution_workers, + ) + .await +} + +#[overseer::contextbounds(ApprovalVotingParallel, prefix = self::overseer)] +async fn run_main_loop( + mut ctx: Context, + subsystem_enabled: bool, + mut to_approval_voting_worker: ToWorker, + mut to_approval_distribution_workers: Vec>, +) -> SubsystemResult<()> { loop { futures::select! { next_msg = ctx.recv().fuse() => { @@ -296,7 +336,7 @@ where break; } }; - if !subsystem.subsystem_enabled { + if !subsystem_enabled { gum::trace!(target: LOG_TARGET, ?next_msg, "Parallel processing is not enabled skipping message"); continue; } @@ -304,7 +344,7 @@ where match next_msg { FromOrchestra::Signal(msg) => { - for worker in approval_distribution_channels.iter_mut() { + for worker in to_approval_distribution_workers.iter_mut() { worker .send_signal(msg.clone()).await?; } @@ -321,7 +361,7 @@ where // Now the message the approval distribution subsystem would've handled and need to // be forwarded to the workers. ApprovalVotingParallelMessage::NewBlocks(msg) => { - for worker in approval_distribution_channels.iter_mut() { + for worker in to_approval_distribution_workers.iter_mut() { worker .send_message( ApprovalDistributionMessage::NewBlocks(msg.clone()), @@ -330,8 +370,8 @@ where } }, ApprovalVotingParallelMessage::DistributeAssignment(assignment, claimed) => { - let worker_index = assignment.validator.0 as usize % approval_distribution_channels.len(); - let worker = approval_distribution_channels.get_mut(worker_index).expect("Worker index is obtained modulo len; qed"); + let worker_index = assignment.validator.0 as usize % to_approval_distribution_workers.len(); + let worker = to_approval_distribution_workers.get_mut(worker_index).expect("Worker index is obtained modulo len; qed"); worker .send_message( ApprovalDistributionMessage::DistributeAssignment(assignment, claimed) @@ -340,8 +380,8 @@ where }, ApprovalVotingParallelMessage::DistributeApproval(vote) => { - let worker_index = vote.validator.0 as usize % approval_distribution_channels.len(); - let worker = approval_distribution_channels.get_mut(worker_index).expect("Worker index is obtained modulo len; qed"); + let worker_index = vote.validator.0 as usize % to_approval_distribution_workers.len(); + let worker = to_approval_distribution_workers.get_mut(worker_index).expect("Worker index is obtained modulo len; qed"); worker .send_message( ApprovalDistributionMessage::DistributeApproval(vote) @@ -357,8 +397,8 @@ where let (all_msgs_from_same_validator, messages_split_by_validator) = validator_index_for_msg(msg); for (validator_index, msg) in all_msgs_from_same_validator.into_iter().chain(messages_split_by_validator.into_iter().flatten()) { - let worker_index = validator_index.0 as usize % approval_distribution_channels.len(); - let worker = approval_distribution_channels.get_mut(worker_index).expect("Worker index is obtained modulo len; qed"); + let worker_index = validator_index.0 as usize % to_approval_distribution_workers.len(); + let worker = to_approval_distribution_workers.get_mut(worker_index).expect("Worker index is obtained modulo len; qed"); worker .send_message( @@ -370,7 +410,7 @@ where ).await; } } else { - for worker in approval_distribution_channels.iter_mut() { + for worker in to_approval_distribution_workers.iter_mut() { worker .send_message_with_priority::( ApprovalDistributionMessage::NetworkBridgeUpdate(msg.clone()), @@ -381,7 +421,7 @@ where ApprovalVotingParallelMessage::GetApprovalSignatures(indices, tx) => { let mut sigs = HashMap::new(); let mut signatures_channels = Vec::new(); - for worker in approval_distribution_channels.iter_mut() { + for worker in to_approval_distribution_workers.iter_mut() { let (tx, rx) = oneshot::channel(); worker .send_message( @@ -413,7 +453,7 @@ where } }, ApprovalVotingParallelMessage::ApprovalCheckingLagUpdate(lag) => { - for worker in approval_distribution_channels.iter_mut() { + for worker in to_approval_distribution_workers.iter_mut() { worker .send_message( ApprovalDistributionMessage::ApprovalCheckingLagUpdate(lag) From f42fbf6b55cca25d741c772cdd45a52e49df842d Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 30 Jul 2024 10:24:26 +0300 Subject: [PATCH 10/45] Move gather of assignments on separate task Signed-off-by: Alexandru Gheorghe --- .../core/approval-voting-parallel/src/lib.rs | 110 ++++++++++++------ 1 file changed, 75 insertions(+), 35 deletions(-) diff --git a/polkadot/node/core/approval-voting-parallel/src/lib.rs b/polkadot/node/core/approval-voting-parallel/src/lib.rs index c02ab58daf0f..49b00ff515f8 100644 --- a/polkadot/node/core/approval-voting-parallel/src/lib.rs +++ b/polkadot/node/core/approval-voting-parallel/src/lib.rs @@ -40,8 +40,8 @@ use polkadot_node_subsystem_util::{ metrics::{self, prometheus}, runtime::{Config as RuntimeInfoConfig, RuntimeInfo}, }; -use polkadot_overseer::{OverseerSignal, Priority, SubsystemSender}; -use polkadot_primitives::ValidatorIndex; +use polkadot_overseer::{OverseerSignal, Priority, SubsystemSender, TimeoutExt}; +use polkadot_primitives::{CandidateIndex, Hash, ValidatorIndex, ValidatorSignature}; use rand::SeedableRng; use sc_keystore::LocalKeystore; @@ -51,10 +51,18 @@ use futures::{channel::oneshot, prelude::*, StreamExt}; use polkadot_node_core_approval_voting::{ approval_db::common::Config as DatabaseConfig, ApprovalVotingWorkProvider, }; -use std::{collections::HashMap, fmt::Debug, sync::Arc}; +use std::{ + collections::{HashMap, HashSet}, + fmt::Debug, + sync::Arc, + time::Duration, +}; use stream::{select_with_strategy, PollNext}; pub(crate) const LOG_TARGET: &str = "parachain::approval-voting-parallel"; +// Value rather arbitrarily: Should not be hit in practice, it exists to more easily diagnose dead +// lock issues for example. +const WAIT_FOR_SIGS_GATHER_TIMEOUT: Duration = Duration::from_millis(2000); /// The approval voting subsystem. pub struct ApprovalVotingParallelSubsystem { @@ -419,38 +427,7 @@ async fn run_main_loop( } }, ApprovalVotingParallelMessage::GetApprovalSignatures(indices, tx) => { - let mut sigs = HashMap::new(); - let mut signatures_channels = Vec::new(); - for worker in to_approval_distribution_workers.iter_mut() { - let (tx, rx) = oneshot::channel(); - worker - .send_message( - ApprovalDistributionMessage::GetApprovalSignatures(indices.clone(), tx) - ).await; - signatures_channels.push(rx); - } - let results = futures::future::join_all(signatures_channels).await; - - for result in results { - let worker_sigs = match result { - Ok(sigs) => sigs, - Err(_) => { - gum::error!( - target: LOG_TARGET, - "Getting approval signatures failed, oneshot got closed" - ); - continue; - }, - }; - sigs.extend(worker_sigs); - } - - if let Err(_) = tx.send(sigs) { - gum::debug!( - target: LOG_TARGET, - "Sending back approval signatures failed, oneshot got closed" - ); - } + handle_get_approval_signatures(&mut ctx, &mut to_approval_distribution_workers, indices, tx).await; }, ApprovalVotingParallelMessage::ApprovalCheckingLagUpdate(lag) => { for worker in to_approval_distribution_workers.iter_mut() { @@ -469,6 +446,69 @@ async fn run_main_loop( Ok(()) } +// It sends a message to all approval workers to get the approval signatures for the requested +// candidates and then merges them altogether and sends them back to the requester. +#[overseer::contextbounds(ApprovalVotingParallel, prefix = self::overseer)] +async fn handle_get_approval_signatures( + ctx: &mut Context, + to_approval_distribution_workers: &mut Vec>, + requested_candidates: HashSet<(Hash, CandidateIndex)>, + result_channel: oneshot::Sender< + HashMap, ValidatorSignature)>, + >, +) { + let mut sigs = HashMap::new(); + let mut signatures_channels = Vec::new(); + for worker in to_approval_distribution_workers.iter_mut() { + let (tx, rx) = oneshot::channel(); + worker + .send_message(ApprovalDistributionMessage::GetApprovalSignatures( + requested_candidates.clone(), + tx, + )) + .await; + signatures_channels.push(rx); + } + + let gather_signatures = async move { + let Some(results) = futures::future::join_all(signatures_channels) + .timeout(WAIT_FOR_SIGS_GATHER_TIMEOUT) + .await + else { + gum::warn!( + target: LOG_TARGET, + "Waiting for approval signatures timed out - dead lock?" + ); + return; + }; + + for result in results { + let worker_sigs = match result { + Ok(sigs) => sigs, + Err(_) => { + gum::error!( + target: LOG_TARGET, + "Getting approval signatures failed, oneshot got closed" + ); + continue; + }, + }; + sigs.extend(worker_sigs); + } + + if let Err(_) = result_channel.send(sigs) { + gum::debug!( + target: LOG_TARGET, + "Sending back approval signatures failed, oneshot got closed" + ); + } + }; + + if let Err(err) = ctx.spawn("approval-voting-gather-signatures", Box::pin(gather_signatures)) { + gum::warn!(target: LOG_TARGET, "Failed to spawn gather signatures task: {:?}", err); + } +} + // Returns the validators that initially created this assignments/votes, the validator index // is later used to decide which approval-distribution worker should receive the message. // From 57534a7332c3dc04156023356da9ad397b7e2ba5 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 30 Jul 2024 10:31:27 +0300 Subject: [PATCH 11/45] Some nits Signed-off-by: Alexandru Gheorghe --- .../core/approval-voting-parallel/src/lib.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/polkadot/node/core/approval-voting-parallel/src/lib.rs b/polkadot/node/core/approval-voting-parallel/src/lib.rs index 49b00ff515f8..1d081cb1b49e 100644 --- a/polkadot/node/core/approval-voting-parallel/src/lib.rs +++ b/polkadot/node/core/approval-voting-parallel/src/lib.rs @@ -862,20 +862,19 @@ impl> 'life0: 'async_trait, Self: 'async_trait, { - match P::priority() { - polkadot_overseer::PriorityLevel::Normal => self.send_message(msg), - polkadot_overseer::PriorityLevel::High => - async { self.send_unbounded_message(msg) }.boxed(), - } + self.0.send_message_with_priority::

(msg.into()) } fn try_send_message_with_priority( &mut self, msg: ApprovalDistributionMessage, ) -> Result<(), metered::TrySendError> { - match P::priority() { - polkadot_overseer::PriorityLevel::Normal => self.try_send_message(msg), - polkadot_overseer::PriorityLevel::High => Ok(self.send_unbounded_message(msg)), - } + self.0.try_send_message_with_priority::

(msg.into()).map_err(|err| match err { + // Safe to unwrap because it was built from the same type. + metered::TrySendError::Closed(msg) => + metered::TrySendError::Closed(msg.try_into().unwrap()), + metered::TrySendError::Full(msg) => + metered::TrySendError::Full(msg.try_into().unwrap()), + }) } } From d9200956ddf014e46f13bcd2e8cc9e2af8049226 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 1 Aug 2024 15:13:13 +0300 Subject: [PATCH 12/45] Add metrics for channels to workers Signed-off-by: Alexandru Gheorghe --- .../core/approval-voting-parallel/src/lib.rs | 100 ++++---- .../approval-voting-parallel/src/metrics.rs | 236 ++++++++++++++++++ .../approval-distribution/src/metrics.rs | 63 ++--- .../approval-distribution/src/tests.rs | 4 +- polkadot/node/service/src/overseer.rs | 4 +- .../subsystem-bench/src/lib/approval/mod.rs | 12 +- .../src/lib/mock/network_bridge.rs | 5 +- 7 files changed, 316 insertions(+), 108 deletions(-) create mode 100644 polkadot/node/core/approval-voting-parallel/src/metrics.rs diff --git a/polkadot/node/core/approval-voting-parallel/src/lib.rs b/polkadot/node/core/approval-voting-parallel/src/lib.rs index 1d081cb1b49e..5e98f25aeb1f 100644 --- a/polkadot/node/core/approval-voting-parallel/src/lib.rs +++ b/polkadot/node/core/approval-voting-parallel/src/lib.rs @@ -21,12 +21,14 @@ //! do their work in parallel, rather than serially, when they are run //! as independent subsystems. use itertools::Itertools; +use metrics::{Meters, MetricsWatcher}; use polkadot_node_core_approval_voting::{ time::{Clock, SystemClock}, Config, RealAssignmentCriteria, }; use polkadot_node_metrics::metered::{ - self, channel, unbounded, MeteredSender, UnboundedMeteredSender, + self, channel, unbounded, MeteredReceiver, MeteredSender, UnboundedMeteredReceiver, + UnboundedMeteredSender, }; use polkadot_node_primitives::DISPUTE_WINDOW; @@ -37,7 +39,6 @@ use polkadot_node_subsystem::{ use polkadot_node_subsystem_util::{ self, database::Database, - metrics::{self, prometheus}, runtime::{Config as RuntimeInfoConfig, RuntimeInfo}, }; use polkadot_overseer::{OverseerSignal, Priority, SubsystemSender, TimeoutExt}; @@ -48,6 +49,7 @@ use sc_keystore::LocalKeystore; use sp_consensus::SyncOracle; use futures::{channel::oneshot, prelude::*, StreamExt}; +pub use metrics::Metrics; use polkadot_node_core_approval_voting::{ approval_db::common::Config as DatabaseConfig, ApprovalVotingWorkProvider, }; @@ -58,6 +60,7 @@ use std::{ time::Duration, }; use stream::{select_with_strategy, PollNext}; +pub mod metrics; pub(crate) const LOG_TARGET: &str = "parachain::approval-voting-parallel"; // Value rather arbitrarily: Should not be hit in practice, it exists to more easily diagnose dead @@ -80,24 +83,6 @@ pub struct ApprovalVotingParallelSubsystem { subsystem_enabled: bool, } -/// Approval Voting metrics. -#[derive(Default, Clone)] -pub struct Metrics( - pub polkadot_approval_distribution::metrics::Metrics, - pub polkadot_node_core_approval_voting::Metrics, -); - -impl metrics::Metrics for Metrics { - fn try_register( - registry: &prometheus::Registry, - ) -> std::result::Result { - Ok(Metrics( - polkadot_approval_distribution::metrics::Metrics::try_register(registry)?, - polkadot_node_core_approval_voting::Metrics::try_register(registry)?, - )) - } -} - impl ApprovalVotingParallelSubsystem { /// Create a new approval voting subsystem with the given keystore, config, and database. pub fn with_config( @@ -174,23 +159,16 @@ fn prio_right<'a>(_val: &'a mut ()) -> PollNext { async fn start_workers( ctx: &mut Context, subsystem: ApprovalVotingParallelSubsystem, + metrics_watcher: &mut MetricsWatcher, ) -> SubsystemResult<(ToWorker, Vec>)> where { // Build approval voting handles. - let (tx_approval_voting_work, rx_approval_voting_work) = - channel::>(WORKERS_CHANNEL_SIZE); - let (tx_approval_voting_work_unbounded, rx_approval_voting_work_unbounded) = - unbounded::>(); - - let to_approval_voting_worker = - ToWorker(tx_approval_voting_work, tx_approval_voting_work_unbounded); + let (to_approval_voting_worker, rx_approval_voting_worker) = + build_channels("approval-voting-parallel-db".into(), WORKERS_CHANNEL_SIZE, metrics_watcher); - let prioritised = select_with_strategy( - rx_approval_voting_work, - rx_approval_voting_work_unbounded, - prio_right, - ); + let prioritised = + select_with_strategy(rx_approval_voting_worker.0, rx_approval_voting_worker.1, prio_right); let approval_voting_work_provider = ApprovalVotingWorkProviderImpl(prioritised); gum::info!(target: LOG_TARGET, "Starting approval distribution workers"); @@ -201,21 +179,17 @@ where for i in 0..APPROVAL_DISTRIBUTION_WORKER_COUNT { let approval_distr_instance = polkadot_approval_distribution::ApprovalDistribution::new_with_clock( - subsystem.metrics.0.clone(), + subsystem.metrics.approval_distribution_metrics(), subsystem.slot_duration_millis, subsystem.clock.clone(), false, ); + let task_name = format!("approval-voting-parallel-{}", i); - let (tx_approval_distribution_work, rx_approval_distribution_work) = - channel::>(WORKERS_CHANNEL_SIZE); - let (tx_approval_distribution_work_unbounded, rx_approval_distribution_unbounded) = - unbounded::>(); - - let to_approval_distribution_worker = - ToWorker(tx_approval_distribution_work, tx_approval_distribution_work_unbounded); + let (to_approval_distribution_worker, rx_approval_distribution_worker) = + build_channels(task_name.clone(), WORKERS_CHANNEL_SIZE, metrics_watcher); - let task_name = format!("approval-voting-parallel-{}", i); + metrics_watcher.watch(task_name.clone(), to_approval_distribution_worker.meter()); let mut network_sender = ctx.sender().clone(); let mut runtime_api_sender = ctx.sender().clone(); @@ -235,8 +209,8 @@ where }); let mut work_channels = select_with_strategy( - rx_approval_distribution_work, - rx_approval_distribution_unbounded, + rx_approval_distribution_worker.0, + rx_approval_distribution_worker.1, prio_right, ); @@ -284,7 +258,7 @@ where subsystem.db.clone(), subsystem.keystore.clone(), subsystem.sync_oracle, - subsystem.metrics.1.clone(), + subsystem.metrics.approval_voting_metrics(), subsystem.spawner.clone(), subsystem.clock.clone(), ) @@ -300,7 +274,7 @@ async fn run( subsystem: ApprovalVotingParallelSubsystem, ) -> SubsystemResult<()> { let subsystem_enabled = subsystem.subsystem_enabled; - + let mut metrics_watcher = MetricsWatcher::new(subsystem.metrics.clone()); gum::info!( target: LOG_TARGET, subsystem_enabled, @@ -308,7 +282,7 @@ async fn run( ); let (to_approval_voting_worker, to_approval_distribution_workers) = - start_workers(&mut ctx, subsystem).await?; + start_workers(&mut ctx, subsystem, &mut metrics_watcher).await?; gum::info!( target: LOG_TARGET, @@ -323,6 +297,7 @@ async fn run( subsystem_enabled, to_approval_voting_worker, to_approval_distribution_workers, + metrics_watcher, ) .await } @@ -333,6 +308,7 @@ async fn run_main_loop( subsystem_enabled: bool, mut to_approval_voting_worker: ToWorker, mut to_approval_distribution_workers: Vec>, + metrics_watcher: MetricsWatcher, ) -> SubsystemResult<()> { loop { futures::select! { @@ -352,12 +328,16 @@ async fn run_main_loop( match next_msg { FromOrchestra::Signal(msg) => { + if matches!(msg, OverseerSignal::ActiveLeaves(_)) { + metrics_watcher.collect_metrics(); + } for worker in to_approval_distribution_workers.iter_mut() { worker .send_signal(msg.clone()).await?; } to_approval_voting_worker.send_signal(msg).await?; + }, FromOrchestra::Communication { msg } => match msg { // The message the approval voting subsystem would've handled. @@ -685,6 +665,10 @@ impl ToWorker { .unbounded_send(FromOrchestra::Signal(signal)) .map_err(|err| SubsystemError::QueueError(err.into_send_error())) } + + fn meter(&self) -> Meters { + Meters::new(self.0.meter(), self.1.meter()) + } } impl overseer::SubsystemSender for ToWorker { @@ -792,6 +776,30 @@ impl overseer::SubsystemSender for ToWorker } } +/// Handles to use by an worker to receive work. +pub struct RxWorker( + MeteredReceiver>, + UnboundedMeteredReceiver>, +); + +/// Build all the necessary channels for sending messages to the workers +/// and for the workers to receive them. +fn build_channels( + channel_name: String, + channel_size: usize, + metrics_watcher: &mut MetricsWatcher, +) -> (ToWorker, RxWorker) { + let (tx_approval_voting_work, rx_approval_voting_work) = + channel::>(channel_size); + let (tx_approval_voting_work_unbounded, rx_approval_voting_work_unbounded) = + unbounded::>(); + let to_worker = ToWorker(tx_approval_voting_work, tx_approval_voting_work_unbounded); + + metrics_watcher.watch(channel_name, to_worker.meter()); + + (to_worker, RxWorker(rx_approval_voting_work, rx_approval_voting_work_unbounded)) +} + /// Just a wrapper for implementing overseer::SubsystemSender, so that /// we can inject into the approval voting subsystem. #[derive(Clone)] diff --git a/polkadot/node/core/approval-voting-parallel/src/metrics.rs b/polkadot/node/core/approval-voting-parallel/src/metrics.rs new file mode 100644 index 000000000000..b5ec6ec01277 --- /dev/null +++ b/polkadot/node/core/approval-voting-parallel/src/metrics.rs @@ -0,0 +1,236 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! The Metrics for Approval Voting Parallel Subsystem. + +use std::collections::HashMap; + +use polkadot_node_metrics::{metered::Meter, metrics}; +use polkadot_overseer::prometheus; + +#[derive(Default, Clone)] +pub struct Metrics(Option); + +/// Approval Voting parallel metrics. +#[derive(Clone)] +pub struct MetricsInner { + // The inner metrics of the approval distribution workers. + approval_distribution: polkadot_approval_distribution::metrics::Metrics, + // The inner metrics of the approval voting workers. + approval_voting: polkadot_node_core_approval_voting::Metrics, + + // Time of flight metrics for bounded channels. + to_worker_bounded_tof: prometheus::HistogramVec, + // Number of elements sent to the worker's bounded queue. + to_worker_bounded_sent: prometheus::GaugeVec, + // Number of elements received by the worker's bounded queue. + to_worker_bounded_received: prometheus::GaugeVec, + // Number of times senders blocked while sending messages to the worker. + to_worker_bounded_blocked: prometheus::GaugeVec, + // Time of flight metrics for unbounded channels. + to_worker_unbounded_tof: prometheus::HistogramVec, + // Number of elements sent to the worker's unbounded queue. + to_worker_unbounded_sent: prometheus::GaugeVec, + // Number of elements received by the worker's unbounded queue. + to_worker_unbounded_received: prometheus::GaugeVec, +} + +impl Metrics { + /// Get the approval distribution metrics. + pub fn approval_distribution_metrics( + &self, + ) -> polkadot_approval_distribution::metrics::Metrics { + self.0 + .as_ref() + .map(|metrics_inner| metrics_inner.approval_distribution.clone()) + .unwrap_or_default() + } + + /// Get the approval voting metrics. + pub fn approval_voting_metrics(&self) -> polkadot_node_core_approval_voting::Metrics { + self.0 + .as_ref() + .map(|metrics_inner| metrics_inner.approval_voting.clone()) + .unwrap_or_default() + } +} + +impl metrics::Metrics for Metrics { + /// Try to register the metrics. + fn try_register( + registry: &prometheus::Registry, + ) -> std::result::Result { + Ok(Metrics(Some(MetricsInner { + approval_distribution: polkadot_approval_distribution::metrics::Metrics::try_register( + registry, + )?, + approval_voting: polkadot_node_core_approval_voting::Metrics::try_register(registry)?, + to_worker_bounded_tof: prometheus::register( + prometheus::HistogramVec::new( + prometheus::HistogramOpts::new( + "polkadot_parachain_worker_bounded_tof", + "Duration spent in a particular channel from entrance to removal", + ) + .buckets(vec![ + 0.0001, 0.0004, 0.0016, 0.0064, 0.0256, 0.1024, 0.4096, 1.6384, 3.2768, + 4.9152, 6.5536, + ]), + &["worker_name"], + )?, + registry, + )?, + to_worker_bounded_sent: prometheus::register( + prometheus::GaugeVec::::new( + prometheus::Opts::new( + "polkadot_parachain_worker_bounded_sent", + "Number of elements sent to subsystems' bounded queues", + ), + &["worker_name"], + )?, + registry, + )?, + to_worker_bounded_received: prometheus::register( + prometheus::GaugeVec::::new( + prometheus::Opts::new( + "polkadot_parachain_worker_bounded_received", + "Number of elements received by subsystems' bounded queues", + ), + &["worker_name"], + )?, + registry, + )?, + to_worker_bounded_blocked: prometheus::register( + prometheus::GaugeVec::::new( + prometheus::Opts::new( + "polkadot_parachain_worker_bounded_blocked", + "Number of times senders blocked while sending messages to a subsystem", + ), + &["worker_name"], + )?, + registry, + )?, + to_worker_unbounded_tof: prometheus::register( + prometheus::HistogramVec::new( + prometheus::HistogramOpts::new( + "polkadot_parachain_worker_unbounded_tof", + "Duration spent in a particular channel from entrance to removal", + ) + .buckets(vec![ + 0.0001, 0.0004, 0.0016, 0.0064, 0.0256, 0.1024, 0.4096, 1.6384, 3.2768, + 4.9152, 6.5536, + ]), + &["worker_name"], + )?, + registry, + )?, + to_worker_unbounded_sent: prometheus::register( + prometheus::GaugeVec::::new( + prometheus::Opts::new( + "polkadot_parachain_worker_unbounded_sent", + "Number of elements sent to subsystems' unbounded queues", + ), + &["worker_name"], + )?, + registry, + )?, + to_worker_unbounded_received: prometheus::register( + prometheus::GaugeVec::::new( + prometheus::Opts::new( + "polkadot_parachain_worker_unbounded_received", + "Number of elements received by subsystems' unbounded queues", + ), + &["worker_name"], + )?, + registry, + )?, + }))) + } +} + +/// The meters to watch. +#[derive(Clone)] +pub struct Meters { + bounded: Meter, + unbounded: Meter, +} + +impl Meters { + pub fn new(bounded: &Meter, unbounded: &Meter) -> Self { + Self { bounded: bounded.clone(), unbounded: unbounded.clone() } + } +} + +/// A metrics watcher that watches the meters and updates the metrics. +pub struct MetricsWatcher { + to_watch: HashMap, + metrics: Metrics, +} + +impl MetricsWatcher { + /// Create a new metrics watcher. + pub fn new(metrics: Metrics) -> Self { + Self { to_watch: HashMap::new(), metrics } + } + + /// Watch the meters of a worker with this name. + pub fn watch(&mut self, worker_name: String, meters: Meters) { + self.to_watch.insert(worker_name, meters); + } + + /// Collect all the metrics. + pub fn collect_metrics(&self) { + for (name, meter) in &self.to_watch { + let bounded_readouts = meter.bounded.read(); + let unbounded_readouts = meter.unbounded.read(); + if let Some(metrics) = self.metrics.0.as_ref() { + metrics + .to_worker_bounded_sent + .with_label_values(&[name]) + .set(bounded_readouts.sent as u64); + + metrics + .to_worker_bounded_received + .with_label_values(&[name]) + .set(bounded_readouts.received as u64); + + metrics + .to_worker_bounded_blocked + .with_label_values(&[name]) + .set(bounded_readouts.blocked as u64); + + metrics + .to_worker_unbounded_sent + .with_label_values(&[name]) + .set(unbounded_readouts.sent as u64); + + metrics + .to_worker_unbounded_received + .with_label_values(&[name]) + .set(unbounded_readouts.received as u64); + + let hist_bounded = metrics.to_worker_bounded_tof.with_label_values(&[name]); + for tof in bounded_readouts.tof { + hist_bounded.observe(tof.as_f64()); + } + + let hist_unbounded = metrics.to_worker_unbounded_tof.with_label_values(&[name]); + for tof in unbounded_readouts.tof { + hist_unbounded.observe(tof.as_f64()); + } + } + } + } +} diff --git a/polkadot/node/network/approval-distribution/src/metrics.rs b/polkadot/node/network/approval-distribution/src/metrics.rs index 10553c352966..2f677ba415e4 100644 --- a/polkadot/node/network/approval-distribution/src/metrics.rs +++ b/polkadot/node/network/approval-distribution/src/metrics.rs @@ -79,31 +79,19 @@ impl Metrics { .map(|metrics| metrics.time_import_pending_now_known.start_timer()) } - pub fn on_approval_already_known(&self) { - if let Some(metrics) = &self.0 { - metrics.approvals_received_result.with_label_values(&["known"]).inc() - } - } - - pub fn on_approval_entry_not_found(&self) { - if let Some(metrics) = &self.0 { - metrics.approvals_received_result.with_label_values(&["noapprovalentry"]).inc() - } - } - - pub fn on_approval_recent_outdated(&self) { + pub(crate) fn on_approval_recent_outdated(&self) { if let Some(metrics) = &self.0 { metrics.approvals_received_result.with_label_values(&["outdated"]).inc() } } - pub fn on_approval_invalid_block(&self) { + pub(crate) fn on_approval_invalid_block(&self) { if let Some(metrics) = &self.0 { metrics.approvals_received_result.with_label_values(&["invalidblock"]).inc() } } - pub fn on_approval_unknown_assignment(&self) { + pub(crate) fn on_approval_unknown_assignment(&self) { if let Some(metrics) = &self.0 { metrics .approvals_received_result @@ -112,94 +100,73 @@ impl Metrics { } } - pub fn on_approval_duplicate(&self) { + pub(crate) fn on_approval_duplicate(&self) { if let Some(metrics) = &self.0 { metrics.approvals_received_result.with_label_values(&["duplicate"]).inc() } } - pub fn on_approval_out_of_view(&self) { + pub(crate) fn on_approval_out_of_view(&self) { if let Some(metrics) = &self.0 { metrics.approvals_received_result.with_label_values(&["outofview"]).inc() } } - pub fn on_approval_good_known(&self) { + pub(crate) fn on_approval_good_known(&self) { if let Some(metrics) = &self.0 { metrics.approvals_received_result.with_label_values(&["goodknown"]).inc() } } - pub fn on_approval_bad(&self) { + pub(crate) fn on_approval_bad(&self) { if let Some(metrics) = &self.0 { metrics.approvals_received_result.with_label_values(&["bad"]).inc() } } - pub fn on_approval_unexpected(&self) { - if let Some(metrics) = &self.0 { - metrics.approvals_received_result.with_label_values(&["unexpected"]).inc() - } - } - - pub fn on_approval_bug(&self) { + pub(crate) fn on_approval_bug(&self) { if let Some(metrics) = &self.0 { metrics.approvals_received_result.with_label_values(&["bug"]).inc() } } - pub fn on_assignment_already_known(&self) { - if let Some(metrics) = &self.0 { - metrics.assignments_received_result.with_label_values(&["known"]).inc() - } - } - - pub fn on_assignment_recent_outdated(&self) { + pub(crate) fn on_assignment_recent_outdated(&self) { if let Some(metrics) = &self.0 { metrics.assignments_received_result.with_label_values(&["outdated"]).inc() } } - pub fn on_assignment_invalid_block(&self) { + pub(crate) fn on_assignment_invalid_block(&self) { if let Some(metrics) = &self.0 { metrics.assignments_received_result.with_label_values(&["invalidblock"]).inc() } } - pub fn on_assignment_duplicate(&self) { + pub(crate) fn on_assignment_duplicate(&self) { if let Some(metrics) = &self.0 { metrics.assignments_received_result.with_label_values(&["duplicate"]).inc() } } - pub fn on_assignment_out_of_view(&self) { + pub(crate) fn on_assignment_out_of_view(&self) { if let Some(metrics) = &self.0 { metrics.assignments_received_result.with_label_values(&["outofview"]).inc() } } - pub fn on_assignment_good_known(&self) { + pub(crate) fn on_assignment_good_known(&self) { if let Some(metrics) = &self.0 { metrics.assignments_received_result.with_label_values(&["goodknown"]).inc() } } - pub fn on_assignment_bad(&self) { + pub(crate) fn on_assignment_bad(&self) { if let Some(metrics) = &self.0 { metrics.assignments_received_result.with_label_values(&["bad"]).inc() } } - pub fn on_assignment_duplicatevoting(&self) { - if let Some(metrics) = &self.0 { - metrics - .assignments_received_result - .with_label_values(&["duplicatevoting"]) - .inc() - } - } - - pub fn on_assignment_far(&self) { + pub(crate) fn on_assignment_far(&self) { if let Some(metrics) = &self.0 { metrics.assignments_received_result.with_label_values(&["far"]).inc() } diff --git a/polkadot/node/network/approval-distribution/src/tests.rs b/polkadot/node/network/approval-distribution/src/tests.rs index 897fb984a63e..2fbdc7e1b34a 100644 --- a/polkadot/node/network/approval-distribution/src/tests.rs +++ b/polkadot/node/network/approval-distribution/src/tests.rs @@ -4465,7 +4465,7 @@ fn subsystem_rejects_wrong_claimed_assignments() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(DummyClock {}), + Arc::new(DummyClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -4549,7 +4549,7 @@ fn subsystem_accepts_tranche0_duplicate_assignments() { let _ = test_harness( &MockAssignmentCriteria { tranche: Ok(0) }, - Box::new(DummyClock {}), + Arc::new(DummyClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; diff --git a/polkadot/node/service/src/overseer.rs b/polkadot/node/service/src/overseer.rs index 60d2a30fcea2..7007fb594fde 100644 --- a/polkadot/node/service/src/overseer.rs +++ b/polkadot/node/service/src/overseer.rs @@ -318,7 +318,7 @@ where rand::rngs::StdRng::from_entropy(), )) .approval_distribution(ApprovalDistributionSubsystem::new( - approval_voting_parallel_metrics.0.clone(), + approval_voting_parallel_metrics.approval_distribution_metrics(), approval_voting_config.slot_duration_millis, enable_approval_voting_parallel, )) @@ -327,7 +327,7 @@ where parachains_db.clone(), keystore.clone(), Box::new(sync_service.clone()), - approval_voting_parallel_metrics.1.clone(), + approval_voting_parallel_metrics.approval_voting_metrics(), Arc::new(spawner.clone()), enable_approval_voting_parallel, )) diff --git a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs index de25c75664ce..b69ce0afd2fd 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs @@ -268,7 +268,7 @@ pub struct ApprovalTestState { /// Total unique sent messages. total_unique_messages: Arc, /// Approval voting metrics. - approval_voting_metrics_rewrite: polkadot_node_core_approval_voting_parallel::Metrics, + approval_voting_parallel_metrics: polkadot_node_core_approval_voting_parallel::Metrics, /// The delta ticks from the tick the messages were generated to the the time we start this /// message. delta_tick_from_generated: Arc, @@ -326,7 +326,7 @@ impl ApprovalTestState { total_sent_messages_from_node: Arc::new(AtomicU64::new(0)), total_unique_messages: Arc::new(AtomicU64::new(0)), options, - approval_voting_metrics_rewrite: + approval_voting_parallel_metrics: polkadot_node_core_approval_voting_parallel::Metrics::try_register( &dependencies.registry, ) @@ -814,15 +814,15 @@ fn build_overseer( db.clone(), keystore.clone(), Box::new(TestSyncOracle {}), - state.approval_voting_metrics_rewrite.1.clone(), + state.approval_voting_parallel_metrics.approval_voting_metrics(), Arc::new(system_clock.clone()), Arc::new(SpawnGlue(spawn_task_handle.clone())), true, ); let approval_distribution = ApprovalDistribution::new_with_clock( - state.approval_voting_metrics_rewrite.0.clone(), - TEST_CONFIG.slot_duration_millis as u64, + state.approval_voting_parallel_metrics.approval_distribution_metrics(), + TEST_CONFIG.slot_duration_millis, Arc::new(system_clock.clone()), true, ); @@ -833,7 +833,7 @@ fn build_overseer( db.clone(), keystore.clone(), Box::new(TestSyncOracle {}), - state.approval_voting_metrics_rewrite.clone(), + state.approval_voting_parallel_metrics.clone(), Arc::new(system_clock.clone()), SpawnGlue(spawn_task_handle.clone()), true, diff --git a/polkadot/node/subsystem-bench/src/lib/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/lib/mock/network_bridge.rs index a32b48705463..b6291784c05e 100644 --- a/polkadot/node/subsystem-bench/src/lib/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/lib/mock/network_bridge.rs @@ -28,10 +28,7 @@ use polkadot_node_subsystem::{ overseer, SpawnedSubsystem, SubsystemError, }; use polkadot_node_subsystem_types::{ - messages::{ - ApprovalDistributionMessage, BitfieldDistributionMessage, NetworkBridgeEvent, - StatementDistributionMessage, - }, + messages::{BitfieldDistributionMessage, NetworkBridgeEvent, StatementDistributionMessage}, OverseerSignal, }; use sc_network::{request_responses::ProtocolConfig, RequestFailure}; From e30f83d7f55fa948d8e6801accf78f45fe9c0343 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 1 Aug 2024 18:04:05 +0300 Subject: [PATCH 13/45] subsystem-bench: Make it report per task cpu usage Signed-off-by: Alexandru Gheorghe --- .../core/approval-voting-parallel/src/lib.rs | 4 +- polkadot/node/core/approval-voting/src/lib.rs | 2 +- .../subsystem-bench/src/lib/approval/mod.rs | 9 ++- .../src/lib/availability/mod.rs | 11 ++-- .../node/subsystem-bench/src/lib/display.rs | 17 ++++++ .../subsystem-bench/src/lib/environment.rs | 57 ++++++++----------- .../subsystem-bench/src/lib/statement/mod.rs | 2 +- .../node/subsystem-bench/src/lib/usage.rs | 6 +- 8 files changed, 57 insertions(+), 51 deletions(-) diff --git a/polkadot/node/core/approval-voting-parallel/src/lib.rs b/polkadot/node/core/approval-voting-parallel/src/lib.rs index 5e98f25aeb1f..dc9e9c5f339d 100644 --- a/polkadot/node/core/approval-voting-parallel/src/lib.rs +++ b/polkadot/node/core/approval-voting-parallel/src/lib.rs @@ -143,7 +143,7 @@ impl ApprovalVotingParallelSubsystem { } /// The number of workers used for running the approval-distribution logic. -pub const APPROVAL_DISTRIBUTION_WORKER_COUNT: usize = 2; +pub const APPROVAL_DISTRIBUTION_WORKER_COUNT: usize = 4; /// The channel size for the workers. pub const WORKERS_CHANNEL_SIZE: usize = 64000 / APPROVAL_DISTRIBUTION_WORKER_COUNT; @@ -197,7 +197,7 @@ where subsystem.spawner.spawn_blocking( task_name.leak(), - Some("approval-voting-parallel-subsystem"), + Some("approval-voting-parallel"), Box::pin(async move { let mut state = polkadot_approval_distribution::State::with_config(slot_duration_millis); diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index b334baa96a95..7c7f314b05af 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -1414,7 +1414,7 @@ pub async fn start_approval_worker< let spawner = approval_voting.spawner.clone(); spawner.spawn_blocking( "approval-voting-parallel-db", - Some("approval-voting-parallel-subsystem"), + Some("approval-voting-parallel"), Box::pin(async move { if let Err(err) = run( work_provider, diff --git a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs index b69ce0afd2fd..dd209f2025e9 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs @@ -1114,9 +1114,8 @@ pub async fn bench_approvals_run( state.total_unique_messages.load(std::sync::atomic::Ordering::SeqCst) ); - env.collect_resource_usage(&[ - "approval-distribution", - "approval-voting", - "approval-voting-parallel", - ]) + env.collect_resource_usage( + &["approval-distribution", "approval-voting", "approval-voting-parallel"], + true, + ) } diff --git a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs index 32dc8ae2c8dc..c28bdba49f54 100644 --- a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs @@ -372,7 +372,7 @@ pub async fn benchmark_availability_read( ); env.stop().await; - env.collect_resource_usage(&["availability-recovery"]) + env.collect_resource_usage(&["availability-recovery"], false) } pub async fn benchmark_availability_write( @@ -506,9 +506,8 @@ pub async fn benchmark_availability_write( ); env.stop().await; - env.collect_resource_usage(&[ - "availability-distribution", - "bitfield-distribution", - "availability-store", - ]) + env.collect_resource_usage( + &["availability-distribution", "bitfield-distribution", "availability-store"], + false, + ) } diff --git a/polkadot/node/subsystem-bench/src/lib/display.rs b/polkadot/node/subsystem-bench/src/lib/display.rs index b153d54a7c36..c47dd9a07900 100644 --- a/polkadot/node/subsystem-bench/src/lib/display.rs +++ b/polkadot/node/subsystem-bench/src/lib/display.rs @@ -96,6 +96,23 @@ pub struct TestMetric { value: f64, } +impl TestMetric { + pub fn name(&self) -> &str { + &self.name + } + + pub fn value(&self) -> f64 { + self.value + } + + pub fn label_value(&self, label_name: &str) -> Option<&str> { + self.label_names + .iter() + .position(|name| name == label_name) + .and_then(|index| self.label_values.get(index).map(|s| s.as_str())) + } +} + impl Display for TestMetric { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( diff --git a/polkadot/node/subsystem-bench/src/lib/environment.rs b/polkadot/node/subsystem-bench/src/lib/environment.rs index 7e3c7f42b613..4de683ad6487 100644 --- a/polkadot/node/subsystem-bench/src/lib/environment.rs +++ b/polkadot/node/subsystem-bench/src/lib/environment.rs @@ -24,7 +24,6 @@ use crate::{ }; use core::time::Duration; use futures::{Future, FutureExt}; -use polkadot_node_core_approval_voting_parallel::APPROVAL_DISTRIBUTION_WORKER_COUNT; use polkadot_node_subsystem::{messages::AllMessages, Overseer, SpawnGlue, TimeoutExt}; use polkadot_node_subsystem_types::Hash; use polkadot_node_subsystem_util::metrics::prometheus::{ @@ -352,10 +351,14 @@ impl TestEnvironment { } } - pub fn collect_resource_usage(&self, subsystems_under_test: &[&str]) -> BenchmarkUsage { + pub fn collect_resource_usage( + &self, + subsystems_under_test: &[&str], + break_down_cpu_usage_per_task: bool, + ) -> BenchmarkUsage { BenchmarkUsage { network_usage: self.network_usage(), - cpu_usage: self.cpu_usage(subsystems_under_test), + cpu_usage: self.cpu_usage(subsystems_under_test, break_down_cpu_usage_per_task), } } @@ -379,7 +382,11 @@ impl TestEnvironment { ] } - fn cpu_usage(&self, subsystems_under_test: &[&str]) -> Vec { + fn cpu_usage( + &self, + subsystems_under_test: &[&str], + break_down_per_task: bool, + ) -> Vec { let test_metrics = super::display::parse_metrics(self.registry()); let mut usage = vec![]; let num_blocks = self.config().num_blocks as f64; @@ -394,36 +401,20 @@ impl TestEnvironment { per_block: total_cpu / num_blocks, }); - if subsystem == &"approval-voting-parallel" { - for i in 0..APPROVAL_DISTRIBUTION_WORKER_COUNT { - let task_name = format!("approval-voting-parallel-{}", i); - - let subsystem_cpu_metrics = - test_metrics.subset_with_label_value("task_name", task_name.as_str()); - - let total_cpu = - subsystem_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); - - usage.push(ResourceUsage { - resource_name: task_name.to_string(), - total: total_cpu, - per_block: total_cpu / num_blocks, - }) + if break_down_per_task { + for metric in subsystem_cpu_metrics.all() { + if metric.name() != "substrate_tasks_polling_duration_sum" { + continue; + } + + if let Some(task_name) = metric.label_value("task_name") { + usage.push(ResourceUsage { + resource_name: format!("{}/{}", subsystem, task_name), + total: metric.value(), + per_block: metric.value() / num_blocks, + }); + } } - - let task_name = format!("approval-voting-parallel-db"); - - let subsystem_cpu_metrics = - test_metrics.subset_with_label_value("task_name", task_name.as_str()); - - let total_cpu = - subsystem_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); - - usage.push(ResourceUsage { - resource_name: task_name.to_string(), - total: total_cpu, - per_block: total_cpu / num_blocks, - }) } } diff --git a/polkadot/node/subsystem-bench/src/lib/statement/mod.rs b/polkadot/node/subsystem-bench/src/lib/statement/mod.rs index bd47505f56ae..710194370140 100644 --- a/polkadot/node/subsystem-bench/src/lib/statement/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/statement/mod.rs @@ -445,5 +445,5 @@ pub async fn benchmark_statement_distribution( ); env.stop().await; - env.collect_resource_usage(&["statement-distribution"]) + env.collect_resource_usage(&["statement-distribution"], false) } diff --git a/polkadot/node/subsystem-bench/src/lib/usage.rs b/polkadot/node/subsystem-bench/src/lib/usage.rs index 883e9aa7ad0a..5f691ae2db39 100644 --- a/polkadot/node/subsystem-bench/src/lib/usage.rs +++ b/polkadot/node/subsystem-bench/src/lib/usage.rs @@ -32,14 +32,14 @@ impl std::fmt::Display for BenchmarkUsage { write!( f, "\n{}\n{}\n\n{}\n{}\n", - format!("{:<32}{:>12}{:>12}", "Network usage, KiB", "total", "per block").blue(), + format!("{:<64}{:>12}{:>12}", "Network usage, KiB", "total", "per block").blue(), self.network_usage .iter() .map(|v| v.to_string()) .sorted() .collect::>() .join("\n"), - format!("{:<32}{:>12}{:>12}", "CPU usage, seconds", "total", "per block").blue(), + format!("{:<64}{:>12}{:>12}", "CPU usage, seconds", "total", "per block").blue(), self.cpu_usage .iter() .map(|v| v.to_string()) @@ -134,7 +134,7 @@ pub struct ResourceUsage { impl std::fmt::Display for ResourceUsage { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{:<32}{:>12.4}{:>12.4}", self.resource_name.cyan(), self.total, self.per_block) + write!(f, "{:<64}{:>12.4}{:>12.4}", self.resource_name.cyan(), self.total, self.per_block) } } From 05c53cd4d90c4fe429b5cd095b7891716af05b24 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 1 Aug 2024 18:47:12 +0300 Subject: [PATCH 14/45] Make subsystem-bench run in both parallel and single thread mode Signed-off-by: Alexandru Gheorghe --- .../node/core/approval-voting/src/import.rs | 2 +- .../examples/approvals_no_shows.yaml | 1 + .../examples/approvals_throughput.yaml | 1 + .../approvals_throughput_best_case.yaml | 1 + .../src/lib/approval/helpers.rs | 30 +++++-- .../subsystem-bench/src/lib/approval/mod.rs | 90 ++++++++++++++----- .../src/lib/availability/mod.rs | 2 +- .../src/lib/mock/network_bridge.rs | 21 +++-- .../subsystem-bench/src/lib/statement/mod.rs | 3 +- 9 files changed, 114 insertions(+), 37 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/import.rs b/polkadot/node/core/approval-voting/src/import.rs index 3531b9d0bca5..9b3173be672c 100644 --- a/polkadot/node/core/approval-voting/src/import.rs +++ b/polkadot/node/core/approval-voting/src/import.rs @@ -41,7 +41,7 @@ use polkadot_node_subsystem::{ ApprovalDistributionMessage, ChainApiMessage, ChainSelectionMessage, RuntimeApiMessage, RuntimeApiRequest, }, - overseer, RuntimeApiError, SubsystemContext, SubsystemError, SubsystemResult, + overseer, RuntimeApiError, SubsystemError, SubsystemResult, }; use polkadot_node_subsystem_util::{determine_new_blocks, runtime::RuntimeInfo}; use polkadot_overseer::SubsystemSender; diff --git a/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml b/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml index cae1a30914da..1423d324df3f 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml @@ -9,6 +9,7 @@ TestConfiguration: coalesce_tranche_diff: 12 num_no_shows_per_candidate: 10 workdir_prefix: "/tmp/" + approval_voting_parallel_enabled: false n_validators: 500 n_cores: 100 min_pov_size: 1120 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml index 7edb48e302a4..87c6103a5d0a 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml @@ -9,6 +9,7 @@ TestConfiguration: coalesce_tranche_diff: 12 num_no_shows_per_candidate: 0 workdir_prefix: "/tmp" + approval_voting_parallel_enabled: true n_validators: 500 n_cores: 100 min_pov_size: 1120 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml index 7c24f50e6af5..5e2ea3817d17 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml @@ -8,6 +8,7 @@ TestConfiguration: stop_when_approved: true coalesce_tranche_diff: 12 num_no_shows_per_candidate: 0 + approval_voting_parallel_enabled: false workdir_prefix: "/tmp/" n_validators: 500 n_cores: 100 diff --git a/polkadot/node/subsystem-bench/src/lib/approval/helpers.rs b/polkadot/node/subsystem-bench/src/lib/approval/helpers.rs index f9eaacfeb148..a1491a1b7860 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/helpers.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/helpers.rs @@ -21,7 +21,9 @@ use polkadot_node_network_protocol::{ grid_topology::{SessionGridTopology, TopologyPeerInfo}, View, }; -use polkadot_node_subsystem::messages::ApprovalVotingParallelMessage; +use polkadot_node_subsystem::messages::{ + ApprovalDistributionMessage, ApprovalVotingParallelMessage, +}; use polkadot_node_subsystem_types::messages::{ network_bridge_event::NewGossipTopology, NetworkBridgeEvent, }; @@ -122,6 +124,7 @@ pub fn generate_topology(test_authorities: &TestAuthorities) -> SessionGridTopol pub fn generate_new_session_topology( test_authorities: &TestAuthorities, test_node: ValidatorIndex, + approval_voting_parallel_enabled: bool, ) -> Vec { let topology = generate_topology(test_authorities); @@ -130,16 +133,29 @@ pub fn generate_new_session_topology( topology, local_index: Some(test_node), }); - vec![AllMessages::ApprovalVotingParallel(ApprovalVotingParallelMessage::NetworkBridgeUpdate( - event, - ))] + vec![if approval_voting_parallel_enabled { + AllMessages::ApprovalVotingParallel(ApprovalVotingParallelMessage::NetworkBridgeUpdate( + event, + )) + } else { + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate(event)) + }] } /// Generates a peer view change for the passed `block_hash` -pub fn generate_peer_view_change_for(block_hash: Hash, peer_id: PeerId) -> AllMessages { +pub fn generate_peer_view_change_for( + block_hash: Hash, + peer_id: PeerId, + approval_voting_parallel_enabled: bool, +) -> AllMessages { let network = NetworkBridgeEvent::PeerViewChange(peer_id, View::new([block_hash], 0)); - - AllMessages::ApprovalVotingParallel(ApprovalVotingParallelMessage::NetworkBridgeUpdate(network)) + if approval_voting_parallel_enabled { + AllMessages::ApprovalVotingParallel(ApprovalVotingParallelMessage::NetworkBridgeUpdate( + network, + )) + } else { + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate(network)) + } } /// Helper function to create a a signature for the block header. diff --git a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs index dd209f2025e9..de77b2201a5d 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs @@ -55,8 +55,8 @@ use polkadot_node_core_approval_voting::{ use polkadot_node_network_protocol::v3 as protocol_v3; use polkadot_node_primitives::approval::{self, v1::RelayVRFStory}; use polkadot_node_subsystem::{ - messages::ApprovalVotingParallelMessage, overseer, AllMessages, Overseer, OverseerConnector, - SpawnGlue, + messages::{ApprovalDistributionMessage, ApprovalVotingMessage, ApprovalVotingParallelMessage}, + overseer, AllMessages, Overseer, OverseerConnector, SpawnGlue, }; use polkadot_node_subsystem_test_helpers::mock::new_block_import_info; use polkadot_overseer::Handle as OverseerHandleReal; @@ -134,6 +134,9 @@ pub struct ApprovalsOptions { /// The number of no shows per candidate #[clap(short, long, default_value_t = 0)] pub num_no_shows_per_candidate: u32, + /// Enable approval voting parallel. + #[clap(short, long, default_value_t = true)] + pub approval_voting_parallel_enabled: bool, } impl ApprovalsOptions { @@ -596,13 +599,16 @@ impl PeerMessageProducer { // so when the approval-distribution answered to it, we know it doesn't have anything // else to process. let (tx, rx) = oneshot::channel(); - let msg = ApprovalVotingParallelMessage::GetApprovalSignatures(HashSet::new(), tx); - self.send_overseer_message( - AllMessages::ApprovalVotingParallel(msg), - ValidatorIndex(0), - None, - ) - .await; + let msg = if self.options.approval_voting_parallel_enabled { + AllMessages::ApprovalVotingParallel( + ApprovalVotingParallelMessage::GetApprovalSignatures(HashSet::new(), tx), + ) + } else { + AllMessages::ApprovalDistribution( + ApprovalDistributionMessage::GetApprovalSignatures(HashSet::new(), tx), + ) + }; + self.send_overseer_message(msg, ValidatorIndex(0), None).await; rx.await.expect("Failed to get signatures"); self.notify_done.send(()).expect("Failed to notify main loop"); gum::info!("All messages processed "); @@ -742,7 +748,11 @@ impl PeerMessageProducer { for validator in 1..self.state.test_authorities.validator_authority_id.len() as u32 { let peer_id = self.state.test_authorities.peer_ids.get(validator as usize).unwrap(); let validator = ValidatorIndex(validator); - let view_update = generate_peer_view_change_for(block_info.hash, *peer_id); + let view_update = generate_peer_view_change_for( + block_info.hash, + *peer_id, + self.state.options.approval_voting_parallel_enabled, + ); self.send_overseer_message(view_update, validator, None).await; } @@ -817,14 +827,14 @@ fn build_overseer( state.approval_voting_parallel_metrics.approval_voting_metrics(), Arc::new(system_clock.clone()), Arc::new(SpawnGlue(spawn_task_handle.clone())), - true, + state.options.approval_voting_parallel_enabled, ); let approval_distribution = ApprovalDistribution::new_with_clock( state.approval_voting_parallel_metrics.approval_distribution_metrics(), TEST_CONFIG.slot_duration_millis, Arc::new(system_clock.clone()), - true, + state.options.approval_voting_parallel_enabled, ); let approval_voting_parallel = @@ -836,7 +846,7 @@ fn build_overseer( state.approval_voting_parallel_metrics.clone(), Arc::new(system_clock.clone()), SpawnGlue(spawn_task_handle.clone()), - true, + state.options.approval_voting_parallel_enabled, ); let mock_chain_api = MockChainApi::new(state.build_chain_api_state()); @@ -855,7 +865,11 @@ fn build_overseer( network_interface.subsystem_sender(), state.test_authorities.clone(), ); - let mock_rx_bridge = MockNetworkBridgeRx::new(network_receiver, None); + let mock_rx_bridge = MockNetworkBridgeRx::new( + network_receiver, + None, + state.options.approval_voting_parallel_enabled, + ); let overseer_metrics = OverseerMetrics::try_register(&dependencies.registry).unwrap(); let dummy = dummy_builder!(spawn_task_handle, overseer_metrics) .replace_approval_distribution(|_| approval_distribution) @@ -959,11 +973,18 @@ pub async fn bench_approvals_run( // First create the initialization messages that make sure that then node under // tests receives notifications about the topology used and the connected peers. let mut initialization_messages = env.network().generate_peer_connected(|e| { - AllMessages::ApprovalVotingParallel(ApprovalVotingParallelMessage::NetworkBridgeUpdate(e)) + if state.options.approval_voting_parallel_enabled { + AllMessages::ApprovalVotingParallel(ApprovalVotingParallelMessage::NetworkBridgeUpdate( + e, + )) + } else { + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate(e)) + } }); initialization_messages.extend(generate_new_session_topology( &state.test_authorities, ValidatorIndex(NODE_UNDER_TEST), + state.options.approval_voting_parallel_enabled, )); for message in initialization_messages { env.send_message(message).await; @@ -1028,7 +1049,14 @@ pub async fn bench_approvals_run( state.total_sent_messages_to_node.load(std::sync::atomic::Ordering::SeqCst) as usize; env.wait_until_metric( "polkadot_parachain_subsystem_bounded_received", - Some(("subsystem_name", "approval-voting-parallel-subsystem")), + Some(( + "subsystem_name", + if state.options.approval_voting_parallel_enabled { + "approval-voting-parallel-subsystem" + } else { + "approval-distribution-subsystem" + }, + )), |value| { gum::debug!(target: LOG_TARGET, ?value, ?at_least_messages, "Waiting metric"); value >= at_least_messages as f64 @@ -1045,11 +1073,22 @@ pub async fn bench_approvals_run( CandidateEvent::CandidateIncluded(receipt_fetch, _head, _, _) => { let (tx, rx) = oneshot::channel(); - let msg = ApprovalVotingParallelMessage::GetApprovalSignaturesForCandidate( - receipt_fetch.hash(), - tx, - ); - env.send_message(AllMessages::ApprovalVotingParallel(msg)).await; + let msg = if state.options.approval_voting_parallel_enabled { + AllMessages::ApprovalVotingParallel( + ApprovalVotingParallelMessage::GetApprovalSignaturesForCandidate( + receipt_fetch.hash(), + tx, + ), + ) + } else { + AllMessages::ApprovalVoting( + ApprovalVotingMessage::GetApprovalSignaturesForCandidate( + receipt_fetch.hash(), + tx, + ), + ) + }; + env.send_message(msg).await; let result = rx.await.unwrap(); @@ -1073,7 +1112,14 @@ pub async fn bench_approvals_run( state.total_sent_messages_to_node.load(std::sync::atomic::Ordering::SeqCst) as usize; env.wait_until_metric( "polkadot_parachain_subsystem_bounded_received", - Some(("subsystem_name", "approval-voting-parallel-subsystem")), + Some(( + "subsystem_name", + if state.options.approval_voting_parallel_enabled { + "approval-voting-parallel-subsystem" + } else { + "approval-distribution-subsystem" + }, + )), |value| { gum::debug!(target: LOG_TARGET, ?value, ?at_least_messages, "Waiting metric"); value >= at_least_messages as f64 diff --git a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs index c28bdba49f54..f28adff315f8 100644 --- a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs @@ -210,7 +210,7 @@ pub fn prepare_test( state.test_authorities.clone(), ); let network_bridge_rx = - network_bridge::MockNetworkBridgeRx::new(network_receiver, Some(chunk_req_v2_cfg)); + network_bridge::MockNetworkBridgeRx::new(network_receiver, Some(chunk_req_v2_cfg), false); let runtime_api = MockRuntimeApi::new( state.config.clone(), diff --git a/polkadot/node/subsystem-bench/src/lib/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/lib/mock/network_bridge.rs index b6291784c05e..f5474a61e3dc 100644 --- a/polkadot/node/subsystem-bench/src/lib/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/lib/mock/network_bridge.rs @@ -24,7 +24,9 @@ use crate::{ use futures::{channel::mpsc::UnboundedSender, FutureExt, StreamExt}; use polkadot_node_network_protocol::Versioned; use polkadot_node_subsystem::{ - messages::{ApprovalVotingParallelMessage, NetworkBridgeTxMessage}, + messages::{ + ApprovalDistributionMessage, ApprovalVotingParallelMessage, NetworkBridgeTxMessage, + }, overseer, SpawnedSubsystem, SubsystemError, }; use polkadot_node_subsystem_types::{ @@ -55,6 +57,8 @@ pub struct MockNetworkBridgeRx { network_receiver: NetworkInterfaceReceiver, /// Chunk request sender chunk_request_sender: Option, + /// Approval voting parallel enabled. + approval_voting_parallel_enabled: bool, } impl MockNetworkBridgeTx { @@ -71,8 +75,9 @@ impl MockNetworkBridgeRx { pub fn new( network_receiver: NetworkInterfaceReceiver, chunk_request_sender: Option, + approval_voting_parallel_enabled: bool, ) -> MockNetworkBridgeRx { - Self { network_receiver, chunk_request_sender } + Self { network_receiver, chunk_request_sender, approval_voting_parallel_enabled } } } @@ -197,9 +202,15 @@ impl MockNetworkBridgeRx { Versioned::V3( polkadot_node_network_protocol::v3::ValidationProtocol::ApprovalDistribution(msg) ) => { - ctx.send_message( - ApprovalVotingParallelMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage(peer_id, polkadot_node_network_protocol::Versioned::V3(msg))) - ).await; + if self.approval_voting_parallel_enabled { + ctx.send_message( + ApprovalVotingParallelMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage(peer_id, polkadot_node_network_protocol::Versioned::V3(msg))) + ).await; + } else { + ctx.send_message( + ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage(peer_id, polkadot_node_network_protocol::Versioned::V3(msg))) + ).await; + } } Versioned::V3( polkadot_node_network_protocol::v3::ValidationProtocol::StatementDistribution(msg) diff --git a/polkadot/node/subsystem-bench/src/lib/statement/mod.rs b/polkadot/node/subsystem-bench/src/lib/statement/mod.rs index 710194370140..e2d50f285687 100644 --- a/polkadot/node/subsystem-bench/src/lib/statement/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/statement/mod.rs @@ -135,7 +135,8 @@ fn build_overseer( network_interface.subsystem_sender(), state.test_authorities.clone(), ); - let network_bridge_rx = MockNetworkBridgeRx::new(network_receiver, Some(candidate_req_cfg)); + let network_bridge_rx = + MockNetworkBridgeRx::new(network_receiver, Some(candidate_req_cfg), false); let dummy = dummy_builder!(spawn_task_handle, overseer_metrics) .replace_runtime_api(|_| mock_runtime_api) From 493116f39fab82d24079a95af7bbc02e5a4cfcea Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 5 Aug 2024 15:00:25 +0300 Subject: [PATCH 15/45] Add some documentation Signed-off-by: Alexandru Gheorghe --- polkadot/node/primitives/src/approval.rs | 2 +- polkadot/node/subsystem-types/src/messages.rs | 43 +++++++++---------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/polkadot/node/primitives/src/approval.rs b/polkadot/node/primitives/src/approval.rs index d1874f760364..aa0278fa2e41 100644 --- a/polkadot/node/primitives/src/approval.rs +++ b/polkadot/node/primitives/src/approval.rs @@ -18,7 +18,7 @@ /// A list of primitives introduced in v1. pub mod v1 { - use sp_consensus_babe::{self as babe_primitives, SlotDuration}; + use sp_consensus_babe::{self as babe_primitives}; pub use sp_consensus_babe::{ Randomness, Slot, VrfPreOutput, VrfProof, VrfSignature, VrfTranscript, }; diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index 9fdba3533620..74b16369b2c2 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -955,48 +955,45 @@ pub struct BlockDescription { pub candidates: Vec, } -/// Message to the Approval Voting subsystem running both approval-distribution and approval-voting -/// logic in parallel. This is a combination of all the messages ApprovalVoting and +/// Message to the approval voting parallel subsystem running both approval-distribution and +/// approval-voting logic in parallel. This is a combination of all the messages ApprovalVoting and /// ApprovalDistribution subsystems can receive. +/// +/// The reason this exists is, so that we can keep both mods of running in the same polkadot binary, +/// based on the value of `--approval-voting-parallel-enabled`, we decide if we run with two +/// different subsystems for approval-distribution and approval-voting or run the approval-voting +/// parallel which has several parallel workers for the approval-distribution and an worker for +/// approval-voting. +/// +/// This is meant to be a temporary state until we can safely remove running the two subsystems +/// individually. #[derive(Debug, derive_more::From)] pub enum ApprovalVotingParallelMessage { - /// Returns the highest possible ancestor hash of the provided block hash which is - /// acceptable to vote on finality for. - /// The `BlockNumber` provided is the number of the block's ancestor which is the - /// earliest possible vote. - /// - /// It can also return the same block hash, if that is acceptable to vote upon. - /// Return `None` if the input hash is unrecognized. + /// Gets mapped into `ApprovalVotingMessage::ApprovedAncestor` ApprovedAncestor(Hash, BlockNumber, oneshot::Sender>), - /// Retrieve all available approval signatures for a candidate from approval-voting. - /// - /// This message involves a linear search for candidates on each relay chain fork and also - /// requires calling into `approval-distribution`: Calls should be infrequent and bounded. + /// Gets mapped into `ApprovalVotingMessage::GetApprovalSignaturesForCandidate` GetApprovalSignaturesForCandidate( CandidateHash, oneshot::Sender, ValidatorSignature)>>, ), - /// Notify the `ApprovalDistribution` subsystem about new blocks - /// and the candidates contained within them. + /// Gets mapped into `ApprovalDistributionMessage::NewBlocks` NewBlocks(Vec), - /// Distribute an assignment cert from the local validator. The cert is assumed - /// to be valid, relevant, and for the given relay-parent and validator index. + /// Gets mapped into `ApprovalDistributionMessage::DistributeAssignment` DistributeAssignment(IndirectAssignmentCertV2, CandidateBitfield), - /// Distribute an approval vote for the local validator. The approval vote is assumed to be - /// valid, relevant, and the corresponding approval already issued. - /// If not, the subsystem is free to drop the message. + /// Gets mapped into `ApprovalDistributionMessage::DistributeApproval` DistributeApproval(IndirectSignedApprovalVoteV2), - /// An update from the network bridge. + /// An update from the network bridge, gets mapped into + /// `ApprovalDistributionMessage::NetworkBridgeUpdate` #[from] NetworkBridgeUpdate(NetworkBridgeEvent), - /// Get all approval signatures for all chains a candidate appeared in. + /// Gets mapped into `ApprovalDistributionMessage::GetApprovalSignatures` GetApprovalSignatures( HashSet<(Hash, CandidateIndex)>, oneshot::Sender, ValidatorSignature)>>, ), - /// Approval checking lag update measured in blocks. + /// Gets mapped into `ApprovalDistributionMessage::ApprovalCheckingLagUpdate` ApprovalCheckingLagUpdate(BlockNumber), } From bb4ca09c2f7e31cedf0f0ed03a22988655590fdf Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 6 Aug 2024 18:21:29 +0300 Subject: [PATCH 16/45] Add subsystem unittests Signed-off-by: Alexandru Gheorghe --- .../core/approval-voting-parallel/src/lib.rs | 103 +- .../approval-voting-parallel/src/tests.rs | 921 ++++++++++++++++++ 2 files changed, 1002 insertions(+), 22 deletions(-) create mode 100644 polkadot/node/core/approval-voting-parallel/src/tests.rs diff --git a/polkadot/node/core/approval-voting-parallel/src/lib.rs b/polkadot/node/core/approval-voting-parallel/src/lib.rs index dc9e9c5f339d..c2f30b182992 100644 --- a/polkadot/node/core/approval-voting-parallel/src/lib.rs +++ b/polkadot/node/core/approval-voting-parallel/src/lib.rs @@ -59,9 +59,12 @@ use std::{ sync::Arc, time::Duration, }; -use stream::{select_with_strategy, PollNext}; +use stream::{select_with_strategy, PollNext, SelectWithStrategy}; pub mod metrics; +#[cfg(test)] +mod tests; + pub(crate) const LOG_TARGET: &str = "parachain::approval-voting-parallel"; // Value rather arbitrarily: Should not be hit in practice, it exists to more easily diagnose dead // lock issues for example. @@ -164,12 +167,12 @@ async fn start_workers( where { // Build approval voting handles. - let (to_approval_voting_worker, rx_approval_voting_worker) = - build_channels("approval-voting-parallel-db".into(), WORKERS_CHANNEL_SIZE, metrics_watcher); - - let prioritised = - select_with_strategy(rx_approval_voting_worker.0, rx_approval_voting_worker.1, prio_right); - let approval_voting_work_provider = ApprovalVotingWorkProviderImpl(prioritised); + let (to_approval_voting_worker, approval_voting_work_provider) = build_worker_handles( + "approval-voting-parallel-db".into(), + WORKERS_CHANNEL_SIZE, + metrics_watcher, + prio_right, + ); gum::info!(target: LOG_TARGET, "Starting approval distribution workers"); @@ -186,8 +189,13 @@ where ); let task_name = format!("approval-voting-parallel-{}", i); - let (to_approval_distribution_worker, rx_approval_distribution_worker) = - build_channels(task_name.clone(), WORKERS_CHANNEL_SIZE, metrics_watcher); + let (to_approval_distribution_worker, mut approval_distribution_work_provider) = + build_worker_handles( + task_name.clone(), + WORKERS_CHANNEL_SIZE, + metrics_watcher, + prio_right, + ); metrics_watcher.watch(task_name.clone(), to_approval_distribution_worker.meter()); @@ -208,14 +216,8 @@ where session_cache_lru_size: DISPUTE_WINDOW.get(), }); - let mut work_channels = select_with_strategy( - rx_approval_distribution_worker.0, - rx_approval_distribution_worker.1, - prio_right, - ); - loop { - let message = match work_channels.next().await { + let message = match approval_distribution_work_provider.next().await { Some(message) => message, None => { gum::info!( @@ -324,20 +326,20 @@ async fn run_main_loop( gum::trace!(target: LOG_TARGET, ?next_msg, "Parallel processing is not enabled skipping message"); continue; } - gum::trace!(target: LOG_TARGET, ?next_msg, "Parallel processing is not enabled skipping message"); match next_msg { FromOrchestra::Signal(msg) => { if matches!(msg, OverseerSignal::ActiveLeaves(_)) { metrics_watcher.collect_metrics(); } + gum::info!(target: LOG_TARGET, "To approval distribution"); + for worker in to_approval_distribution_workers.iter_mut() { worker .send_signal(msg.clone()).await?; } to_approval_voting_worker.send_signal(msg).await?; - }, FromOrchestra::Communication { msg } => match msg { // The message the approval voting subsystem would've handled. @@ -628,12 +630,37 @@ fn validator_index_for_msg( } } -/// Just a wrapper over a channel Receiver, that is injected into approval-voting worker for -/// providing the messages to be processed. -pub struct ApprovalVotingWorkProviderImpl(T); +/// A handler object that both type of workers use for receiving work. +/// +/// In practive this just a wrapper over a channel Receiver, that is injected into approval-voting +/// worker and approval-distribution workers. +type WorkProvider = WorkProviderImpl< + SelectWithStrategy< + MeteredReceiver>, + UnboundedMeteredReceiver>, + Clos, + State, + >, +>; + +pub struct WorkProviderImpl(T); + +impl Stream for WorkProviderImpl +where + T: Stream> + Unpin + Send, +{ + type Item = FromOrchestra; + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.0.poll_next_unpin(cx) + } +} #[async_trait::async_trait] -impl ApprovalVotingWorkProvider for ApprovalVotingWorkProviderImpl +impl ApprovalVotingWorkProvider for WorkProviderImpl where T: Stream> + Unpin + Send, { @@ -644,6 +671,19 @@ where } } +impl WorkProvider +where + M: Send + Sync + 'static, + Clos: FnMut(&mut State) -> PollNext, + State: Default, +{ + // Constructs a work providers from the channels handles. + fn from_rx_worker(rx: RxWorker, prio: Clos) -> Self { + let prioritised = select_with_strategy(rx.0, rx.1, prio); + WorkProviderImpl(prioritised) + } +} + /// Just a wrapper for implementing overseer::SubsystemSender and /// overseer::SubsystemSender, so that we can inject into the /// workers, so they can talke directly with each other without intermediating in this subsystem @@ -800,6 +840,25 @@ fn build_channels( (to_worker, RxWorker(rx_approval_voting_work, rx_approval_voting_work_unbounded)) } +/// Build the worker handles used for interacting with the workers. +/// +/// `ToWorker` is used for sending messages to the workers. +/// `WorkProvider` is used by the workers for receiving the messages. +fn build_worker_handles( + channel_name: String, + channel_size: usize, + metrics_watcher: &mut MetricsWatcher, + prio_right: Clos, +) -> (ToWorker, WorkProvider) +where + M: Send + Sync + 'static, + Clos: FnMut(&mut State) -> PollNext, + State: Default, +{ + let (to_worker, rx_worker) = build_channels(channel_name, channel_size, metrics_watcher); + (to_worker, WorkProviderImpl::from_rx_worker(rx_worker, prio_right)) +} + /// Just a wrapper for implementing overseer::SubsystemSender, so that /// we can inject into the approval voting subsystem. #[derive(Clone)] diff --git a/polkadot/node/core/approval-voting-parallel/src/tests.rs b/polkadot/node/core/approval-voting-parallel/src/tests.rs new file mode 100644 index 000000000000..f18a5300c0e9 --- /dev/null +++ b/polkadot/node/core/approval-voting-parallel/src/tests.rs @@ -0,0 +1,921 @@ +use std::{ + collections::{HashMap, HashSet}, + future::Future, + sync::Arc, + time::Duration, +}; + +use crate::{ + build_worker_handles, metrics::MetricsWatcher, prio_right, run_main_loop, start_workers, + validator_index_for_msg, ApprovalVotingParallelSubsystem, Metrics, WorkProvider, LOG_TARGET, +}; +use assert_matches::assert_matches; +use futures::{channel::oneshot, future, stream::PollNext, StreamExt}; +use itertools::Itertools; +use polkadot_node_core_approval_voting::{time::SystemClock, ApprovalVotingWorkProvider, Config}; +use polkadot_node_network_protocol::{peer_set::ValidationVersion, ObservedRole, PeerId}; +use polkadot_node_primitives::approval::{ + v1::{ + AssignmentCert, AssignmentCertKind, IndirectAssignmentCert, IndirectSignedApprovalVote, + RELAY_VRF_MODULO_CONTEXT, + }, + v2::{ + AssignmentCertKindV2, AssignmentCertV2, CoreBitfield, IndirectAssignmentCertV2, + IndirectSignedApprovalVoteV2, + }, +}; +use polkadot_node_subsystem::{ + messages::{ApprovalDistributionMessage, ApprovalVotingMessage, ApprovalVotingParallelMessage}, + FromOrchestra, +}; +use polkadot_node_subsystem_test_helpers::{mock::new_leaf, TestSubsystemContext}; +use polkadot_overseer::{ActiveLeavesUpdate, OverseerSignal, SpawnGlue, TimeoutExt}; +use polkadot_primitives::{CandidateHash, CoreIndex, Hash, ValidatorIndex}; +use sc_keystore::{Keystore, LocalKeystore}; +use sp_consensus::SyncOracle; +use sp_consensus_babe::{VrfPreOutput, VrfProof, VrfSignature}; +use sp_core::{testing::TaskExecutor, H256}; +use sp_keyring::Sr25519Keyring; +type VirtualOverseer = + polkadot_node_subsystem_test_helpers::TestSubsystemContextHandle; + +const SLOT_DURATION_MILLIS: u64 = 6000; + +pub mod test_constants { + pub(crate) const DATA_COL: u32 = 0; + pub(crate) const NUM_COLUMNS: u32 = 1; +} + +fn fake_assignment_cert(block_hash: Hash, validator: ValidatorIndex) -> IndirectAssignmentCert { + let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); + let msg = b"WhenParachains?"; + let mut prng = rand_core::OsRng; + let keypair = schnorrkel::Keypair::generate_with(&mut prng); + let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); + let preout = inout.to_preout(); + + IndirectAssignmentCert { + block_hash, + validator, + cert: AssignmentCert { + kind: AssignmentCertKind::RelayVRFModulo { sample: 1 }, + vrf: VrfSignature { pre_output: VrfPreOutput(preout), proof: VrfProof(proof) }, + }, + } +} + +fn fake_assignment_cert_v2( + block_hash: Hash, + validator: ValidatorIndex, + core_bitfield: CoreBitfield, +) -> IndirectAssignmentCertV2 { + let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); + let msg = b"WhenParachains?"; + let mut prng = rand_core::OsRng; + let keypair = schnorrkel::Keypair::generate_with(&mut prng); + let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); + let preout = inout.to_preout(); + + IndirectAssignmentCertV2 { + block_hash, + validator, + cert: AssignmentCertV2 { + kind: AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield }, + vrf: VrfSignature { pre_output: VrfPreOutput(preout), proof: VrfProof(proof) }, + }, + } +} + +/// Creates a meaningless signature +pub fn dummy_signature() -> polkadot_primitives::ValidatorSignature { + sp_core::crypto::UncheckedFrom::unchecked_from([1u8; 64]) +} + +fn build_subsystem( + sync_oracle: Box, +) -> ( + ApprovalVotingParallelSubsystem, + TestSubsystemContext>, + VirtualOverseer, +) { + let _ = env_logger::builder() + .is_test(true) + .filter(Some("polkadot_node_core_approval_voting"), log::LevelFilter::Trace) + .filter(Some(LOG_TARGET), log::LevelFilter::Trace) + .try_init(); + + let pool = sp_core::testing::TaskExecutor::new(); + let (context, virtual_overseer) = polkadot_node_subsystem_test_helpers::make_subsystem_context::< + ApprovalVotingParallelMessage, + _, + >(pool.clone()); + + let keystore = LocalKeystore::in_memory(); + let _ = keystore.sr25519_generate_new( + polkadot_primitives::PARACHAIN_KEY_TYPE_ID, + Some(&Sr25519Keyring::Alice.to_seed()), + ); + + let clock = Arc::new(SystemClock {}); + let db = kvdb_memorydb::create(test_constants::NUM_COLUMNS); + let db = polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[]); + + ( + ApprovalVotingParallelSubsystem::with_config_and_clock( + Config { + col_approval_data: test_constants::DATA_COL, + slot_duration_millis: SLOT_DURATION_MILLIS, + }, + Arc::new(db), + Arc::new(keystore), + sync_oracle, + Metrics::default(), + clock.clone(), + SpawnGlue(pool), + false, + ), + context, + virtual_overseer, + ) +} + +#[derive(Clone)] +struct TestSyncOracle {} + +impl SyncOracle for TestSyncOracle { + fn is_major_syncing(&self) -> bool { + false + } + + fn is_offline(&self) -> bool { + unimplemented!("not used in network bridge") + } +} + +fn test_harness( + num_approval_distro_workers: usize, + prio_right: Clos, + test_fn: impl FnOnce( + VirtualOverseer, + WorkProvider, + Vec>, + ) -> T, +) where + T: Future, + Clos: Clone + FnMut(&mut State) -> PollNext, + State: Default, +{ + let (subsystem, context, virtual_overseer) = build_subsystem(Box::new(TestSyncOracle {})); + let mut metrics_watcher = MetricsWatcher::new(subsystem.metrics.clone()); + let channel_size = 1; + + let (to_approval_voting_worker, approval_voting_work_provider) = + build_worker_handles::( + "to_approval_voting_worker".into(), + channel_size, + &mut metrics_watcher, + prio_right.clone(), + ); + + let approval_distribution_channels = { 0..num_approval_distro_workers } + .into_iter() + .map(|worker_index| { + build_worker_handles::( + format!("to_approval_distro/{}", worker_index), + channel_size, + &mut metrics_watcher, + prio_right.clone(), + ) + }) + .collect_vec(); + + let to_approval_distribution_workers = + approval_distribution_channels.iter().map(|(tx, _)| tx.clone()).collect_vec(); + let approval_distribution_work_providers = + approval_distribution_channels.into_iter().map(|(_, rx)| rx).collect_vec(); + + let subsystem = run_main_loop( + context, + true, + to_approval_voting_worker, + to_approval_distribution_workers, + metrics_watcher, + ); + + let test_fut = test_fn( + virtual_overseer, + approval_voting_work_provider, + approval_distribution_work_providers, + ); + + futures::pin_mut!(test_fut); + futures::pin_mut!(subsystem); + + futures::executor::block_on(future::join( + async move { + let _overseer = test_fut.await; + }, + subsystem, + )) + .1 + .unwrap(); +} + +const TIMEOUT: Duration = Duration::from_millis(2000); + +async fn overseer_signal(overseer: &mut VirtualOverseer, signal: OverseerSignal) { + overseer + .send(FromOrchestra::Signal(signal)) + .timeout(TIMEOUT) + .await + .expect(&format!("{:?} is more than enough for sending signals.", TIMEOUT)); +} + +async fn overseer_message(overseer: &mut VirtualOverseer, msg: ApprovalVotingParallelMessage) { + overseer + .send(FromOrchestra::Communication { msg }) + .timeout(TIMEOUT) + .await + .expect(&format!("{:?} is more than enough for sending signals.", TIMEOUT)); +} + +async fn run_start_workers() { + let (subsystem, mut context, _) = build_subsystem(Box::new(TestSyncOracle {})); + let mut metrics_watcher = MetricsWatcher::new(subsystem.metrics.clone()); + let _workers = start_workers(&mut context, subsystem, &mut metrics_watcher).await.unwrap(); +} + +// Test starting the workers succeeds. +#[test] +fn start_workers_succeeds() { + futures::executor::block_on(run_start_workers()); +} + +// Test main loop forwards messages to the correct worker for all type of messages. +#[test] +fn test_main_loop_forwards_correctly() { + let num_approval_distro_workers = 4; + test_harness( + num_approval_distro_workers, + prio_right, + |mut overseer, mut approval_voting_work_provider, mut rx_approval_distribution_workers| async move { + // 1. Check Signals are correctly forwarded to the workers. + let signal = OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + Hash::random(), + 1, + ))); + overseer_signal(&mut overseer, signal.clone()).await; + let approval_voting_receives = approval_voting_work_provider.recv().await.unwrap(); + assert!(matches!(approval_voting_receives, FromOrchestra::Signal(_))); + for rx_approval_distribution_worker in rx_approval_distribution_workers.iter_mut() { + let approval_distribution_receives = + rx_approval_distribution_worker.next().await.unwrap(); + assert!(matches!(approval_distribution_receives, FromOrchestra::Signal(_))); + } + + let (test_tx, _rx) = oneshot::channel(); + let test_hash = Hash::random(); + let test_block_nr = 2; + overseer_message( + &mut overseer, + ApprovalVotingParallelMessage::ApprovedAncestor(test_hash, test_block_nr, test_tx), + ) + .await; + assert_matches!( + approval_voting_work_provider.recv().await.unwrap(), + FromOrchestra::Communication { + msg: ApprovalVotingMessage::ApprovedAncestor(hash, block_nr, _) + } => { + assert_eq!(hash, test_hash); + assert_eq!(block_nr, test_block_nr); + } + ); + for rx_approval_distribution_worker in rx_approval_distribution_workers.iter_mut() { + assert!(rx_approval_distribution_worker + .next() + .timeout(Duration::from_millis(200)) + .await + .is_none()); + } + + // 2. Check GetApprovalSignaturesForCandidate is correctly forwarded to the workers. + let (test_tx, _rx) = oneshot::channel(); + let test_hash = CandidateHash(Hash::random()); + overseer_message( + &mut overseer, + ApprovalVotingParallelMessage::GetApprovalSignaturesForCandidate( + test_hash, test_tx, + ), + ) + .await; + + assert_matches!( + approval_voting_work_provider.recv().await.unwrap(), + FromOrchestra::Communication { + msg: ApprovalVotingMessage::GetApprovalSignaturesForCandidate(hash, _) + } => { + assert_eq!(hash, test_hash); + } + ); + + for rx_approval_distribution_worker in rx_approval_distribution_workers.iter_mut() { + assert!(rx_approval_distribution_worker + .next() + .timeout(Duration::from_millis(200)) + .await + .is_none()); + } + + // 3. Check NewBlocks is correctly forwarded to the workers. + overseer_message(&mut overseer, ApprovalVotingParallelMessage::NewBlocks(vec![])).await; + for rx_approval_distribution_worker in rx_approval_distribution_workers.iter_mut() { + assert_matches!(rx_approval_distribution_worker.next().await.unwrap(), + FromOrchestra::Communication { + msg: ApprovalDistributionMessage::NewBlocks(blocks) + } => { + assert!(blocks.is_empty()); + } + ); + } + assert!(approval_voting_work_provider + .recv() + .timeout(Duration::from_millis(200)) + .await + .is_none()); + + // 4. Check DistributeAssignment is correctly forwarded to the workers. + let validator_index = ValidatorIndex(17); + let assignment = + fake_assignment_cert_v2(Hash::random(), validator_index, CoreIndex(1).into()); + overseer_message( + &mut overseer, + ApprovalVotingParallelMessage::DistributeAssignment(assignment.clone(), 1.into()), + ) + .await; + + for (index, rx_approval_distribution_worker) in + rx_approval_distribution_workers.iter_mut().enumerate() + { + if index == validator_index.0 as usize % num_approval_distro_workers { + assert_matches!(rx_approval_distribution_worker.next().await.unwrap(), + FromOrchestra::Communication { + msg: ApprovalDistributionMessage::DistributeAssignment(cert, bitfield) + } => { + assert_eq!(cert, assignment); + assert_eq!(bitfield, 1.into()); + } + ); + } else { + assert!(rx_approval_distribution_worker + .next() + .timeout(Duration::from_millis(200)) + .await + .is_none()); + } + } + assert!(approval_voting_work_provider + .recv() + .timeout(Duration::from_millis(200)) + .await + .is_none()); + + // 5. Check DistributeApproval is correctly forwarded to the workers. + let validator_index = ValidatorIndex(26); + let expected_vote = IndirectSignedApprovalVoteV2 { + block_hash: H256::random(), + candidate_indices: 1.into(), + validator: validator_index, + signature: dummy_signature(), + }; + + overseer_message( + &mut overseer, + ApprovalVotingParallelMessage::DistributeApproval(expected_vote.clone()), + ) + .await; + + for (index, rx_approval_distribution_worker) in + rx_approval_distribution_workers.iter_mut().enumerate() + { + if index == validator_index.0 as usize % num_approval_distro_workers { + assert_matches!(rx_approval_distribution_worker.next().await.unwrap(), + FromOrchestra::Communication { + msg: ApprovalDistributionMessage::DistributeApproval(vote) + } => { + assert_eq!(vote, expected_vote); + } + ); + } else { + assert!(rx_approval_distribution_worker + .next() + .timeout(Duration::from_millis(200)) + .await + .is_none()); + } + } + + // 6. Check NetworkBridgeUpdate::PeerMessage is correctly forwarded just to one of the + // workers. + let approvals = vec![ + IndirectSignedApprovalVoteV2 { + block_hash: H256::random(), + candidate_indices: 1.into(), + validator: validator_index, + signature: dummy_signature(), + }, + IndirectSignedApprovalVoteV2 { + block_hash: H256::random(), + candidate_indices: 2.into(), + validator: validator_index, + signature: dummy_signature(), + }, + ]; + let expected_msg = polkadot_node_network_protocol::Versioned::V3( + polkadot_node_network_protocol::v3::ApprovalDistributionMessage::Approvals( + approvals.clone(), + ), + ); + overseer_message( + &mut overseer, + ApprovalVotingParallelMessage::NetworkBridgeUpdate( + polkadot_node_subsystem::messages::NetworkBridgeEvent::PeerMessage( + PeerId::random(), + expected_msg.clone(), + ), + ), + ) + .await; + + for (index, rx_approval_distribution_worker) in + rx_approval_distribution_workers.iter_mut().enumerate() + { + if index == validator_index.0 as usize % num_approval_distro_workers { + assert_matches!(rx_approval_distribution_worker.next().await.unwrap(), + FromOrchestra::Communication { + msg: ApprovalDistributionMessage::NetworkBridgeUpdate( + polkadot_node_subsystem::messages::NetworkBridgeEvent::PeerMessage( + _, + msg, + ), + ) + } => { + assert_eq!(msg, expected_msg); + } + ); + } else { + assert!(rx_approval_distribution_worker + .next() + .timeout(Duration::from_millis(200)) + .await + .is_none()); + } + } + assert!(approval_voting_work_provider + .recv() + .timeout(Duration::from_millis(200)) + .await + .is_none()); + + assert!(approval_voting_work_provider + .recv() + .timeout(Duration::from_millis(200)) + .await + .is_none()); + + // 7. Check NetworkBridgeUpdate::PeerConnected is correctly forwarded to all workers. + let expected_peer_id = PeerId::random(); + overseer_message( + &mut overseer, + ApprovalVotingParallelMessage::NetworkBridgeUpdate( + polkadot_node_subsystem::messages::NetworkBridgeEvent::PeerConnected( + expected_peer_id, + ObservedRole::Authority, + ValidationVersion::V3.into(), + None, + ), + ), + ) + .await; + + for rx_approval_distribution_worker in rx_approval_distribution_workers.iter_mut() { + assert_matches!(rx_approval_distribution_worker.next().await.unwrap(), + FromOrchestra::Communication { + msg: ApprovalDistributionMessage::NetworkBridgeUpdate( + polkadot_node_subsystem::messages::NetworkBridgeEvent::PeerConnected( + peer_id, + role, + version, + authority_id, + ), + ) + } => { + assert_eq!(peer_id, expected_peer_id); + assert_eq!(role, ObservedRole::Authority); + assert_eq!(version, ValidationVersion::V3.into()); + assert_eq!(authority_id, None); + } + ); + } + assert!(approval_voting_work_provider + .recv() + .timeout(Duration::from_millis(200)) + .await + .is_none()); + + // 8. Check ApprovalCheckingLagUpdate is correctly forwarded to all workers. + overseer_message( + &mut overseer, + ApprovalVotingParallelMessage::ApprovalCheckingLagUpdate(7), + ) + .await; + + for rx_approval_distribution_worker in rx_approval_distribution_workers.iter_mut() { + assert_matches!(rx_approval_distribution_worker.next().await.unwrap(), + FromOrchestra::Communication { + msg: ApprovalDistributionMessage::ApprovalCheckingLagUpdate( + lag + ) + } => { + assert_eq!(lag, 7); + } + ); + } + assert!(approval_voting_work_provider + .recv() + .timeout(Duration::from_millis(200)) + .await + .is_none()); + + overseer + }, + ); +} + +/// Test GetApprovalSignatures correctly gatheres the signatures from all workers. +#[test] +fn test_handle_get_approval_signatures() { + let num_approval_distro_workers = 4; + + test_harness( + num_approval_distro_workers, + prio_right, + |mut overseer, mut approval_voting_work_provider, mut rx_approval_distribution_workers| async move { + let (tx, rx) = oneshot::channel(); + let first_block = Hash::random(); + let second_block = Hash::random(); + let expected_candidates: HashSet<_> = + vec![(first_block, 2), (second_block, 3)].into_iter().collect(); + + overseer_message( + &mut overseer, + ApprovalVotingParallelMessage::GetApprovalSignatures( + expected_candidates.clone(), + tx, + ), + ) + .await; + + assert!(approval_voting_work_provider + .recv() + .timeout(Duration::from_millis(200)) + .await + .is_none()); + let mut all_votes = HashMap::new(); + for (index, rx_approval_distribution_worker) in + rx_approval_distribution_workers.iter_mut().enumerate() + { + assert_matches!(rx_approval_distribution_worker.next().await.unwrap(), + FromOrchestra::Communication { + msg: ApprovalDistributionMessage::GetApprovalSignatures( + candidates, tx + ) + } => { + assert_eq!(candidates, expected_candidates); + let to_send: HashMap<_, _> = {0..10}.into_iter().map(|validator| { + let validator_index = ValidatorIndex(validator as u32 * num_approval_distro_workers as u32 + index as u32); + (validator_index, (first_block, vec![2, 4], dummy_signature())) + }).collect(); + tx.send(to_send.clone()).unwrap(); + all_votes.extend(to_send.clone()); + + } + ); + } + + let received_votes = rx.await.unwrap(); + assert_eq!(received_votes, all_votes); + overseer + }, + ) +} + +// Test validator_index_for_msg with empty messages. +#[test] +fn test_validator_index_with_empty_message() { + let result = validator_index_for_msg(polkadot_node_network_protocol::Versioned::V1( + polkadot_node_network_protocol::v1::ApprovalDistributionMessage::Assignments(vec![]), + )); + + assert_eq!(result, (None, Some(vec![]))); + + let result = validator_index_for_msg(polkadot_node_network_protocol::Versioned::V2( + polkadot_node_network_protocol::v2::ApprovalDistributionMessage::Assignments(vec![]), + )); + + assert_eq!(result, (None, Some(vec![]))); + + let result = validator_index_for_msg(polkadot_node_network_protocol::Versioned::V3( + polkadot_node_network_protocol::v3::ApprovalDistributionMessage::Assignments(vec![]), + )); + + assert_eq!(result, (None, Some(vec![]))); + + let result = validator_index_for_msg(polkadot_node_network_protocol::Versioned::V1( + polkadot_node_network_protocol::v1::ApprovalDistributionMessage::Approvals(vec![]), + )); + + assert_eq!(result, (None, Some(vec![]))); + + let result = validator_index_for_msg(polkadot_node_network_protocol::Versioned::V2( + polkadot_node_network_protocol::v2::ApprovalDistributionMessage::Approvals(vec![]), + )); + + assert_eq!(result, (None, Some(vec![]))); + + let result = validator_index_for_msg(polkadot_node_network_protocol::Versioned::V3( + polkadot_node_network_protocol::v3::ApprovalDistributionMessage::Approvals(vec![]), + )); + + assert_eq!(result, (None, Some(vec![]))); +} + +// Test validator_index_for_msg when all the messages are originating from the same validator. +#[test] +fn test_validator_index_with_all_messages_from_the_same_validator() { + let validator_index = ValidatorIndex(3); + let v1_assignment = polkadot_node_network_protocol::Versioned::V1( + polkadot_node_network_protocol::v1::ApprovalDistributionMessage::Assignments(vec![ + (fake_assignment_cert(H256::random(), validator_index), 1), + (fake_assignment_cert(H256::random(), validator_index), 3), + ]), + ); + let result = validator_index_for_msg(v1_assignment.clone()); + + assert_eq!(result, (Some((validator_index, v1_assignment)), None)); + + let v1_approval = polkadot_node_network_protocol::Versioned::V1( + polkadot_node_network_protocol::v1::ApprovalDistributionMessage::Approvals(vec![ + IndirectSignedApprovalVote { + block_hash: H256::random(), + candidate_index: 1, + validator: validator_index, + signature: dummy_signature(), + }, + IndirectSignedApprovalVote { + block_hash: H256::random(), + candidate_index: 1, + validator: validator_index, + signature: dummy_signature(), + }, + ]), + ); + let result = validator_index_for_msg(v1_approval.clone()); + + assert_eq!(result, (Some((validator_index, v1_approval)), None)); + + let validator_index = ValidatorIndex(3); + let v2_assignment = polkadot_node_network_protocol::Versioned::V2( + polkadot_node_network_protocol::v2::ApprovalDistributionMessage::Assignments(vec![ + (fake_assignment_cert(H256::random(), validator_index), 1), + (fake_assignment_cert(H256::random(), validator_index), 3), + ]), + ); + let result = validator_index_for_msg(v2_assignment.clone()); + + assert_eq!(result, (Some((validator_index, v2_assignment)), None)); + + let v2_approval = polkadot_node_network_protocol::Versioned::V2( + polkadot_node_network_protocol::v2::ApprovalDistributionMessage::Approvals(vec![ + IndirectSignedApprovalVote { + block_hash: H256::random(), + candidate_index: 1, + validator: validator_index, + signature: dummy_signature(), + }, + IndirectSignedApprovalVote { + block_hash: H256::random(), + candidate_index: 1, + validator: validator_index, + signature: dummy_signature(), + }, + ]), + ); + let result = validator_index_for_msg(v2_approval.clone()); + + assert_eq!(result, (Some((validator_index, v2_approval)), None)); + + let validator_index = ValidatorIndex(3); + let v3_assignment = polkadot_node_network_protocol::Versioned::V3( + polkadot_node_network_protocol::v3::ApprovalDistributionMessage::Assignments(vec![ + ( + fake_assignment_cert_v2(H256::random(), validator_index, CoreIndex(1).into()), + 1.into(), + ), + ( + fake_assignment_cert_v2(H256::random(), validator_index, CoreIndex(3).into()), + 3.into(), + ), + ]), + ); + let result = validator_index_for_msg(v3_assignment.clone()); + + assert_eq!(result, (Some((validator_index, v3_assignment)), None)); + + let v3_approval = polkadot_node_network_protocol::Versioned::V3( + polkadot_node_network_protocol::v3::ApprovalDistributionMessage::Approvals(vec![ + IndirectSignedApprovalVoteV2 { + block_hash: H256::random(), + candidate_indices: 1.into(), + validator: validator_index, + signature: dummy_signature(), + }, + IndirectSignedApprovalVoteV2 { + block_hash: H256::random(), + candidate_indices: 1.into(), + validator: validator_index, + signature: dummy_signature(), + }, + ]), + ); + let result = validator_index_for_msg(v3_approval.clone()); + + assert_eq!(result, (Some((validator_index, v3_approval)), None)); +} + +// Test validator_index_for_msg when all the messages are originating from different validators, +// so the function should split them by validator index, so we can forward them separately to the +// worker they are assigned to. +#[test] +fn test_validator_index_with_messages_from_different_validators() { + let first_validator_index = ValidatorIndex(3); + let second_validator_index = ValidatorIndex(4); + let assignments = vec![ + (fake_assignment_cert(H256::random(), first_validator_index), 1), + (fake_assignment_cert(H256::random(), second_validator_index), 3), + ]; + let v1_assignment = polkadot_node_network_protocol::Versioned::V1( + polkadot_node_network_protocol::v1::ApprovalDistributionMessage::Assignments( + assignments.clone(), + ), + ); + let result = validator_index_for_msg(v1_assignment.clone()); + + assert!(matches!(result, (None, Some(_)))); + let messsages_split_by_validator = result.1.unwrap(); + assert_eq!(messsages_split_by_validator.len(), assignments.len()); + for (index, (validator_index, message)) in messsages_split_by_validator.into_iter().enumerate() + { + assert_eq!(validator_index, assignments[index].0.validator); + assert_eq!( + message, + polkadot_node_network_protocol::Versioned::V1( + polkadot_node_network_protocol::v1::ApprovalDistributionMessage::Assignments( + assignments.get(index).into_iter().cloned().collect(), + ), + ) + ); + } + + let v2_assignment = polkadot_node_network_protocol::Versioned::V2( + polkadot_node_network_protocol::v2::ApprovalDistributionMessage::Assignments( + assignments.clone(), + ), + ); + let result = validator_index_for_msg(v2_assignment.clone()); + + assert!(matches!(result, (None, Some(_)))); + let messsages_split_by_validator = result.1.unwrap(); + assert_eq!(messsages_split_by_validator.len(), assignments.len()); + for (index, (validator_index, message)) in messsages_split_by_validator.into_iter().enumerate() + { + assert_eq!(validator_index, assignments[index].0.validator); + assert_eq!( + message, + polkadot_node_network_protocol::Versioned::V2( + polkadot_node_network_protocol::v2::ApprovalDistributionMessage::Assignments( + assignments.get(index).into_iter().cloned().collect(), + ), + ) + ); + } + + let first_validator_index = ValidatorIndex(3); + let second_validator_index = ValidatorIndex(4); + let v2_assignments = vec![ + ( + fake_assignment_cert_v2(H256::random(), first_validator_index, CoreIndex(1).into()), + 1.into(), + ), + ( + fake_assignment_cert_v2(H256::random(), second_validator_index, CoreIndex(3).into()), + 3.into(), + ), + ]; + + let approvals = vec![ + IndirectSignedApprovalVote { + block_hash: H256::random(), + candidate_index: 1, + validator: first_validator_index, + signature: dummy_signature(), + }, + IndirectSignedApprovalVote { + block_hash: H256::random(), + candidate_index: 2, + validator: second_validator_index, + signature: dummy_signature(), + }, + ]; + let v2_approvals = polkadot_node_network_protocol::Versioned::V2( + polkadot_node_network_protocol::v2::ApprovalDistributionMessage::Approvals( + approvals.clone(), + ), + ); + let result = validator_index_for_msg(v2_approvals.clone()); + + assert!(matches!(result, (None, Some(_)))); + let messsages_split_by_validator = result.1.unwrap(); + assert_eq!(messsages_split_by_validator.len(), approvals.len()); + for (index, (validator_index, message)) in messsages_split_by_validator.into_iter().enumerate() + { + assert_eq!(validator_index, approvals[index].validator); + assert_eq!( + message, + polkadot_node_network_protocol::Versioned::V2( + polkadot_node_network_protocol::v2::ApprovalDistributionMessage::Approvals( + approvals.get(index).into_iter().cloned().collect(), + ), + ) + ); + } + + let v3_assignment = polkadot_node_network_protocol::Versioned::V3( + polkadot_node_network_protocol::v3::ApprovalDistributionMessage::Assignments( + v2_assignments.clone(), + ), + ); + let result = validator_index_for_msg(v3_assignment.clone()); + + assert!(matches!(result, (None, Some(_)))); + let messsages_split_by_validator = result.1.unwrap(); + assert_eq!(messsages_split_by_validator.len(), v2_assignments.len()); + for (index, (validator_index, message)) in messsages_split_by_validator.into_iter().enumerate() + { + assert_eq!(validator_index, v2_assignments[index].0.validator); + assert_eq!( + message, + polkadot_node_network_protocol::Versioned::V3( + polkadot_node_network_protocol::v3::ApprovalDistributionMessage::Assignments( + v2_assignments.get(index).into_iter().cloned().collect(), + ), + ) + ); + } + + let approvals = vec![ + IndirectSignedApprovalVoteV2 { + block_hash: H256::random(), + candidate_indices: 1.into(), + validator: first_validator_index, + signature: dummy_signature(), + }, + IndirectSignedApprovalVoteV2 { + block_hash: H256::random(), + candidate_indices: 2.into(), + validator: second_validator_index, + signature: dummy_signature(), + }, + ]; + let v3_approvals = polkadot_node_network_protocol::Versioned::V3( + polkadot_node_network_protocol::v3::ApprovalDistributionMessage::Approvals( + approvals.clone(), + ), + ); + let result = validator_index_for_msg(v3_approvals.clone()); + + assert!(matches!(result, (None, Some(_)))); + let messsages_split_by_validator = result.1.unwrap(); + assert_eq!(messsages_split_by_validator.len(), approvals.len()); + for (index, (validator_index, message)) in messsages_split_by_validator.into_iter().enumerate() + { + assert_eq!(validator_index, approvals[index].validator); + assert_eq!( + message, + polkadot_node_network_protocol::Versioned::V3( + polkadot_node_network_protocol::v3::ApprovalDistributionMessage::Approvals( + approvals.get(index).into_iter().cloned().collect(), + ), + ) + ); + } +} From 83d65f7acfcabd2f56f14e1a7be7e1674d792739 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 7 Aug 2024 14:08:23 +0300 Subject: [PATCH 17/45] Add more unittests Signed-off-by: Alexandru Gheorghe --- .../core/approval-voting-parallel/src/lib.rs | 8 +- .../approval-voting-parallel/src/tests.rs | 263 +++++++++++++++++- .../node/core/dispute-coordinator/src/lib.rs | 2 +- 3 files changed, 260 insertions(+), 13 deletions(-) diff --git a/polkadot/node/core/approval-voting-parallel/src/lib.rs b/polkadot/node/core/approval-voting-parallel/src/lib.rs index c2f30b182992..41855b27d18c 100644 --- a/polkadot/node/core/approval-voting-parallel/src/lib.rs +++ b/polkadot/node/core/approval-voting-parallel/src/lib.rs @@ -319,7 +319,7 @@ async fn run_main_loop( Ok(msg) => msg, Err(err) => { gum::info!(target: LOG_TARGET, ?err, "Approval voting parallel subsystem received an error"); - break; + return Err(err); } }; if !subsystem_enabled { @@ -332,14 +332,16 @@ async fn run_main_loop( if matches!(msg, OverseerSignal::ActiveLeaves(_)) { metrics_watcher.collect_metrics(); } - gum::info!(target: LOG_TARGET, "To approval distribution"); for worker in to_approval_distribution_workers.iter_mut() { worker .send_signal(msg.clone()).await?; } - to_approval_voting_worker.send_signal(msg).await?; + to_approval_voting_worker.send_signal(msg.clone()).await?; + if matches!(msg, OverseerSignal::Conclude) { + break; + } }, FromOrchestra::Communication { msg } => match msg { // The message the approval voting subsystem would've handled. diff --git a/polkadot/node/core/approval-voting-parallel/src/tests.rs b/polkadot/node/core/approval-voting-parallel/src/tests.rs index f18a5300c0e9..2c6a2c014274 100644 --- a/polkadot/node/core/approval-voting-parallel/src/tests.rs +++ b/polkadot/node/core/approval-voting-parallel/src/tests.rs @@ -13,7 +13,7 @@ use assert_matches::assert_matches; use futures::{channel::oneshot, future, stream::PollNext, StreamExt}; use itertools::Itertools; use polkadot_node_core_approval_voting::{time::SystemClock, ApprovalVotingWorkProvider, Config}; -use polkadot_node_network_protocol::{peer_set::ValidationVersion, ObservedRole, PeerId}; +use polkadot_node_network_protocol::{peer_set::ValidationVersion, ObservedRole, PeerId, View}; use polkadot_node_primitives::approval::{ v1::{ AssignmentCert, AssignmentCertKind, IndirectAssignmentCert, IndirectSignedApprovalVote, @@ -155,6 +155,7 @@ impl SyncOracle for TestSyncOracle { fn test_harness( num_approval_distro_workers: usize, prio_right: Clos, + subsystem_gracefully_exits: bool, test_fn: impl FnOnce( VirtualOverseer, WorkProvider, @@ -167,7 +168,7 @@ fn test_harness( { let (subsystem, context, virtual_overseer) = build_subsystem(Box::new(TestSyncOracle {})); let mut metrics_watcher = MetricsWatcher::new(subsystem.metrics.clone()); - let channel_size = 1; + let channel_size = 5; let (to_approval_voting_worker, approval_voting_work_provider) = build_worker_handles::( @@ -194,13 +195,22 @@ fn test_harness( let approval_distribution_work_providers = approval_distribution_channels.into_iter().map(|(_, rx)| rx).collect_vec(); - let subsystem = run_main_loop( - context, - true, - to_approval_voting_worker, - to_approval_distribution_workers, - metrics_watcher, - ); + let subsystem = async move { + let result = run_main_loop( + context, + true, + to_approval_voting_worker, + to_approval_distribution_workers, + metrics_watcher, + ) + .await; + + if subsystem_gracefully_exits && result.is_err() { + result + } else { + Ok(()) + } + }; let test_fut = test_fn( virtual_overseer, @@ -258,6 +268,7 @@ fn test_main_loop_forwards_correctly() { test_harness( num_approval_distro_workers, prio_right, + true, |mut overseer, mut approval_voting_work_provider, mut rx_approval_distribution_workers| async move { // 1. Check Signals are correctly forwarded to the workers. let signal = OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( @@ -546,6 +557,8 @@ fn test_main_loop_forwards_correctly() { .await .is_none()); + overseer_signal(&mut overseer, OverseerSignal::Conclude).await; + overseer }, ); @@ -559,6 +572,7 @@ fn test_handle_get_approval_signatures() { test_harness( num_approval_distro_workers, prio_right, + true, |mut overseer, mut approval_voting_work_provider, mut rx_approval_distribution_workers| async move { let (tx, rx) = oneshot::channel(); let first_block = Hash::random(); @@ -604,6 +618,237 @@ fn test_handle_get_approval_signatures() { let received_votes = rx.await.unwrap(); assert_eq!(received_votes, all_votes); + overseer_signal(&mut overseer, OverseerSignal::Conclude).await; + + overseer + }, + ) +} + +/// Test subsystem exits with error when approval_voting_work_provider exits. +#[test] +fn test_subsystem_exits_with_error_if_approval_voting_worker_errors() { + let num_approval_distro_workers = 4; + + test_harness( + num_approval_distro_workers, + prio_right, + false, + |overseer, approval_voting_work_provider, _rx_approval_distribution_workers| async move { + // Drop the approval_voting_work_provider to simulate an error. + std::mem::drop(approval_voting_work_provider); + + overseer + }, + ) +} + +/// Test subsystem exits with error when approval_distribution_workers exits. +#[test] +fn test_subsystem_exits_with_error_if_approval_distribution_worker_errors() { + let num_approval_distro_workers = 4; + + test_harness( + num_approval_distro_workers, + prio_right, + false, + |overseer, _approval_voting_work_provider, rx_approval_distribution_workers| async move { + // Drop the approval_distribution_workers to simulate an error. + std::mem::drop(rx_approval_distribution_workers.into_iter().next().unwrap()); + overseer + }, + ) +} + +/// Test signals sent before messages are processed in order. +#[test] +fn test_signal_before_message_keeps_receive_order() { + let num_approval_distro_workers = 4; + + test_harness( + num_approval_distro_workers, + prio_right, + true, + |mut overseer, mut approval_voting_work_provider, mut rx_approval_distribution_workers| async move { + let signal = OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + Hash::random(), + 1, + ))); + overseer_signal(&mut overseer, signal.clone()).await; + + let validator_index = ValidatorIndex(17); + let assignment = + fake_assignment_cert_v2(Hash::random(), validator_index, CoreIndex(1).into()); + overseer_message( + &mut overseer, + ApprovalVotingParallelMessage::DistributeAssignment(assignment.clone(), 1.into()), + ) + .await; + + let approval_voting_receives = approval_voting_work_provider.recv().await.unwrap(); + assert!(matches!(approval_voting_receives, FromOrchestra::Signal(_))); + let rx_approval_distribution_worker = rx_approval_distribution_workers + .get_mut(validator_index.0 as usize % num_approval_distro_workers) + .unwrap(); + let approval_distribution_receives = + rx_approval_distribution_worker.next().await.unwrap(); + assert!(matches!(approval_distribution_receives, FromOrchestra::Signal(_))); + assert_matches!( + rx_approval_distribution_worker.next().await.unwrap(), + FromOrchestra::Communication { + msg: ApprovalDistributionMessage::DistributeAssignment(_, _) + } => { + } + ); + + overseer_signal(&mut overseer, OverseerSignal::Conclude).await; + overseer + }, + ) +} + +/// Test signals sent after messages are processed with the highest priority. +#[test] +fn test_signal_is_prioritized_when_unread_messages_in_the_queue() { + let num_approval_distro_workers = 4; + + test_harness( + num_approval_distro_workers, + prio_right, + true, + |mut overseer, mut approval_voting_work_provider, mut rx_approval_distribution_workers| async move { + let validator_index = ValidatorIndex(17); + let assignment = + fake_assignment_cert_v2(Hash::random(), validator_index, CoreIndex(1).into()); + overseer_message( + &mut overseer, + ApprovalVotingParallelMessage::DistributeAssignment(assignment.clone(), 1.into()), + ) + .await; + + let signal = OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + Hash::random(), + 1, + ))); + overseer_signal(&mut overseer, signal.clone()).await; + + let approval_voting_receives = approval_voting_work_provider.recv().await.unwrap(); + assert!(matches!(approval_voting_receives, FromOrchestra::Signal(_))); + let rx_approval_distribution_worker = rx_approval_distribution_workers + .get_mut(validator_index.0 as usize % num_approval_distro_workers) + .unwrap(); + let approval_distribution_receives = + rx_approval_distribution_worker.next().await.unwrap(); + assert!(matches!(approval_distribution_receives, FromOrchestra::Signal(_))); + assert_matches!( + rx_approval_distribution_worker.next().await.unwrap(), + FromOrchestra::Communication { + msg: ApprovalDistributionMessage::DistributeAssignment(_, _) + } => { + } + ); + + overseer_signal(&mut overseer, OverseerSignal::Conclude).await; + overseer + }, + ) +} + +/// Test peer view updates have higher priority than normal messages. +#[test] +fn test_peer_view_is_prioritized_when_unread_messages_in_the_queue() { + let num_approval_distro_workers = 4; + + test_harness( + num_approval_distro_workers, + prio_right, + true, + |mut overseer, mut approval_voting_work_provider, mut rx_approval_distribution_workers| async move { + let validator_index = ValidatorIndex(17); + let approvals = vec![ + IndirectSignedApprovalVoteV2 { + block_hash: H256::random(), + candidate_indices: 1.into(), + validator: validator_index, + signature: dummy_signature(), + }, + IndirectSignedApprovalVoteV2 { + block_hash: H256::random(), + candidate_indices: 2.into(), + validator: validator_index, + signature: dummy_signature(), + }, + ]; + let expected_msg = polkadot_node_network_protocol::Versioned::V3( + polkadot_node_network_protocol::v3::ApprovalDistributionMessage::Approvals( + approvals.clone(), + ), + ); + overseer_message( + &mut overseer, + ApprovalVotingParallelMessage::NetworkBridgeUpdate( + polkadot_node_subsystem::messages::NetworkBridgeEvent::PeerMessage( + PeerId::random(), + expected_msg.clone(), + ), + ), + ) + .await; + + overseer_message( + &mut overseer, + ApprovalVotingParallelMessage::NetworkBridgeUpdate( + polkadot_node_subsystem::messages::NetworkBridgeEvent::PeerViewChange( + PeerId::random(), + View::default(), + ), + ), + ) + .await; + + for (index, rx_approval_distribution_worker) in + rx_approval_distribution_workers.iter_mut().enumerate() + { + assert_matches!(rx_approval_distribution_worker.next().await.unwrap(), + FromOrchestra::Communication { + msg: ApprovalDistributionMessage::NetworkBridgeUpdate( + polkadot_node_subsystem::messages::NetworkBridgeEvent::PeerViewChange( + _, + _, + ), + ) + } => { + } + ); + if index == validator_index.0 as usize % num_approval_distro_workers { + assert_matches!(rx_approval_distribution_worker.next().await.unwrap(), + FromOrchestra::Communication { + msg: ApprovalDistributionMessage::NetworkBridgeUpdate( + polkadot_node_subsystem::messages::NetworkBridgeEvent::PeerMessage( + _, + msg, + ), + ) + } => { + assert_eq!(msg, expected_msg); + } + ); + } else { + assert!(rx_approval_distribution_worker + .next() + .timeout(Duration::from_millis(200)) + .await + .is_none()); + } + } + + assert!(approval_voting_work_provider + .recv() + .timeout(Duration::from_millis(200)) + .await + .is_none()); + + overseer_signal(&mut overseer, OverseerSignal::Conclude).await; overseer }, ) diff --git a/polkadot/node/core/dispute-coordinator/src/lib.rs b/polkadot/node/core/dispute-coordinator/src/lib.rs index fcac596f4b8f..702adf3e6c91 100644 --- a/polkadot/node/core/dispute-coordinator/src/lib.rs +++ b/polkadot/node/core/dispute-coordinator/src/lib.rs @@ -34,7 +34,7 @@ use gum::CandidateHash; use sc_keystore::LocalKeystore; use polkadot_node_primitives::{ - approval, CandidateVotes, DisputeMessage, DisputeMessageCheckError, SignedDisputeStatement, + CandidateVotes, DisputeMessage, DisputeMessageCheckError, SignedDisputeStatement, DISPUTE_WINDOW, }; use polkadot_node_subsystem::{ From 3e4821efd1abfda6e81e529e22d4467f9d7beeb5 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 7 Aug 2024 17:49:55 +0300 Subject: [PATCH 18/45] Add zombienet for running approval-voting-in parallel Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/zombienet/polkadot.yml | 8 ++ .../0016-approval-voting-parallel.toml | 120 ++++++++++++++++++ .../0016-approval-voting-parallel.zndsl | 35 +++++ 3 files changed, 163 insertions(+) create mode 100644 polkadot/zombienet_tests/functional/0016-approval-voting-parallel.toml create mode 100644 polkadot/zombienet_tests/functional/0016-approval-voting-parallel.zndsl diff --git a/.gitlab/pipeline/zombienet/polkadot.yml b/.gitlab/pipeline/zombienet/polkadot.yml index b4ef4bb7446c..3152b30c0e72 100644 --- a/.gitlab/pipeline/zombienet/polkadot.yml +++ b/.gitlab/pipeline/zombienet/polkadot.yml @@ -216,6 +216,14 @@ zombienet-polkadot-functional-0015-coretime-shared-core: --local-dir="${LOCAL_DIR}/functional" --test="0015-coretime-shared-core.zndsl" +zombienet-polkadot-functional-0016-approval-voting-parallel: + extends: + - .zombienet-polkadot-common + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh + --local-dir="${LOCAL_DIR}/functional" + --test="0016-approval-voting-parallel.zndsl" + zombienet-polkadot-smoke-0001-parachains-smoke-test: extends: - .zombienet-polkadot-common diff --git a/polkadot/zombienet_tests/functional/0016-approval-voting-parallel.toml b/polkadot/zombienet_tests/functional/0016-approval-voting-parallel.toml new file mode 100644 index 000000000000..eb7435658962 --- /dev/null +++ b/polkadot/zombienet_tests/functional/0016-approval-voting-parallel.toml @@ -0,0 +1,120 @@ +[settings] +timeout = 1000 + +[relaychain] +default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" +chain = "rococo-local" + +[relaychain.genesis.runtimeGenesis.patch.configuration.config] + needed_approvals = 4 + relay_vrf_modulo_samples = 6 + +[relaychain.genesis.runtimeGenesis.patch.configuration.config.approval_voting_params] + max_approval_coalesce_count = 5 + +[relaychain.default_resources] +limits = { memory = "4G", cpu = "2" } +requests = { memory = "2G", cpu = "1" } + + [[relaychain.node_groups]] + name = "alice" + args = ["-lparachain=trace,runtime=debug", "--enable-approval-voting-parallel"] + count = 8 + + [[relaychain.node_groups]] + name = "bob" + args = ["-lparachain=trace,runtime=debug"] + count = 7 + +[[parachains]] +id = 2000 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=1" + + [parachains.collator] + name = "collator01" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--pvf-complexity=1", "--parachain-id=2000"] + +[[parachains]] +id = 2001 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=10" + + [parachains.collator] + name = "collator02" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--parachain-id=2001", "--pvf-complexity=10"] + +[[parachains]] +id = 2002 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=100" + + [parachains.collator] + name = "collator03" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--parachain-id=2002", "--pvf-complexity=100"] + +[[parachains]] +id = 2003 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=20000 --pvf-complexity=300" + + [parachains.collator] + name = "collator04" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=20000", "--parachain-id=2003", "--pvf-complexity=300"] + +[[parachains]] +id = 2004 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=300" + + [parachains.collator] + name = "collator05" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--parachain-id=2004", "--pvf-complexity=300"] + +[[parachains]] +id = 2005 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=20000 --pvf-complexity=400" + + [parachains.collator] + name = "collator06" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=20000", "--pvf-complexity=400", "--parachain-id=2005"] + +[[parachains]] +id = 2006 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=300" + + [parachains.collator] + name = "collator07" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--pvf-complexity=300", "--parachain-id=2006"] + +[[parachains]] +id = 2007 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=300" + + [parachains.collator] + name = "collator08" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--pvf-complexity=300", "--parachain-id=2007"] + +[types.Header] +number = "u64" +parent_hash = "Hash" +post_state = "Hash" \ No newline at end of file diff --git a/polkadot/zombienet_tests/functional/0016-approval-voting-parallel.zndsl b/polkadot/zombienet_tests/functional/0016-approval-voting-parallel.zndsl new file mode 100644 index 000000000000..2be0dd05a9d8 --- /dev/null +++ b/polkadot/zombienet_tests/functional/0016-approval-voting-parallel.zndsl @@ -0,0 +1,35 @@ +Description: Check finality works with approval voting parallel enabled +Network: ./0016-approval-voting-parallel.toml +Creds: config + +# Check authority status. +alice: reports node_roles is 4 + +# Ensure parachains are registered. +alice: parachain 2000 is registered within 60 seconds +alice: parachain 2001 is registered within 60 seconds +alice: parachain 2002 is registered within 60 seconds +alice: parachain 2003 is registered within 60 seconds +alice: parachain 2004 is registered within 60 seconds +alice: parachain 2005 is registered within 60 seconds +alice: parachain 2006 is registered within 60 seconds +alice: parachain 2007 is registered within 60 seconds + +# Ensure parachains made progress. +alice: parachain 2000 block height is at least 10 within 300 seconds +alice: parachain 2001 block height is at least 10 within 300 seconds +alice: parachain 2002 block height is at least 10 within 300 seconds +alice: parachain 2003 block height is at least 10 within 300 seconds +alice: parachain 2004 block height is at least 10 within 300 seconds +alice: parachain 2005 block height is at least 10 within 300 seconds +alice: parachain 2006 block height is at least 10 within 300 seconds +alice: parachain 2007 block height is at least 10 within 300 seconds + +alice: reports substrate_block_height{status="finalized"} is at least 30 within 40000 seconds +bob: reports substrate_block_height{status="finalized"} is at least 30 within 40000 seconds + +alice: reports polkadot_parachain_approval_checking_finality_lag < 3 +bob: reports polkadot_parachain_approval_checking_finality_lag < 3 + +alice: reports polkadot_parachain_approvals_no_shows_total < 3 within 10 seconds +bob: reports polkadot_parachain_approvals_no_shows_total < 3 within 10 seconds From ead49d2c847d97c8fd57b74ef89000129f4f98e9 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 8 Aug 2024 12:07:26 +0300 Subject: [PATCH 19/45] Review feedback Signed-off-by: Alexandru Gheorghe --- .../node/network/approval-distribution/src/lib.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 5e910b8201df..17af2bad15d9 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -369,6 +369,7 @@ pub struct State { /// Aggregated reputation change reputation: ReputationAggregator, + /// Slot duration in millis slot_duration_millis: u64, } @@ -672,7 +673,7 @@ enum MessageSource { // Encountered error while validating an assignment. #[derive(Debug)] enum InvalidAssignmentError { - // The the vrf check for the assignment failed. + // The vrf check for the assignment failed. #[allow(dead_code)] CryptoCheckFailed(InvalidAssignment), // The assignment did not claim any valid candidate. @@ -685,7 +686,7 @@ enum InvalidAssignmentError { }, // The assignment claimes more candidates than the maximum allowed. OversizedClaimedBitfield, - // Session Info was not found for the block hash in the assignment. + // `SessionInfo` was not found for the block hash in the assignment. #[allow(dead_code)] SessionInfoNotFound(polkadot_node_subsystem_util::runtime::Error), } @@ -699,7 +700,7 @@ enum InvalidVoteError { ValidatorIndexOutOfBounds, // The signature of the vote was invalid. InvalidSignature, - // Session Info was not found for the block hash in the approval. + // `SessionInfo` was not found for the block hash in the approval. #[allow(dead_code)] SessionInfoNotFound(polkadot_node_subsystem_util::runtime::Error), } @@ -1708,9 +1709,9 @@ impl State { }) .collect::, InvalidAssignmentError>>()?; - if claimed_cores.is_empty() { + let Ok(claimed_cores) = claimed_cores.try_into() else { return Err(InvalidAssignmentError::NoClaimedCandidates) - } + }; let backing_groups = claimed_candidate_indices .iter_ones() @@ -1721,7 +1722,7 @@ impl State { assignment_criteria .check_assignment_cert( - claimed_cores.try_into().expect("Non-empty vec; qed"), + claimed_cores, assignment.validator, &polkadot_node_core_approval_voting::AssignmentConfig::from(session_info), entry.vrf_story.clone(), From c67d99d9e784a72db748537dd0d07b9e2dd971ae Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 8 Aug 2024 13:33:07 +0300 Subject: [PATCH 20/45] Add checked indirect assignment Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 21 ++- .../node/core/approval-voting/src/tests.rs | 122 ++++++++------- .../network/approval-distribution/src/lib.rs | 29 ++-- .../approval-distribution/src/tests.rs | 145 +++++++----------- polkadot/node/subsystem-types/src/messages.rs | 59 ++++++- 5 files changed, 205 insertions(+), 171 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 2359bb5dfbd0..4fb4df953e14 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -40,8 +40,9 @@ use polkadot_node_subsystem::{ ApprovalCheckError, ApprovalCheckResult, ApprovalDistributionMessage, ApprovalVotingMessage, AssignmentCheckError, AssignmentCheckResult, AvailabilityRecoveryMessage, BlockDescription, CandidateValidationMessage, ChainApiMessage, - ChainSelectionMessage, DisputeCoordinatorMessage, HighestApprovedAncestorBlock, - RuntimeApiMessage, RuntimeApiRequest, + ChainSelectionMessage, CheckedIndirectAssignment, CheckedIndirectSignedApprovalVote, + DisputeCoordinatorMessage, HighestApprovedAncestorBlock, RuntimeApiMessage, + RuntimeApiRequest, }, overseer, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, SubsystemResult, SubsystemSender, @@ -1883,15 +1884,13 @@ async fn handle_from_overseer( vec![Action::Conclude] }, FromOrchestra::Communication { msg } => match msg { - ApprovalVotingMessage::ImportAssignment(a, claimed_cores, tranche, tx) => { + ApprovalVotingMessage::ImportAssignment(checked_assignment, tx) => { let (check_outcome, actions) = import_assignment( ctx.sender(), state, db, session_info_provider, - a, - claimed_cores, - tranche, + checked_assignment, ) .await?; // approval-distribution makes sure this assignment is valid and expected, @@ -2472,15 +2471,15 @@ async fn import_assignment( state: &State, db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, - assignment: IndirectAssignmentCertV2, - candidate_indices: CandidateBitfield, - tranche: DelayTranche, + checked_assignment: CheckedIndirectAssignment, ) -> SubsystemResult<(AssignmentCheckResult, Vec)> where Sender: SubsystemSender, { let tick_now = state.clock.tick_now(); - + let assignment = checked_assignment.assignment(); + let candidate_indices = checked_assignment.candidate_indices(); + let tranche = checked_assignment.tranche(); let mut import_assignment_span = state .spans .get(&assignment.block_hash) @@ -2698,7 +2697,7 @@ async fn import_approval( db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, metrics: &Metrics, - approval: IndirectSignedApprovalVoteV2, + approval: CheckedIndirectSignedApprovalVote, wakeups: &Wakeups, ) -> SubsystemResult<(Vec, ApprovalCheckResult)> where diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 4687b35a0024..ca0a07cd1d95 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -673,12 +673,12 @@ async fn import_approval( overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::ImportApproval( - IndirectSignedApprovalVoteV2 { + CheckedIndirectSignedApprovalVote::from_checked(IndirectSignedApprovalVoteV2 { block_hash, candidate_indices: candidate_index.into(), validator, signature, - }, + }), Some(tx), ), }, @@ -707,14 +707,18 @@ async fn import_assignment( overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::ImportAssignment( - IndirectAssignmentCertV2 { - block_hash, - validator, - cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + CheckedIndirectAssignment::from_checked( + IndirectAssignmentCertV2 { + block_hash, + validator, + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { + sample: 0, + }) .into(), - }, - candidate_index.into(), - tranche, + }, + candidate_index.into(), + tranche, + ), Some(tx), ), }, @@ -734,21 +738,25 @@ async fn import_assignment_v2( overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::ImportAssignment( - IndirectAssignmentCertV2 { - block_hash, - validator, - cert: garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { - core_bitfield: core_indices - .clone() - .into_iter() - .map(|c| CoreIndex(c)) - .collect::>() - .try_into() - .unwrap(), - }), - }, - core_indices.try_into().unwrap(), - 0, + CheckedIndirectAssignment::from_checked( + IndirectAssignmentCertV2 { + block_hash, + validator, + cert: garbage_assignment_cert_v2( + AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: core_indices + .clone() + .into_iter() + .map(|c| CoreIndex(c)) + .collect::>() + .try_into() + .unwrap(), + }, + ), + }, + core_indices.try_into().unwrap(), + 0, + ), Some(tx), ), }, @@ -1171,16 +1179,18 @@ fn blank_subsystem_act_on_bad_block() { &mut virtual_overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::ImportAssignment( - IndirectAssignmentCertV2 { - block_hash: bad_block_hash, - validator: 0u32.into(), - cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { - sample: 0, - }) - .into(), - }, - 0u32.into(), - 0, + CheckedIndirectAssignment::from_checked( + IndirectAssignmentCertV2 { + block_hash: bad_block_hash, + validator: 0u32.into(), + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { + sample: 0, + }) + .into(), + }, + 0u32.into(), + 0, + ), Some(tx), ), }, @@ -2054,16 +2064,18 @@ fn linear_import_act_on_leaf() { &mut virtual_overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::ImportAssignment( - IndirectAssignmentCertV2 { - block_hash: head, - validator: 0u32.into(), - cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { - sample: 0, - }) - .into(), - }, - 0u32.into(), - 0, + CheckedIndirectAssignment::from_checked( + IndirectAssignmentCertV2 { + block_hash: head, + validator: 0u32.into(), + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { + sample: 0, + }) + .into(), + }, + 0u32.into(), + 0, + ), Some(tx), ), }, @@ -2126,16 +2138,18 @@ fn forkful_import_at_same_height_act_on_leaf() { &mut virtual_overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::ImportAssignment( - IndirectAssignmentCertV2 { - block_hash: head, - validator: 0u32.into(), - cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { - sample: 0, - }) - .into(), - }, - 0u32.into(), - 0, + CheckedIndirectAssignment::from_checked( + IndirectAssignmentCertV2 { + block_hash: head, + validator: 0u32.into(), + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { + sample: 0, + }) + .into(), + }, + 0u32.into(), + 0, + ), Some(tx), ), }, diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 17af2bad15d9..8023d10d297b 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -55,8 +55,9 @@ use polkadot_node_primitives::{ }; use polkadot_node_subsystem::{ messages::{ - ApprovalDistributionMessage, ApprovalVotingMessage, NetworkBridgeEvent, - NetworkBridgeTxMessage, RuntimeApiMessage, + ApprovalDistributionMessage, ApprovalVotingMessage, CheckedIndirectAssignment, + CheckedIndirectSignedApprovalVote, NetworkBridgeEvent, NetworkBridgeTxMessage, + RuntimeApiMessage, }, overseer, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, }; @@ -1490,12 +1491,12 @@ impl State { .await; match result { - Ok(tranche) => { + Ok(checked_assignment) => { let current_tranche = clock.tranche_now(self.slot_duration_millis, entry.slot); let too_far_in_future = current_tranche + TICK_TOO_FAR_IN_FUTURE as DelayTranche; - if tranche >= too_far_in_future { + if checked_assignment.tranche() >= too_far_in_future { gum::debug!( target: LOG_TARGET, hash = ?block_hash, @@ -1516,9 +1517,7 @@ impl State { approval_voting_sender .send_message(ApprovalVotingMessage::ImportAssignment( - assignment.clone(), - claimed_candidate_indices.clone(), - tranche, + checked_assignment, None, )) .await; @@ -1687,7 +1686,7 @@ impl State { claimed_candidate_indices: &CandidateBitfield, runtime_info: &mut RuntimeInfo, runtime_api_sender: &mut RA, - ) -> Result { + ) -> Result { let ExtendedSessionInfo { ref session_info, .. } = runtime_info .get_session_info_by_index(runtime_api_sender, assignment.block_hash, entry.session) .await @@ -1730,6 +1729,13 @@ impl State { backing_groups, ) .map_err(|err| InvalidAssignmentError::CryptoCheckFailed(err)) + .map(|tranche| { + CheckedIndirectAssignment::from_checked( + assignment.clone(), + claimed_candidate_indices.clone(), + tranche, + ) + }) } // Checks if an approval can be processed. // Returns true if we can continue with processing the approval and false otherwise. @@ -1902,9 +1908,9 @@ impl State { .await; match result { - Ok(_) => { + Ok(vote) => { approval_voting_sender - .send_message(ApprovalVotingMessage::ImportApproval(vote.clone(), None)) + .send_message(ApprovalVotingMessage::ImportApproval(vote, None)) .await; modify_reputation( @@ -2040,7 +2046,7 @@ impl State { entry: &BlockEntry, runtime_info: &mut RuntimeInfo, runtime_api_sender: &mut RA, - ) -> Result<(), InvalidVoteError> { + ) -> Result { if vote.candidate_indices.len() > entry.candidates_metadata.len() { return Err(InvalidVoteError::CandidateIndexOutOfBounds) } @@ -2075,6 +2081,7 @@ impl State { &vote.signature, ) .map_err(|_| InvalidVoteError::InvalidSignature) + .map(|_| CheckedIndirectSignedApprovalVote::from_checked(vote.clone())) } /// Retrieve approval signatures from state for the given relay block/indices: diff --git a/polkadot/node/network/approval-distribution/src/tests.rs b/polkadot/node/network/approval-distribution/src/tests.rs index 62cf0ec5317e..a0400f91b762 100644 --- a/polkadot/node/network/approval-distribution/src/tests.rs +++ b/polkadot/node/network/approval-distribution/src/tests.rs @@ -600,13 +600,11 @@ fn try_import_the_same_assignment() { overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( assignment, - claimed_indices, - tranche, _, )) => { - assert_eq!(claimed_indices, 0u32.into()); - assert_eq!(assignment, cert.into()); - assert_eq!(tranche, 0); + assert_eq!(assignment.candidate_indices(), &0u32.into()); + assert_eq!(assignment.assignment(), &cert.into()); + assert_eq!(assignment.tranche(), 0); } ); @@ -716,13 +714,11 @@ fn try_import_the_same_assignment_v2() { overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( assignment, - claimed_indices, - tranche, _, )) => { - assert_eq!(claimed_indices, cores.try_into().unwrap()); - assert_eq!(assignment, cert.into()); - assert_eq!(tranche, 0); + assert_eq!(assignment.candidate_indices(), &cores.try_into().unwrap()); + assert_eq!(assignment.assignment(), &cert.into()); + assert_eq!(assignment.tranche(), 0); } ); @@ -807,13 +803,11 @@ fn delay_reputation_change() { overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( assignment, - claimed_candidates, - tranche, _, )) => { - assert_eq!(assignment.cert, cert.cert.into()); - assert_eq!(claimed_candidates, vec![0u32].try_into().unwrap()); - assert_eq!(tranche, 0); + assert_eq!(assignment.assignment().cert, cert.cert.into()); + assert_eq!(assignment.candidate_indices(), &vec![0u32].try_into().unwrap()); + assert_eq!(assignment.tranche(), 0); } ); expect_reputation_changes( @@ -891,13 +885,11 @@ fn spam_attack_results_in_negative_reputation_change() { overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( assignment, - claimed_candidate_index, - tranche, _, )) => { - assert_eq!(assignment, assignments[i].0.clone().into()); - assert_eq!(claimed_candidate_index, assignments[i].1.into()); - assert_eq!(tranche, 0); + assert_eq!(assignment.assignment(), &assignments[i].0.clone().into()); + assert_eq!(assignment.candidate_indices(), &assignments[i].1.into()); + assert_eq!(assignment.tranche(), 0); } ); @@ -1146,7 +1138,7 @@ fn import_approval_happy_path_v1_v2_peers() { AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportApproval( vote, _, )) => { - assert_eq!(vote, approval); + assert_eq!(Into::::into(vote), approval); } ); @@ -1275,7 +1267,7 @@ fn import_approval_happy_path_v2() { AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportApproval( vote, _, )) => { - assert_eq!(vote, approval); + assert_eq!(Into::::into(vote), approval); } ); @@ -1377,11 +1369,10 @@ fn multiple_assignments_covered_with_one_approval_vote() { assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( - _, _, - tranche, + assignment, _, )) => { - assert_eq!(tranche, 0); + assert_eq!(assignment.tranche(), 0); } ); expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; @@ -1420,10 +1411,9 @@ fn multiple_assignments_covered_with_one_approval_vote() { assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( - _, _, - tranche, _, + assignment, _, )) => { - assert_eq!(tranche, 0); + assert_eq!(assignment.tranche(), 0); } ); expect_reputation_change(overseer, &peer_c, BENEFIT_VALID_MESSAGE_FIRST).await; @@ -1463,7 +1453,7 @@ fn multiple_assignments_covered_with_one_approval_vote() { AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportApproval( vote, _, )) => { - assert_eq!(vote, approval); + assert_eq!(Into::::into(vote), approval); } ); @@ -1584,10 +1574,9 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( - _, _, - tranche, _, + assignment, _, )) => { - assert_eq!(tranche, 0); + assert_eq!(assignment.tranche(), 0); } ); expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; @@ -1611,10 +1600,9 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( - _, _, - tranche, _, + assignment, _, )) => { - assert_eq!(tranche, 0); + assert_eq!(assignment.tranche(), 0); } ); expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; @@ -1639,7 +1627,7 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportApproval( vote, _, )) => { - assert_eq!(vote, approval); + assert_eq!(Into::::into(vote), approval); } ); @@ -1784,12 +1772,11 @@ fn import_approval_bad() { overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( assignment, - i, - tranche, _, + _, )) => { - assert_eq!(assignment, cert.into()); - assert_eq!(i, candidate_index.into()); - assert_eq!(tranche, 0); + assert_eq!(assignment.assignment(), &cert.into()); + assert_eq!(assignment.candidate_indices(), &candidate_index.into()); + assert_eq!(assignment.tranche(), 0); } ); @@ -2336,12 +2323,11 @@ fn import_remotely_then_locally() { overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( assignment, - i, - tranche, _, + _, )) => { - assert_eq!(assignment, cert.clone().into()); - assert_eq!(i, candidate_index.into()); - assert_eq!(tranche, 0); + assert_eq!(assignment.assignment(), &cert.clone().into()); + assert_eq!(assignment.candidate_indices(), &candidate_index.into()); + assert_eq!(assignment.tranche(), 0); } ); @@ -2374,7 +2360,7 @@ fn import_remotely_then_locally() { AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportApproval( vote, _, )) => { - assert_eq!(vote, approval); + assert_eq!(Into::::into(vote), approval); } ); expect_reputation_change(overseer, peer, BENEFIT_VALID_MESSAGE_FIRST).await; @@ -2691,12 +2677,11 @@ fn race_condition_in_local_vs_remote_view_update() { overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( assignment, - claimed_candidate_index, - tranche, _, + _, )) => { - assert_eq!(assignment, assignments[i].0.clone().into()); - assert_eq!(claimed_candidate_index, assignments[i].1.into()); - assert_eq!(tranche, 0); + assert_eq!(assignment.assignment(), &assignments[i].0.clone().into()); + assert_eq!(assignment.candidate_indices(), &assignments[i].1.into()); + assert_eq!(assignment.tranche(), 0); } ); @@ -2913,11 +2898,9 @@ fn propagates_assignments_along_unshared_dimension() { assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( - _, - _, - tranche, _, + assignment, _, )) => { - assert_eq!(tranche, 0); + assert_eq!(assignment.tranche(), 0); } ); expect_reputation_change(overseer, &peers[99].0, BENEFIT_VALID_MESSAGE_FIRST).await; @@ -2963,11 +2946,9 @@ fn propagates_assignments_along_unshared_dimension() { assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( - _, - _, - tranche, _, + assignment, _, )) => { - assert_eq!(tranche, 0); + assert_eq!(assignment.tranche(), 0); } ); expect_reputation_change(overseer, &peers[99].0, BENEFIT_VALID_MESSAGE_FIRST).await; @@ -3556,11 +3537,9 @@ fn non_originator_aggression_l1() { assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( - _, - _, - tranche, _, + assignment, _, )) => { - assert_eq!(tranche, 0); + assert_eq!(assignment.tranche(), 0); } ); @@ -3682,11 +3661,9 @@ fn non_originator_aggression_l2() { assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( - _, - _, - tranche, _, + assignment, _, )) => { - assert_eq!(tranche, 0); + assert_eq!(assignment.tranche(), 0); } ); @@ -3872,11 +3849,9 @@ fn resends_messages_periodically() { assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( - _, - _, - tranche, _, + assignment, _, )) => { - assert_eq!(tranche, 0); + assert_eq!(assignment.tranche(), 0); } ); expect_reputation_change(overseer, &peers[99].0, BENEFIT_VALID_MESSAGE_FIRST).await; @@ -4074,7 +4049,7 @@ fn import_versioned_approval() { AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportApproval( vote, _, )) => { - assert_eq!(vote, approval.into()); + assert_eq!(Into::::into(vote), approval.into()); } ); @@ -4612,13 +4587,11 @@ fn subsystem_accepts_tranche0_duplicate_assignments() { overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( assignment, - claimed_indices, - tranche, _, )) => { - assert_eq!(claimed_indices, candidate_indices); - assert_eq!(assignment, cert.into()); - assert_eq!(tranche, 0); + assert_eq!(assignment.candidate_indices(), &candidate_indices); + assert_eq!(assignment.assignment(), &cert.into()); + assert_eq!(assignment.tranche(), 0); } ); @@ -4652,13 +4625,11 @@ fn subsystem_accepts_tranche0_duplicate_assignments() { overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( assignment, - claimed_indices, - tranche, _, )) => { - assert_eq!(claimed_indices, candidate_indices); - assert_eq!(assignment, cert.into()); - assert_eq!(tranche, 0); + assert_eq!(assignment.candidate_indices(), &candidate_indices); + assert_eq!(assignment.assignment(), &cert.into()); + assert_eq!(assignment.tranche(), 0); } ); @@ -4692,13 +4663,11 @@ fn subsystem_accepts_tranche0_duplicate_assignments() { overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::ImportAssignment( assignment, - claimed_indices, - tranche, _, )) => { - assert_eq!(claimed_indices, candidate_indices); - assert_eq!(assignment, cert.into()); - assert_eq!(tranche, 0); + assert_eq!(assignment.candidate_indices(), &candidate_indices); + assert_eq!(assignment.assignment(), &cert.into()); + assert_eq!(assignment.tranche(), 0); } ); diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index 57342e14b5b7..67a0dd951616 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -969,23 +969,68 @@ pub struct HighestApprovedAncestorBlock { pub descriptions: Vec, } +/// A checked indirect assignment, the crypto for the cert has been validated +/// and the `candidate_bitfield` is correctly claimed at `delay_tranche`. +#[derive(Debug)] +pub struct CheckedIndirectAssignment { + assignment: IndirectAssignmentCertV2, + candidate_indices: CandidateBitfield, + tranche: DelayTranche, +} + +impl CheckedIndirectAssignment { + /// Builds a checked assignment from an assignment that was checked to be valid for the + /// `claimed_candidate_indices` at the give tranche + pub fn from_checked( + assignment: IndirectAssignmentCertV2, + claimed_candidate_indices: CandidateBitfield, + tranche: DelayTranche, + ) -> Self { + Self { assignment, candidate_indices: claimed_candidate_indices, tranche } + } + + /// Returns the indirect assignment. + pub fn assignment(&self) -> &IndirectAssignmentCertV2 { + &self.assignment + } + + /// Returns the candidate bitfield claimed by the assignment. + pub fn candidate_indices(&self) -> &CandidateBitfield { + &self.candidate_indices + } + + /// Returns the tranche this assignment is claimed at. + pub fn tranche(&self) -> DelayTranche { + self.tranche + } +} + +/// A checked indirect signed approval vote. +/// +/// The crypto for the vote has been validated and the signature can be trusted as being valid and +/// to correspond to the `validator_index` inside the structure. +#[derive(Debug, derive_more::Deref, derive_more::Into)] +pub struct CheckedIndirectSignedApprovalVote(IndirectSignedApprovalVoteV2); + +impl CheckedIndirectSignedApprovalVote { + /// Builds a checked vote from a vote that was checked to be valid and correctly signed. + pub fn from_checked(vote: IndirectSignedApprovalVoteV2) -> Self { + Self(vote) + } +} + /// Message to the Approval Voting subsystem. #[derive(Debug)] pub enum ApprovalVotingMessage { /// Import an assignment into the approval-voting database. /// /// Should not be sent unless the block hash is known and the VRF assignment checks out. - ImportAssignment( - IndirectAssignmentCertV2, - CandidateBitfield, - DelayTranche, - Option>, - ), + ImportAssignment(CheckedIndirectAssignment, Option>), /// Import an approval vote into approval-voting database /// /// Should not be sent unless the block hash within the indirect vote is known, vote is /// correctly signed and we had a previous assignment for the candidate. - ImportApproval(IndirectSignedApprovalVoteV2, Option>), + ImportApproval(CheckedIndirectSignedApprovalVote, Option>), /// Returns the highest possible ancestor hash of the provided block hash which is /// acceptable to vote on finality for. /// The `BlockNumber` provided is the number of the block's ancestor which is the From 0b140cc2dabc6e395d2d698173a724dc36994869 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 13 Aug 2024 11:31:34 +0300 Subject: [PATCH 21/45] Change OurViewChange order Signed-off-by: Alexandru Gheorghe --- polkadot/node/network/bridge/src/rx/mod.rs | 30 +++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/polkadot/node/network/bridge/src/rx/mod.rs b/polkadot/node/network/bridge/src/rx/mod.rs index 56965ce6ba40..7745c42f78a1 100644 --- a/polkadot/node/network/bridge/src/rx/mod.rs +++ b/polkadot/node/network/bridge/src/rx/mod.rs @@ -962,6 +962,21 @@ fn update_our_view( ) }; + let our_view = OurView::new( + live_heads.iter().take(MAX_VIEW_HEADS).cloned().map(|a| (a.hash, a.span)), + finalized_number, + ); + + dispatch_validation_event_to_all_unbounded( + NetworkBridgeEvent::OurViewChange(our_view.clone()), + ctx.sender(), + ); + + dispatch_collation_event_to_all_unbounded( + NetworkBridgeEvent::OurViewChange(our_view), + ctx.sender(), + ); + let v1_validation_peers = filter_by_peer_version(&validation_peers, ValidationVersion::V1.into()); let v1_collation_peers = filter_by_peer_version(&collation_peers, CollationVersion::V1.into()); @@ -1007,21 +1022,6 @@ fn update_our_view( metrics, notification_sinks, ); - - let our_view = OurView::new( - live_heads.iter().take(MAX_VIEW_HEADS).cloned().map(|a| (a.hash, a.span)), - finalized_number, - ); - - dispatch_validation_event_to_all_unbounded( - NetworkBridgeEvent::OurViewChange(our_view.clone()), - ctx.sender(), - ); - - dispatch_collation_event_to_all_unbounded( - NetworkBridgeEvent::OurViewChange(our_view), - ctx.sender(), - ); } // Handle messages on a specific v1 peer-set. The peer is expected to be connected on that From c6157b6c314c58c804669d6e5619817d6935a204 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 14 Aug 2024 19:18:58 +0300 Subject: [PATCH 22/45] Update Cargo.lock Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 54bd0231cfd2..ab6f29e6afe7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5348,6 +5348,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" dependencies = [ "log", + "regex", ] [[package]] @@ -5382,6 +5383,7 @@ dependencies = [ "anstream", "anstyle", "env_filter", + "humantime", "log", ] From 7539184e84f2a8d604cd3709e6acfb2a267512a2 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 27 Aug 2024 15:50:44 +0300 Subject: [PATCH 23/45] Remove crate dependency between approval-distribution and approval-voting Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 4 +- .../approval-voting/src/approval_checking.rs | 8 +- .../node/core/approval-voting/src/criteria.rs | 172 +++-------------- .../node/core/approval-voting/src/import.rs | 6 +- polkadot/node/core/approval-voting/src/lib.rs | 6 +- .../approval-voting/src/persisted_entries.rs | 4 +- .../node/core/approval-voting/src/tests.rs | 4 +- .../network/approval-distribution/Cargo.toml | 1 - .../network/approval-distribution/src/lib.rs | 51 ++--- .../approval-distribution/src/tests.rs | 89 ++++----- polkadot/node/primitives/Cargo.toml | 3 + .../node/primitives/src/approval/criteria.rs | 177 ++++++++++++++++++ .../src/{approval.rs => approval/mod.rs} | 6 + .../src => primitives/src/approval}/time.rs | 13 +- polkadot/node/service/src/overseer.rs | 3 +- .../src/lib/approval/helpers.rs | 2 +- .../src/lib/approval/message_generator.rs | 7 +- .../src/lib/approval/mock_chain_selection.rs | 2 +- .../subsystem-bench/src/lib/approval/mod.rs | 7 +- 19 files changed, 329 insertions(+), 236 deletions(-) create mode 100644 polkadot/node/primitives/src/approval/criteria.rs rename polkadot/node/primitives/src/{approval.rs => approval/mod.rs} (99%) rename polkadot/node/{core/approval-voting/src => primitives/src/approval}/time.rs (95%) diff --git a/Cargo.lock b/Cargo.lock index b570906c0f04..ce9921026dea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12989,7 +12989,6 @@ dependencies = [ "futures-timer", "itertools 0.11.0", "log", - "polkadot-node-core-approval-voting", "polkadot-node-jaeger", "polkadot-node-metrics", "polkadot-node-network-protocol", @@ -13817,14 +13816,17 @@ dependencies = [ "bitvec", "bounded-vec", "futures", + "futures-timer", "parity-scale-codec", "polkadot-erasure-coding", "polkadot-parachain-primitives", "polkadot-primitives", + "sc-keystore", "schnorrkel 0.11.4", "serde", "sp-application-crypto", "sp-consensus-babe", + "sp-consensus-slots", "sp-core", "sp-keystore", "sp-maybe-compressed-blob", diff --git a/polkadot/node/core/approval-voting/src/approval_checking.rs b/polkadot/node/core/approval-voting/src/approval_checking.rs index 96eb25626de8..3774edc69981 100644 --- a/polkadot/node/core/approval-voting/src/approval_checking.rs +++ b/polkadot/node/core/approval-voting/src/approval_checking.rs @@ -22,9 +22,9 @@ use polkadot_primitives::ValidatorIndex; use crate::{ persisted_entries::{ApprovalEntry, CandidateEntry, TrancheEntry}, - time::Tick, MAX_RECORDED_NO_SHOW_VALIDATORS_PER_CANDIDATE, }; +use polkadot_node_primitives::approval::time::Tick; /// Result of counting the necessary tranches needed for approving a block. #[derive(Debug, PartialEq, Clone)] @@ -1195,9 +1195,9 @@ mod tests { struct NoShowTest { assignments: Vec<(ValidatorIndex, Tick)>, approvals: Vec, - clock_drift: crate::time::Tick, - no_show_duration: crate::time::Tick, - drifted_tick_now: crate::time::Tick, + clock_drift: Tick, + no_show_duration: Tick, + drifted_tick_now: Tick, exp_no_shows: usize, exp_next_no_show: Option, } diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 4a1e87868c2d..669b6001538e 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -16,8 +16,11 @@ //! Assignment criteria VRF generation and checking. -use codec::{Decode, Encode}; +use codec::Encode; use itertools::Itertools; +pub use polkadot_node_primitives::approval::criteria::{ + AssignmentCriteria, Config, InvalidAssignment, InvalidAssignmentReason, OurAssignment, +}; use polkadot_node_primitives::approval::{ self as approval_types, v1::{AssignmentCert, AssignmentCertKind, DelayTranche, RelayVRFStory}, @@ -25,9 +28,9 @@ use polkadot_node_primitives::approval::{ AssignmentCertKindV2, AssignmentCertV2, CoreBitfield, VrfPreOutput, VrfProof, VrfSignature, }, }; + use polkadot_primitives::{ - AssignmentId, AssignmentPair, CandidateHash, CoreIndex, GroupIndex, IndexedVec, SessionInfo, - ValidatorIndex, + AssignmentPair, CandidateHash, CoreIndex, GroupIndex, IndexedVec, ValidatorIndex, }; use rand::{seq::SliceRandom, SeedableRng}; use rand_chacha::ChaCha20Rng; @@ -44,56 +47,19 @@ use std::{ use super::LOG_TARGET; -/// Details pertaining to our assignment on a block. -#[derive(Debug, Clone, Encode, Decode, PartialEq)] -pub struct OurAssignment { - cert: AssignmentCertV2, - tranche: DelayTranche, - validator_index: ValidatorIndex, - // Whether the assignment has been triggered already. - triggered: bool, -} - -impl OurAssignment { - pub fn cert(&self) -> &AssignmentCertV2 { - &self.cert - } - - pub fn tranche(&self) -> DelayTranche { - self.tranche - } - - pub(crate) fn validator_index(&self) -> ValidatorIndex { - self.validator_index - } - - pub(crate) fn triggered(&self) -> bool { - self.triggered - } - - pub(crate) fn mark_triggered(&mut self) { - self.triggered = true; - } -} - impl From for OurAssignment { fn from(entry: crate::approval_db::v2::OurAssignment) -> Self { - OurAssignment { - cert: entry.cert, - tranche: entry.tranche, - validator_index: entry.validator_index, - triggered: entry.triggered, - } + OurAssignment::new(entry.cert, entry.tranche, entry.validator_index, entry.triggered) } } impl From for crate::approval_db::v2::OurAssignment { fn from(entry: OurAssignment) -> Self { Self { - cert: entry.cert, - tranche: entry.tranche, - validator_index: entry.validator_index, - triggered: entry.triggered, + tranche: entry.tranche(), + validator_index: entry.validator_index(), + triggered: entry.triggered(), + cert: entry.into_cert(), } } } @@ -223,59 +189,6 @@ fn assigned_core_transcript(core_index: CoreIndex) -> Transcript { t } -/// Information about the world assignments are being produced in. -#[derive(Clone, Debug)] -pub struct Config { - /// The assignment public keys for validators. - assignment_keys: Vec, - /// The groups of validators assigned to each core. - validator_groups: IndexedVec>, - /// The number of availability cores used by the protocol during this session. - n_cores: u32, - /// The zeroth delay tranche width. - zeroth_delay_tranche_width: u32, - /// The number of samples we do of `relay_vrf_modulo`. - relay_vrf_modulo_samples: u32, - /// The number of delay tranches in total. - n_delay_tranches: u32, -} - -impl<'a> From<&'a SessionInfo> for Config { - fn from(s: &'a SessionInfo) -> Self { - Config { - assignment_keys: s.assignment_keys.clone(), - validator_groups: s.validator_groups.clone(), - n_cores: s.n_cores, - zeroth_delay_tranche_width: s.zeroth_delay_tranche_width, - relay_vrf_modulo_samples: s.relay_vrf_modulo_samples, - n_delay_tranches: s.n_delay_tranches, - } - } -} - -/// A trait for producing and checking assignments. Used to mock. -pub trait AssignmentCriteria { - fn compute_assignments( - &self, - keystore: &LocalKeystore, - relay_vrf_story: RelayVRFStory, - config: &Config, - leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>, - enable_v2_assignments: bool, - ) -> HashMap; - - fn check_assignment_cert( - &self, - claimed_core_bitfield: CoreBitfield, - validator_index: ValidatorIndex, - config: &Config, - relay_vrf_story: RelayVRFStory, - assignment: &AssignmentCertV2, - // Backing groups for each "leaving core". - backing_groups: Vec, - ) -> Result; -} - pub struct RealAssignmentCriteria; impl AssignmentCriteria for RealAssignmentCriteria { @@ -469,12 +382,12 @@ fn compute_relay_vrf_modulo_assignments_v1( }; // All assignments of type RelayVRFModulo have tranche 0. - assignments.entry(core).or_insert(OurAssignment { - cert: cert.into(), - tranche: 0, + assignments.entry(core).or_insert(OurAssignment::new( + cert.into(), + 0, validator_index, - triggered: false, - }); + false, + )); } } } @@ -549,7 +462,7 @@ fn compute_relay_vrf_modulo_assignments_v2( }; // All assignments of type RelayVRFModulo have tranche 0. - OurAssignment { cert, tranche: 0, validator_index, triggered: false } + OurAssignment::new(cert, 0, validator_index, false) }) { for core_index in assigned_cores { assignments.insert(core_index, assignment.clone()); @@ -583,7 +496,7 @@ fn compute_relay_vrf_delay_assignments( }, }; - let our_assignment = OurAssignment { cert, tranche, validator_index, triggered: false }; + let our_assignment = OurAssignment::new(cert, tranche, validator_index, false); let used = match assignments.entry(core) { Entry::Vacant(e) => { @@ -591,7 +504,7 @@ fn compute_relay_vrf_delay_assignments( true }, Entry::Occupied(mut e) => - if e.get().tranche > our_assignment.tranche { + if e.get().tranche() > our_assignment.tranche() { e.insert(our_assignment); true } else { @@ -612,35 +525,6 @@ fn compute_relay_vrf_delay_assignments( } } -/// Assignment invalid. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct InvalidAssignment(pub InvalidAssignmentReason); - -impl std::fmt::Display for InvalidAssignment { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "Invalid Assignment: {:?}", self.0) - } -} - -impl std::error::Error for InvalidAssignment {} - -/// Failure conditions when checking an assignment cert. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum InvalidAssignmentReason { - ValidatorIndexOutOfBounds, - SampleOutOfBounds, - CoreIndexOutOfBounds, - InvalidAssignmentKey, - IsInBackingGroup, - VRFModuloCoreIndexMismatch, - VRFModuloOutputMismatch, - VRFDelayCoreIndexMismatch, - VRFDelayOutputMismatch, - InvalidArguments, - /// Assignment vrf check resulted in 0 assigned cores. - NullAssignment, -} - /// Checks the crypto of an assignment cert. Failure conditions: /// * Validator index out of bounds /// * VRF signature check fails @@ -820,13 +704,13 @@ fn is_in_backing_group( /// Migration helpers. impl From for OurAssignment { fn from(value: crate::approval_db::v1::OurAssignment) -> Self { - Self { - cert: value.cert.into(), - tranche: value.tranche, - validator_index: value.validator_index, + Self::new( + value.cert.into(), + value.tranche, + value.validator_index, // Whether the assignment has been triggered already. - triggered: value.triggered, - } + value.triggered, + ) } } @@ -834,7 +718,7 @@ impl From for OurAssignment { mod tests { use super::*; use crate::import::tests::garbage_vrf_signature; - use polkadot_primitives::{Hash, ASSIGNMENT_KEY_TYPE_ID}; + use polkadot_primitives::{AssignmentId, Hash, ASSIGNMENT_KEY_TYPE_ID}; use sp_application_crypto::sr25519; use sp_core::crypto::Pair as PairT; use sp_keyring::sr25519::Keyring as Sr25519Keyring; @@ -1053,7 +937,7 @@ mod tests { let mut counted = 0; for (core, assignment) in assignments { - let cores = match assignment.cert.kind.clone() { + let cores = match assignment.cert().kind.clone() { AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => core_bitfield, AssignmentCertKindV2::RelayVRFModulo { sample: _ } => core.into(), AssignmentCertKindV2::RelayVRFDelay { core_index } => core_index.into(), @@ -1062,7 +946,7 @@ mod tests { let mut mutated = MutatedAssignment { cores: cores.clone(), groups: cores.iter_ones().map(|core| group_for_core(core)).collect(), - cert: assignment.cert, + cert: assignment.into_cert(), own_group: GroupIndex(0), val_index: ValidatorIndex(0), config: config.clone(), diff --git a/polkadot/node/core/approval-voting/src/import.rs b/polkadot/node/core/approval-voting/src/import.rs index a138af1188d2..b163d718eb25 100644 --- a/polkadot/node/core/approval-voting/src/import.rs +++ b/polkadot/node/core/approval-voting/src/import.rs @@ -62,9 +62,10 @@ use crate::{ criteria::{AssignmentCriteria, OurAssignment}, get_extended_session_info, get_session_info, persisted_entries::CandidateEntry, - time::{slot_number_to_tick, Tick}, }; +use polkadot_node_primitives::approval::time::{slot_number_to_tick, Tick}; + use super::{State, LOG_TARGET}; #[derive(Debug)] @@ -613,6 +614,7 @@ pub(crate) mod tests { approval_db::common::{load_block_entry, DbBackend}, RuntimeInfo, RuntimeInfoConfig, MAX_BLOCKS_WITH_ASSIGNMENT_TIMESTAMPS, }; + use approval_types::time::Clock; use assert_matches::assert_matches; use polkadot_node_primitives::{ approval::v1::{VrfSignature, VrfTranscript}, @@ -646,7 +648,7 @@ pub(crate) mod tests { #[derive(Default)] struct MockClock; - impl crate::time::Clock for MockClock { + impl Clock for MockClock { fn tick_now(&self) -> Tick { 42 // chosen by fair dice roll } diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 4fb4df953e14..7c628f4f2ad9 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -93,7 +93,9 @@ use approval_checking::RequiredTranches; use bitvec::{order::Lsb0, vec::BitVec}; pub use criteria::{AssignmentCriteria, Config as AssignmentConfig, RealAssignmentCriteria}; use persisted_entries::{ApprovalEntry, BlockEntry, CandidateEntry}; -use time::{slot_number_to_tick, Clock, ClockExt, DelayedApprovalTimer, SystemClock, Tick}; +use polkadot_node_primitives::approval::time::{ + slot_number_to_tick, Clock, ClockExt, DelayedApprovalTimer, SystemClock, Tick, +}; mod approval_checking; pub mod approval_db; @@ -102,7 +104,6 @@ pub mod criteria; mod import; mod ops; mod persisted_entries; -pub mod time; use crate::{ approval_checking::{Check, TranchesToApproveResult}, @@ -123,7 +124,6 @@ const APPROVAL_CHECKING_TIMEOUT: Duration = Duration::from_secs(120); const WAIT_FOR_SIGS_TIMEOUT: Duration = Duration::from_millis(500); const APPROVAL_CACHE_SIZE: u32 = 1024; -pub const TICK_TOO_FAR_IN_FUTURE: Tick = 20; // 10 seconds. const APPROVAL_DELAY: Tick = 2; pub(crate) const LOG_TARGET: &str = "parachain::approval-voting"; diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index 59a461810051..16e231aa1a2d 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -36,7 +36,9 @@ use std::collections::BTreeMap; use crate::approval_db::v2::Bitfield; -use super::{criteria::OurAssignment, time::Tick}; +use super::criteria::OurAssignment; + +use polkadot_node_primitives::approval::time::Tick; /// Metadata regarding a specific tranche of assignments for a specific candidate. #[derive(Debug, Clone, PartialEq)] diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 2f2083a7d880..8a3102f22d0d 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -140,8 +140,8 @@ impl HeadSupportsParachains for MockSupportsParachains { } } -fn slot_to_tick(t: impl Into) -> crate::time::Tick { - crate::time::slot_number_to_tick(SLOT_DURATION_MILLIS, t.into()) +fn slot_to_tick(t: impl Into) -> Tick { + slot_number_to_tick(SLOT_DURATION_MILLIS, t.into()) } #[derive(Default, Clone)] diff --git a/polkadot/node/network/approval-distribution/Cargo.toml b/polkadot/node/network/approval-distribution/Cargo.toml index d421d41435f9..51478dfa4a4f 100644 --- a/polkadot/node/network/approval-distribution/Cargo.toml +++ b/polkadot/node/network/approval-distribution/Cargo.toml @@ -10,7 +10,6 @@ license.workspace = true workspace = true [dependencies] -polkadot-node-core-approval-voting = { workspace = true, default-features = true } polkadot-node-metrics = { workspace = true, default-features = true } polkadot-node-network-protocol = { workspace = true, default-features = true } polkadot-node-primitives = { workspace = true, default-features = true } diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 81657d7cc534..971b6de5f8f6 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -27,11 +27,6 @@ use self::metrics::Metrics; use futures::{select, FutureExt as _}; use itertools::Itertools; use net_protocol::peer_set::{ProtocolVersion, ValidationVersion}; -use polkadot_node_core_approval_voting::{ - criteria::InvalidAssignment, - time::{Clock, ClockExt, SystemClock}, - AssignmentCriteria, RealAssignmentCriteria, TICK_TOO_FAR_IN_FUTURE, -}; use polkadot_node_jaeger as jaeger; use polkadot_node_network_protocol::{ self as net_protocol, filter_by_peer_version, @@ -42,6 +37,8 @@ use polkadot_node_network_protocol::{ }; use polkadot_node_primitives::{ approval::{ + criteria::{AssignmentCriteria, InvalidAssignment}, + time::{Clock, ClockExt, SystemClock, TICK_TOO_FAR_IN_FUTURE}, v1::{ AssignmentCertKind, BlockApprovalMeta, DelayTranche, IndirectAssignmentCert, IndirectSignedApprovalVote, RelayVRFStory, @@ -72,6 +69,7 @@ use polkadot_primitives::{ use rand::{CryptoRng, Rng, SeedableRng}; use std::{ collections::{hash_map, BTreeMap, HashMap, HashSet, VecDeque}, + sync::Arc, time::Duration, }; @@ -102,6 +100,7 @@ pub struct ApprovalDistribution { metrics: Metrics, slot_duration_millis: u64, clock: Box, + assignment_criteria: Arc, } /// Contains recently finalized @@ -515,7 +514,7 @@ struct BlockEntry { /// candidates being claimed by assignments. approval_entries: HashMap<(ValidatorIndex, CandidateBitfield), ApprovalEntry>, /// The block vrf story. - pub vrf_story: RelayVRFStory, + vrf_story: RelayVRFStory, /// The block slot. slot: Slot, } @@ -739,7 +738,7 @@ impl State { metrics: &Metrics, event: NetworkBridgeEvent, rng: &mut (impl CryptoRng + Rng), - assignment_criteria: &impl AssignmentCriteria, + assignment_criteria: &(impl AssignmentCriteria + ?Sized), clock: &(impl Clock + ?Sized), session_info_provider: &mut RuntimeInfo, ) { @@ -859,7 +858,7 @@ impl State { metrics: &Metrics, metas: Vec, rng: &mut (impl CryptoRng + Rng), - assignment_criteria: &impl AssignmentCriteria, + assignment_criteria: &(impl AssignmentCriteria + ?Sized), clock: &(impl Clock + ?Sized), session_info_provider: &mut RuntimeInfo, ) { @@ -1041,7 +1040,7 @@ impl State { peer_id: PeerId, assignments: Vec<(IndirectAssignmentCertV2, CandidateBitfield)>, rng: &mut R, - assignment_criteria: &impl AssignmentCriteria, + assignment_criteria: &(impl AssignmentCriteria + ?Sized), clock: &(impl Clock + ?Sized), session_info_provider: &mut RuntimeInfo, ) where @@ -1152,7 +1151,7 @@ impl State { protocol_v3::ApprovalDistributionMessage, >, rng: &mut R, - assignment_criteria: &impl AssignmentCriteria, + assignment_criteria: &(impl AssignmentCriteria + ?Sized), clock: &(impl Clock + ?Sized), session_info_provider: &mut RuntimeInfo, ) where @@ -1348,7 +1347,7 @@ impl State { assignment: IndirectAssignmentCertV2, claimed_candidate_indices: CandidateBitfield, rng: &mut R, - assignment_criteria: &impl AssignmentCriteria, + assignment_criteria: &(impl AssignmentCriteria + ?Sized), clock: &(impl Clock + ?Sized), session_info_provider: &mut RuntimeInfo, ) where @@ -1680,7 +1679,7 @@ impl State { } async fn check_assignment_valid>( - assignment_criteria: &impl AssignmentCriteria, + assignment_criteria: &(impl AssignmentCriteria + ?Sized), entry: &BlockEntry, assignment: &IndirectAssignmentCertV2, claimed_candidate_indices: &CandidateBitfield, @@ -1723,7 +1722,7 @@ impl State { .check_assignment_cert( claimed_cores, assignment.validator, - &polkadot_node_core_approval_voting::AssignmentConfig::from(session_info), + &polkadot_node_primitives::approval::criteria::Config::from(session_info), entry.vrf_story.clone(), &assignment.cert, backing_groups, @@ -2661,8 +2660,17 @@ async fn modify_reputation( #[overseer::contextbounds(ApprovalDistribution, prefix = self::overseer)] impl ApprovalDistribution { /// Create a new instance of the [`ApprovalDistribution`] subsystem. - pub fn new(metrics: Metrics, slot_duration_millis: u64) -> Self { - Self::new_with_clock(metrics, slot_duration_millis, Box::new(SystemClock)) + pub fn new( + metrics: Metrics, + slot_duration_millis: u64, + assignment_criteria: Arc, + ) -> Self { + Self::new_with_clock( + metrics, + slot_duration_millis, + Box::new(SystemClock), + assignment_criteria, + ) } /// Create a new instance of the [`ApprovalDistribution`] subsystem, with a custom clock. @@ -2670,8 +2678,9 @@ impl ApprovalDistribution { metrics: Metrics, slot_duration_millis: u64, clock: Box, + assignment_criteria: Arc, ) -> Self { - Self { metrics, slot_duration_millis, clock } + Self { metrics, slot_duration_millis, clock, assignment_criteria } } async fn run(self, ctx: Context) { @@ -2680,7 +2689,6 @@ impl ApprovalDistribution { // According to the docs of `rand`, this is a ChaCha12 RNG in practice // and will always be chosen for strong performance and security properties. let mut rng = rand::rngs::StdRng::from_entropy(); - let assignment_criteria = RealAssignmentCriteria {}; let mut session_info_provider = RuntimeInfo::new_with_config(RuntimeInfoConfig { keystore: None, session_cache_lru_size: DISPUTE_WINDOW.get(), @@ -2691,7 +2699,6 @@ impl ApprovalDistribution { &mut state, REPUTATION_CHANGE_INTERVAL, &mut rng, - &assignment_criteria, &mut session_info_provider, ) .await @@ -2704,7 +2711,6 @@ impl ApprovalDistribution { state: &mut State, reputation_interval: Duration, rng: &mut (impl CryptoRng + Rng), - assignment_criteria: &impl AssignmentCriteria, session_info_provider: &mut RuntimeInfo, ) { let new_reputation_delay = || futures_timer::Delay::new(reputation_interval).fuse(); @@ -2728,7 +2734,7 @@ impl ApprovalDistribution { }, }; - if self.handle_from_orchestra(message, &mut approval_voting_sender, &mut network_sender, &mut runtime_api_sender, state, rng, assignment_criteria, session_info_provider).await { + if self.handle_from_orchestra(message, &mut approval_voting_sender, &mut network_sender, &mut runtime_api_sender, state, rng, session_info_provider).await { return; } @@ -2752,7 +2758,6 @@ impl ApprovalDistribution { runtime_api_sender: &mut RA, state: &mut State, rng: &mut (impl CryptoRng + Rng), - assignment_criteria: &impl AssignmentCriteria, session_info_provider: &mut RuntimeInfo, ) -> bool { match message { @@ -2765,7 +2770,7 @@ impl ApprovalDistribution { msg, &self.metrics, rng, - assignment_criteria, + self.assignment_criteria.as_ref(), self.clock.as_ref(), session_info_provider, ) @@ -2804,7 +2809,7 @@ impl ApprovalDistribution { msg: ApprovalDistributionMessage, metrics: &Metrics, rng: &mut (impl CryptoRng + Rng), - assignment_criteria: &impl AssignmentCriteria, + assignment_criteria: &(impl AssignmentCriteria + ?Sized), clock: &(impl Clock + ?Sized), session_info_provider: &mut RuntimeInfo, ) { diff --git a/polkadot/node/network/approval-distribution/src/tests.rs b/polkadot/node/network/approval-distribution/src/tests.rs index 3b92af0e7ab1..83b5eb7684b9 100644 --- a/polkadot/node/network/approval-distribution/src/tests.rs +++ b/polkadot/node/network/approval-distribution/src/tests.rs @@ -17,7 +17,6 @@ use super::*; use assert_matches::assert_matches; use futures::{channel::oneshot, executor, future, Future}; -use polkadot_node_core_approval_voting::criteria; use polkadot_node_network_protocol::{ grid_topology::{SessionGridTopology, TopologyPeerInfo}, our_view, @@ -25,6 +24,7 @@ use polkadot_node_network_protocol::{ view, ObservedRole, }; use polkadot_node_primitives::approval::{ + criteria, v1::{ AssignmentCert, AssignmentCertKind, IndirectAssignmentCert, IndirectSignedApprovalVote, VrfPreOutput, VrfProof, VrfSignature, @@ -53,7 +53,7 @@ type VirtualOverseer = polkadot_node_subsystem_test_helpers::TestSubsystemContextHandle; fn test_harness>( - assignment_criteria: &impl AssignmentCriteria, + assignment_criteria: Arc, clock: Box, mut state: State, test_fn: impl FnOnce(VirtualOverseer) -> T, @@ -64,8 +64,12 @@ fn test_harness>( let (context, virtual_overseer) = polkadot_node_subsystem_test_helpers::make_subsystem_context(pool.clone()); - let subsystem = - ApprovalDistribution::new_with_clock(Metrics::default(), Default::default(), clock); + let subsystem = ApprovalDistribution::new_with_clock( + Metrics::default(), + Default::default(), + clock, + assignment_criteria, + ); { let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(12345); let mut session_info_provider = RuntimeInfo::new_with_config(RuntimeInfoConfig { @@ -81,7 +85,6 @@ fn test_harness>( &mut state, REPUTATION_CHANGE_TEST_INTERVAL, &mut rng, - assignment_criteria, &mut session_info_provider, ) .await; @@ -552,7 +555,7 @@ fn try_import_the_same_assignment() { let hash = Hash::repeat_byte(0xAA); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { @@ -658,7 +661,7 @@ fn try_import_the_same_assignment_v2() { let hash = Hash::repeat_byte(0xAA); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { @@ -769,7 +772,7 @@ fn delay_reputation_change() { let hash = Hash::repeat_byte(0xAA); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state_with_reputation_delay(), |mut virtual_overseer| async move { @@ -842,7 +845,7 @@ fn spam_attack_results_in_negative_reputation_change() { let hash_b = Hash::repeat_byte(0xBB); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { @@ -939,7 +942,7 @@ fn peer_sending_us_the_same_we_just_sent_them_is_ok() { let peer_a = peers.first().unwrap().0; let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { @@ -1040,7 +1043,7 @@ fn import_approval_happy_path_v1_v2_peers() { let candidate_hash = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xBB)); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { @@ -1180,7 +1183,7 @@ fn import_approval_happy_path_v2() { let candidate_hash_first = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xBB)); let candidate_hash_second = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xCC)); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { @@ -1311,7 +1314,7 @@ fn multiple_assignments_covered_with_one_approval_vote() { let candidate_hash_second = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xCC)); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { @@ -1521,7 +1524,7 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { let candidate_hash_second = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xCC)); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { @@ -1720,7 +1723,7 @@ fn import_approval_bad() { let diff_candidate_hash = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xCC)); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { @@ -1807,7 +1810,7 @@ fn update_our_view() { let hash_c = Hash::repeat_byte(0xCC); let state = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), State::default(), |mut virtual_overseer| async move { @@ -1855,7 +1858,7 @@ fn update_our_view() { assert!(state.blocks.get(&hash_c).is_some()); let state = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state, |mut virtual_overseer| async move { @@ -1874,7 +1877,7 @@ fn update_our_view() { assert!(state.blocks.get(&hash_c).is_some()); let state = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state, |mut virtual_overseer| async move { @@ -1902,7 +1905,7 @@ fn update_peer_view() { let peer = &peer_a; let state = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), State::default(), |mut virtual_overseer| async move { @@ -2001,7 +2004,7 @@ fn update_peer_view() { ); let state = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state, |mut virtual_overseer| async move { @@ -2061,7 +2064,7 @@ fn update_peer_view() { let finalized_number = 4_000_000_000; let state = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state, |mut virtual_overseer| async move { @@ -2103,7 +2106,7 @@ fn update_peer_authority_id() { let neighbour_y = peers.get(neighbour_y_index).unwrap().0; let _state = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), State::default(), |mut virtual_overseer| async move { @@ -2284,7 +2287,7 @@ fn import_remotely_then_locally() { let peer = &peer_a; let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { @@ -2390,7 +2393,7 @@ fn sends_assignments_even_when_state_is_approved() { let peer = &peer_a; let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), State::default(), |mut virtual_overseer| async move { @@ -2496,7 +2499,7 @@ fn sends_assignments_even_when_state_is_approved_v2() { let peer = &peer_a; let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), State::default(), |mut virtual_overseer| async move { @@ -2622,7 +2625,7 @@ fn race_condition_in_local_vs_remote_view_update() { let hash_b = Hash::repeat_byte(0xBB); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { @@ -2708,7 +2711,7 @@ fn propagates_locally_generated_assignment_to_both_dimensions() { let peers = make_peers_and_authority_ids(100); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), State::default(), |mut virtual_overseer| async move { @@ -2838,7 +2841,7 @@ fn propagates_assignments_along_unshared_dimension() { let peers = make_peers_and_authority_ids(100); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { @@ -2997,7 +3000,7 @@ fn propagates_to_required_after_connect() { let peers = make_peers_and_authority_ids(100); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), State::default(), |mut virtual_overseer| async move { @@ -3162,7 +3165,7 @@ fn sends_to_more_peers_after_getting_topology() { let peers = make_peers_and_authority_ids(100); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), State::default(), |mut virtual_overseer| async move { @@ -3300,7 +3303,7 @@ fn originator_aggression_l1() { let aggression_l1_threshold = state.aggression_config.l1_threshold.unwrap(); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state, |mut virtual_overseer| async move { @@ -3481,7 +3484,7 @@ fn non_originator_aggression_l1() { let aggression_l1_threshold = state.aggression_config.l1_threshold.unwrap(); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state, |mut virtual_overseer| async move { @@ -3606,7 +3609,7 @@ fn non_originator_aggression_l2() { let aggression_l1_threshold = state.aggression_config.l1_threshold.unwrap(); let aggression_l2_threshold = state.aggression_config.l2_threshold.unwrap(); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state, |mut virtual_overseer| async move { @@ -3791,7 +3794,7 @@ fn resends_messages_periodically() { state.aggression_config.l2_threshold = None; state.aggression_config.resend_unfinalized_period = Some(2); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state, |mut virtual_overseer| async move { @@ -3955,7 +3958,7 @@ fn import_versioned_approval() { let state = state_without_reputation_delay(); let candidate_hash = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xBB)); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(SystemClock {}), state, |mut virtual_overseer| async move { @@ -4130,6 +4133,7 @@ fn batch_test_round(message_count: usize) { Default::default(), Default::default(), Box::new(SystemClock {}), + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), ); let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(12345); let mut sender = context.sender().clone(); @@ -4143,7 +4147,6 @@ fn batch_test_round(message_count: usize) { &mut state, REPUTATION_CHANGE_TEST_INTERVAL, &mut rng, - &MockAssignmentCriteria { tranche: Ok(0) }, &mut session_info_provider, ); @@ -4294,13 +4297,13 @@ fn const_ensure_size_not_zero() { struct DummyClock; impl Clock for DummyClock { - fn tick_now(&self) -> polkadot_node_core_approval_voting::time::Tick { + fn tick_now(&self) -> polkadot_node_primitives::approval::time::Tick { 0 } fn wait( &self, - _tick: polkadot_node_core_approval_voting::time::Tick, + _tick: polkadot_node_primitives::approval::time::Tick, ) -> std::pin::Pin + Send + 'static>> { todo!() } @@ -4315,7 +4318,7 @@ fn subsystem_rejects_assignment_in_future() { let hash = Hash::repeat_byte(0xAA); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(89) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(89) }), Box::new(DummyClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { @@ -4379,9 +4382,9 @@ fn subsystem_rejects_bad_assignments() { let hash = Hash::repeat_byte(0xAA); let _ = test_harness( - &MockAssignmentCriteria { + Arc::new(MockAssignmentCriteria { tranche: Err(InvalidAssignment(criteria::InvalidAssignmentReason::NullAssignment)), - }, + }), Box::new(DummyClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { @@ -4444,7 +4447,7 @@ fn subsystem_rejects_wrong_claimed_assignments() { let hash = Hash::repeat_byte(0xAA); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(DummyClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { @@ -4528,7 +4531,7 @@ fn subsystem_accepts_tranche0_duplicate_assignments() { let candidate_hash_fourth = polkadot_primitives::CandidateHash(Hash::repeat_byte(0xBB)); let _ = test_harness( - &MockAssignmentCriteria { tranche: Ok(0) }, + Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), Box::new(DummyClock {}), state_without_reputation_delay(), |mut virtual_overseer| async move { diff --git a/polkadot/node/primitives/Cargo.toml b/polkadot/node/primitives/Cargo.toml index cd642bf16ff9..7185205f905b 100644 --- a/polkadot/node/primitives/Cargo.toml +++ b/polkadot/node/primitives/Cargo.toml @@ -12,11 +12,13 @@ workspace = true [dependencies] bounded-vec = { workspace = true } futures = { workspace = true } +futures-timer = { workspace = true } polkadot-primitives = { workspace = true, default-features = true } codec = { features = ["derive"], workspace = true } sp-core = { workspace = true, default-features = true } sp-application-crypto = { workspace = true, default-features = true } sp-consensus-babe = { workspace = true, default-features = true } +sp-consensus-slots = { workspace = true } sp-keystore = { workspace = true, default-features = true } sp-maybe-compressed-blob = { workspace = true, default-features = true } sp-runtime = { workspace = true, default-features = true } @@ -25,6 +27,7 @@ schnorrkel = { workspace = true, default-features = true } thiserror = { workspace = true } bitvec = { features = ["alloc"], workspace = true } serde = { features = ["derive"], workspace = true, default-features = true } +sc-keystore = { workspace = true } [target.'cfg(not(target_os = "unknown"))'.dependencies] zstd = { version = "0.12.4", default-features = false } diff --git a/polkadot/node/primitives/src/approval/criteria.rs b/polkadot/node/primitives/src/approval/criteria.rs new file mode 100644 index 000000000000..0a1a0ee2367f --- /dev/null +++ b/polkadot/node/primitives/src/approval/criteria.rs @@ -0,0 +1,177 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Assignment criteria VRF generation and checking interfaces. + +use crate::approval::{ + v1::{DelayTranche, RelayVRFStory}, + v2::{AssignmentCertV2, CoreBitfield}, +}; +use codec::{Decode, Encode}; +use polkadot_primitives::{ + AssignmentId, CandidateHash, CoreIndex, GroupIndex, IndexedVec, SessionInfo, ValidatorIndex, +}; +use sc_keystore::LocalKeystore; + +use std::collections::HashMap; + +/// Details pertaining to our assignment on a block. +#[derive(Debug, Clone, Encode, Decode, PartialEq)] +pub struct OurAssignment { + cert: AssignmentCertV2, + tranche: DelayTranche, + validator_index: ValidatorIndex, + // Whether the assignment has been triggered already. + triggered: bool, +} + +impl OurAssignment { + /// Create a new `OurAssignment`. + pub fn new( + cert: AssignmentCertV2, + tranche: DelayTranche, + validator_index: ValidatorIndex, + triggered: bool, + ) -> Self { + OurAssignment { cert, tranche, validator_index, triggered } + } + /// Returns a reference to the assignment cert. + pub fn cert(&self) -> &AssignmentCertV2 { + &self.cert + } + + /// Returns the assignment cert. + pub fn into_cert(self) -> AssignmentCertV2 { + self.cert + } + + /// Returns the delay tranche of the assignment. + pub fn tranche(&self) -> DelayTranche { + self.tranche + } + + /// Returns the validator index of the assignment. + pub fn validator_index(&self) -> ValidatorIndex { + self.validator_index + } + + /// Returns whether the assignment has been triggered. + pub fn triggered(&self) -> bool { + self.triggered + } + + /// Marks the assignment as triggered. + pub fn mark_triggered(&mut self) { + self.triggered = true; + } +} + +/// Information about the world assignments are being produced in. +#[derive(Clone, Debug)] +pub struct Config { + /// The assignment public keys for validators. + pub assignment_keys: Vec, + /// The groups of validators assigned to each core. + pub validator_groups: IndexedVec>, + /// The number of availability cores used by the protocol during this session. + pub n_cores: u32, + /// The zeroth delay tranche width. + pub zeroth_delay_tranche_width: u32, + /// The number of samples we do of `relay_vrf_modulo`. + pub relay_vrf_modulo_samples: u32, + /// The number of delay tranches in total. + pub n_delay_tranches: u32, +} + +impl<'a> From<&'a SessionInfo> for Config { + fn from(s: &'a SessionInfo) -> Self { + Config { + assignment_keys: s.assignment_keys.clone(), + validator_groups: s.validator_groups.clone(), + n_cores: s.n_cores, + zeroth_delay_tranche_width: s.zeroth_delay_tranche_width, + relay_vrf_modulo_samples: s.relay_vrf_modulo_samples, + n_delay_tranches: s.n_delay_tranches, + } + } +} + +/// A trait for producing and checking assignments. +/// +/// Approval voting subsystem implements a a real implemention +/// for it and tests use a mock implementation. +pub trait AssignmentCriteria { + /// Compute the assignments for the given relay VRF story. + fn compute_assignments( + &self, + keystore: &LocalKeystore, + relay_vrf_story: RelayVRFStory, + config: &Config, + leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>, + enable_v2_assignments: bool, + ) -> HashMap; + + /// Check the assignment cert for the given relay VRF story and returns the delay tranche. + fn check_assignment_cert( + &self, + claimed_core_bitfield: CoreBitfield, + validator_index: ValidatorIndex, + config: &Config, + relay_vrf_story: RelayVRFStory, + assignment: &AssignmentCertV2, + // Backing groups for each "leaving core". + backing_groups: Vec, + ) -> Result; +} + +/// Assignment invalid. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct InvalidAssignment(pub InvalidAssignmentReason); + +impl std::fmt::Display for InvalidAssignment { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "Invalid Assignment: {:?}", self.0) + } +} + +impl std::error::Error for InvalidAssignment {} + +/// Failure conditions when checking an assignment cert. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InvalidAssignmentReason { + /// The validator index is out of bounds. + ValidatorIndexOutOfBounds, + /// Sample index is out of bounds. + SampleOutOfBounds, + /// Core index is out of bounds. + CoreIndexOutOfBounds, + /// Invalid assignment key. + InvalidAssignmentKey, + /// Node is in backing group. + IsInBackingGroup, + /// Modulo core index mismatch. + VRFModuloCoreIndexMismatch, + /// Modulo output mismatch. + VRFModuloOutputMismatch, + /// Delay core index mismatch. + VRFDelayCoreIndexMismatch, + /// Delay output mismatch. + VRFDelayOutputMismatch, + /// Invalid arguments + InvalidArguments, + /// Assignment vrf check resulted in 0 assigned cores. + NullAssignment, +} diff --git a/polkadot/node/primitives/src/approval.rs b/polkadot/node/primitives/src/approval/mod.rs similarity index 99% rename from polkadot/node/primitives/src/approval.rs rename to polkadot/node/primitives/src/approval/mod.rs index ec41647fc3d1..79f4cfa9e0be 100644 --- a/polkadot/node/primitives/src/approval.rs +++ b/polkadot/node/primitives/src/approval/mod.rs @@ -16,6 +16,12 @@ //! Types relevant for approval. +/// Criteria for assignment. +pub mod criteria; + +/// Time utilities for approval voting. +pub mod time; + /// A list of primitives introduced in v1. pub mod v1 { use sp_consensus_babe as babe_primitives; diff --git a/polkadot/node/core/approval-voting/src/time.rs b/polkadot/node/primitives/src/approval/time.rs similarity index 95% rename from polkadot/node/core/approval-voting/src/time.rs rename to polkadot/node/primitives/src/approval/time.rs index 5c3e7e85a17a..465aae23c90e 100644 --- a/polkadot/node/core/approval-voting/src/time.rs +++ b/polkadot/node/primitives/src/approval/time.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -//! Time utilities for approval voting. +//! Time utilities for approval voting subsystems. use futures::{ future::BoxFuture, @@ -23,7 +23,7 @@ use futures::{ Stream, StreamExt, }; -use polkadot_node_primitives::approval::v1::DelayTranche; +use crate::approval::v1::DelayTranche; use sp_consensus_slots::Slot; use std::{ collections::HashSet, @@ -33,11 +33,15 @@ use std::{ }; use polkadot_primitives::{Hash, ValidatorIndex}; +/// The duration of a single tick in milliseconds. pub const TICK_DURATION_MILLIS: u64 = 500; /// A base unit of time, starting from the Unix epoch, split into half-second intervals. pub type Tick = u64; +/// How far in the future a tick can be accepted. +pub const TICK_TOO_FAR_IN_FUTURE: Tick = 20; // 10 seconds. + /// A clock which allows querying of the current tick as well as /// waiting for a tick to be reached. pub trait Clock { @@ -50,6 +54,7 @@ pub trait Clock { /// Extension methods for clocks. pub trait ClockExt { + /// Returns the current tranche. fn tranche_now(&self, slot_duration_millis: u64, base_slot: Slot) -> DelayTranche; } @@ -124,7 +129,7 @@ impl DelayedApprovalTimer { /// /// Guarantees that if a timer already exits for the give block hash, /// no additional timer is started. - pub(crate) fn maybe_arm_timer( + pub fn maybe_arm_timer( &mut self, wait_until: Tick, clock: &dyn Clock, @@ -173,7 +178,7 @@ mod tests { use futures_timer::Delay; use polkadot_primitives::{Hash, ValidatorIndex}; - use crate::time::{Clock, SystemClock}; + use crate::approval::time::{Clock, SystemClock}; use super::DelayedApprovalTimer; diff --git a/polkadot/node/service/src/overseer.rs b/polkadot/node/service/src/overseer.rs index 5dab221b040d..0b57ff6e395f 100644 --- a/polkadot/node/service/src/overseer.rs +++ b/polkadot/node/service/src/overseer.rs @@ -20,7 +20,7 @@ use polkadot_overseer::{DummySubsystem, InitializedOverseerBuilder, SubsystemErr use sp_core::traits::SpawnNamed; use polkadot_availability_distribution::IncomingRequestReceivers; -use polkadot_node_core_approval_voting::Config as ApprovalVotingConfig; +use polkadot_node_core_approval_voting::{Config as ApprovalVotingConfig, RealAssignmentCriteria}; use polkadot_node_core_av_store::Config as AvailabilityConfig; use polkadot_node_core_candidate_validation::Config as CandidateValidationConfig; use polkadot_node_core_chain_selection::Config as ChainSelectionConfig; @@ -312,6 +312,7 @@ where .approval_distribution(ApprovalDistributionSubsystem::new( Metrics::register(registry)?, approval_voting_config.slot_duration_millis, + Arc::new(RealAssignmentCriteria {}), )) .approval_voting(ApprovalVotingSubsystem::with_config( approval_voting_config, diff --git a/polkadot/node/subsystem-bench/src/lib/approval/helpers.rs b/polkadot/node/subsystem-bench/src/lib/approval/helpers.rs index ca58875c8139..4b2b91696824 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/helpers.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/helpers.rs @@ -16,11 +16,11 @@ use crate::configuration::TestAuthorities; use itertools::Itertools; -use polkadot_node_core_approval_voting::time::{Clock, SystemClock, Tick}; use polkadot_node_network_protocol::{ grid_topology::{SessionGridTopology, TopologyPeerInfo}, View, }; +use polkadot_node_primitives::approval::time::{Clock, SystemClock, Tick}; use polkadot_node_subsystem_types::messages::{ network_bridge_event::NewGossipTopology, ApprovalDistributionMessage, NetworkBridgeEvent, }; diff --git a/polkadot/node/subsystem-bench/src/lib/approval/message_generator.rs b/polkadot/node/subsystem-bench/src/lib/approval/message_generator.rs index 6d3e7dd92db1..da25a3bf3b79 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/message_generator.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/message_generator.rs @@ -28,16 +28,15 @@ use crate::{ use codec::Encode; use futures::SinkExt; use itertools::Itertools; -use polkadot_node_core_approval_voting::{ - criteria::{compute_assignments, Config}, - time::tranche_to_tick, -}; +use polkadot_node_core_approval_voting::criteria::{compute_assignments, Config}; + use polkadot_node_network_protocol::{ grid_topology::{GridNeighbors, RandomRouting, RequiredRouting, SessionGridTopology}, v3 as protocol_v3, }; use polkadot_node_primitives::approval::{ self, + time::tranche_to_tick, v2::{CoreBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2}, }; use polkadot_primitives::{ diff --git a/polkadot/node/subsystem-bench/src/lib/approval/mock_chain_selection.rs b/polkadot/node/subsystem-bench/src/lib/approval/mock_chain_selection.rs index 77ba80d4b2bb..709d56d52f0b 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/mock_chain_selection.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/mock_chain_selection.rs @@ -16,7 +16,7 @@ use crate::approval::{ApprovalTestState, PastSystemClock, LOG_TARGET, SLOT_DURATION_MILLIS}; use futures::FutureExt; -use polkadot_node_core_approval_voting::time::{slot_number_to_tick, Clock, TICK_DURATION_MILLIS}; +use polkadot_node_primitives::approval::time::{slot_number_to_tick, Clock, TICK_DURATION_MILLIS}; use polkadot_node_subsystem::{overseer, SpawnedSubsystem, SubsystemError}; use polkadot_node_subsystem_types::messages::ChainSelectionMessage; diff --git a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs index 73a305dae4fb..f05d061f3fde 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs @@ -49,9 +49,13 @@ use itertools::Itertools; use orchestra::TimeoutExt; use overseer::{metrics::Metrics as OverseerMetrics, MetricsTrait}; use polkadot_approval_distribution::ApprovalDistribution; +use polkadot_node_primitives::approval::time::{ + slot_number_to_tick, tick_to_slot_number, Clock, ClockExt, SystemClock, +}; + use polkadot_node_core_approval_voting::{ - time::{slot_number_to_tick, tick_to_slot_number, Clock, ClockExt, SystemClock}, ApprovalVotingSubsystem, Config as ApprovalVotingConfig, Metrics as ApprovalVotingMetrics, + RealAssignmentCriteria, }; use polkadot_node_network_protocol::v3 as protocol_v3; use polkadot_node_primitives::approval::{self, v1::RelayVRFStory}; @@ -817,6 +821,7 @@ fn build_overseer( Metrics::register(Some(&dependencies.registry)).unwrap(), SLOT_DURATION_MILLIS, Box::new(system_clock.clone()), + Arc::new(RealAssignmentCriteria {}), ); let mock_chain_api = MockChainApi::new(state.build_chain_api_state()); let mock_chain_selection = MockChainSelection { state: state.clone(), clock: system_clock }; From 377f4e1bcd1434a5dc3ecaef2f554afa2e9f168c Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 27 Aug 2024 16:11:56 +0300 Subject: [PATCH 24/45] Address review feedback Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 5abc9b5252f7..8a6e64570096 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -1388,6 +1388,8 @@ pub async fn start_approval_worker< sync_oracle: Box, metrics: Metrics, spawner: Arc, + task_name: &'static str, + group_name: &'static str, clock: Arc, ) -> SubsystemResult<()> { let approval_voting = ApprovalVotingSubsystem::with_config_and_clock( @@ -1402,8 +1404,8 @@ pub async fn start_approval_worker< let backend = DbBackend::new(db.clone(), approval_voting.db_config); let spawner = approval_voting.spawner.clone(); spawner.spawn_blocking( - "approval-voting-rewrite-db", - Some("approval-voting-rewrite-subsystem"), + task_name, + Some(group_name), Box::pin(async move { if let Err(err) = run( work_provider, @@ -2013,14 +2015,9 @@ async fn handle_from_overseer< }, FromOrchestra::Communication { msg } => match msg { ApprovalVotingMessage::ImportAssignment(checked_assignment, tx) => { - let (check_outcome, actions) = import_assignment( - sender, - state, - db, - session_info_provider, - checked_assignment, - ) - .await?; + let (check_outcome, actions) = + import_assignment(sender, state, db, session_info_provider, checked_assignment) + .await?; // approval-distribution makes sure this assignment is valid and expected, // so this import should never fail, if it does it might mean one of two things, // there is a bug in the code or the two subsystems got out of sync. From 3b3299e37df83951df1cd0d184444997eb22d638 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 27 Aug 2024 17:38:05 +0300 Subject: [PATCH 25/45] Make subsystem-bench happy Signed-off-by: Alexandru Gheorghe --- .../approval-voting/benches/approval-voting-regression-bench.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/polkadot/node/core/approval-voting/benches/approval-voting-regression-bench.rs b/polkadot/node/core/approval-voting/benches/approval-voting-regression-bench.rs index a18e667253d0..941ed71d0166 100644 --- a/polkadot/node/core/approval-voting/benches/approval-voting-regression-bench.rs +++ b/polkadot/node/core/approval-voting/benches/approval-voting-regression-bench.rs @@ -53,6 +53,7 @@ fn main() -> Result<(), String> { stop_when_approved: false, workdir_prefix: "/tmp".to_string(), num_no_shows_per_candidate: 0, + approval_voting_parallel_enabled: false, }; println!("Benchmarking..."); From 3fe407668df726f5369148b310d289684ad3b6ff Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 9 Sep 2024 15:07:35 +0300 Subject: [PATCH 26/45] Address review feedback Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 4 +-- .../core/approval-voting-parallel/Cargo.toml | 2 +- .../core/approval-voting-parallel/src/lib.rs | 27 ++++++++++------ .../approval-voting-parallel/src/tests.rs | 32 ++++++++----------- .../approval-voting-regression-bench.rs | 2 +- polkadot/node/primitives/src/approval/mod.rs | 2 +- .../subsystem-bench/src/lib/approval/mod.rs | 18 ++++++----- polkadot/node/subsystem-types/src/messages.rs | 2 +- 8 files changed, 46 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9582b294aa84..5b0621d0f27c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5310,7 +5310,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" dependencies = [ "log", - "regex", ] [[package]] @@ -5345,7 +5344,6 @@ dependencies = [ "anstream", "anstyle", "env_filter", - "humantime", "log", ] @@ -13399,7 +13397,6 @@ dependencies = [ "async-trait", "bitvec", "derive_more", - "env_logger 0.11.3", "futures", "futures-timer", "itertools 0.11.0", @@ -13436,6 +13433,7 @@ dependencies = [ "sp-keyring", "sp-keystore", "sp-runtime", + "sp-tracing 16.0.0", "thiserror", "tracing-gum", ] diff --git a/polkadot/node/core/approval-voting-parallel/Cargo.toml b/polkadot/node/core/approval-voting-parallel/Cargo.toml index 10653cd21891..77c0bd0d6972 100644 --- a/polkadot/node/core/approval-voting-parallel/Cargo.toml +++ b/polkadot/node/core/approval-voting-parallel/Cargo.toml @@ -56,12 +56,12 @@ sp-keyring = { workspace = true, default-features = true } sp-keystore = {workspace = true, default-features = true} sp-core = { workspace = true, default-features = true} sp-consensus-babe = { workspace = true, default-features = true } +sp-tracing = { workspace = true } polkadot-node-subsystem-test-helpers = { workspace = true, default-features = true} assert_matches = "1.4.0" kvdb-memorydb = "0.13.0" polkadot-primitives-test-helpers = { workspace = true, default-features = true } log = { workspace = true, default-features = true } -env_logger = "0.11" polkadot-subsystem-bench = { workspace = true, default-features = true} diff --git a/polkadot/node/core/approval-voting-parallel/src/lib.rs b/polkadot/node/core/approval-voting-parallel/src/lib.rs index 4582a2b04351..f42df8714808 100644 --- a/polkadot/node/core/approval-voting-parallel/src/lib.rs +++ b/polkadot/node/core/approval-voting-parallel/src/lib.rs @@ -70,7 +70,7 @@ pub(crate) const LOG_TARGET: &str = "parachain::approval-voting-parallel"; // lock issues for example. const WAIT_FOR_SIGS_GATHER_TIMEOUT: Duration = Duration::from_millis(2000); -/// The approval voting subsystem. +/// The approval voting parallel subsystem. pub struct ApprovalVotingParallelSubsystem { /// `LocalKeystore` is needed for assignment keys, but not necessarily approval keys. /// @@ -109,7 +109,7 @@ impl ApprovalVotingParallelSubsystem { ) } - /// Create a new approval voting subsystem with the given keystore, config, and database. + /// Create a new approval voting subsystem with the given keystore, config, clock, and database. pub fn with_config_and_clock( config: Config, db: Arc, @@ -150,6 +150,7 @@ pub const APPROVAL_DISTRIBUTION_WORKER_COUNT: usize = 4; /// The channel size for the workers. pub const WORKERS_CHANNEL_SIZE: usize = 64000 / APPROVAL_DISTRIBUTION_WORKER_COUNT; + fn prio_right<'a>(_val: &'a mut ()) -> PollNext { PollNext::Right } @@ -363,8 +364,7 @@ async fn run_main_loop( } }, ApprovalVotingParallelMessage::DistributeAssignment(assignment, claimed) => { - let worker_index = assignment.validator.0 as usize % to_approval_distribution_workers.len(); - let worker = to_approval_distribution_workers.get_mut(worker_index).expect("Worker index is obtained modulo len; qed"); + let worker = assigned_worker_for_validator(assignment.validator, &mut to_approval_distribution_workers); worker .send_message( ApprovalDistributionMessage::DistributeAssignment(assignment, claimed) @@ -373,8 +373,7 @@ async fn run_main_loop( }, ApprovalVotingParallelMessage::DistributeApproval(vote) => { - let worker_index = vote.validator.0 as usize % to_approval_distribution_workers.len(); - let worker = to_approval_distribution_workers.get_mut(worker_index).expect("Worker index is obtained modulo len; qed"); + let worker = assigned_worker_for_validator(vote.validator, &mut to_approval_distribution_workers); worker .send_message( ApprovalDistributionMessage::DistributeApproval(vote) @@ -390,8 +389,7 @@ async fn run_main_loop( let (all_msgs_from_same_validator, messages_split_by_validator) = validator_index_for_msg(msg); for (validator_index, msg) in all_msgs_from_same_validator.into_iter().chain(messages_split_by_validator.into_iter().flatten()) { - let worker_index = validator_index.0 as usize % to_approval_distribution_workers.len(); - let worker = to_approval_distribution_workers.get_mut(worker_index).expect("Worker index is obtained modulo len; qed"); + let worker = assigned_worker_for_validator(validator_index, &mut to_approval_distribution_workers); worker .send_message( @@ -494,11 +492,22 @@ async fn handle_get_approval_signatures( } } +// Returns the worker that should receive the message for the given validator. +fn assigned_worker_for_validator( + validator: ValidatorIndex, + to_approval_distribution_workers: &mut Vec>, +) -> &mut ToWorker { + let worker_index = validator.0 as usize % to_approval_distribution_workers.len(); + to_approval_distribution_workers + .get_mut(worker_index) + .expect("Worker index is obtained modulo len; qed") +} + // Returns the validators that initially created this assignments/votes, the validator index // is later used to decide which approval-distribution worker should receive the message. // // Because this is on the hot path and we don't want to be unnecessarily slow, it contains two logic -// paths. The ultra fast path where all messages have the same validator index and we don't don't do +// paths. The ultra fast path where all messages have the same validator index and we don't do // any cloning or allocation and the path where we need to split the messages into multiple // messages, because they have different validator indices, where we do need to clone and allocate. // In practice most of the message will fall on the ultra fast path. diff --git a/polkadot/node/core/approval-voting-parallel/src/tests.rs b/polkadot/node/core/approval-voting-parallel/src/tests.rs index 9a16b66dd615..bed591b5f210 100644 --- a/polkadot/node/core/approval-voting-parallel/src/tests.rs +++ b/polkadot/node/core/approval-voting-parallel/src/tests.rs @@ -7,7 +7,7 @@ use std::{ use crate::{ build_worker_handles, metrics::MetricsWatcher, prio_right, run_main_loop, start_workers, - validator_index_for_msg, ApprovalVotingParallelSubsystem, Metrics, WorkProvider, LOG_TARGET, + validator_index_for_msg, ApprovalVotingParallelSubsystem, Metrics, WorkProvider, }; use assert_matches::assert_matches; use futures::{channel::oneshot, future, stream::PollNext, StreamExt}; @@ -99,11 +99,7 @@ fn build_subsystem( TestSubsystemContext>, VirtualOverseer, ) { - let _ = env_logger::builder() - .is_test(true) - .filter(Some("polkadot_node_core_approval_voting"), log::LevelFilter::Trace) - .filter(Some(LOG_TARGET), log::LevelFilter::Trace) - .try_init(); + sp_tracing::init_for_tests(); let pool = sp_core::testing::TaskExecutor::new(); let (context, virtual_overseer) = polkadot_node_subsystem_test_helpers::make_subsystem_context::< @@ -278,11 +274,11 @@ fn test_main_loop_forwards_correctly() { ))); overseer_signal(&mut overseer, signal.clone()).await; let approval_voting_receives = approval_voting_work_provider.recv().await.unwrap(); - assert!(matches!(approval_voting_receives, FromOrchestra::Signal(_))); + assert_matches!(approval_voting_receives, FromOrchestra::Signal(_)); for rx_approval_distribution_worker in rx_approval_distribution_workers.iter_mut() { let approval_distribution_receives = rx_approval_distribution_worker.next().await.unwrap(); - assert!(matches!(approval_distribution_receives, FromOrchestra::Signal(_))); + assert_matches!(approval_distribution_receives, FromOrchestra::Signal(_)); } let (test_tx, _rx) = oneshot::channel(); @@ -687,18 +683,17 @@ fn test_signal_before_message_keeps_receive_order() { .await; let approval_voting_receives = approval_voting_work_provider.recv().await.unwrap(); - assert!(matches!(approval_voting_receives, FromOrchestra::Signal(_))); + assert_matches!(approval_voting_receives, FromOrchestra::Signal(_)); let rx_approval_distribution_worker = rx_approval_distribution_workers .get_mut(validator_index.0 as usize % num_approval_distro_workers) .unwrap(); let approval_distribution_receives = rx_approval_distribution_worker.next().await.unwrap(); - assert!(matches!(approval_distribution_receives, FromOrchestra::Signal(_))); + assert_matches!(approval_distribution_receives, FromOrchestra::Signal(_)); assert_matches!( rx_approval_distribution_worker.next().await.unwrap(), FromOrchestra::Communication { msg: ApprovalDistributionMessage::DistributeAssignment(_, _) - } => { } ); @@ -734,18 +729,17 @@ fn test_signal_is_prioritized_when_unread_messages_in_the_queue() { overseer_signal(&mut overseer, signal.clone()).await; let approval_voting_receives = approval_voting_work_provider.recv().await.unwrap(); - assert!(matches!(approval_voting_receives, FromOrchestra::Signal(_))); + assert_matches!(approval_voting_receives, FromOrchestra::Signal(_)); let rx_approval_distribution_worker = rx_approval_distribution_workers .get_mut(validator_index.0 as usize % num_approval_distro_workers) .unwrap(); let approval_distribution_receives = rx_approval_distribution_worker.next().await.unwrap(); - assert!(matches!(approval_distribution_receives, FromOrchestra::Signal(_))); + assert_matches!(approval_distribution_receives, FromOrchestra::Signal(_)); assert_matches!( rx_approval_distribution_worker.next().await.unwrap(), FromOrchestra::Communication { msg: ApprovalDistributionMessage::DistributeAssignment(_, _) - } => { } ); @@ -1016,7 +1010,7 @@ fn test_validator_index_with_messages_from_different_validators() { ); let result = validator_index_for_msg(v1_assignment.clone()); - assert!(matches!(result, (None, Some(_)))); + assert_matches!(result, (None, Some(_))); let messsages_split_by_validator = result.1.unwrap(); assert_eq!(messsages_split_by_validator.len(), assignments.len()); for (index, (validator_index, message)) in messsages_split_by_validator.into_iter().enumerate() @@ -1039,7 +1033,7 @@ fn test_validator_index_with_messages_from_different_validators() { ); let result = validator_index_for_msg(v2_assignment.clone()); - assert!(matches!(result, (None, Some(_)))); + assert_matches!(result, (None, Some(_))); let messsages_split_by_validator = result.1.unwrap(); assert_eq!(messsages_split_by_validator.len(), assignments.len()); for (index, (validator_index, message)) in messsages_split_by_validator.into_iter().enumerate() @@ -1089,7 +1083,7 @@ fn test_validator_index_with_messages_from_different_validators() { ); let result = validator_index_for_msg(v2_approvals.clone()); - assert!(matches!(result, (None, Some(_)))); + assert_matches!(result, (None, Some(_))); let messsages_split_by_validator = result.1.unwrap(); assert_eq!(messsages_split_by_validator.len(), approvals.len()); for (index, (validator_index, message)) in messsages_split_by_validator.into_iter().enumerate() @@ -1112,7 +1106,7 @@ fn test_validator_index_with_messages_from_different_validators() { ); let result = validator_index_for_msg(v3_assignment.clone()); - assert!(matches!(result, (None, Some(_)))); + assert_matches!(result, (None, Some(_))); let messsages_split_by_validator = result.1.unwrap(); assert_eq!(messsages_split_by_validator.len(), v2_assignments.len()); for (index, (validator_index, message)) in messsages_split_by_validator.into_iter().enumerate() @@ -1149,7 +1143,7 @@ fn test_validator_index_with_messages_from_different_validators() { ); let result = validator_index_for_msg(v3_approvals.clone()); - assert!(matches!(result, (None, Some(_)))); + assert_matches!(result, (None, Some(_))); let messsages_split_by_validator = result.1.unwrap(); assert_eq!(messsages_split_by_validator.len(), approvals.len()); for (index, (validator_index, message)) in messsages_split_by_validator.into_iter().enumerate() diff --git a/polkadot/node/core/approval-voting/benches/approval-voting-regression-bench.rs b/polkadot/node/core/approval-voting/benches/approval-voting-regression-bench.rs index bc813f663743..db0396a83193 100644 --- a/polkadot/node/core/approval-voting/benches/approval-voting-regression-bench.rs +++ b/polkadot/node/core/approval-voting/benches/approval-voting-regression-bench.rs @@ -53,7 +53,7 @@ fn main() -> Result<(), String> { stop_when_approved: false, workdir_prefix: "/tmp".to_string(), num_no_shows_per_candidate: 0, - approval_voting_parallel_enabled: false, + approval_voting_parallel_enabled: true, }; println!("Benchmarking..."); diff --git a/polkadot/node/primitives/src/approval/mod.rs b/polkadot/node/primitives/src/approval/mod.rs index acdbc2098571..42342f9889a9 100644 --- a/polkadot/node/primitives/src/approval/mod.rs +++ b/polkadot/node/primitives/src/approval/mod.rs @@ -24,7 +24,7 @@ pub mod time; /// A list of primitives introduced in v1. pub mod v1 { - use sp_consensus_babe::{self as babe_primitives}; + use sp_consensus_babe as babe_primitives; pub use sp_consensus_babe::{ Randomness, Slot, VrfPreOutput, VrfProof, VrfSignature, VrfTranscript, }; diff --git a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs index f817cb3b2aae..423a7be567a7 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs @@ -13,6 +13,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . + use crate::{ approval::{ helpers::{ @@ -461,6 +462,14 @@ impl ApprovalTestState { }) .collect() } + + fn subsystem_name(&self) -> &'static str { + if self.options.approval_voting_parallel_enabled { + "approval-voting-parallel-subsystem" + } else { + "approval-distribution-subsystem" + } + } } impl ApprovalTestState { @@ -1116,14 +1125,7 @@ pub async fn bench_approvals_run( state.total_sent_messages_to_node.load(std::sync::atomic::Ordering::SeqCst) as usize; env.wait_until_metric( "polkadot_parachain_subsystem_bounded_received", - Some(( - "subsystem_name", - if state.options.approval_voting_parallel_enabled { - "approval-voting-parallel-subsystem" - } else { - "approval-distribution-subsystem" - }, - )), + Some(("subsystem_name", state.subsystem_name())), |value| { gum::debug!(target: LOG_TARGET, ?value, ?at_least_messages, "Waiting metric"); value >= at_least_messages as f64 diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index 26f74c8a626b..38bdb7fd5d52 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -962,7 +962,7 @@ pub struct BlockDescription { /// The reason this exists is, so that we can keep both mods of running in the same polkadot binary, /// based on the value of `--approval-voting-parallel-enabled`, we decide if we run with two /// different subsystems for approval-distribution and approval-voting or run the approval-voting -/// parallel which has several parallel workers for the approval-distribution and an worker for +/// parallel which has several parallel workers for the approval-distribution and a worker for /// approval-voting. /// /// This is meant to be a temporary state until we can safely remove running the two subsystems From df08769ffc7306419d0fb77e73e5583688729d8b Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 10 Sep 2024 14:55:18 +0300 Subject: [PATCH 27/45] Remove subsystem_enabled and subystem_disabled flags Signed-off-by: Alexandru Gheorghe --- .../core/approval-voting-parallel/src/lib.rs | 24 +- .../approval-voting-parallel/src/metrics.rs | 28 +-- .../approval-voting-parallel/src/tests.rs | 2 - polkadot/node/core/approval-voting/src/lib.rs | 36 +-- .../node/core/approval-voting/src/tests.rs | 1 - .../network/approval-distribution/src/lib.rs | 10 +- .../approval-distribution/src/tests.rs | 5 +- polkadot/node/service/src/lib.rs | 1 - polkadot/node/service/src/overseer.rs | 235 +++++++++++++++++- .../subsystem-bench/src/lib/approval/mod.rs | 3 - polkadot/node/subsystem-types/src/messages.rs | 4 +- .../0016-approval-voting-parallel.toml | 2 +- 12 files changed, 257 insertions(+), 94 deletions(-) diff --git a/polkadot/node/core/approval-voting-parallel/src/lib.rs b/polkadot/node/core/approval-voting-parallel/src/lib.rs index f42df8714808..7d1889965a0f 100644 --- a/polkadot/node/core/approval-voting-parallel/src/lib.rs +++ b/polkadot/node/core/approval-voting-parallel/src/lib.rs @@ -83,7 +83,6 @@ pub struct ApprovalVotingParallelSubsystem { metrics: Metrics, spawner: Arc, clock: Arc, - subsystem_enabled: bool, } impl ApprovalVotingParallelSubsystem { @@ -95,7 +94,6 @@ impl ApprovalVotingParallelSubsystem { sync_oracle: Box, metrics: Metrics, spawner: impl overseer::gen::Spawner + 'static + Clone, - subsystem_enabled: bool, ) -> Self { ApprovalVotingParallelSubsystem::with_config_and_clock( config, @@ -105,7 +103,6 @@ impl ApprovalVotingParallelSubsystem { metrics, Arc::new(SystemClock {}), spawner, - subsystem_enabled, ) } @@ -118,7 +115,6 @@ impl ApprovalVotingParallelSubsystem { metrics: Metrics, clock: Arc, spawner: impl overseer::gen::Spawner + 'static, - subsystem_enabled: bool, ) -> Self { ApprovalVotingParallelSubsystem { keystore, @@ -129,7 +125,6 @@ impl ApprovalVotingParallelSubsystem { metrics, spawner: Arc::new(spawner), clock, - subsystem_enabled, } } } @@ -187,7 +182,6 @@ where subsystem.slot_duration_millis, subsystem.clock.clone(), Arc::new(RealAssignmentCriteria {}), - false, ); let task_name = format!("approval-voting-parallel-{}", i); @@ -277,11 +271,9 @@ async fn run( mut ctx: Context, subsystem: ApprovalVotingParallelSubsystem, ) -> SubsystemResult<()> { - let subsystem_enabled = subsystem.subsystem_enabled; let mut metrics_watcher = MetricsWatcher::new(subsystem.metrics.clone()); gum::info!( target: LOG_TARGET, - subsystem_enabled, "Starting workers" ); @@ -290,26 +282,18 @@ async fn run( gum::info!( target: LOG_TARGET, - subsystem_enabled, "Starting main subsystem loop" ); // Main loop of the subsystem, it shouldn't include any logic just dispatching of messages to // the workers. - run_main_loop( - ctx, - subsystem_enabled, - to_approval_voting_worker, - to_approval_distribution_workers, - metrics_watcher, - ) - .await + run_main_loop(ctx, to_approval_voting_worker, to_approval_distribution_workers, metrics_watcher) + .await } #[overseer::contextbounds(ApprovalVotingParallel, prefix = self::overseer)] async fn run_main_loop( mut ctx: Context, - subsystem_enabled: bool, mut to_approval_voting_worker: ToWorker, mut to_approval_distribution_workers: Vec>, metrics_watcher: MetricsWatcher, @@ -324,10 +308,6 @@ async fn run_main_loop( return Err(err); } }; - if !subsystem_enabled { - gum::trace!(target: LOG_TARGET, ?next_msg, "Parallel processing is not enabled skipping message"); - continue; - } match next_msg { FromOrchestra::Signal(msg) => { diff --git a/polkadot/node/core/approval-voting-parallel/src/metrics.rs b/polkadot/node/core/approval-voting-parallel/src/metrics.rs index b5ec6ec01277..1b4ab4bd9b88 100644 --- a/polkadot/node/core/approval-voting-parallel/src/metrics.rs +++ b/polkadot/node/core/approval-voting-parallel/src/metrics.rs @@ -81,8 +81,8 @@ impl metrics::Metrics for Metrics { to_worker_bounded_tof: prometheus::register( prometheus::HistogramVec::new( prometheus::HistogramOpts::new( - "polkadot_parachain_worker_bounded_tof", - "Duration spent in a particular channel from entrance to removal", + "polkadot_approval_voting_parallel_worker_bounded_tof", + "Duration spent in a particular approval voting worker channel from entrance to removal", ) .buckets(vec![ 0.0001, 0.0004, 0.0016, 0.0064, 0.0256, 0.1024, 0.4096, 1.6384, 3.2768, @@ -95,8 +95,8 @@ impl metrics::Metrics for Metrics { to_worker_bounded_sent: prometheus::register( prometheus::GaugeVec::::new( prometheus::Opts::new( - "polkadot_parachain_worker_bounded_sent", - "Number of elements sent to subsystems' bounded queues", + "polkadot_approval_voting_parallel_worker_bounded_sent", + "Number of elements sent to approval voting workers' bounded queues", ), &["worker_name"], )?, @@ -105,8 +105,8 @@ impl metrics::Metrics for Metrics { to_worker_bounded_received: prometheus::register( prometheus::GaugeVec::::new( prometheus::Opts::new( - "polkadot_parachain_worker_bounded_received", - "Number of elements received by subsystems' bounded queues", + "polkadot_approval_voting_parallel_worker_bounded_received", + "Number of elements received by approval voting workers' bounded queues", ), &["worker_name"], )?, @@ -115,8 +115,8 @@ impl metrics::Metrics for Metrics { to_worker_bounded_blocked: prometheus::register( prometheus::GaugeVec::::new( prometheus::Opts::new( - "polkadot_parachain_worker_bounded_blocked", - "Number of times senders blocked while sending messages to a subsystem", + "polkadot_approval_voting_parallel_worker_bounded_blocked", + "Number of times approval voting workers blocked while sending messages to a subsystem", ), &["worker_name"], )?, @@ -125,8 +125,8 @@ impl metrics::Metrics for Metrics { to_worker_unbounded_tof: prometheus::register( prometheus::HistogramVec::new( prometheus::HistogramOpts::new( - "polkadot_parachain_worker_unbounded_tof", - "Duration spent in a particular channel from entrance to removal", + "polkadot_approval_voting_parallel_worker_unbounded_tof", + "Duration spent in a particular approval voting worker channel from entrance to removal", ) .buckets(vec![ 0.0001, 0.0004, 0.0016, 0.0064, 0.0256, 0.1024, 0.4096, 1.6384, 3.2768, @@ -139,8 +139,8 @@ impl metrics::Metrics for Metrics { to_worker_unbounded_sent: prometheus::register( prometheus::GaugeVec::::new( prometheus::Opts::new( - "polkadot_parachain_worker_unbounded_sent", - "Number of elements sent to subsystems' unbounded queues", + "polkadot_approval_voting_parallel_worker_unbounded_sent", + "Number of elements sent to approval voting workers' unbounded queues", ), &["worker_name"], )?, @@ -149,8 +149,8 @@ impl metrics::Metrics for Metrics { to_worker_unbounded_received: prometheus::register( prometheus::GaugeVec::::new( prometheus::Opts::new( - "polkadot_parachain_worker_unbounded_received", - "Number of elements received by subsystems' unbounded queues", + "polkadot_approval_voting_parallel_worker_unbounded_received", + "Number of elements received by approval voting workers' unbounded queues", ), &["worker_name"], )?, diff --git a/polkadot/node/core/approval-voting-parallel/src/tests.rs b/polkadot/node/core/approval-voting-parallel/src/tests.rs index bed591b5f210..721552ef4519 100644 --- a/polkadot/node/core/approval-voting-parallel/src/tests.rs +++ b/polkadot/node/core/approval-voting-parallel/src/tests.rs @@ -129,7 +129,6 @@ fn build_subsystem( Metrics::default(), clock.clone(), SpawnGlue(pool), - false, ), context, virtual_overseer, @@ -195,7 +194,6 @@ fn test_harness( let subsystem = async move { let result = run_main_loop( context, - true, to_approval_voting_worker, to_approval_distribution_workers, metrics_watcher, diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index e5b82a059dc7..7edf8d450ac1 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -167,7 +167,6 @@ pub struct ApprovalVotingSubsystem { metrics: Metrics, clock: Arc, spawner: Arc, - subsystem_disabled: bool, } #[derive(Clone)] @@ -487,7 +486,6 @@ impl ApprovalVotingSubsystem { sync_oracle: Box, metrics: Metrics, spawner: Arc, - subsystem_disabled: bool, ) -> Self { ApprovalVotingSubsystem::with_config_and_clock( config, @@ -497,7 +495,6 @@ impl ApprovalVotingSubsystem { metrics, Arc::new(SystemClock {}), spawner, - subsystem_disabled, ) } @@ -510,7 +507,6 @@ impl ApprovalVotingSubsystem { metrics: Metrics, clock: Arc, spawner: Arc, - subsystem_disabled: bool, ) -> Self { ApprovalVotingSubsystem { keystore, @@ -521,7 +517,6 @@ impl ApprovalVotingSubsystem { metrics, clock, spawner, - subsystem_disabled, } } @@ -1257,12 +1252,6 @@ where ).await? } next_msg = work_provider.recv().fuse() => { - if subsystem.subsystem_disabled { - // If we are running in parallel mode, we need to ensure that we are not - // processing messages, but also consume the messages, so that the system - // is not marked as stalled because of the signals it receives. - continue; - } let mut actions = handle_from_overseer( &mut to_other_subsystems, &mut to_approval_distr, @@ -1411,7 +1400,6 @@ pub async fn start_approval_worker< metrics, clock, spawner, - false, ); let backend = DbBackend::new(db.clone(), approval_voting.db_config); let spawner = approval_voting.spawner.clone(); @@ -2036,14 +2024,9 @@ async fn handle_from_overseer< }, FromOrchestra::Communication { msg } => match msg { ApprovalVotingMessage::ImportAssignment(checked_assignment, tx) => { - let (check_outcome, actions) = import_assignment( - sender, - state, - db, - session_info_provider, - checked_assignment, - ) - .await?; + let (check_outcome, actions) = + import_assignment(sender, state, db, session_info_provider, checked_assignment) + .await?; // approval-distribution makes sure this assignment is valid and expected, // so this import should never fail, if it does it might mean one of two things, // there is a bug in the code or the two subsystems got out of sync. @@ -2054,16 +2037,9 @@ async fn handle_from_overseer< actions }, ApprovalVotingMessage::ImportApproval(a, tx) => { - let result = import_approval( - sender, - state, - db, - session_info_provider, - metrics, - a, - &wakeups, - ) - .await?; + let result = + import_approval(sender, state, db, session_info_provider, metrics, a, &wakeups) + .await?; // approval-distribution makes sure this vote is valid and expected, // so this import should never fail, if it does it might mean one of two things, // there is a bug in the code or the two subsystems got out of sync. diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 73aee8cf6a8d..4f205eab34f4 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -579,7 +579,6 @@ fn test_harness>( Metrics::default(), clock.clone(), Arc::new(SpawnGlue(pool)), - false, ), assignment_criteria, backend, diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 8de4bc96383c..2fcb639338ef 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -102,7 +102,6 @@ pub struct ApprovalDistribution { slot_duration_millis: u64, clock: Arc, assignment_criteria: Arc, - subsystem_disabled: bool, } /// Contains recently finalized @@ -2666,14 +2665,12 @@ impl ApprovalDistribution { metrics: Metrics, slot_duration_millis: u64, assignment_criteria: Arc, - subsystem_disabled: bool, ) -> Self { Self::new_with_clock( metrics, slot_duration_millis, Arc::new(SystemClock), assignment_criteria, - subsystem_disabled, ) } @@ -2683,9 +2680,8 @@ impl ApprovalDistribution { slot_duration_millis: u64, clock: Arc, assignment_criteria: Arc, - subsystem_disabled: bool, ) -> Self { - Self { metrics, slot_duration_millis, clock, assignment_criteria, subsystem_disabled } + Self { metrics, slot_duration_millis, clock, assignment_criteria } } async fn run(self, ctx: Context) { @@ -2731,10 +2727,6 @@ impl ApprovalDistribution { reputation_delay = new_reputation_delay(); }, message = ctx.recv().fuse() => { - if self.subsystem_disabled { - gum::trace!(target: LOG_TARGET, "Approval voting parallel is enabled skipping messages"); - continue; - } let message = match message { Ok(message) => message, Err(e) => { diff --git a/polkadot/node/network/approval-distribution/src/tests.rs b/polkadot/node/network/approval-distribution/src/tests.rs index 59dfa67d83c4..063e71f2f528 100644 --- a/polkadot/node/network/approval-distribution/src/tests.rs +++ b/polkadot/node/network/approval-distribution/src/tests.rs @@ -69,7 +69,6 @@ fn test_harness>( Default::default(), clock, assignment_criteria, - false, ); { let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(12345); @@ -536,7 +535,8 @@ impl AssignmentCriteria for MockAssignmentCriteria { _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, _assignment: &polkadot_node_primitives::approval::v2::AssignmentCertV2, _backing_groups: Vec, - ) -> Result { + ) -> Result + { self.tranche } } @@ -4133,7 +4133,6 @@ fn batch_test_round(message_count: usize) { Default::default(), Arc::new(SystemClock {}), Arc::new(MockAssignmentCriteria { tranche: Ok(0) }), - false, ); let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(12345); let mut sender = context.sender().clone(); diff --git a/polkadot/node/service/src/lib.rs b/polkadot/node/service/src/lib.rs index fb289843590d..dc0061a2226d 100644 --- a/polkadot/node/service/src/lib.rs +++ b/polkadot/node/service/src/lib.rs @@ -1567,7 +1567,6 @@ fn revert_approval_voting( Box::new(sp_consensus::NoNetwork), approval_voting_subsystem::Metrics::default(), Arc::new(SpawnGlue(task_handle)), - false, ); approval_voting diff --git a/polkadot/node/service/src/overseer.rs b/polkadot/node/service/src/overseer.rs index bf87c62c25ac..d622e8368d24 100644 --- a/polkadot/node/service/src/overseer.rs +++ b/polkadot/node/service/src/overseer.rs @@ -142,10 +142,16 @@ pub struct ExtendedOverseerGenArgs { /// than the value put in here we always try to recovery availability from backers. /// The presence of this parameter here is needed to have different values per chain. pub fetch_chunks_threshold: Option, + /// Enable approval-voting-parallel subsystem and disable the standalone approval-voting and + /// approval-distribution subsystems. pub enable_approval_voting_parallel: bool, } /// Obtain a prepared validator `Overseer`, that is initialized with all default values. +/// +/// The difference between this function and `validator_with_parallel_overseer_builder` is that this +/// function enables the standalone approval-voting and approval-distribution subsystems +/// and disables the approval-voting-parallel subsystem. pub fn validator_overseer_builder( OverseerGenArgs { runtime_client, @@ -208,7 +214,7 @@ pub fn validator_overseer_builder( CollatorProtocolSubsystem, ApprovalDistributionSubsystem, ApprovalVotingSubsystem, - ApprovalVotingParallelSubsystem, + DummySubsystem, GossipSupportSubsystem, DisputeCoordinatorSubsystem, DisputeDistributionSubsystem, @@ -321,7 +327,6 @@ where approval_voting_parallel_metrics.approval_distribution_metrics(), approval_voting_config.slot_duration_millis, Arc::new(RealAssignmentCriteria {}), - enable_approval_voting_parallel, )) .approval_voting(ApprovalVotingSubsystem::with_config( approval_voting_config.clone(), @@ -330,8 +335,221 @@ where Box::new(sync_service.clone()), approval_voting_parallel_metrics.approval_voting_metrics(), Arc::new(spawner.clone()), + )) + .approval_voting_parallel(DummySubsystem) + .gossip_support(GossipSupportSubsystem::new( + keystore.clone(), + authority_discovery_service.clone(), + Metrics::register(registry)?, + )) + .dispute_coordinator(DisputeCoordinatorSubsystem::new( + parachains_db.clone(), + dispute_coordinator_config, + keystore.clone(), + Metrics::register(registry)?, + enable_approval_voting_parallel, + )) + .dispute_distribution(DisputeDistributionSubsystem::new( + keystore.clone(), + dispute_req_receiver, + authority_discovery_service.clone(), + Metrics::register(registry)?, + )) + .chain_selection(ChainSelectionSubsystem::new(chain_selection_config, parachains_db)) + .prospective_parachains(ProspectiveParachainsSubsystem::new(Metrics::register(registry)?)) + .activation_external_listeners(Default::default()) + .span_per_active_leaf(Default::default()) + .active_leaves(Default::default()) + .supports_parachains(runtime_client) + .metrics(metrics) + .spawner(spawner); + + let builder = if let Some(capacity) = overseer_message_channel_capacity_override { + builder.message_channel_capacity(capacity) + } else { + builder + }; + Ok(builder) +} + +/// Obtain a prepared validator `Overseer`, that is initialized with all default values. +/// +/// The difference between this function and `validator_overseer_builder` is that this +/// function enables the approval-voting-parallel subsystem and disables the standalone +/// approval-voting and approval-distribution subsystems. +pub fn validator_with_parallel_overseer_builder( + OverseerGenArgs { + runtime_client, + network_service, + sync_service, + authority_discovery_service, + collation_req_v1_receiver: _, + collation_req_v2_receiver: _, + available_data_req_receiver, + registry, + spawner, + is_parachain_node, + overseer_message_channel_capacity_override, + req_protocol_names, + peerset_protocol_names, + notification_services, + }: OverseerGenArgs, + ExtendedOverseerGenArgs { + keystore, + parachains_db, + candidate_validation_config, + availability_config, + pov_req_receiver, + chunk_req_v1_receiver, + chunk_req_v2_receiver, + statement_req_receiver, + candidate_req_v2_receiver, + approval_voting_config, + dispute_req_receiver, + dispute_coordinator_config, + chain_selection_config, + fetch_chunks_threshold, + enable_approval_voting_parallel, + }: ExtendedOverseerGenArgs, +) -> Result< + InitializedOverseerBuilder< + SpawnGlue, + Arc, + CandidateValidationSubsystem, + PvfCheckerSubsystem, + CandidateBackingSubsystem, + StatementDistributionSubsystem, + AvailabilityDistributionSubsystem, + AvailabilityRecoverySubsystem, + BitfieldSigningSubsystem, + BitfieldDistributionSubsystem, + ProvisionerSubsystem, + RuntimeApiSubsystem, + AvailabilityStoreSubsystem, + NetworkBridgeRxSubsystem< + Arc, + AuthorityDiscoveryService, + >, + NetworkBridgeTxSubsystem< + Arc, + AuthorityDiscoveryService, + >, + ChainApiSubsystem, + CollationGenerationSubsystem, + CollatorProtocolSubsystem, + DummySubsystem, + DummySubsystem, + ApprovalVotingParallelSubsystem, + GossipSupportSubsystem, + DisputeCoordinatorSubsystem, + DisputeDistributionSubsystem, + ChainSelectionSubsystem, + ProspectiveParachainsSubsystem, + >, + Error, +> +where + RuntimeClient: RuntimeApiSubsystemClient + ChainApiBackend + AuxStore + 'static, + Spawner: 'static + SpawnNamed + Clone + Unpin, +{ + use polkadot_node_subsystem_util::metrics::Metrics; + + let metrics = ::register(registry)?; + let notification_sinks = Arc::new(Mutex::new(HashMap::new())); + + let spawner = SpawnGlue(spawner); + + let network_bridge_metrics: NetworkBridgeMetrics = Metrics::register(registry)?; + let approval_voting_parallel_metrics: ApprovalVotingParallelMetrics = + Metrics::register(registry)?; + let builder = Overseer::builder() + .network_bridge_tx(NetworkBridgeTxSubsystem::new( + network_service.clone(), + authority_discovery_service.clone(), + network_bridge_metrics.clone(), + req_protocol_names.clone(), + peerset_protocol_names.clone(), + notification_sinks.clone(), + )) + .network_bridge_rx(NetworkBridgeRxSubsystem::new( + network_service.clone(), + authority_discovery_service.clone(), + Box::new(sync_service.clone()), + network_bridge_metrics, + peerset_protocol_names, + notification_services, + notification_sinks, enable_approval_voting_parallel, )) + .availability_distribution(AvailabilityDistributionSubsystem::new( + keystore.clone(), + IncomingRequestReceivers { + pov_req_receiver, + chunk_req_v1_receiver, + chunk_req_v2_receiver, + }, + req_protocol_names.clone(), + Metrics::register(registry)?, + )) + .availability_recovery(AvailabilityRecoverySubsystem::for_validator( + fetch_chunks_threshold, + available_data_req_receiver, + &req_protocol_names, + Metrics::register(registry)?, + )) + .availability_store(AvailabilityStoreSubsystem::new( + parachains_db.clone(), + availability_config, + Box::new(sync_service.clone()), + Metrics::register(registry)?, + )) + .bitfield_distribution(BitfieldDistributionSubsystem::new(Metrics::register(registry)?)) + .bitfield_signing(BitfieldSigningSubsystem::new( + keystore.clone(), + Metrics::register(registry)?, + )) + .candidate_backing(CandidateBackingSubsystem::new( + keystore.clone(), + Metrics::register(registry)?, + )) + .candidate_validation(CandidateValidationSubsystem::with_config( + candidate_validation_config, + keystore.clone(), + Metrics::register(registry)?, // candidate-validation metrics + Metrics::register(registry)?, // validation host metrics + )) + .pvf_checker(PvfCheckerSubsystem::new(keystore.clone(), Metrics::register(registry)?)) + .chain_api(ChainApiSubsystem::new(runtime_client.clone(), Metrics::register(registry)?)) + .collation_generation(CollationGenerationSubsystem::new(Metrics::register(registry)?)) + .collator_protocol({ + let side = match is_parachain_node { + IsParachainNode::Collator(_) | IsParachainNode::FullNode => + return Err(Error::Overseer(SubsystemError::Context( + "build validator overseer for parachain node".to_owned(), + ))), + IsParachainNode::No => ProtocolSide::Validator { + keystore: keystore.clone(), + eviction_policy: Default::default(), + metrics: Metrics::register(registry)?, + }, + }; + CollatorProtocolSubsystem::new(side) + }) + .provisioner(ProvisionerSubsystem::new(Metrics::register(registry)?)) + .runtime_api(RuntimeApiSubsystem::new( + runtime_client.clone(), + Metrics::register(registry)?, + spawner.clone(), + )) + .statement_distribution(StatementDistributionSubsystem::new( + keystore.clone(), + statement_req_receiver, + candidate_req_v2_receiver, + Metrics::register(registry)?, + rand::rngs::StdRng::from_entropy(), + )) + .approval_distribution(DummySubsystem) + .approval_voting(DummySubsystem) .approval_voting_parallel(ApprovalVotingParallelSubsystem::with_config( approval_voting_config, parachains_db.clone(), @@ -339,7 +557,6 @@ where Box::new(sync_service.clone()), approval_voting_parallel_metrics, spawner.clone(), - enable_approval_voting_parallel, )) .gossip_support(GossipSupportSubsystem::new( keystore.clone(), @@ -560,9 +777,15 @@ impl OverseerGen for ValidatorOverseerGen { "create validator overseer as mandatory extended arguments were not provided" .to_owned(), )))?; - validator_overseer_builder(args, ext_args)? - .build_with_connector(connector) - .map_err(|e| e.into()) + if ext_args.enable_approval_voting_parallel { + validator_with_parallel_overseer_builder(args, ext_args)? + .build_with_connector(connector) + .map_err(|e| e.into()) + } else { + validator_overseer_builder(args, ext_args)? + .build_with_connector(connector) + .map_err(|e| e.into()) + } } } diff --git a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs index 423a7be567a7..c5f6411f32fd 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs @@ -839,7 +839,6 @@ fn build_overseer( state.approval_voting_parallel_metrics.approval_voting_metrics(), Arc::new(system_clock.clone()), Arc::new(SpawnGlue(spawn_task_handle.clone())), - state.options.approval_voting_parallel_enabled, ); let approval_distribution = ApprovalDistribution::new_with_clock( @@ -847,7 +846,6 @@ fn build_overseer( TEST_CONFIG.slot_duration_millis, Arc::new(system_clock.clone()), Arc::new(RealAssignmentCriteria {}), - state.options.approval_voting_parallel_enabled, ); let approval_voting_parallel = @@ -859,7 +857,6 @@ fn build_overseer( state.approval_voting_parallel_metrics.clone(), Arc::new(system_clock.clone()), SpawnGlue(spawn_task_handle.clone()), - state.options.approval_voting_parallel_enabled, ); let mock_chain_api = MockChainApi::new(state.build_chain_api_state()); diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index 38bdb7fd5d52..fafc700e739f 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -959,8 +959,8 @@ pub struct BlockDescription { /// approval-voting logic in parallel. This is a combination of all the messages ApprovalVoting and /// ApprovalDistribution subsystems can receive. /// -/// The reason this exists is, so that we can keep both mods of running in the same polkadot binary, -/// based on the value of `--approval-voting-parallel-enabled`, we decide if we run with two +/// The reason this exists is, so that we can keep both modes of running in the same polkadot +/// binary, based on the value of `--approval-voting-parallel-enabled`, we decide if we run with two /// different subsystems for approval-distribution and approval-voting or run the approval-voting /// parallel which has several parallel workers for the approval-distribution and a worker for /// approval-voting. diff --git a/polkadot/zombienet_tests/functional/0016-approval-voting-parallel.toml b/polkadot/zombienet_tests/functional/0016-approval-voting-parallel.toml index eb7435658962..97fb6261131d 100644 --- a/polkadot/zombienet_tests/functional/0016-approval-voting-parallel.toml +++ b/polkadot/zombienet_tests/functional/0016-approval-voting-parallel.toml @@ -7,7 +7,7 @@ chain = "rococo-local" [relaychain.genesis.runtimeGenesis.patch.configuration.config] needed_approvals = 4 - relay_vrf_modulo_samples = 6 + relay_vrf_modulo_samples = 2 [relaychain.genesis.runtimeGenesis.patch.configuration.config.approval_voting_params] max_approval_coalesce_count = 5 From fab18b1923b1f2a974ecf319a97a6d566bb13bb0 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 11 Sep 2024 14:40:20 +0300 Subject: [PATCH 28/45] Fix benchmark Signed-off-by: Alexandru Gheorghe --- .../subsystem-bench/src/lib/approval/mod.rs | 77 +++++++++++-------- 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs index c5f6411f32fd..2395d57ed3e5 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs @@ -49,6 +49,7 @@ use itertools::Itertools; use orchestra::TimeoutExt; use overseer::{metrics::Metrics as OverseerMetrics, MetricsTrait}; use polkadot_approval_distribution::ApprovalDistribution; +use polkadot_node_core_approval_voting_parallel::ApprovalVotingParallelSubsystem; use polkadot_node_primitives::approval::time::{ slot_number_to_tick, tick_to_slot_number, Clock, ClockExt, SystemClock, }; @@ -831,36 +832,10 @@ fn build_overseer( PastSystemClock::new(SystemClock {}, state.delta_tick_from_generated.clone()); let keystore = Arc::new(keystore); let db = Arc::new(db); - let approval_voting = ApprovalVotingSubsystem::with_config_and_clock( - TEST_CONFIG, - db.clone(), - keystore.clone(), - Box::new(TestSyncOracle {}), - state.approval_voting_parallel_metrics.approval_voting_metrics(), - Arc::new(system_clock.clone()), - Arc::new(SpawnGlue(spawn_task_handle.clone())), - ); - - let approval_distribution = ApprovalDistribution::new_with_clock( - state.approval_voting_parallel_metrics.approval_distribution_metrics(), - TEST_CONFIG.slot_duration_millis, - Arc::new(system_clock.clone()), - Arc::new(RealAssignmentCriteria {}), - ); - - let approval_voting_parallel = - polkadot_node_core_approval_voting_parallel::ApprovalVotingParallelSubsystem::with_config_and_clock( - TEST_CONFIG, - db.clone(), - keystore.clone(), - Box::new(TestSyncOracle {}), - state.approval_voting_parallel_metrics.clone(), - Arc::new(system_clock.clone()), - SpawnGlue(spawn_task_handle.clone()), - ); let mock_chain_api = MockChainApi::new(state.build_chain_api_state()); - let mock_chain_selection = MockChainSelection { state: state.clone(), clock: system_clock }; + let mock_chain_selection = + MockChainSelection { state: state.clone(), clock: system_clock.clone() }; let mock_runtime_api = MockRuntimeApi::new( config.clone(), state.test_authorities.clone(), @@ -881,10 +856,8 @@ fn build_overseer( state.options.approval_voting_parallel_enabled, ); let overseer_metrics = OverseerMetrics::try_register(&dependencies.registry).unwrap(); - let dummy = dummy_builder!(spawn_task_handle, overseer_metrics) - .replace_approval_distribution(|_| approval_distribution) - .replace_approval_voting(|_| approval_voting) - .replace_approval_voting_parallel(|_| approval_voting_parallel) + let task_handle = spawn_task_handle.clone(); + let dummy = dummy_builder!(task_handle, overseer_metrics) .replace_chain_api(|_| mock_chain_api) .replace_chain_selection(|_| mock_chain_selection) .replace_runtime_api(|_| mock_runtime_api) @@ -893,8 +866,44 @@ fn build_overseer( .replace_availability_recovery(|_| MockAvailabilityRecovery::new()) .replace_candidate_validation(|_| MockCandidateValidation::new()); - let (overseer, raw_handle) = - dummy.build_with_connector(overseer_connector).expect("Should not fail"); + let (overseer, raw_handle) = if state.options.approval_voting_parallel_enabled { + let approval_voting_parallel = ApprovalVotingParallelSubsystem::with_config_and_clock( + TEST_CONFIG, + db.clone(), + keystore.clone(), + Box::new(TestSyncOracle {}), + state.approval_voting_parallel_metrics.clone(), + Arc::new(system_clock.clone()), + SpawnGlue(spawn_task_handle.clone()), + ); + dummy + .replace_approval_voting_parallel(|_| approval_voting_parallel) + .build_with_connector(overseer_connector) + .expect("Should not fail") + } else { + let approval_voting = ApprovalVotingSubsystem::with_config_and_clock( + TEST_CONFIG, + db.clone(), + keystore.clone(), + Box::new(TestSyncOracle {}), + state.approval_voting_parallel_metrics.approval_voting_metrics(), + Arc::new(system_clock.clone()), + Arc::new(SpawnGlue(spawn_task_handle.clone())), + ); + + let approval_distribution = ApprovalDistribution::new_with_clock( + state.approval_voting_parallel_metrics.approval_distribution_metrics(), + TEST_CONFIG.slot_duration_millis, + Arc::new(system_clock.clone()), + Arc::new(RealAssignmentCriteria {}), + ); + + dummy + .replace_approval_voting(|_| approval_voting) + .replace_approval_distribution(|_| approval_distribution) + .build_with_connector(overseer_connector) + .expect("Should not fail") + }; let overseer_handle = OverseerHandleReal::new(raw_handle); (overseer, overseer_handle) From a7d5f85e1999f6c1bb8992d948c7da681a03ba12 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 11 Sep 2024 15:12:21 +0300 Subject: [PATCH 29/45] Enable it by default on all test networks except kusama and polkadot Signed-off-by: Alexandru Gheorghe --- polkadot/cli/src/cli.rs | 3 ++- polkadot/node/service/src/lib.rs | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/polkadot/cli/src/cli.rs b/polkadot/cli/src/cli.rs index 722521f11efa..1445ade08e20 100644 --- a/polkadot/cli/src/cli.rs +++ b/polkadot/cli/src/cli.rs @@ -154,7 +154,8 @@ pub struct RunCmd { /// Enable approval-voting message processing in parallel. /// - ///**Dangerous!** This is an experimental feature and should not be used in production. + ///**Dangerous!** This is an experimental feature and should not be used in production, unless + /// explicitly advised to. #[arg(long)] pub enable_approval_voting_parallel: bool, } diff --git a/polkadot/node/service/src/lib.rs b/polkadot/node/service/src/lib.rs index dc0061a2226d..34bec2a410a9 100644 --- a/polkadot/node/service/src/lib.rs +++ b/polkadot/node/service/src/lib.rs @@ -794,6 +794,14 @@ pub fn new_full< Some(backoff) }; + // Running approval voting in parallel is enabled by default on all networks except Polkadot and + // Kusama, unless explicitly enabled by the commandline option. + // This is meant to be temporary until we have enough confidence in the new system to enable it + // by default on all networks. + let enable_approval_voting_parallel = (!config.chain_spec.is_kusama() && + !config.chain_spec.is_polkadot()) || + enable_approval_voting_parallel; + let disable_grandpa = config.disable_grandpa; let name = config.network.node_name.clone(); From 11cc9772c8bc79d9acf82c293c414cf0d2fe8273 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 11 Sep 2024 16:57:24 +0300 Subject: [PATCH 30/45] Cosmetic updates Signed-off-by: Alexandru Gheorghe --- .../core/approval-voting-parallel/src/lib.rs | 71 +++++++++---------- .../approval-voting-parallel/src/tests.rs | 18 +++++ polkadot/node/core/approval-voting/src/lib.rs | 24 +++++-- 3 files changed, 71 insertions(+), 42 deletions(-) diff --git a/polkadot/node/core/approval-voting-parallel/src/lib.rs b/polkadot/node/core/approval-voting-parallel/src/lib.rs index 7d1889965a0f..5a76813319bf 100644 --- a/polkadot/node/core/approval-voting-parallel/src/lib.rs +++ b/polkadot/node/core/approval-voting-parallel/src/lib.rs @@ -70,6 +70,16 @@ pub(crate) const LOG_TARGET: &str = "parachain::approval-voting-parallel"; // lock issues for example. const WAIT_FOR_SIGS_GATHER_TIMEOUT: Duration = Duration::from_millis(2000); +/// The number of workers used for running the approval-distribution logic. +pub const APPROVAL_DISTRIBUTION_WORKER_COUNT: usize = 4; + +/// The channel size for the workers. +pub const WORKERS_CHANNEL_SIZE: usize = 64000 / APPROVAL_DISTRIBUTION_WORKER_COUNT; + +fn prio_right<'a>(_val: &'a mut ()) -> PollNext { + PollNext::Right +} + /// The approval voting parallel subsystem. pub struct ApprovalVotingParallelSubsystem { /// `LocalKeystore` is needed for assignment keys, but not necessarily approval keys. @@ -140,16 +150,6 @@ impl ApprovalVotingParallelSubsystem { } } -/// The number of workers used for running the approval-distribution logic. -pub const APPROVAL_DISTRIBUTION_WORKER_COUNT: usize = 4; - -/// The channel size for the workers. -pub const WORKERS_CHANNEL_SIZE: usize = 64000 / APPROVAL_DISTRIBUTION_WORKER_COUNT; - -fn prio_right<'a>(_val: &'a mut ()) -> PollNext { - PollNext::Right -} - // It starts worker for the approval voting subsystem and the `APPROVAL_DISTRIBUTION_WORKER_COUNT` // workers for the approval distribution subsystem. // @@ -162,6 +162,8 @@ async fn start_workers( ) -> SubsystemResult<(ToWorker, Vec>)> where { + gum::info!(target: LOG_TARGET, "Starting approval distribution workers"); + // Build approval voting handles. let (to_approval_voting_worker, approval_voting_work_provider) = build_worker_handles( "approval-voting-parallel-db".into(), @@ -169,13 +171,14 @@ where metrics_watcher, prio_right, ); - - gum::info!(target: LOG_TARGET, "Starting approval distribution workers"); - let mut to_approval_distribution_workers = Vec::new(); let slot_duration_millis = subsystem.slot_duration_millis; for i in 0..APPROVAL_DISTRIBUTION_WORKER_COUNT { + let mut network_sender = ctx.sender().clone(); + let mut runtime_api_sender = ctx.sender().clone(); + let mut approval_distribution_to_approval_voting = to_approval_voting_worker.clone(); + let approval_distr_instance = polkadot_approval_distribution::ApprovalDistribution::new_with_clock( subsystem.metrics.approval_distribution_metrics(), @@ -184,7 +187,6 @@ where Arc::new(RealAssignmentCriteria {}), ); let task_name = format!("approval-voting-parallel-{}", i); - let (to_approval_distribution_worker, mut approval_distribution_work_provider) = build_worker_handles( task_name.clone(), @@ -195,10 +197,6 @@ where metrics_watcher.watch(task_name.clone(), to_approval_distribution_worker.meter()); - let mut network_sender = ctx.sender().clone(); - let mut runtime_api_sender = ctx.sender().clone(); - let mut approval_distribution_to_approval_voting = to_approval_voting_worker.clone(); - subsystem.spawner.spawn_blocking( task_name.leak(), Some("approval-voting-parallel"), @@ -240,9 +238,9 @@ where } gum::info!(target: LOG_TARGET, "Starting approval voting workers"); + let sender = ctx.sender().clone(); let to_approval_distribution = ApprovalVotingToApprovalDistribution(sender.clone()); - polkadot_node_core_approval_voting::start_approval_worker( approval_voting_work_provider, sender.clone(), @@ -265,7 +263,7 @@ where Ok((to_approval_voting_worker, to_approval_distribution_workers)) } -// The main run function of the approval voting subsystem. +// The main run function of the approval parallel voting subsystem. #[overseer::contextbounds(ApprovalVotingParallel, prefix = self::overseer)] async fn run( mut ctx: Context, @@ -285,12 +283,14 @@ async fn run( "Starting main subsystem loop" ); - // Main loop of the subsystem, it shouldn't include any logic just dispatching of messages to - // the workers. run_main_loop(ctx, to_approval_voting_worker, to_approval_distribution_workers, metrics_watcher) .await } +// Main loop of the subsystem, it shouldn't include any logic just dispatching of messages to +// the workers. +// +// It listens for messages from the overseer and dispatches them to the workers. #[overseer::contextbounds(ApprovalVotingParallel, prefix = self::overseer)] async fn run_main_loop( mut ctx: Context, @@ -624,8 +624,8 @@ fn validator_index_for_msg( /// A handler object that both type of workers use for receiving work. /// -/// In practive this just a wrapper over a channel Receiver, that is injected into approval-voting -/// worker and approval-distribution workers. +/// In practive this is just a wrapper over two channels Receiver, that is injected into +/// approval-voting worker and approval-distribution workers. type WorkProvider = WorkProviderImpl< SelectWithStrategy< MeteredReceiver>, @@ -677,9 +677,10 @@ where } /// Just a wrapper for implementing overseer::SubsystemSender and -/// overseer::SubsystemSender, so that we can inject into the -/// workers, so they can talke directly with each other without intermediating in this subsystem -/// loop. +/// overseer::SubsystemSender. +/// +/// The instance of this struct can be injected into the workers, so they can talk +/// directly with each other without intermediating in this subsystem loop. pub struct ToWorker( MeteredSender>, UnboundedMeteredSender>, @@ -808,28 +809,26 @@ impl overseer::SubsystemSender for ToWorker } } -/// Handles to use by an worker to receive work. +/// Handles that are used by an worker to receive work. pub struct RxWorker( MeteredReceiver>, UnboundedMeteredReceiver>, ); -/// Build all the necessary channels for sending messages to the workers -/// and for the workers to receive them. +// Build all the necessary channels for sending messages to an worker +// and for the worker to receive them. fn build_channels( channel_name: String, channel_size: usize, metrics_watcher: &mut MetricsWatcher, ) -> (ToWorker, RxWorker) { - let (tx_approval_voting_work, rx_approval_voting_work) = - channel::>(channel_size); - let (tx_approval_voting_work_unbounded, rx_approval_voting_work_unbounded) = - unbounded::>(); - let to_worker = ToWorker(tx_approval_voting_work, tx_approval_voting_work_unbounded); + let (tx_work, rx_work) = channel::>(channel_size); + let (tx_work_unbounded, rx_work_unbounded) = unbounded::>(); + let to_worker = ToWorker(tx_work, tx_work_unbounded); metrics_watcher.watch(channel_name, to_worker.meter()); - (to_worker, RxWorker(rx_approval_voting_work, rx_approval_voting_work_unbounded)) + (to_worker, RxWorker(rx_work, rx_work_unbounded)) } /// Build the worker handles used for interacting with the workers. diff --git a/polkadot/node/core/approval-voting-parallel/src/tests.rs b/polkadot/node/core/approval-voting-parallel/src/tests.rs index 721552ef4519..d70c72238295 100644 --- a/polkadot/node/core/approval-voting-parallel/src/tests.rs +++ b/polkadot/node/core/approval-voting-parallel/src/tests.rs @@ -1,3 +1,21 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! The tests for Approval Voting Parallel Subsystem. + use std::{ collections::{HashMap, HashSet}, future::Future, diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 7edf8d450ac1..d3eeeea16dc1 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -2024,9 +2024,14 @@ async fn handle_from_overseer< }, FromOrchestra::Communication { msg } => match msg { ApprovalVotingMessage::ImportAssignment(checked_assignment, tx) => { - let (check_outcome, actions) = - import_assignment(sender, state, db, session_info_provider, checked_assignment) - .await?; + let (check_outcome, actions) = import_assignment( + sender, + state, + db, + session_info_provider, + checked_assignment, + ) + .await?; // approval-distribution makes sure this assignment is valid and expected, // so this import should never fail, if it does it might mean one of two things, // there is a bug in the code or the two subsystems got out of sync. @@ -2037,9 +2042,16 @@ async fn handle_from_overseer< actions }, ApprovalVotingMessage::ImportApproval(a, tx) => { - let result = - import_approval(sender, state, db, session_info_provider, metrics, a, &wakeups) - .await?; + let result = import_approval( + sender, + state, + db, + session_info_provider, + metrics, + a, + &wakeups, + ) + .await?; // approval-distribution makes sure this vote is valid and expected, // so this import should never fail, if it does it might mean one of two things, // there is a bug in the code or the two subsystems got out of sync. From bb3887f9b61f7c0dbc9a34b839868778a60a1ac5 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 11 Sep 2024 17:49:44 +0300 Subject: [PATCH 31/45] Reduce timeout Signed-off-by: Alexandru Gheorghe --- .../functional/0016-approval-voting-parallel.zndsl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/zombienet_tests/functional/0016-approval-voting-parallel.zndsl b/polkadot/zombienet_tests/functional/0016-approval-voting-parallel.zndsl index 2be0dd05a9d8..d70707747474 100644 --- a/polkadot/zombienet_tests/functional/0016-approval-voting-parallel.zndsl +++ b/polkadot/zombienet_tests/functional/0016-approval-voting-parallel.zndsl @@ -25,8 +25,8 @@ alice: parachain 2005 block height is at least 10 within 300 seconds alice: parachain 2006 block height is at least 10 within 300 seconds alice: parachain 2007 block height is at least 10 within 300 seconds -alice: reports substrate_block_height{status="finalized"} is at least 30 within 40000 seconds -bob: reports substrate_block_height{status="finalized"} is at least 30 within 40000 seconds +alice: reports substrate_block_height{status="finalized"} is at least 30 within 180 seconds +bob: reports substrate_block_height{status="finalized"} is at least 30 within 180 seconds alice: reports polkadot_parachain_approval_checking_finality_lag < 3 bob: reports polkadot_parachain_approval_checking_finality_lag < 3 From b817677f6ea05280017e244096422622dd1c9c41 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 11 Sep 2024 18:37:37 +0300 Subject: [PATCH 32/45] Cleanup Cargo.toml Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b0621d0f27c..a824d8a86f05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13395,16 +13395,11 @@ version = "7.0.0" dependencies = [ "assert_matches", "async-trait", - "bitvec", - "derive_more", "futures", "futures-timer", "itertools 0.11.0", - "kvdb", "kvdb-memorydb", "log", - "merlin", - "parity-scale-codec", "parking_lot 0.12.3", "polkadot-approval-distribution", "polkadot-node-core-approval-voting", @@ -13423,8 +13418,6 @@ dependencies = [ "rand_chacha", "rand_core", "sc-keystore", - "schnellru", - "schnorrkel 0.11.4", "sp-application-crypto", "sp-consensus", "sp-consensus-babe", From 6521bb9c483dbf6431dfcdfaf913d4538e445c34 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 12 Sep 2024 11:49:41 +0300 Subject: [PATCH 33/45] Add option to capacity override the workers channels Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 1 + .../core/approval-voting-parallel/Cargo.toml | 71 ++++++++----------- .../core/approval-voting-parallel/src/lib.rs | 20 ++++-- .../approval-voting-parallel/src/tests.rs | 1 + polkadot/node/service/src/overseer.rs | 1 + .../subsystem-bench/src/lib/approval/mod.rs | 1 + 6 files changed, 50 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a824d8a86f05..07e0c0971cdd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13418,6 +13418,7 @@ dependencies = [ "rand_chacha", "rand_core", "sc-keystore", + "schnorrkel 0.11.4", "sp-application-crypto", "sp-consensus", "sp-consensus-babe", diff --git a/polkadot/node/core/approval-voting-parallel/Cargo.toml b/polkadot/node/core/approval-voting-parallel/Cargo.toml index 77c0bd0d6972..5bfec329ee89 100644 --- a/polkadot/node/core/approval-voting-parallel/Cargo.toml +++ b/polkadot/node/core/approval-voting-parallel/Cargo.toml @@ -10,58 +10,47 @@ description = "Approval Voting Subsystem running approval work in parallel" workspace = true [dependencies] -futures = "0.3.30" -futures-timer = "3.0.2" -codec = { package = "parity-scale-codec", version = "3.6.12", default-features = false, features = ["bit-vec", "derive"] } +async-trait = { workspace = true } +futures = { workspace = true } +futures-timer = { workspace = true } gum = { package = "tracing-gum", path = "../../gum" } -bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } -schnellru = "0.2.1" -merlin = "3.0" -schnorrkel = "0.11.4" -kvdb = "0.13.0" -derive_more = "0.99.17" +itertools = { workspace = true } thiserror = { workspace = true } -itertools = "0.11" -async-trait = { workspace = true } - -polkadot-node-core-approval-voting = { workspace = true, default-features = true } -polkadot-approval-distribution = { workspace = true, default-features = true } - - -polkadot-node-subsystem = { workspace = true, default-features = true } -polkadot-node-subsystem-util = { workspace = true, default-features = true } -polkadot-overseer = { workspace = true, default-features = true } -polkadot-primitives = { workspace = true, default-features = true } -polkadot-node-primitives = { workspace = true, default-features = true } -polkadot-node-jaeger = { workspace = true, default-features = true } -sc-keystore = { workspace = true, default-features = false } -sp-consensus = { workspace = true, default-features = false } -sp-consensus-slots = { workspace = true, default-features = false } -sp-application-crypto = { workspace = true, default-features = false, features = ["full_crypto"] } -sp-runtime = { workspace = true, default-features = false } +polkadot-node-core-approval-voting = { workspace = true, default-features = true } +polkadot-approval-distribution = { workspace = true, default-features = true } +polkadot-node-subsystem = { workspace = true, default-features = true } +polkadot-node-subsystem-util = { workspace = true, default-features = true } +polkadot-overseer = { workspace = true, default-features = true } +polkadot-primitives = { workspace = true, default-features = true } +polkadot-node-primitives = { workspace = true, default-features = true } +polkadot-node-jaeger = { workspace = true, default-features = true } polkadot-node-network-protocol = { workspace = true, default-features = true } -polkadot-node-metrics = { workspace = true, default-features = true} +polkadot-node-metrics = { workspace = true, default-features = true } -rand = "0.8.5" +sc-keystore = { workspace = true, default-features = false } +sp-consensus = { workspace = true, default-features = false } +sp-consensus-slots = { workspace = true, default-features = false } +sp-application-crypto = { workspace = true, default-features = false, features = ["full_crypto"] } +sp-runtime = { workspace = true, default-features = false } -# rand_core should match schnorrkel -rand_core = "0.6.2" -rand_chacha = { version = "0.3.1" } +rand = { workspace = true } +rand_core = { workspace = true } +rand_chacha = { workspace = true } [dev-dependencies] -async-trait = "0.1.79" -parking_lot = "0.12.1" +async-trait = { workspace = true } +parking_lot = { workspace = true } sp-keyring = { workspace = true, default-features = true } -sp-keystore = {workspace = true, default-features = true} -sp-core = { workspace = true, default-features = true} +sp-keystore = { workspace = true, default-features = true } +sp-core = { workspace = true, default-features = true } sp-consensus-babe = { workspace = true, default-features = true } sp-tracing = { workspace = true } -polkadot-node-subsystem-test-helpers = { workspace = true, default-features = true} -assert_matches = "1.4.0" -kvdb-memorydb = "0.13.0" +polkadot-node-subsystem-test-helpers = { workspace = true, default-features = true } +assert_matches = { workspace = true } +kvdb-memorydb = { workspace = true } polkadot-primitives-test-helpers = { workspace = true, default-features = true } log = { workspace = true, default-features = true } - -polkadot-subsystem-bench = { workspace = true, default-features = true} +polkadot-subsystem-bench = { workspace = true, default-features = true } +schnorrkel = { workspace = true, default-features = true } diff --git a/polkadot/node/core/approval-voting-parallel/src/lib.rs b/polkadot/node/core/approval-voting-parallel/src/lib.rs index 5a76813319bf..7d3237e83064 100644 --- a/polkadot/node/core/approval-voting-parallel/src/lib.rs +++ b/polkadot/node/core/approval-voting-parallel/src/lib.rs @@ -73,8 +73,9 @@ const WAIT_FOR_SIGS_GATHER_TIMEOUT: Duration = Duration::from_millis(2000); /// The number of workers used for running the approval-distribution logic. pub const APPROVAL_DISTRIBUTION_WORKER_COUNT: usize = 4; -/// The channel size for the workers. -pub const WORKERS_CHANNEL_SIZE: usize = 64000 / APPROVAL_DISTRIBUTION_WORKER_COUNT; +/// The default channel size for the workers, can be overridden by the user through +/// `overseer_channel_capacity_override` +pub const DEFAULT_WORKERS_CHANNEL_SIZE: usize = 64000 / APPROVAL_DISTRIBUTION_WORKER_COUNT; fn prio_right<'a>(_val: &'a mut ()) -> PollNext { PollNext::Right @@ -93,6 +94,7 @@ pub struct ApprovalVotingParallelSubsystem { metrics: Metrics, spawner: Arc, clock: Arc, + overseer_message_channel_capacity_override: Option, } impl ApprovalVotingParallelSubsystem { @@ -104,6 +106,7 @@ impl ApprovalVotingParallelSubsystem { sync_oracle: Box, metrics: Metrics, spawner: impl overseer::gen::Spawner + 'static + Clone, + overseer_message_channel_capacity_override: Option, ) -> Self { ApprovalVotingParallelSubsystem::with_config_and_clock( config, @@ -113,6 +116,7 @@ impl ApprovalVotingParallelSubsystem { metrics, Arc::new(SystemClock {}), spawner, + overseer_message_channel_capacity_override, ) } @@ -125,6 +129,7 @@ impl ApprovalVotingParallelSubsystem { metrics: Metrics, clock: Arc, spawner: impl overseer::gen::Spawner + 'static, + overseer_message_channel_capacity_override: Option, ) -> Self { ApprovalVotingParallelSubsystem { keystore, @@ -135,8 +140,15 @@ impl ApprovalVotingParallelSubsystem { metrics, spawner: Arc::new(spawner), clock, + overseer_message_channel_capacity_override, } } + + /// The size of the channel used for the workers. + fn workers_channel_size(&self) -> usize { + self.overseer_message_channel_capacity_override + .unwrap_or(DEFAULT_WORKERS_CHANNEL_SIZE) + } } #[overseer::subsystem(ApprovalVotingParallel, error = SubsystemError, prefix = self::overseer)] @@ -167,7 +179,7 @@ where // Build approval voting handles. let (to_approval_voting_worker, approval_voting_work_provider) = build_worker_handles( "approval-voting-parallel-db".into(), - WORKERS_CHANNEL_SIZE, + subsystem.workers_channel_size(), metrics_watcher, prio_right, ); @@ -190,7 +202,7 @@ where let (to_approval_distribution_worker, mut approval_distribution_work_provider) = build_worker_handles( task_name.clone(), - WORKERS_CHANNEL_SIZE, + subsystem.workers_channel_size(), metrics_watcher, prio_right, ); diff --git a/polkadot/node/core/approval-voting-parallel/src/tests.rs b/polkadot/node/core/approval-voting-parallel/src/tests.rs index d70c72238295..215a707147fc 100644 --- a/polkadot/node/core/approval-voting-parallel/src/tests.rs +++ b/polkadot/node/core/approval-voting-parallel/src/tests.rs @@ -147,6 +147,7 @@ fn build_subsystem( Metrics::default(), clock.clone(), SpawnGlue(pool), + None, ), context, virtual_overseer, diff --git a/polkadot/node/service/src/overseer.rs b/polkadot/node/service/src/overseer.rs index d622e8368d24..a98b6bcb3083 100644 --- a/polkadot/node/service/src/overseer.rs +++ b/polkadot/node/service/src/overseer.rs @@ -557,6 +557,7 @@ where Box::new(sync_service.clone()), approval_voting_parallel_metrics, spawner.clone(), + overseer_message_channel_capacity_override, )) .gossip_support(GossipSupportSubsystem::new( keystore.clone(), diff --git a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs index 2395d57ed3e5..29ebc4a419ae 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs @@ -875,6 +875,7 @@ fn build_overseer( state.approval_voting_parallel_metrics.clone(), Arc::new(system_clock.clone()), SpawnGlue(spawn_task_handle.clone()), + None, ); dummy .replace_approval_voting_parallel(|_| approval_voting_parallel) From a2bfe6bd08eef23cb59ee2968699853e81c9efab Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 12 Sep 2024 11:54:39 +0300 Subject: [PATCH 34/45] Add qed to expect. Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting-parallel/src/lib.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/polkadot/node/core/approval-voting-parallel/src/lib.rs b/polkadot/node/core/approval-voting-parallel/src/lib.rs index 7d3237e83064..5f9d78a035cf 100644 --- a/polkadot/node/core/approval-voting-parallel/src/lib.rs +++ b/polkadot/node/core/approval-voting-parallel/src/lib.rs @@ -341,8 +341,12 @@ async fn run_main_loop( // The message the approval voting subsystem would've handled. ApprovalVotingParallelMessage::ApprovedAncestor(_, _,_) | ApprovalVotingParallelMessage::GetApprovalSignaturesForCandidate(_, _) => { - // Safe to unwrap because we know the message is of the right type. - to_approval_voting_worker.send_message(msg.try_into().unwrap()).await; + to_approval_voting_worker.send_message( + msg.try_into().expect( + "Message is one of ApprovedAncestor, GetApprovalSignaturesForCandidate + and that can be safely converted to ApprovalVotingMessage; qed" + ) + ).await; }, // Now the message the approval distribution subsystem would've handled and need to // be forwarded to the workers. From e4f8771bb3989d3c8f378a6c14f72bf20f1dd98e Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 18 Sep 2024 15:50:00 +0300 Subject: [PATCH 35/45] Fix some typos Signed-off-by: Alexandru Gheorghe --- .../node/core/approval-voting-parallel/src/lib.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/polkadot/node/core/approval-voting-parallel/src/lib.rs b/polkadot/node/core/approval-voting-parallel/src/lib.rs index 5f9d78a035cf..172c131c6f47 100644 --- a/polkadot/node/core/approval-voting-parallel/src/lib.rs +++ b/polkadot/node/core/approval-voting-parallel/src/lib.rs @@ -232,7 +232,7 @@ where break; }, }; - approval_distr_instance + if approval_distr_instance .handle_from_orchestra( message, &mut approval_distribution_to_approval_voting, @@ -242,7 +242,13 @@ where &mut rng, &mut session_info_provider, ) - .await; + .await + { + gum::info!( + target: LOG_TARGET, + "Approval distribution worker {}, exiting because of shutdown", i + ); + }; } }), ); @@ -426,7 +432,7 @@ async fn run_main_loop( } // It sends a message to all approval workers to get the approval signatures for the requested -// candidates and then merges them altogether and sends them back to the requester. +// candidates and then merges them all together and sends them back to the requester. #[overseer::contextbounds(ApprovalVotingParallel, prefix = self::overseer)] async fn handle_get_approval_signatures( ctx: &mut Context, @@ -874,8 +880,7 @@ pub struct ApprovalVotingToApprovalDistribution> - overseer::SubsystemSender - for ApprovalVotingToApprovalDistribution + overseer::SubsystemSender for ApprovalVotingToApprovalDistribution { #[allow(clippy::type_complexity, clippy::type_repetition_in_bounds)] fn send_message<'life0, 'async_trait>( From f6c58b9d1ac4ed1a2691e30505936e8e4d187d5e Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 20 Sep 2024 14:17:00 +0300 Subject: [PATCH 36/45] Fix cargo fmt Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/tests.rs | 3 +-- polkadot/node/network/approval-distribution/src/tests.rs | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 4f205eab34f4..65aa4f894c23 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -263,8 +263,7 @@ where _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, _assignment: &polkadot_node_primitives::approval::v2::AssignmentCertV2, _backing_groups: Vec, - ) -> Result - { + ) -> Result { self.1(validator_index) } } diff --git a/polkadot/node/network/approval-distribution/src/tests.rs b/polkadot/node/network/approval-distribution/src/tests.rs index 063e71f2f528..068559dea762 100644 --- a/polkadot/node/network/approval-distribution/src/tests.rs +++ b/polkadot/node/network/approval-distribution/src/tests.rs @@ -535,8 +535,7 @@ impl AssignmentCriteria for MockAssignmentCriteria { _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, _assignment: &polkadot_node_primitives::approval::v2::AssignmentCertV2, _backing_groups: Vec, - ) -> Result - { + ) -> Result { self.tranche } } From f356912eb0210c1d4c47598b745ac2892bcc804d Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 20 Sep 2024 14:42:22 +0300 Subject: [PATCH 37/45] Add prdoc Signed-off-by: Alexandru Gheorghe --- prdoc/pr_4849.prdoc | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 prdoc/pr_4849.prdoc diff --git a/prdoc/pr_4849.prdoc b/prdoc/pr_4849.prdoc new file mode 100644 index 000000000000..e65e7904ec01 --- /dev/null +++ b/prdoc/pr_4849.prdoc @@ -0,0 +1,45 @@ +title: Introduce approval-voting-parallel subsystem + +doc: + - audience: Node Dev + description: | + This introduces a new subsystem called approval-voting-parallel. It combines the tasks + previously handled by the approval-voting and approval-distribution subsystems. + + The new subsystem is enabled by default on all test networks. On production networks + like Polkadot and Kusama, the legacy system with two separate subsystems is still in use. + However, there is a CLI option --enable-approval-voting-parallel to gradually roll out + the new subsystem on specific nodes. Once we are confident that it works as expected, + it will be enabled by default on all networks. + + The approval-voting-parallel subsystem coordinates two groups of workers: + - Four approval-distribution workers that operate in parallel, each handling tasks based + on the validator_index of the message originator. + - One approval-voting worker that performs the tasks previously managed by the standalone + approval-voting subsystem. + +crates: + - name: polkadot-overseer + bump: major + - name: polkadot-node-primitives + bump: major + - name: polkadot-node-subsystem-types + bump: major + - name: polkadot-service + bump: major + - name: polkadot-approval-distribution + bump: major + - name: polkadot-node-core-approval-voting + bump: major + - name: polkadot-node-core-approval-voting-parallel + bump: major + - name: polkadot-network-bridge + bump: major + - name: polkadot-node-core-dispute-coordinator + bump: major + - name: cumulus-relay-chain-inprocess-interface + bump: major + - name: polkadot-cli + bump: major + - name: polkadot + bump: major From e19d8dc88066ae4aef7534a7fdce0c33c0df3f1f Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 20 Sep 2024 15:44:37 +0300 Subject: [PATCH 38/45] Add implementers guide documenation for the new subsystem Signed-off-by: Alexandru Gheorghe --- .../node/approval/approval-voting-parallel.md | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 polkadot/roadmap/implementers-guide/src/node/approval/approval-voting-parallel.md diff --git a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting-parallel.md b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting-parallel.md new file mode 100644 index 000000000000..84661b7bf9b3 --- /dev/null +++ b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting-parallel.md @@ -0,0 +1,30 @@ +# Approval voting parallel + +The approval-voting-parallel subsystem acts as an orchestrator for the tasks handled by the [Approval Voting](approval-voting.md) +and [Approval Distribution](approval-distribution.md) subsystems. Initially, these two systems operated separately and interacted +with each other and other subsystems through orchestra. + +With approval-voting-parallel, we have a single subsystem that creates two types of workers: +- Four approval-distribution workers that operate in parallel, each handling tasks based on the validator_index of the message + originator. +- One approval-voting worker that performs the tasks previously managed by the standalone approval-voting subsystem. + +This subsystem does not maintain any state. Instead, it functions as an orchestrator that: +- Spawns and initializes each workers. +- Forwards each message and signal to the appropriate worker. +- Aggregates results for messages that require input from more than one worker, such as GetApprovalSignatures. + +## Forwarding logic + +The messages received and forwarded by approval-voting-parallel split in three categories: +- Signals which need to be forwarded to all workers. +- Messages that only the `approval-voting` worker needs to handle, `ApprovalVotingParallelMessage::ApprovedAncestor` + and `ApprovalVotingParallelMessage::GetApprovalSignaturesForCandidate` +- Control messages that all `approval-distribution` workers need to receive `ApprovalVotingParallelMessage::NewBlocks`, + `ApprovalVotingParallelMessage::ApprovalCheckingLagUpdate` and all network bridge variants `ApprovalVotingParallelMessage::NetworkBridgeUpdate` + except `ApprovalVotingParallelMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage)` +- Data messages `ApprovalVotingParallelMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage)` which need to be sent + just to a single `approval-distribution` worker based on the ValidatorIndex. The logic for assigning the work is: + ``` + assigned_worker_index = validator_index % number_of_workers; + ``` From ae0a087340b130f89dd7cb983812e65c6500425f Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 20 Sep 2024 16:20:40 +0300 Subject: [PATCH 39/45] Fix gum reference Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting-parallel/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/core/approval-voting-parallel/Cargo.toml b/polkadot/node/core/approval-voting-parallel/Cargo.toml index 5bfec329ee89..e9c94ba48425 100644 --- a/polkadot/node/core/approval-voting-parallel/Cargo.toml +++ b/polkadot/node/core/approval-voting-parallel/Cargo.toml @@ -13,7 +13,7 @@ workspace = true async-trait = { workspace = true } futures = { workspace = true } futures-timer = { workspace = true } -gum = { package = "tracing-gum", path = "../../gum" } +gum = { workspace = true } itertools = { workspace = true } thiserror = { workspace = true } From 2b6e5c65f587f8a5a3e124ee0149f987229ff230 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 20 Sep 2024 17:12:07 +0300 Subject: [PATCH 40/45] Make cargo doc happy Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting-parallel/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/polkadot/node/core/approval-voting-parallel/src/lib.rs b/polkadot/node/core/approval-voting-parallel/src/lib.rs index 172c131c6f47..c6e7c0788463 100644 --- a/polkadot/node/core/approval-voting-parallel/src/lib.rs +++ b/polkadot/node/core/approval-voting-parallel/src/lib.rs @@ -698,8 +698,8 @@ where } } -/// Just a wrapper for implementing overseer::SubsystemSender and -/// overseer::SubsystemSender. +/// Just a wrapper for implementing `overseer::SubsystemSender` and +/// `overseer::SubsystemSender`. /// /// The instance of this struct can be injected into the workers, so they can talk /// directly with each other without intermediating in this subsystem loop. @@ -872,8 +872,8 @@ where (to_worker, WorkProviderImpl::from_rx_worker(rx_worker, prio_right)) } -/// Just a wrapper for implementing overseer::SubsystemSender, so that -/// we can inject into the approval voting subsystem. +/// Just a wrapper for implementing `overseer::SubsystemSender`, so +/// that we can inject into the approval voting subsystem. #[derive(Clone)] pub struct ApprovalVotingToApprovalDistribution>( S, From cecb78d6de3176365c0a946d7e9cfd4ba51c6136 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 20 Sep 2024 17:14:21 +0300 Subject: [PATCH 41/45] Update Cargo.lock Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4e28e2f64da2..d4a44fc6785d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14281,17 +14281,17 @@ dependencies = [ "polkadot-subsystem-bench", "rand", "rand_chacha", - "rand_core", + "rand_core 0.6.4", "sc-keystore", "schnorrkel 0.11.4", - "sp-application-crypto", + "sp-application-crypto 30.0.0", "sp-consensus", "sp-consensus-babe", "sp-consensus-slots", - "sp-core", + "sp-core 28.0.0", "sp-keyring", - "sp-keystore", - "sp-runtime", + "sp-keystore 0.34.0", + "sp-runtime 31.0.1", "sp-tracing 16.0.0", "thiserror", "tracing-gum", From 18facc1c1e2107f2210451c7487cbb20e0ffe828 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 20 Sep 2024 17:22:11 +0300 Subject: [PATCH 42/45] Make taplo happy Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting-parallel/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/polkadot/node/core/approval-voting-parallel/Cargo.toml b/polkadot/node/core/approval-voting-parallel/Cargo.toml index e9c94ba48425..e62062eab409 100644 --- a/polkadot/node/core/approval-voting-parallel/Cargo.toml +++ b/polkadot/node/core/approval-voting-parallel/Cargo.toml @@ -53,4 +53,3 @@ polkadot-primitives-test-helpers = { workspace = true, default-features = true } log = { workspace = true, default-features = true } polkadot-subsystem-bench = { workspace = true, default-features = true } schnorrkel = { workspace = true, default-features = true } - From 89ef3fa5bec11535341046d5a43e9752de2d15c9 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Sat, 21 Sep 2024 10:22:43 +0300 Subject: [PATCH 43/45] Send GetApprovalSignatures on unbounded Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting-parallel/src/lib.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/polkadot/node/core/approval-voting-parallel/src/lib.rs b/polkadot/node/core/approval-voting-parallel/src/lib.rs index c6e7c0788463..18c73cfba1fc 100644 --- a/polkadot/node/core/approval-voting-parallel/src/lib.rs +++ b/polkadot/node/core/approval-voting-parallel/src/lib.rs @@ -446,12 +446,10 @@ async fn handle_get_approval_signatures( let mut signatures_channels = Vec::new(); for worker in to_approval_distribution_workers.iter_mut() { let (tx, rx) = oneshot::channel(); - worker - .send_message(ApprovalDistributionMessage::GetApprovalSignatures( - requested_candidates.clone(), - tx, - )) - .await; + worker.send_unbounded_message(ApprovalDistributionMessage::GetApprovalSignatures( + requested_candidates.clone(), + tx, + )); signatures_channels.push(rx); } From 64be040d1b05114af447c282d4f601b347cb3d2e Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Sat, 21 Sep 2024 11:00:07 +0300 Subject: [PATCH 44/45] Fix failing unittest Signed-off-by: Alexandru Gheorghe --- polkadot/node/overseer/src/tests.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/polkadot/node/overseer/src/tests.rs b/polkadot/node/overseer/src/tests.rs index 8833b93c0d1b..cb0add03e2e7 100644 --- a/polkadot/node/overseer/src/tests.rs +++ b/polkadot/node/overseer/src/tests.rs @@ -950,7 +950,7 @@ fn test_prospective_parachains_msg() -> ProspectiveParachainsMessage { // Checks that `stop`, `broadcast_signal` and `broadcast_message` are implemented correctly. #[test] fn overseer_all_subsystems_receive_signals_and_messages() { - const NUM_SUBSYSTEMS: usize = 23; + const NUM_SUBSYSTEMS: usize = 24; // -4 for BitfieldSigning, GossipSupport, AvailabilityDistribution and PvfCheckerSubsystem. const NUM_SUBSYSTEMS_MESSAGED: usize = NUM_SUBSYSTEMS - 4; @@ -1028,6 +1028,11 @@ fn overseer_all_subsystems_receive_signals_and_messages() { handle .send_msg_anon(AllMessages::ApprovalDistribution(test_approval_distribution_msg())) .await; + handle + .send_msg_anon(AllMessages::ApprovalVotingParallel( + test_approval_distribution_msg().into(), + )) + .await; handle .send_msg_anon(AllMessages::ApprovalVoting(test_approval_voting_msg())) .await; From 19816a7ba38866604a186e38418be7c290f944f8 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 23 Sep 2024 09:20:54 +0300 Subject: [PATCH 45/45] Make umbrella check happy Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 1 + umbrella/Cargo.toml | 7 ++++++- umbrella/src/lib.rs | 4 ++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index d4a44fc6785d..85e2adfabd3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15426,6 +15426,7 @@ dependencies = [ "polkadot-network-bridge", "polkadot-node-collation-generation", "polkadot-node-core-approval-voting", + "polkadot-node-core-approval-voting-parallel", "polkadot-node-core-av-store", "polkadot-node-core-backing", "polkadot-node-core-bitfield-signing", diff --git a/umbrella/Cargo.toml b/umbrella/Cargo.toml index b7c1c375094f..83cbebbc61cd 100644 --- a/umbrella/Cargo.toml +++ b/umbrella/Cargo.toml @@ -600,7 +600,7 @@ runtime = [ "sp-wasm-interface", "sp-weights", ] -node = ["asset-test-utils", "bridge-hub-test-utils", "cumulus-client-cli", "cumulus-client-collator", "cumulus-client-consensus-aura", "cumulus-client-consensus-common", "cumulus-client-consensus-proposer", "cumulus-client-consensus-relay-chain", "cumulus-client-network", "cumulus-client-parachain-inherent", "cumulus-client-pov-recovery", "cumulus-client-service", "cumulus-relay-chain-inprocess-interface", "cumulus-relay-chain-interface", "cumulus-relay-chain-minimal-node", "cumulus-relay-chain-rpc-interface", "cumulus-test-relay-sproof-builder", "emulated-integration-tests-common", "fork-tree", "frame-benchmarking-cli", "frame-remote-externalities", "frame-support-procedural-tools", "generate-bags", "mmr-gadget", "mmr-rpc", "pallet-contracts-mock-network", "pallet-revive-mock-network", "pallet-transaction-payment-rpc", "parachains-runtimes-test-utils", "polkadot-approval-distribution", "polkadot-availability-bitfield-distribution", "polkadot-availability-distribution", "polkadot-availability-recovery", "polkadot-cli", "polkadot-collator-protocol", "polkadot-dispute-distribution", "polkadot-erasure-coding", "polkadot-gossip-support", "polkadot-network-bridge", "polkadot-node-collation-generation", "polkadot-node-core-approval-voting", "polkadot-node-core-av-store", "polkadot-node-core-backing", "polkadot-node-core-bitfield-signing", "polkadot-node-core-candidate-validation", "polkadot-node-core-chain-api", "polkadot-node-core-chain-selection", "polkadot-node-core-dispute-coordinator", "polkadot-node-core-parachains-inherent", "polkadot-node-core-prospective-parachains", "polkadot-node-core-provisioner", "polkadot-node-core-pvf", "polkadot-node-core-pvf-checker", "polkadot-node-core-pvf-common", "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", "polkadot-node-core-runtime-api", "polkadot-node-jaeger", "polkadot-node-metrics", "polkadot-node-network-protocol", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-types", "polkadot-node-subsystem-util", "polkadot-overseer", "polkadot-parachain-lib", "polkadot-rpc", "polkadot-service", "polkadot-statement-distribution", "polkadot-statement-table", "sc-allocator", "sc-authority-discovery", "sc-basic-authorship", "sc-block-builder", "sc-chain-spec", "sc-cli", "sc-client-api", "sc-client-db", "sc-consensus", "sc-consensus-aura", "sc-consensus-babe", "sc-consensus-babe-rpc", "sc-consensus-beefy", "sc-consensus-beefy-rpc", "sc-consensus-epochs", "sc-consensus-grandpa", "sc-consensus-grandpa-rpc", "sc-consensus-manual-seal", "sc-consensus-pow", "sc-consensus-slots", "sc-executor", "sc-executor-common", "sc-executor-polkavm", "sc-executor-wasmtime", "sc-informant", "sc-keystore", "sc-mixnet", "sc-network", "sc-network-common", "sc-network-gossip", "sc-network-light", "sc-network-statement", "sc-network-sync", "sc-network-transactions", "sc-network-types", "sc-offchain", "sc-proposer-metrics", "sc-rpc", "sc-rpc-api", "sc-rpc-server", "sc-rpc-spec-v2", "sc-service", "sc-state-db", "sc-statement-store", "sc-storage-monitor", "sc-sync-state-rpc", "sc-sysinfo", "sc-telemetry", "sc-tracing", "sc-transaction-pool", "sc-transaction-pool-api", "sc-utils", "snowbridge-runtime-test-common", "sp-blockchain", "sp-consensus", "sp-core-hashing", "sp-core-hashing-proc-macro", "sp-database", "sp-maybe-compressed-blob", "sp-panic-handler", "sp-rpc", "staging-chain-spec-builder", "staging-node-inspect", "staging-tracking-allocator", "std", "subkey", "substrate-build-script-utils", "substrate-frame-rpc-support", "substrate-frame-rpc-system", "substrate-prometheus-endpoint", "substrate-rpc-client", "substrate-state-trie-migration-rpc", "substrate-wasm-builder", "tracing-gum", "xcm-emulator", "xcm-simulator"] +node = ["asset-test-utils", "bridge-hub-test-utils", "cumulus-client-cli", "cumulus-client-collator", "cumulus-client-consensus-aura", "cumulus-client-consensus-common", "cumulus-client-consensus-proposer", "cumulus-client-consensus-relay-chain", "cumulus-client-network", "cumulus-client-parachain-inherent", "cumulus-client-pov-recovery", "cumulus-client-service", "cumulus-relay-chain-inprocess-interface", "cumulus-relay-chain-interface", "cumulus-relay-chain-minimal-node", "cumulus-relay-chain-rpc-interface", "cumulus-test-relay-sproof-builder", "emulated-integration-tests-common", "fork-tree", "frame-benchmarking-cli", "frame-remote-externalities", "frame-support-procedural-tools", "generate-bags", "mmr-gadget", "mmr-rpc", "pallet-contracts-mock-network", "pallet-revive-mock-network", "pallet-transaction-payment-rpc", "parachains-runtimes-test-utils", "polkadot-approval-distribution", "polkadot-availability-bitfield-distribution", "polkadot-availability-distribution", "polkadot-availability-recovery", "polkadot-cli", "polkadot-collator-protocol", "polkadot-dispute-distribution", "polkadot-erasure-coding", "polkadot-gossip-support", "polkadot-network-bridge", "polkadot-node-collation-generation", "polkadot-node-core-approval-voting", "polkadot-node-core-approval-voting-parallel", "polkadot-node-core-av-store", "polkadot-node-core-backing", "polkadot-node-core-bitfield-signing", "polkadot-node-core-candidate-validation", "polkadot-node-core-chain-api", "polkadot-node-core-chain-selection", "polkadot-node-core-dispute-coordinator", "polkadot-node-core-parachains-inherent", "polkadot-node-core-prospective-parachains", "polkadot-node-core-provisioner", "polkadot-node-core-pvf", "polkadot-node-core-pvf-checker", "polkadot-node-core-pvf-common", "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", "polkadot-node-core-runtime-api", "polkadot-node-jaeger", "polkadot-node-metrics", "polkadot-node-network-protocol", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-types", "polkadot-node-subsystem-util", "polkadot-overseer", "polkadot-parachain-lib", "polkadot-rpc", "polkadot-service", "polkadot-statement-distribution", "polkadot-statement-table", "sc-allocator", "sc-authority-discovery", "sc-basic-authorship", "sc-block-builder", "sc-chain-spec", "sc-cli", "sc-client-api", "sc-client-db", "sc-consensus", "sc-consensus-aura", "sc-consensus-babe", "sc-consensus-babe-rpc", "sc-consensus-beefy", "sc-consensus-beefy-rpc", "sc-consensus-epochs", "sc-consensus-grandpa", "sc-consensus-grandpa-rpc", "sc-consensus-manual-seal", "sc-consensus-pow", "sc-consensus-slots", "sc-executor", "sc-executor-common", "sc-executor-polkavm", "sc-executor-wasmtime", "sc-informant", "sc-keystore", "sc-mixnet", "sc-network", "sc-network-common", "sc-network-gossip", "sc-network-light", "sc-network-statement", "sc-network-sync", "sc-network-transactions", "sc-network-types", "sc-offchain", "sc-proposer-metrics", "sc-rpc", "sc-rpc-api", "sc-rpc-server", "sc-rpc-spec-v2", "sc-service", "sc-state-db", "sc-statement-store", "sc-storage-monitor", "sc-sync-state-rpc", "sc-sysinfo", "sc-telemetry", "sc-tracing", "sc-transaction-pool", "sc-transaction-pool-api", "sc-utils", "snowbridge-runtime-test-common", "sp-blockchain", "sp-consensus", "sp-core-hashing", "sp-core-hashing-proc-macro", "sp-database", "sp-maybe-compressed-blob", "sp-panic-handler", "sp-rpc", "staging-chain-spec-builder", "staging-node-inspect", "staging-tracking-allocator", "std", "subkey", "substrate-build-script-utils", "substrate-frame-rpc-support", "substrate-frame-rpc-system", "substrate-prometheus-endpoint", "substrate-rpc-client", "substrate-state-trie-migration-rpc", "substrate-wasm-builder", "tracing-gum", "xcm-emulator", "xcm-simulator"] tuples-96 = [ "frame-support-procedural?/tuples-96", "frame-support?/tuples-96", @@ -1967,6 +1967,11 @@ path = "../polkadot/node/core/approval-voting" default-features = false optional = true +[dependencies.polkadot-node-core-approval-voting-parallel] +path = "../polkadot/node/core/approval-voting-parallel" +default-features = false +optional = true + [dependencies.polkadot-node-core-av-store] path = "../polkadot/node/core/av-store" default-features = false diff --git a/umbrella/src/lib.rs b/umbrella/src/lib.rs index b7b9c15fe588..4a653dab99b6 100644 --- a/umbrella/src/lib.rs +++ b/umbrella/src/lib.rs @@ -796,6 +796,10 @@ pub use polkadot_node_collation_generation; #[cfg(feature = "polkadot-node-core-approval-voting")] pub use polkadot_node_core_approval_voting; +/// Approval Voting Subsystem running approval work in parallel. +#[cfg(feature = "polkadot-node-core-approval-voting-parallel")] +pub use polkadot_node_core_approval_voting_parallel; + /// The Availability Store subsystem. Wrapper over the DB that stores availability data and /// chunks. #[cfg(feature = "polkadot-node-core-av-store")]