From 7230df4a818458f3eb5244c142092d52811c28d0 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 25 Aug 2023 19:15:22 +0300 Subject: [PATCH 001/192] merge from archived repo Signed-off-by: Andrei Sandu --- .gitignore | 16 + polkadot/node/core/approval-voting/Cargo.toml | 6 + .../approval-voting/src/approval_checking.rs | 60 +- .../approval-voting/src/approval_db/mod.rs | 1 + .../approval-voting/src/approval_db/v1/mod.rs | 268 +--- .../src/approval_db/v2/migration_helpers.rs | 241 +++ .../approval-voting/src/approval_db/v2/mod.rs | 394 +++++ .../src/approval_db/v2/tests.rs | 570 +++++++ .../node/core/approval-voting/src/backend.rs | 16 +- .../node/core/approval-voting/src/criteria.rs | 498 ++++-- .../node/core/approval-voting/src/import.rs | 38 +- polkadot/node/core/approval-voting/src/lib.rs | 518 +++++-- polkadot/node/core/approval-voting/src/ops.rs | 2 +- .../approval-voting/src/persisted_entries.rs | 135 +- .../node/core/approval-voting/src/tests.rs | 395 ++++- .../node/core/approval-voting/src/time.rs | 2 +- .../core/dispute-coordinator/src/import.rs | 2 +- .../network/approval-distribution/Cargo.toml | 2 + .../network/approval-distribution/src/lib.rs | 1352 ++++++++++------- .../approval-distribution/src/metrics.rs | 45 +- .../approval-distribution/src/tests.rs | 597 +++++--- .../network/bitfield-distribution/src/lib.rs | 12 +- polkadot/node/network/bridge/src/network.rs | 107 +- polkadot/node/network/bridge/src/rx/mod.rs | 110 +- polkadot/node/network/bridge/src/tx/mod.rs | 91 +- polkadot/node/network/bridge/src/tx/tests.rs | 5 +- polkadot/node/network/protocol/src/lib.rs | 25 +- polkadot/node/primitives/Cargo.toml | 1 + polkadot/node/primitives/src/approval.rs | 655 ++++++-- .../node/service/src/parachains_db/mod.rs | 36 +- .../node/service/src/parachains_db/upgrade.rs | 367 ++++- polkadot/node/service/src/tests.rs | 2 +- polkadot/node/subsystem-types/Cargo.toml | 1 + polkadot/node/subsystem-types/src/messages.rs | 13 +- .../node/subsystem-util/src/reputation.rs | 5 + .../src/node/approval/approval-voting.md | 31 +- .../src/protocol-approval.md | 10 +- .../implementers-guide/src/types/approval.md | 29 + .../implementers-guide/src/types/runtime.md | 2 +- .../scripts/ci/gitlab/pipeline/zombienet.yml | 30 +- .../0005-parachains-max-tranche0.toml | 40 + .../0005-parachains-max-tranche0.zndsl | 27 + .../0002-parachains-upgrade-smoke-test.toml | 2 +- 43 files changed, 4986 insertions(+), 1773 deletions(-) create mode 100644 polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs create mode 100644 polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs create mode 100644 polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs create mode 100644 polkadot/zombienet_tests/functional/0005-parachains-max-tranche0.toml create mode 100644 polkadot/zombienet_tests/functional/0005-parachains-max-tranche0.zndsl diff --git a/.gitignore b/.gitignore index e69de29bb2d1..61ef9e91a55e 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1,16 @@ +**/target/ +**/*.rs.bk +*.swp +.wasm-binaries +runtime/wasm/target/ +**/._* +.idea +.vscode +polkadot.* +!polkadot.service +.DS_Store +.env + +artifacts +release-artifacts +release.json diff --git a/polkadot/node/core/approval-voting/Cargo.toml b/polkadot/node/core/approval-voting/Cargo.toml index 5ae99d44d0a5..983189a682c5 100644 --- a/polkadot/node/core/approval-voting/Cargo.toml +++ b/polkadot/node/core/approval-voting/Cargo.toml @@ -17,6 +17,7 @@ schnorrkel = "0.9.1" kvdb = "0.13.0" derive_more = "0.99.17" thiserror = "1.0.31" +itertools = "0.10.5" polkadot-node-subsystem = { path = "../../subsystem" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } @@ -30,6 +31,9 @@ sp-consensus = { path = "../../../../substrate/primitives/consensus/common", def sp-consensus-slots = { path = "../../../../substrate/primitives/consensus/slots", default-features = false } sp-application-crypto = { path = "../../../../substrate/primitives/application-crypto", default-features = false, features = ["full_crypto"] } sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } +# Needed for migration test helpers +test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" } +rand_core = "0.5.1" [dev-dependencies] async-trait = "0.1.57" @@ -43,3 +47,5 @@ polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } assert_matches = "1.4.0" kvdb-memorydb = "0.13.0" test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" } +log = "0.4.17" +env_logger = "0.9.0" diff --git a/polkadot/node/core/approval-voting/src/approval_checking.rs b/polkadot/node/core/approval-voting/src/approval_checking.rs index f345b57029b5..5d24ff164193 100644 --- a/polkadot/node/core/approval-voting/src/approval_checking.rs +++ b/polkadot/node/core/approval-voting/src/approval_checking.rs @@ -17,7 +17,7 @@ //! Utilities for checking whether a candidate has been approved under a given block. use bitvec::{order::Lsb0 as BitOrderLsb0, slice::BitSlice}; -use polkadot_node_primitives::approval::DelayTranche; +use polkadot_node_primitives::approval::v1::DelayTranche; use polkadot_primitives::ValidatorIndex; use crate::{ @@ -472,9 +472,9 @@ mod tests { } .into(); - let approval_entry = approval_db::v1::ApprovalEntry { + let approval_entry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: BitVec::default(), + assigned_validators: BitVec::default(), our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -509,22 +509,22 @@ mod tests { candidate.mark_approval(ValidatorIndex(i)); } - let approval_entry = approval_db::v1::ApprovalEntry { + let approval_entry = approval_db::v2::ApprovalEntry { tranches: vec![ - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 0, assignments: (0..2).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 1, assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(), }, - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 2, assignments: (5..10).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, ], - assignments: bitvec![u8, BitOrderLsb0; 1; 10], + assigned_validators: bitvec![u8, BitOrderLsb0; 1; 10], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -581,22 +581,22 @@ mod tests { candidate.mark_approval(ValidatorIndex(i)); } - let approval_entry = approval_db::v1::ApprovalEntry { + let approval_entry = approval_db::v2::ApprovalEntry { tranches: vec![ - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 0, assignments: (0..4).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 1, assignments: (4..6).map(|i| (ValidatorIndex(i), 1.into())).collect(), }, - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 2, assignments: (6..10).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, ], - assignments: bitvec![u8, BitOrderLsb0; 1; 10], + assigned_validators: bitvec![u8, BitOrderLsb0; 1; 10], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -647,9 +647,9 @@ mod tests { let no_show_duration = 10; let needed_approvals = 4; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; 5], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; 5], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -691,9 +691,9 @@ mod tests { let no_show_duration = 10; let needed_approvals = 4; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; 10], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; 10], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -731,9 +731,9 @@ mod tests { let no_show_duration = 10; let needed_approvals = 4; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; 10], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; 10], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -776,9 +776,9 @@ mod tests { let needed_approvals = 4; let n_validators = 8; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; n_validators], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -843,9 +843,9 @@ mod tests { let needed_approvals = 4; let n_validators = 8; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; n_validators], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -934,9 +934,9 @@ mod tests { let needed_approvals = 4; let n_validators = 8; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; n_validators], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -1041,15 +1041,15 @@ mod tests { candidate.mark_approval(ValidatorIndex(i)); } - let approval_entry = approval_db::v1::ApprovalEntry { + let approval_entry = approval_db::v2::ApprovalEntry { tranches: vec![ // Assignments with invalid validator indexes. - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 1, assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(), }, ], - assignments: bitvec![u8, BitOrderLsb0; 1; 3], + assigned_validators: bitvec![u8, BitOrderLsb0; 1; 3], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -1094,12 +1094,12 @@ mod tests { ]; for test_tranche in test_tranches { - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), backing_group: GroupIndex(0), our_assignment: None, our_approval_sig: None, - assignments: bitvec![u8, BitOrderLsb0; 0; 3], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; 3], approved: false, } .into(); diff --git a/polkadot/node/core/approval-voting/src/approval_db/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/mod.rs index f0ace95613da..20fb6aa82d8d 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/mod.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/mod.rs @@ -31,3 +31,4 @@ //! time being we share the same DB with the rest of Substrate. pub mod v1; +pub mod v2; diff --git a/polkadot/node/core/approval-voting/src/approval_db/v1/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/v1/mod.rs index c31389269d2e..011d0a559c02 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v1/mod.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v1/mod.rs @@ -23,144 +23,15 @@ //! require a db migration (check `node/service/src/parachains_db/upgrade.rs`). use parity_scale_codec::{Decode, Encode}; -use polkadot_node_primitives::approval::{AssignmentCert, DelayTranche}; -use polkadot_node_subsystem::{SubsystemError, SubsystemResult}; -use polkadot_node_subsystem_util::database::{DBTransaction, Database}; +use polkadot_node_primitives::approval::v1::{AssignmentCert, DelayTranche}; use polkadot_primitives::{ BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, SessionIndex, ValidatorIndex, ValidatorSignature, }; use sp_consensus_slots::Slot; +use std::collections::BTreeMap; -use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; -use std::{collections::BTreeMap, sync::Arc}; - -use crate::{ - backend::{Backend, BackendWriteOp}, - persisted_entries, -}; - -const STORED_BLOCKS_KEY: &[u8] = b"Approvals_StoredBlocks"; - -#[cfg(test)] -pub mod tests; - -/// `DbBackend` is a concrete implementation of the higher-level Backend trait -pub struct DbBackend { - inner: Arc, - config: Config, -} - -impl DbBackend { - /// Create a new [`DbBackend`] with the supplied key-value store and - /// config. - pub fn new(db: Arc, config: Config) -> Self { - DbBackend { inner: db, config } - } -} - -impl Backend for DbBackend { - fn load_block_entry( - &self, - block_hash: &Hash, - ) -> SubsystemResult> { - load_block_entry(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) - } - - fn load_candidate_entry( - &self, - candidate_hash: &CandidateHash, - ) -> SubsystemResult> { - load_candidate_entry(&*self.inner, &self.config, candidate_hash).map(|e| e.map(Into::into)) - } - - fn load_blocks_at_height(&self, block_height: &BlockNumber) -> SubsystemResult> { - load_blocks_at_height(&*self.inner, &self.config, block_height) - } - - fn load_all_blocks(&self) -> SubsystemResult> { - load_all_blocks(&*self.inner, &self.config) - } - - fn load_stored_blocks(&self) -> SubsystemResult> { - load_stored_blocks(&*self.inner, &self.config) - } - - /// Atomically write the list of operations, with later operations taking precedence over prior. - fn write(&mut self, ops: I) -> SubsystemResult<()> - where - I: IntoIterator, - { - let mut tx = DBTransaction::new(); - for op in ops { - match op { - BackendWriteOp::WriteStoredBlockRange(stored_block_range) => { - tx.put_vec( - self.config.col_approval_data, - &STORED_BLOCKS_KEY, - stored_block_range.encode(), - ); - }, - BackendWriteOp::DeleteStoredBlockRange => { - tx.delete(self.config.col_approval_data, &STORED_BLOCKS_KEY); - }, - BackendWriteOp::WriteBlocksAtHeight(h, blocks) => { - tx.put_vec( - self.config.col_approval_data, - &blocks_at_height_key(h), - blocks.encode(), - ); - }, - BackendWriteOp::DeleteBlocksAtHeight(h) => { - tx.delete(self.config.col_approval_data, &blocks_at_height_key(h)); - }, - BackendWriteOp::WriteBlockEntry(block_entry) => { - let block_entry: BlockEntry = block_entry.into(); - tx.put_vec( - self.config.col_approval_data, - &block_entry_key(&block_entry.block_hash), - block_entry.encode(), - ); - }, - BackendWriteOp::DeleteBlockEntry(hash) => { - tx.delete(self.config.col_approval_data, &block_entry_key(&hash)); - }, - BackendWriteOp::WriteCandidateEntry(candidate_entry) => { - let candidate_entry: CandidateEntry = candidate_entry.into(); - tx.put_vec( - self.config.col_approval_data, - &candidate_entry_key(&candidate_entry.candidate.hash()), - candidate_entry.encode(), - ); - }, - BackendWriteOp::DeleteCandidateEntry(candidate_hash) => { - tx.delete(self.config.col_approval_data, &candidate_entry_key(&candidate_hash)); - }, - } - } - - self.inner.write(tx).map_err(|e| e.into()) - } -} - -/// A range from earliest..last block number stored within the DB. -#[derive(Encode, Decode, Debug, Clone, PartialEq)] -pub struct StoredBlockRange(pub BlockNumber, pub BlockNumber); - -// slot_duration * 2 + DelayTranche gives the number of delay tranches since the -// unix epoch. -#[derive(Encode, Decode, Clone, Copy, Debug, PartialEq)] -pub struct Tick(u64); - -/// Convenience type definition -pub type Bitfield = BitVec; - -/// The database config. -#[derive(Debug, Clone, Copy)] -pub struct Config { - /// The column family in the database where data is stored. - pub col_approval_data: u32, -} +use super::v2::Bitfield; /// Details pertaining to our assignment on a block. #[derive(Encode, Decode, Debug, Clone, PartialEq)] @@ -171,15 +42,7 @@ pub struct OurAssignment { // Whether the assignment has been triggered already. pub triggered: bool, } - -/// Metadata regarding a specific tranche of assignments for a specific candidate. -#[derive(Encode, Decode, Debug, Clone, PartialEq)] -pub struct TrancheEntry { - pub tranche: DelayTranche, - // Assigned validators, and the instant we received their assignment, rounded - // to the nearest tick. - pub assignments: Vec<(ValidatorIndex, Tick)>, -} +use super::v2::TrancheEntry; /// Metadata regarding approval of a particular candidate within the context of some /// particular block. @@ -226,126 +89,3 @@ pub struct BlockEntry { pub approved_bitfield: Bitfield, pub children: Vec, } - -impl From for Tick { - fn from(tick: crate::Tick) -> Tick { - Tick(tick) - } -} - -impl From for crate::Tick { - fn from(tick: Tick) -> crate::Tick { - tick.0 - } -} - -/// Errors while accessing things from the DB. -#[derive(Debug, derive_more::From, derive_more::Display)] -pub enum Error { - Io(std::io::Error), - InvalidDecoding(parity_scale_codec::Error), -} - -impl std::error::Error for Error {} - -/// Result alias for DB errors. -pub type Result = std::result::Result; - -pub(crate) fn load_decode( - store: &dyn Database, - col_approval_data: u32, - key: &[u8], -) -> Result> { - match store.get(col_approval_data, key)? { - None => Ok(None), - Some(raw) => D::decode(&mut &raw[..]).map(Some).map_err(Into::into), - } -} - -/// The key a given block entry is stored under. -pub(crate) fn block_entry_key(block_hash: &Hash) -> [u8; 46] { - const BLOCK_ENTRY_PREFIX: [u8; 14] = *b"Approvals_blck"; - - let mut key = [0u8; 14 + 32]; - key[0..14].copy_from_slice(&BLOCK_ENTRY_PREFIX); - key[14..][..32].copy_from_slice(block_hash.as_ref()); - - key -} - -/// The key a given candidate entry is stored under. -pub(crate) fn candidate_entry_key(candidate_hash: &CandidateHash) -> [u8; 46] { - const CANDIDATE_ENTRY_PREFIX: [u8; 14] = *b"Approvals_cand"; - - let mut key = [0u8; 14 + 32]; - key[0..14].copy_from_slice(&CANDIDATE_ENTRY_PREFIX); - key[14..][..32].copy_from_slice(candidate_hash.0.as_ref()); - - key -} - -/// The key a set of block hashes corresponding to a block number is stored under. -pub(crate) fn blocks_at_height_key(block_number: BlockNumber) -> [u8; 16] { - const BLOCKS_AT_HEIGHT_PREFIX: [u8; 12] = *b"Approvals_at"; - - let mut key = [0u8; 12 + 4]; - key[0..12].copy_from_slice(&BLOCKS_AT_HEIGHT_PREFIX); - block_number.using_encoded(|s| key[12..16].copy_from_slice(s)); - - key -} - -/// Return all blocks which have entries in the DB, ascending, by height. -pub fn load_all_blocks(store: &dyn Database, config: &Config) -> SubsystemResult> { - let mut hashes = Vec::new(); - if let Some(stored_blocks) = load_stored_blocks(store, config)? { - for height in stored_blocks.0..stored_blocks.1 { - let blocks = load_blocks_at_height(store, config, &height)?; - hashes.extend(blocks); - } - } - - Ok(hashes) -} - -/// Load the stored-blocks key from the state. -pub fn load_stored_blocks( - store: &dyn Database, - config: &Config, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, STORED_BLOCKS_KEY) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - -/// Load a blocks-at-height entry for a given block number. -pub fn load_blocks_at_height( - store: &dyn Database, - config: &Config, - block_number: &BlockNumber, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, &blocks_at_height_key(*block_number)) - .map(|x| x.unwrap_or_default()) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - -/// Load a block entry from the aux store. -pub fn load_block_entry( - store: &dyn Database, - config: &Config, - block_hash: &Hash, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, &block_entry_key(block_hash)) - .map(|u: Option| u.map(|v| v.into())) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - -/// Load a candidate entry from the aux store. -pub fn load_candidate_entry( - store: &dyn Database, - config: &Config, - candidate_hash: &CandidateHash, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash)) - .map(|u: Option| u.map(|v| v.into())) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs new file mode 100644 index 000000000000..3d1f28ee1937 --- /dev/null +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs @@ -0,0 +1,241 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Approval DB migration helpers. +use super::{StoredBlockRange, *}; +use crate::backend::Backend; +use polkadot_node_primitives::approval::v1::{ + AssignmentCert, AssignmentCertKind, VrfOutput, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT, +}; +use polkadot_node_subsystem_util::database::Database; +use std::{collections::HashSet, sync::Arc}; + +use ::test_helpers::dummy_candidate_receipt; + +fn dummy_assignment_cert(kind: AssignmentCertKind) -> AssignmentCert { + let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); + let msg = b"test-garbage"; + let mut prng = rand_core::OsRng; + let keypair = schnorrkel::Keypair::generate_with(&mut prng); + let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); + let out = inout.to_output(); + + AssignmentCert { kind, vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) } } +} + +fn make_block_entry_v1( + block_hash: Hash, + parent_hash: Hash, + block_number: BlockNumber, + candidates: Vec<(CoreIndex, CandidateHash)>, +) -> crate::approval_db::v1::BlockEntry { + crate::approval_db::v1::BlockEntry { + block_hash, + parent_hash, + block_number, + session: 1, + slot: Slot::from(1), + relay_vrf_story: [0u8; 32], + approved_bitfield: make_bitvec(candidates.len()), + candidates, + children: Vec::new(), + } +} + +fn make_bitvec(len: usize) -> BitVec { + bitvec::bitvec![u8, BitOrderLsb0; 0; len] +} + +/// Migrates `OurAssignment`, `CandidateEntry` and `ApprovalEntry` to version 2. +/// Returns on any error. +/// Must only be used in parachains DB migration code - `polkadot-service` crate. +pub fn v1_to_v2(db: Arc, config: Config) -> Result<()> { + let mut backend = crate::DbBackend::new(db, config); + let all_blocks = backend + .load_all_blocks() + .map_err(|e| Error::InternalError(e))? + .iter() + .filter_map(|block_hash| { + backend + .load_block_entry_v1(block_hash) + .map_err(|e| Error::InternalError(e)) + .ok()? + }) + .collect::>(); + + gum::info!( + target: crate::LOG_TARGET, + "Migrating candidate entries on top of {} blocks", + all_blocks.len() + ); + + let mut overlay = crate::OverlayedBackend::new(&backend); + let mut counter = 0; + // Get all candidate entries, approval entries and convert each of them. + for block in all_blocks { + for (_core_index, candidate_hash) in block.candidates() { + // Loading the candidate will also perform the conversion to the updated format and + // return that represantation. + if let Some(candidate_entry) = backend + .load_candidate_entry_v1(&candidate_hash) + .map_err(|e| Error::InternalError(e))? + { + // Write the updated representation. + overlay.write_candidate_entry(candidate_entry); + counter += 1; + } + } + overlay.write_block_entry(block); + } + + gum::info!(target: crate::LOG_TARGET, "Migrated {} entries", counter); + + // Commit all changes to DB. + let write_ops = overlay.into_write_ops(); + backend.write(write_ops).unwrap(); + + Ok(()) +} + +// Checks if the migration doesn't leave the DB in an unsane state. +// This function is to be used in tests. +pub fn v1_to_v2_sanity_check( + db: Arc, + config: Config, + expected_candidates: HashSet, +) -> Result<()> { + let backend = crate::DbBackend::new(db, config); + + let all_blocks = backend + .load_all_blocks() + .unwrap() + .iter() + .map(|block_hash| backend.load_block_entry(block_hash).unwrap().unwrap()) + .collect::>(); + + let mut candidates = HashSet::new(); + + // Iterate all blocks and approval entries. + for block in all_blocks { + for (_core_index, candidate_hash) in block.candidates() { + // Loading the candidate will also perform the conversion to the updated format and + // return that represantation. + if let Some(candidate_entry) = backend.load_candidate_entry(&candidate_hash).unwrap() { + candidates.insert(candidate_entry.candidate.hash()); + } + } + } + + assert_eq!(candidates, expected_candidates); + + Ok(()) +} + +// Fills the db with dummy data in v1 scheme. +pub fn v1_to_v2_fill_test_data( + db: Arc, + config: Config, +) -> Result> { + let mut backend = crate::DbBackend::new(db.clone(), config); + let mut overlay_db = crate::OverlayedBackend::new(&backend); + let mut expected_candidates = HashSet::new(); + + const RELAY_BLOCK_COUNT: u32 = 10; + + let range = StoredBlockRange(1, 11); + overlay_db.write_stored_block_range(range.clone()); + + for relay_number in 1..=RELAY_BLOCK_COUNT { + let relay_hash = Hash::repeat_byte(relay_number as u8); + let assignment_core_index = CoreIndex(relay_number); + let candidate = dummy_candidate_receipt(relay_hash); + let candidate_hash = candidate.hash(); + + let at_height = vec![relay_hash]; + + let block_entry = make_block_entry_v1( + relay_hash, + Default::default(), + relay_number, + vec![(assignment_core_index, candidate_hash)], + ); + + let dummy_assignment = crate::approval_db::v1::OurAssignment { + cert: dummy_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }).into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + }; + + let candidate_entry = crate::approval_db::v1::CandidateEntry { + candidate, + session: 123, + block_assignments: vec![( + relay_hash, + crate::approval_db::v1::ApprovalEntry { + tranches: Vec::new(), + backing_group: GroupIndex(1), + our_assignment: Some(dummy_assignment), + our_approval_sig: None, + assignments: Default::default(), + approved: false, + }, + )] + .into_iter() + .collect(), + approvals: Default::default(), + }; + + overlay_db.write_blocks_at_height(relay_number, at_height.clone()); + expected_candidates.insert(candidate_entry.candidate.hash()); + + db.write(write_candidate_entry_v1(candidate_entry, config)).unwrap(); + db.write(write_block_entry_v1(block_entry, config)).unwrap(); + } + + let write_ops = overlay_db.into_write_ops(); + backend.write(write_ops).unwrap(); + + Ok(expected_candidates) +} + +// Low level DB helper to write a candidate entry in v1 scheme. +fn write_candidate_entry_v1( + candidate_entry: crate::approval_db::v1::CandidateEntry, + config: Config, +) -> DBTransaction { + let mut tx = DBTransaction::new(); + tx.put_vec( + config.col_approval_data, + &candidate_entry_key(&candidate_entry.candidate.hash()), + candidate_entry.encode(), + ); + tx +} + +// Low level DB helper to write a block entry in v1 scheme. +fn write_block_entry_v1( + block_entry: crate::approval_db::v1::BlockEntry, + config: Config, +) -> DBTransaction { + let mut tx = DBTransaction::new(); + tx.put_vec( + config.col_approval_data, + &block_entry_key(&block_entry.block_hash), + block_entry.encode(), + ); + tx +} diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs new file mode 100644 index 000000000000..66df6ee8f653 --- /dev/null +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs @@ -0,0 +1,394 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Version 2 of the DB schema. + +use parity_scale_codec::{Decode, Encode}; +use polkadot_node_primitives::approval::{v1::DelayTranche, v2::AssignmentCertV2}; +use polkadot_node_subsystem::{SubsystemError, SubsystemResult}; +use polkadot_node_subsystem_util::database::{DBTransaction, Database}; +use polkadot_primitives::{ + BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, SessionIndex, + ValidatorIndex, ValidatorSignature, +}; + +use sp_consensus_slots::Slot; + +use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; +use std::{collections::BTreeMap, sync::Arc}; + +use crate::{ + backend::{Backend, BackendWriteOp, V1ReadBackend}, + persisted_entries, +}; + +const STORED_BLOCKS_KEY: &[u8] = b"Approvals_StoredBlocks"; + +pub mod migration_helpers; +#[cfg(test)] +pub mod tests; + +/// `DbBackend` is a concrete implementation of the higher-level Backend trait +pub struct DbBackend { + inner: Arc, + config: Config, +} + +impl DbBackend { + /// Create a new [`DbBackend`] with the supplied key-value store and + /// config. + pub fn new(db: Arc, config: Config) -> Self { + DbBackend { inner: db, config } + } +} + +impl V1ReadBackend for DbBackend { + fn load_candidate_entry_v1( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult> { + load_candidate_entry_v1(&*self.inner, &self.config, candidate_hash) + .map(|e| e.map(Into::into)) + } + + fn load_block_entry_v1( + &self, + block_hash: &Hash, + ) -> SubsystemResult> { + load_block_entry_v1(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) + } +} + +impl Backend for DbBackend { + fn load_block_entry( + &self, + block_hash: &Hash, + ) -> SubsystemResult> { + load_block_entry(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) + } + + fn load_candidate_entry( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult> { + load_candidate_entry(&*self.inner, &self.config, candidate_hash).map(|e| e.map(Into::into)) + } + + fn load_blocks_at_height(&self, block_height: &BlockNumber) -> SubsystemResult> { + load_blocks_at_height(&*self.inner, &self.config, block_height) + } + + fn load_all_blocks(&self) -> SubsystemResult> { + load_all_blocks(&*self.inner, &self.config) + } + + fn load_stored_blocks(&self) -> SubsystemResult> { + load_stored_blocks(&*self.inner, &self.config) + } + + /// Atomically write the list of operations, with later operations taking precedence over prior. + fn write(&mut self, ops: I) -> SubsystemResult<()> + where + I: IntoIterator, + { + let mut tx = DBTransaction::new(); + for op in ops { + match op { + BackendWriteOp::WriteStoredBlockRange(stored_block_range) => { + tx.put_vec( + self.config.col_approval_data, + &STORED_BLOCKS_KEY, + stored_block_range.encode(), + ); + }, + BackendWriteOp::DeleteStoredBlockRange => { + tx.delete(self.config.col_approval_data, &STORED_BLOCKS_KEY); + }, + BackendWriteOp::WriteBlocksAtHeight(h, blocks) => { + tx.put_vec( + self.config.col_approval_data, + &blocks_at_height_key(h), + blocks.encode(), + ); + }, + BackendWriteOp::DeleteBlocksAtHeight(h) => { + tx.delete(self.config.col_approval_data, &blocks_at_height_key(h)); + }, + BackendWriteOp::WriteBlockEntry(block_entry) => { + let block_entry: BlockEntry = block_entry.into(); + tx.put_vec( + self.config.col_approval_data, + &block_entry_key(&block_entry.block_hash), + block_entry.encode(), + ); + }, + BackendWriteOp::DeleteBlockEntry(hash) => { + tx.delete(self.config.col_approval_data, &block_entry_key(&hash)); + }, + BackendWriteOp::WriteCandidateEntry(candidate_entry) => { + let candidate_entry: CandidateEntry = candidate_entry.into(); + tx.put_vec( + self.config.col_approval_data, + &candidate_entry_key(&candidate_entry.candidate.hash()), + candidate_entry.encode(), + ); + }, + BackendWriteOp::DeleteCandidateEntry(candidate_hash) => { + tx.delete(self.config.col_approval_data, &candidate_entry_key(&candidate_hash)); + }, + } + } + + self.inner.write(tx).map_err(|e| e.into()) + } +} + +/// A range from earliest..last block number stored within the DB. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct StoredBlockRange(pub BlockNumber, pub BlockNumber); + +// slot_duration * 2 + DelayTranche gives the number of delay tranches since the +// unix epoch. +#[derive(Encode, Decode, Clone, Copy, Debug, PartialEq)] +pub struct Tick(u64); + +/// Convenience type definition +pub type Bitfield = BitVec; + +/// The database config. +#[derive(Debug, Clone, Copy)] +pub struct Config { + /// The column family in the database where data is stored. + pub col_approval_data: u32, +} + +/// Details pertaining to our assignment on a block. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct OurAssignment { + /// Our assignment certificate. + pub cert: AssignmentCertV2, + /// The tranche for which the assignment refers to. + pub tranche: DelayTranche, + /// Our validator index for the session in which the candidates were included. + pub validator_index: ValidatorIndex, + /// Whether the assignment has been triggered already. + pub triggered: bool, +} + +/// Metadata regarding a specific tranche of assignments for a specific candidate. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct TrancheEntry { + pub tranche: DelayTranche, + // Assigned validators, and the instant we received their assignment, rounded + // to the nearest tick. + pub assignments: Vec<(ValidatorIndex, Tick)>, +} + +/// Metadata regarding approval of a particular candidate within the context of some +/// particular block. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct ApprovalEntry { + pub tranches: Vec, + pub backing_group: GroupIndex, + pub our_assignment: Option, + pub our_approval_sig: Option, + // `n_validators` bits. + pub assigned_validators: Bitfield, + pub approved: bool, +} + +/// Metadata regarding approval of a particular candidate. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct CandidateEntry { + pub candidate: CandidateReceipt, + pub session: SessionIndex, + // Assignments are based on blocks, so we need to track assignments separately + // based on the block we are looking at. + pub block_assignments: BTreeMap, + pub approvals: Bitfield, +} + +/// Metadata regarding approval of a particular block, by way of approval of the +/// candidates contained within it. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct BlockEntry { + pub block_hash: Hash, + pub block_number: BlockNumber, + pub parent_hash: Hash, + pub session: SessionIndex, + pub slot: Slot, + /// Random bytes derived from the VRF submitted within the block by the block + /// author as a credential and used as input to approval assignment criteria. + pub relay_vrf_story: [u8; 32], + // The candidates included as-of this block and the index of the core they are + // leaving. Sorted ascending by core index. + pub candidates: Vec<(CoreIndex, CandidateHash)>, + // A bitfield where the i'th bit corresponds to the i'th candidate in `candidates`. + // The i'th bit is `true` iff the candidate has been approved in the context of this + // block. The block can be considered approved if the bitfield has all bits set to `true`. + pub approved_bitfield: Bitfield, + pub children: Vec, + // Assignments we already distributed. A 1 bit means the candidate index for which + // we already have sent out an assignment. We need this to avoid distributing + // multiple core assignments more than once. + pub distributed_assignments: Bitfield, +} + +impl From for Tick { + fn from(tick: crate::Tick) -> Tick { + Tick(tick) + } +} + +impl From for crate::Tick { + fn from(tick: Tick) -> crate::Tick { + tick.0 + } +} + +/// Errors while accessing things from the DB. +#[derive(Debug, derive_more::From, derive_more::Display)] +pub enum Error { + Io(std::io::Error), + InvalidDecoding(parity_scale_codec::Error), + InternalError(SubsystemError), +} + +impl std::error::Error for Error {} + +/// Result alias for DB errors. +pub type Result = std::result::Result; + +pub(crate) fn load_decode( + store: &dyn Database, + col_approval_data: u32, + key: &[u8], +) -> Result> { + match store.get(col_approval_data, key)? { + None => Ok(None), + Some(raw) => D::decode(&mut &raw[..]).map(Some).map_err(Into::into), + } +} + +/// The key a given block entry is stored under. +pub(crate) fn block_entry_key(block_hash: &Hash) -> [u8; 46] { + const BLOCK_ENTRY_PREFIX: [u8; 14] = *b"Approvals_blck"; + + let mut key = [0u8; 14 + 32]; + key[0..14].copy_from_slice(&BLOCK_ENTRY_PREFIX); + key[14..][..32].copy_from_slice(block_hash.as_ref()); + + key +} + +/// The key a given candidate entry is stored under. +pub(crate) fn candidate_entry_key(candidate_hash: &CandidateHash) -> [u8; 46] { + const CANDIDATE_ENTRY_PREFIX: [u8; 14] = *b"Approvals_cand"; + + let mut key = [0u8; 14 + 32]; + key[0..14].copy_from_slice(&CANDIDATE_ENTRY_PREFIX); + key[14..][..32].copy_from_slice(candidate_hash.0.as_ref()); + + key +} + +/// The key a set of block hashes corresponding to a block number is stored under. +pub(crate) fn blocks_at_height_key(block_number: BlockNumber) -> [u8; 16] { + const BLOCKS_AT_HEIGHT_PREFIX: [u8; 12] = *b"Approvals_at"; + + let mut key = [0u8; 12 + 4]; + key[0..12].copy_from_slice(&BLOCKS_AT_HEIGHT_PREFIX); + block_number.using_encoded(|s| key[12..16].copy_from_slice(s)); + + key +} + +/// Return all blocks which have entries in the DB, ascending, by height. +pub fn load_all_blocks(store: &dyn Database, config: &Config) -> SubsystemResult> { + let mut hashes = Vec::new(); + if let Some(stored_blocks) = load_stored_blocks(store, config)? { + for height in stored_blocks.0..stored_blocks.1 { + let blocks = load_blocks_at_height(store, config, &height)?; + hashes.extend(blocks); + } + } + + Ok(hashes) +} + +/// Load the stored-blocks key from the state. +pub fn load_stored_blocks( + store: &dyn Database, + config: &Config, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, STORED_BLOCKS_KEY) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a blocks-at-height entry for a given block number. +pub fn load_blocks_at_height( + store: &dyn Database, + config: &Config, + block_number: &BlockNumber, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &blocks_at_height_key(*block_number)) + .map(|x| x.unwrap_or_default()) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a block entry from the aux store. +pub fn load_block_entry( + store: &dyn Database, + config: &Config, + block_hash: &Hash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &block_entry_key(block_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a candidate entry from the aux store in current version format. +pub fn load_candidate_entry( + store: &dyn Database, + config: &Config, + candidate_hash: &CandidateHash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a candidate entry from the aux store in v1 format. +pub fn load_candidate_entry_v1( + store: &dyn Database, + config: &Config, + candidate_hash: &CandidateHash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a block entry from the aux store in v1 format. +pub fn load_block_entry_v1( + store: &dyn Database, + config: &Config, + block_hash: &Hash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &block_entry_key(block_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs new file mode 100644 index 000000000000..50a5a924ca8d --- /dev/null +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs @@ -0,0 +1,570 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Tests for the aux-schema of approval voting. + +use super::{DbBackend, StoredBlockRange, *}; +use crate::{ + backend::{Backend, OverlayedBackend}, + ops::{add_block_entry, canonicalize, force_approve, NewCandidateInfo}, +}; +use polkadot_node_subsystem_util::database::Database; +use polkadot_primitives::Id as ParaId; +use std::{collections::HashMap, sync::Arc}; + +use ::test_helpers::{dummy_candidate_receipt, dummy_candidate_receipt_bad_sig, dummy_hash}; + +const DATA_COL: u32 = 0; + +const NUM_COLUMNS: u32 = 1; + +const TEST_CONFIG: Config = Config { col_approval_data: DATA_COL }; + +fn make_db() -> (DbBackend, Arc) { + let db = kvdb_memorydb::create(NUM_COLUMNS); + let db = polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[]); + let db_writer: Arc = Arc::new(db); + (DbBackend::new(db_writer.clone(), TEST_CONFIG), db_writer) +} + +fn make_block_entry( + block_hash: Hash, + parent_hash: Hash, + block_number: BlockNumber, + candidates: Vec<(CoreIndex, CandidateHash)>, +) -> BlockEntry { + BlockEntry { + block_hash, + parent_hash, + block_number, + session: 1, + slot: Slot::from(1), + relay_vrf_story: [0u8; 32], + approved_bitfield: make_bitvec(candidates.len()), + candidates, + children: Vec::new(), + distributed_assignments: Default::default(), + } +} + +fn make_bitvec(len: usize) -> BitVec { + bitvec::bitvec![u8, BitOrderLsb0; 0; len] +} + +fn make_candidate(para_id: ParaId, relay_parent: Hash) -> CandidateReceipt { + let mut c = dummy_candidate_receipt(dummy_hash()); + + c.descriptor.para_id = para_id; + c.descriptor.relay_parent = relay_parent; + + c +} + +#[test] +fn read_write() { + let (mut db, store) = make_db(); + + let hash_a = Hash::repeat_byte(1); + let hash_b = Hash::repeat_byte(2); + let candidate_hash = dummy_candidate_receipt_bad_sig(dummy_hash(), None).hash(); + + let range = StoredBlockRange(10, 20); + let at_height = vec![hash_a, hash_b]; + + let block_entry = + make_block_entry(hash_a, Default::default(), 1, vec![(CoreIndex(0), candidate_hash)]); + + let candidate_entry = CandidateEntry { + candidate: dummy_candidate_receipt_bad_sig(dummy_hash(), None), + session: 5, + block_assignments: vec![( + hash_a, + ApprovalEntry { + tranches: Vec::new(), + backing_group: GroupIndex(1), + our_assignment: None, + our_approval_sig: None, + assigned_validators: Default::default(), + approved: false, + }, + )] + .into_iter() + .collect(), + approvals: Default::default(), + }; + + let mut overlay_db = OverlayedBackend::new(&db); + overlay_db.write_stored_block_range(range.clone()); + overlay_db.write_blocks_at_height(1, at_height.clone()); + overlay_db.write_block_entry(block_entry.clone().into()); + overlay_db.write_candidate_entry(candidate_entry.clone().into()); + + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert_eq!(load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap(), Some(range)); + assert_eq!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap(), at_height); + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap(), + Some(block_entry.into()) + ); + assert_eq!( + load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash).unwrap(), + Some(candidate_entry.into()), + ); + + let mut overlay_db = OverlayedBackend::new(&db); + overlay_db.delete_blocks_at_height(1); + overlay_db.delete_block_entry(&hash_a); + overlay_db.delete_candidate_entry(&candidate_hash); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap().is_empty()); + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap().is_none()); + assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash) + .unwrap() + .is_none()); +} + +#[test] +fn add_block_entry_works() { + let (mut db, store) = make_db(); + + let parent_hash = Hash::repeat_byte(1); + let block_hash_a = Hash::repeat_byte(2); + let block_hash_b = Hash::repeat_byte(69); + + let candidate_receipt_a = make_candidate(ParaId::from(1_u32), parent_hash); + let candidate_receipt_b = make_candidate(ParaId::from(2_u32), parent_hash); + + let candidate_hash_a = candidate_receipt_a.hash(); + let candidate_hash_b = candidate_receipt_b.hash(); + + let block_number = 10; + + let block_entry_a = make_block_entry( + block_hash_a, + parent_hash, + block_number, + vec![(CoreIndex(0), candidate_hash_a)], + ); + + let block_entry_b = make_block_entry( + block_hash_b, + parent_hash, + block_number, + vec![(CoreIndex(0), candidate_hash_a), (CoreIndex(1), candidate_hash_b)], + ); + + let n_validators = 10; + + let mut new_candidate_info = HashMap::new(); + new_candidate_info + .insert(candidate_hash_a, NewCandidateInfo::new(candidate_receipt_a, GroupIndex(0), None)); + + let mut overlay_db = OverlayedBackend::new(&db); + add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |h| { + new_candidate_info.get(h).map(|x| x.clone()) + }) + .unwrap(); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + new_candidate_info + .insert(candidate_hash_b, NewCandidateInfo::new(candidate_receipt_b, GroupIndex(1), None)); + + let mut overlay_db = OverlayedBackend::new(&db); + add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |h| { + new_candidate_info.get(h).map(|x| x.clone()) + }) + .unwrap(); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(), + Some(block_entry_a.into()) + ); + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(), + Some(block_entry_b.into()) + ); + + let candidate_entry_a = load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash_a) + .unwrap() + .unwrap(); + assert_eq!( + candidate_entry_a.block_assignments.keys().collect::>(), + vec![&block_hash_a, &block_hash_b] + ); + + let candidate_entry_b = load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash_b) + .unwrap() + .unwrap(); + assert_eq!(candidate_entry_b.block_assignments.keys().collect::>(), vec![&block_hash_b]); +} + +#[test] +fn add_block_entry_adds_child() { + let (mut db, store) = make_db(); + + let parent_hash = Hash::repeat_byte(1); + let block_hash_a = Hash::repeat_byte(2); + let block_hash_b = Hash::repeat_byte(69); + + let mut block_entry_a = make_block_entry(block_hash_a, parent_hash, 1, Vec::new()); + + let block_entry_b = make_block_entry(block_hash_b, block_hash_a, 2, Vec::new()); + + let n_validators = 10; + + let mut overlay_db = OverlayedBackend::new(&db); + add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |_| None).unwrap(); + + add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |_| None).unwrap(); + + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + block_entry_a.children.push(block_hash_b); + + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(), + Some(block_entry_a.into()) + ); + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(), + Some(block_entry_b.into()) + ); +} + +#[test] +fn canonicalize_works() { + let (mut db, store) = make_db(); + + // -> B1 -> C1 -> D1 + // A -> B2 -> C2 -> D2 + // + // We'll canonicalize C1. Everytning except D1 should disappear. + // + // Candidates: + // Cand1 in B2 + // Cand2 in C2 + // Cand3 in C2 and D1 + // Cand4 in D1 + // Cand5 in D2 + // Only Cand3 and Cand4 should remain after canonicalize. + + let n_validators = 10; + + let mut overlay_db = OverlayedBackend::new(&db); + overlay_db.write_stored_block_range(StoredBlockRange(1, 5)); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + let genesis = Hash::repeat_byte(0); + + let block_hash_a = Hash::repeat_byte(1); + let block_hash_b1 = Hash::repeat_byte(2); + let block_hash_b2 = Hash::repeat_byte(3); + let block_hash_c1 = Hash::repeat_byte(4); + let block_hash_c2 = Hash::repeat_byte(5); + let block_hash_d1 = Hash::repeat_byte(6); + let block_hash_d2 = Hash::repeat_byte(7); + + let candidate_receipt_genesis = make_candidate(ParaId::from(1_u32), genesis); + let candidate_receipt_a = make_candidate(ParaId::from(2_u32), block_hash_a); + let candidate_receipt_b = make_candidate(ParaId::from(3_u32), block_hash_a); + let candidate_receipt_b1 = make_candidate(ParaId::from(4_u32), block_hash_b1); + let candidate_receipt_c1 = make_candidate(ParaId::from(5_u32), block_hash_c1); + + let cand_hash_1 = candidate_receipt_genesis.hash(); + let cand_hash_2 = candidate_receipt_a.hash(); + let cand_hash_3 = candidate_receipt_b.hash(); + let cand_hash_4 = candidate_receipt_b1.hash(); + let cand_hash_5 = candidate_receipt_c1.hash(); + + let block_entry_a = make_block_entry(block_hash_a, genesis, 1, Vec::new()); + let block_entry_b1 = make_block_entry(block_hash_b1, block_hash_a, 2, Vec::new()); + let block_entry_b2 = + make_block_entry(block_hash_b2, block_hash_a, 2, vec![(CoreIndex(0), cand_hash_1)]); + let block_entry_c1 = make_block_entry(block_hash_c1, block_hash_b1, 3, Vec::new()); + let block_entry_c2 = make_block_entry( + block_hash_c2, + block_hash_b2, + 3, + vec![(CoreIndex(0), cand_hash_2), (CoreIndex(1), cand_hash_3)], + ); + let block_entry_d1 = make_block_entry( + block_hash_d1, + block_hash_c1, + 4, + vec![(CoreIndex(0), cand_hash_3), (CoreIndex(1), cand_hash_4)], + ); + let block_entry_d2 = + make_block_entry(block_hash_d2, block_hash_c2, 4, vec![(CoreIndex(0), cand_hash_5)]); + + let candidate_info = { + let mut candidate_info = HashMap::new(); + candidate_info.insert( + cand_hash_1, + NewCandidateInfo::new(candidate_receipt_genesis, GroupIndex(1), None), + ); + + candidate_info + .insert(cand_hash_2, NewCandidateInfo::new(candidate_receipt_a, GroupIndex(2), None)); + + candidate_info + .insert(cand_hash_3, NewCandidateInfo::new(candidate_receipt_b, GroupIndex(3), None)); + + candidate_info + .insert(cand_hash_4, NewCandidateInfo::new(candidate_receipt_b1, GroupIndex(4), None)); + + candidate_info + .insert(cand_hash_5, NewCandidateInfo::new(candidate_receipt_c1, GroupIndex(5), None)); + + candidate_info + }; + + // now insert all the blocks. + let blocks = vec![ + block_entry_a.clone(), + block_entry_b1.clone(), + block_entry_b2.clone(), + block_entry_c1.clone(), + block_entry_c2.clone(), + block_entry_d1.clone(), + block_entry_d2.clone(), + ]; + + let mut overlay_db = OverlayedBackend::new(&db); + for block_entry in blocks { + add_block_entry(&mut overlay_db, block_entry.into(), n_validators, |h| { + candidate_info.get(h).map(|x| x.clone()) + }) + .unwrap(); + } + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + let check_candidates_in_store = |expected: Vec<(CandidateHash, Option>)>| { + for (c_hash, in_blocks) in expected { + let (entry, in_blocks) = match in_blocks { + None => { + assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash) + .unwrap() + .is_none()); + continue + }, + Some(i) => ( + load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash).unwrap().unwrap(), + i, + ), + }; + + assert_eq!(entry.block_assignments.len(), in_blocks.len()); + + for x in in_blocks { + assert!(entry.block_assignments.contains_key(&x)); + } + } + }; + + let check_blocks_in_store = |expected: Vec<(Hash, Option>)>| { + for (hash, with_candidates) in expected { + let (entry, with_candidates) = match with_candidates { + None => { + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash) + .unwrap() + .is_none()); + continue + }, + Some(i) => + (load_block_entry(store.as_ref(), &TEST_CONFIG, &hash).unwrap().unwrap(), i), + }; + + assert_eq!(entry.candidates.len(), with_candidates.len()); + + for x in with_candidates { + assert!(entry.candidates.iter().any(|(_, c)| c == &x)); + } + } + }; + + check_candidates_in_store(vec![ + (cand_hash_1, Some(vec![block_hash_b2])), + (cand_hash_2, Some(vec![block_hash_c2])), + (cand_hash_3, Some(vec![block_hash_c2, block_hash_d1])), + (cand_hash_4, Some(vec![block_hash_d1])), + (cand_hash_5, Some(vec![block_hash_d2])), + ]); + + check_blocks_in_store(vec![ + (block_hash_a, Some(vec![])), + (block_hash_b1, Some(vec![])), + (block_hash_b2, Some(vec![cand_hash_1])), + (block_hash_c1, Some(vec![])), + (block_hash_c2, Some(vec![cand_hash_2, cand_hash_3])), + (block_hash_d1, Some(vec![cand_hash_3, cand_hash_4])), + (block_hash_d2, Some(vec![cand_hash_5])), + ]); + + let mut overlay_db = OverlayedBackend::new(&db); + canonicalize(&mut overlay_db, 3, block_hash_c1).unwrap(); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert_eq!( + load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap().unwrap(), + StoredBlockRange(4, 5) + ); + + check_candidates_in_store(vec![ + (cand_hash_1, None), + (cand_hash_2, None), + (cand_hash_3, Some(vec![block_hash_d1])), + (cand_hash_4, Some(vec![block_hash_d1])), + (cand_hash_5, None), + ]); + + check_blocks_in_store(vec![ + (block_hash_a, None), + (block_hash_b1, None), + (block_hash_b2, None), + (block_hash_c1, None), + (block_hash_c2, None), + (block_hash_d1, Some(vec![cand_hash_3, cand_hash_4])), + (block_hash_d2, None), + ]); +} + +#[test] +fn force_approve_works() { + let (mut db, store) = make_db(); + let n_validators = 10; + + let mut overlay_db = OverlayedBackend::new(&db); + overlay_db.write_stored_block_range(StoredBlockRange(1, 4)); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + let single_candidate_vec = vec![(CoreIndex(0), candidate_hash)]; + let candidate_info = { + let mut candidate_info = HashMap::new(); + candidate_info.insert( + candidate_hash, + NewCandidateInfo::new( + make_candidate(ParaId::from(1_u32), Default::default()), + GroupIndex(1), + None, + ), + ); + + candidate_info + }; + + let block_hash_a = Hash::repeat_byte(1); // 1 + let block_hash_b = Hash::repeat_byte(2); + let block_hash_c = Hash::repeat_byte(3); + let block_hash_d = Hash::repeat_byte(4); // 4 + + let block_entry_a = + make_block_entry(block_hash_a, Default::default(), 1, single_candidate_vec.clone()); + let block_entry_b = + make_block_entry(block_hash_b, block_hash_a, 2, single_candidate_vec.clone()); + let block_entry_c = + make_block_entry(block_hash_c, block_hash_b, 3, single_candidate_vec.clone()); + let block_entry_d = + make_block_entry(block_hash_d, block_hash_c, 4, single_candidate_vec.clone()); + + let blocks = vec![ + block_entry_a.clone(), + block_entry_b.clone(), + block_entry_c.clone(), + block_entry_d.clone(), + ]; + + let mut overlay_db = OverlayedBackend::new(&db); + for block_entry in blocks { + add_block_entry(&mut overlay_db, block_entry.into(), n_validators, |h| { + candidate_info.get(h).map(|x| x.clone()) + }) + .unwrap(); + } + let approved_hashes = force_approve(&mut overlay_db, block_hash_d, 2).unwrap(); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a,) + .unwrap() + .unwrap() + .approved_bitfield + .all()); + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b,) + .unwrap() + .unwrap() + .approved_bitfield + .all()); + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_c,) + .unwrap() + .unwrap() + .approved_bitfield + .not_any()); + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_d,) + .unwrap() + .unwrap() + .approved_bitfield + .not_any()); + assert_eq!(approved_hashes, vec![block_hash_b, block_hash_a]); +} + +#[test] +fn load_all_blocks_works() { + let (mut db, store) = make_db(); + + let parent_hash = Hash::repeat_byte(1); + let block_hash_a = Hash::repeat_byte(2); + let block_hash_b = Hash::repeat_byte(69); + let block_hash_c = Hash::repeat_byte(42); + + let block_number = 10; + + let block_entry_a = make_block_entry(block_hash_a, parent_hash, block_number, vec![]); + + let block_entry_b = make_block_entry(block_hash_b, parent_hash, block_number, vec![]); + + let block_entry_c = make_block_entry(block_hash_c, block_hash_a, block_number + 1, vec![]); + + let n_validators = 10; + + let mut overlay_db = OverlayedBackend::new(&db); + add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |_| None).unwrap(); + + // add C before B to test sorting. + add_block_entry(&mut overlay_db, block_entry_c.clone().into(), n_validators, |_| None).unwrap(); + + add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |_| None).unwrap(); + + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert_eq!( + load_all_blocks(store.as_ref(), &TEST_CONFIG).unwrap(), + vec![block_hash_a, block_hash_b, block_hash_c], + ) +} diff --git a/polkadot/node/core/approval-voting/src/backend.rs b/polkadot/node/core/approval-voting/src/backend.rs index 87d67c52c467..374e7a826d19 100644 --- a/polkadot/node/core/approval-voting/src/backend.rs +++ b/polkadot/node/core/approval-voting/src/backend.rs @@ -27,7 +27,7 @@ use polkadot_primitives::{BlockNumber, CandidateHash, Hash}; use std::collections::HashMap; use super::{ - approval_db::v1::StoredBlockRange, + approval_db::v2::StoredBlockRange, persisted_entries::{BlockEntry, CandidateEntry}, }; @@ -44,6 +44,7 @@ pub enum BackendWriteOp { } /// An abstraction over backend storage for the logic of this subsystem. +/// Implementation must always target latest storage version. pub trait Backend { /// Load a block entry from the DB. fn load_block_entry(&self, hash: &Hash) -> SubsystemResult>; @@ -52,6 +53,7 @@ pub trait Backend { &self, candidate_hash: &CandidateHash, ) -> SubsystemResult>; + /// Load all blocks at a specific height. fn load_blocks_at_height(&self, height: &BlockNumber) -> SubsystemResult>; /// Load all block from the DB. @@ -64,6 +66,18 @@ pub trait Backend { I: IntoIterator; } +/// A read only backed to enable db migration from version 1 of DB. +pub trait V1ReadBackend: Backend { + /// Load a candidate entry from the DB with scheme version 1. + fn load_candidate_entry_v1( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult>; + + /// Load a block entry from the DB with scheme version 1. + fn load_block_entry_v1(&self, block_hash: &Hash) -> SubsystemResult>; +} + // Status of block range in the `OverlayedBackend`. #[derive(PartialEq)] enum BlockRangeStatus { diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 0e1d18198c21..1f751e2bf140 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -18,7 +18,9 @@ use parity_scale_codec::{Decode, Encode}; use polkadot_node_primitives::approval::{ - self as approval_types, AssignmentCert, AssignmentCertKind, DelayTranche, RelayVRFStory, + self as approval_types, + v1::{AssignmentCert, AssignmentCertKind, DelayTranche, RelayVRFStory}, + v2::{AssignmentCertKindV2, AssignmentCertV2, CoreBitfield, VrfOutput, VrfProof, VrfSignature}, }; use polkadot_primitives::{ AssignmentId, AssignmentPair, CandidateHash, CoreIndex, GroupIndex, IndexedVec, SessionInfo, @@ -30,6 +32,7 @@ use sp_application_crypto::ByteArray; use merlin::Transcript; use schnorrkel::vrf::VRFInOut; +use itertools::Itertools; use std::collections::{hash_map::Entry, HashMap}; use super::LOG_TARGET; @@ -37,7 +40,7 @@ use super::LOG_TARGET; /// Details pertaining to our assignment on a block. #[derive(Debug, Clone, Encode, Decode, PartialEq)] pub struct OurAssignment { - cert: AssignmentCert, + cert: AssignmentCertV2, tranche: DelayTranche, validator_index: ValidatorIndex, // Whether the assignment has been triggered already. @@ -45,7 +48,7 @@ pub struct OurAssignment { } impl OurAssignment { - pub(crate) fn cert(&self) -> &AssignmentCert { + pub(crate) fn cert(&self) -> &AssignmentCertV2 { &self.cert } @@ -66,8 +69,8 @@ impl OurAssignment { } } -impl From for OurAssignment { - fn from(entry: crate::approval_db::v1::OurAssignment) -> Self { +impl From for OurAssignment { + fn from(entry: crate::approval_db::v2::OurAssignment) -> Self { OurAssignment { cert: entry.cert, tranche: entry.tranche, @@ -77,7 +80,7 @@ impl From for OurAssignment { } } -impl From for crate::approval_db::v1::OurAssignment { +impl From for crate::approval_db::v2::OurAssignment { fn from(entry: OurAssignment) -> Self { Self { cert: entry.cert, @@ -88,17 +91,97 @@ impl From for crate::approval_db::v1::OurAssignment { } } -fn relay_vrf_modulo_transcript(relay_vrf_story: RelayVRFStory, sample: u32) -> Transcript { - // combine the relay VRF story with a sample number. - let mut t = Transcript::new(approval_types::RELAY_VRF_MODULO_CONTEXT); - t.append_message(b"RC-VRF", &relay_vrf_story.0); - sample.using_encoded(|s| t.append_message(b"sample", s)); +// Combines the relay VRF story with a sample number if any. +fn relay_vrf_modulo_transcript_inner( + mut transcript: Transcript, + relay_vrf_story: RelayVRFStory, + sample: Option, +) -> Transcript { + transcript.append_message(b"RC-VRF", &relay_vrf_story.0); - t + if let Some(sample) = sample { + sample.using_encoded(|s| transcript.append_message(b"sample", s)); + } + + transcript +} + +fn relay_vrf_modulo_transcript_v1(relay_vrf_story: RelayVRFStory, sample: u32) -> Transcript { + relay_vrf_modulo_transcript_inner( + Transcript::new(approval_types::v1::RELAY_VRF_MODULO_CONTEXT), + relay_vrf_story, + Some(sample), + ) +} + +fn relay_vrf_modulo_transcript_v2(relay_vrf_story: RelayVRFStory) -> Transcript { + relay_vrf_modulo_transcript_inner( + Transcript::new(approval_types::v2::RELAY_VRF_MODULO_CONTEXT), + relay_vrf_story, + None, + ) +} + +/// A hard upper bound on num_cores * target_checkers / num_validators +const MAX_MODULO_SAMPLES: usize = 40; + +use std::convert::AsMut; + +fn clone_into_array(slice: &[T]) -> A +where + A: Default + AsMut<[T]>, + T: Clone, +{ + let mut a = A::default(); + >::as_mut(&mut a).clone_from_slice(slice); + a +} + +struct BigArray(pub [u8; MAX_MODULO_SAMPLES * 4]); + +impl Default for BigArray { + fn default() -> Self { + BigArray([0u8; MAX_MODULO_SAMPLES * 4]) + } +} + +impl AsMut<[u8]> for BigArray { + fn as_mut(&mut self) -> &mut [u8] { + self.0.as_mut() + } +} + +/// Takes the VRF output as input and returns a Vec of cores the validator is assigned +/// to as a tranche0 checker. +fn relay_vrf_modulo_cores( + vrf_in_out: &VRFInOut, + // Configuration - `relay_vrf_modulo_samples`. + num_samples: u32, + // Configuration - `n_cores`. + max_cores: u32, +) -> Vec { + if num_samples as usize > MAX_MODULO_SAMPLES { + gum::warn!( + target: LOG_TARGET, + n_cores = max_cores, + num_samples, + max_modulo_samples = MAX_MODULO_SAMPLES, + "`num_samples` is greater than `MAX_MODULO_SAMPLES`", + ); + } + + vrf_in_out + .make_bytes::(approval_types::v2::CORE_RANDOMNESS_CONTEXT) + .0 + .chunks_exact(4) + .take(num_samples as usize) + .map(move |sample| CoreIndex(u32::from_le_bytes(clone_into_array(&sample)) % max_cores)) + .unique() + .collect::>() } fn relay_vrf_modulo_core(vrf_in_out: &VRFInOut, n_cores: u32) -> CoreIndex { - let bytes: [u8; 4] = vrf_in_out.make_bytes(approval_types::CORE_RANDOMNESS_CONTEXT); + let bytes: [u8; 4] = vrf_in_out.make_bytes(approval_types::v1::CORE_RANDOMNESS_CONTEXT); // interpret as little-endian u32. let random_core = u32::from_le_bytes(bytes) % n_cores; @@ -106,7 +189,7 @@ fn relay_vrf_modulo_core(vrf_in_out: &VRFInOut, n_cores: u32) -> CoreIndex { } fn relay_vrf_delay_transcript(relay_vrf_story: RelayVRFStory, core_index: CoreIndex) -> Transcript { - let mut t = Transcript::new(approval_types::RELAY_VRF_DELAY_CONTEXT); + let mut t = Transcript::new(approval_types::v1::RELAY_VRF_DELAY_CONTEXT); t.append_message(b"RC-VRF", &relay_vrf_story.0); core_index.0.using_encoded(|s| t.append_message(b"core", s)); t @@ -117,7 +200,7 @@ fn relay_vrf_delay_tranche( num_delay_tranches: u32, zeroth_delay_tranche_width: u32, ) -> DelayTranche { - let bytes: [u8; 4] = vrf_in_out.make_bytes(approval_types::TRANCHE_RANDOMNESS_CONTEXT); + let bytes: [u8; 4] = vrf_in_out.make_bytes(approval_types::v1::TRANCHE_RANDOMNESS_CONTEXT); // interpret as little-endian u32 and reduce by the number of tranches. let wide_tranche = @@ -128,13 +211,13 @@ fn relay_vrf_delay_tranche( } fn assigned_core_transcript(core_index: CoreIndex) -> Transcript { - let mut t = Transcript::new(approval_types::ASSIGNED_CORE_CONTEXT); + let mut t = Transcript::new(approval_types::v1::ASSIGNED_CORE_CONTEXT); core_index.0.using_encoded(|s| t.append_message(b"core", s)); t } /// Information about the world assignments are being produced in. -#[derive(Clone)] +#[derive(Clone, Debug)] pub(crate) struct Config { /// The assignment public keys for validators. assignment_keys: Vec, @@ -175,12 +258,13 @@ pub(crate) trait AssignmentCriteria { fn check_assignment_cert( &self, - claimed_core_index: CoreIndex, + claimed_core_bitfield: CoreBitfield, validator_index: ValidatorIndex, config: &Config, relay_vrf_story: RelayVRFStory, - assignment: &AssignmentCert, - backing_group: GroupIndex, + assignment: &AssignmentCertV2, + // Backing groups for each "leaving core". + backing_groups: Vec, ) -> Result; } @@ -194,25 +278,25 @@ impl AssignmentCriteria for RealAssignmentCriteria { config: &Config, leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>, ) -> HashMap { - compute_assignments(keystore, relay_vrf_story, config, leaving_cores) + compute_assignments(keystore, relay_vrf_story, config, leaving_cores, false) } fn check_assignment_cert( &self, - claimed_core_index: CoreIndex, + claimed_core_bitfield: CoreBitfield, validator_index: ValidatorIndex, config: &Config, relay_vrf_story: RelayVRFStory, - assignment: &AssignmentCert, - backing_group: GroupIndex, + assignment: &AssignmentCertV2, + backing_groups: Vec, ) -> Result { check_assignment_cert( - claimed_core_index, + claimed_core_bitfield, validator_index, config, relay_vrf_story, assignment, - backing_group, + backing_groups, ) } } @@ -233,6 +317,7 @@ pub(crate) fn compute_assignments( relay_vrf_story: RelayVRFStory, config: &Config, leaving_cores: impl IntoIterator + Clone, + enable_v2_assignments: bool, ) -> HashMap { if config.n_cores == 0 || config.assignment_keys.is_empty() || @@ -291,14 +376,25 @@ pub(crate) fn compute_assignments( let mut assignments = HashMap::new(); // First run `RelayVRFModulo` for each sample. - compute_relay_vrf_modulo_assignments( - &assignments_key, - index, - config, - relay_vrf_story.clone(), - leaving_cores.iter().cloned(), - &mut assignments, - ); + if enable_v2_assignments { + compute_relay_vrf_modulo_assignments_v2( + &assignments_key, + index, + config, + relay_vrf_story.clone(), + leaving_cores.clone(), + &mut assignments, + ); + } else { + compute_relay_vrf_modulo_assignments_v1( + &assignments_key, + index, + config, + relay_vrf_story.clone(), + leaving_cores.clone(), + &mut assignments, + ); + } // Then run `RelayVRFDelay` once for the whole block. compute_relay_vrf_delay_assignments( @@ -313,7 +409,7 @@ pub(crate) fn compute_assignments( assignments } -fn compute_relay_vrf_modulo_assignments( +fn compute_relay_vrf_modulo_assignments_v1( assignments_key: &schnorrkel::Keypair, validator_index: ValidatorIndex, config: &Config, @@ -329,7 +425,7 @@ fn compute_relay_vrf_modulo_assignments( // into closure. let core = &mut core; assignments_key.vrf_sign_extra_after_check( - relay_vrf_modulo_transcript(relay_vrf_story.clone(), rvm_sample), + relay_vrf_modulo_transcript_v1(relay_vrf_story.clone(), rvm_sample), |vrf_in_out| { *core = relay_vrf_modulo_core(&vrf_in_out, config.n_cores); if let Some((candidate_hash, _)) = @@ -357,15 +453,15 @@ fn compute_relay_vrf_modulo_assignments( // has been executed. let cert = AssignmentCert { kind: AssignmentCertKind::RelayVRFModulo { sample: rvm_sample }, - vrf: approval_types::VrfSignature { - output: approval_types::VrfOutput(vrf_in_out.to_output()), - proof: approval_types::VrfProof(vrf_proof), + vrf: VrfSignature { + output: VrfOutput(vrf_in_out.to_output()), + proof: VrfProof(vrf_proof), }, }; // All assignments of type RelayVRFModulo have tranche 0. assignments.entry(core).or_insert(OurAssignment { - cert, + cert: cert.into(), tranche: 0, validator_index, triggered: false, @@ -374,6 +470,84 @@ fn compute_relay_vrf_modulo_assignments( } } +fn assigned_cores_transcript(core_bitfield: &CoreBitfield) -> Transcript { + let mut t = Transcript::new(approval_types::v2::ASSIGNED_CORE_CONTEXT); + core_bitfield.using_encoded(|s| t.append_message(b"cores", s)); + t +} + +fn compute_relay_vrf_modulo_assignments_v2( + assignments_key: &schnorrkel::Keypair, + validator_index: ValidatorIndex, + config: &Config, + relay_vrf_story: RelayVRFStory, + leaving_cores: Vec<(CandidateHash, CoreIndex)>, + assignments: &mut HashMap, +) { + let mut assigned_cores = Vec::new(); + let leaving_cores = leaving_cores.iter().map(|(_, core)| core).collect::>(); + + let maybe_assignment = { + let assigned_cores = &mut assigned_cores; + assignments_key.vrf_sign_extra_after_check( + relay_vrf_modulo_transcript_v2(relay_vrf_story.clone()), + |vrf_in_out| { + *assigned_cores = relay_vrf_modulo_cores( + &vrf_in_out, + config.relay_vrf_modulo_samples, + config.n_cores, + ) + .into_iter() + .filter(|core| leaving_cores.contains(&core)) + .collect::>(); + + if !assigned_cores.is_empty() { + gum::trace!( + target: LOG_TARGET, + ?assigned_cores, + ?validator_index, + tranche = 0, + "RelayVRFModuloCompact Assignment." + ); + + let assignment_bitfield: CoreBitfield = assigned_cores + .clone() + .try_into() + .expect("Just checked `!assigned_cores.is_empty()`; qed"); + + Some(assigned_cores_transcript(&assignment_bitfield)) + } else { + None + } + }, + ) + }; + + if let Some(assignment) = maybe_assignment.map(|(vrf_in_out, vrf_proof, _)| { + let assignment_bitfield: CoreBitfield = assigned_cores + .clone() + .try_into() + .expect("Just checked `!assigned_cores.is_empty()`; qed"); + + let cert = AssignmentCertV2 { + kind: AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: assignment_bitfield.clone(), + }, + vrf: VrfSignature { + output: VrfOutput(vrf_in_out.to_output()), + proof: VrfProof(vrf_proof), + }, + }; + + // All assignments of type RelayVRFModulo have tranche 0. + OurAssignment { cert, tranche: 0, validator_index, triggered: false } + }) { + for core_index in assigned_cores { + assignments.insert(core_index, assignment.clone()); + } + } +} + fn compute_relay_vrf_delay_assignments( assignments_key: &schnorrkel::Keypair, validator_index: ValidatorIndex, @@ -392,11 +566,11 @@ fn compute_relay_vrf_delay_assignments( config.zeroth_delay_tranche_width, ); - let cert = AssignmentCert { - kind: AssignmentCertKind::RelayVRFDelay { core_index: core }, - vrf: approval_types::VrfSignature { - output: approval_types::VrfOutput(vrf_in_out.to_output()), - proof: approval_types::VrfProof(vrf_proof), + let cert = AssignmentCertV2 { + kind: AssignmentCertKindV2::RelayVRFDelay { core_index: core }, + vrf: VrfSignature { + output: VrfOutput(vrf_in_out.to_output()), + proof: VrfProof(vrf_proof), }, }; @@ -453,12 +627,15 @@ pub(crate) enum InvalidAssignmentReason { VRFModuloOutputMismatch, VRFDelayCoreIndexMismatch, VRFDelayOutputMismatch, + InvalidArguments, + /// Assignment vrf check resulted in 0 assigned cores. + NullAssignment, } /// Checks the crypto of an assignment cert. Failure conditions: /// * Validator index out of bounds /// * VRF signature check fails -/// * VRF output doesn't match assigned core +/// * VRF output doesn't match assigned cores /// * Core is not covered by extra data in signature /// * Core index out of bounds /// * Sample is out of bounds @@ -467,12 +644,12 @@ pub(crate) enum InvalidAssignmentReason { /// This function does not check whether the core is actually a valid assignment or not. That should /// be done outside the scope of this function. pub(crate) fn check_assignment_cert( - claimed_core_index: CoreIndex, + claimed_core_indices: CoreBitfield, validator_index: ValidatorIndex, config: &Config, relay_vrf_story: RelayVRFStory, - assignment: &AssignmentCert, - backing_group: GroupIndex, + assignment: &AssignmentCertV2, + backing_groups: Vec, ) -> Result { use InvalidAssignmentReason as Reason; @@ -484,52 +661,133 @@ pub(crate) fn check_assignment_cert( let public = schnorrkel::PublicKey::from_bytes(validator_public.as_slice()) .map_err(|_| InvalidAssignment(Reason::InvalidAssignmentKey))?; - if claimed_core_index.0 >= config.n_cores { - return Err(InvalidAssignment(Reason::CoreIndexOutOfBounds)) + // Check that we have all backing groups for claimed cores. + if claimed_core_indices.count_ones() == 0 || + claimed_core_indices.count_ones() != backing_groups.len() + { + return Err(InvalidAssignment(Reason::InvalidArguments)) } // Check that the validator was not part of the backing group // and not already assigned. - let is_in_backing = - is_in_backing_group(&config.validator_groups, validator_index, backing_group); + for (claimed_core, backing_group) in claimed_core_indices.iter_ones().zip(backing_groups.iter()) + { + if claimed_core >= config.n_cores as usize { + return Err(InvalidAssignment(Reason::CoreIndexOutOfBounds)) + } - if is_in_backing { - return Err(InvalidAssignment(Reason::IsInBackingGroup)) + let is_in_backing = + is_in_backing_group(&config.validator_groups, validator_index, *backing_group); + + if is_in_backing { + return Err(InvalidAssignment(Reason::IsInBackingGroup)) + } } - let vrf_signature = &assignment.vrf; - match assignment.kind { - AssignmentCertKind::RelayVRFModulo { sample } => { - if sample >= config.relay_vrf_modulo_samples { + let vrf_output = &assignment.vrf.output; + let vrf_proof = &assignment.vrf.proof; + let first_claimed_core_index = + claimed_core_indices.first_one().expect("Checked above; qed") as u32; + + match &assignment.kind { + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => { + // Check that claimed core bitfield match the one from certificate. + if &claimed_core_indices != core_bitfield { + return Err(InvalidAssignment(Reason::VRFModuloCoreIndexMismatch)) + } + + let (vrf_in_out, _) = public + .vrf_verify_extra( + relay_vrf_modulo_transcript_v2(relay_vrf_story), + &vrf_output.0, + &vrf_proof.0, + assigned_cores_transcript(core_bitfield), + ) + .map_err(|_| InvalidAssignment(Reason::VRFModuloOutputMismatch))?; + + let resulting_cores = relay_vrf_modulo_cores( + &vrf_in_out, + config.relay_vrf_modulo_samples, + config.n_cores, + ); + + // Currently validators can opt out of checking specific cores. + // This is the same issue to how validator can opt out and not send their assignments in + // the first place. Ensure that the `vrf_in_out` actually includes all of the claimed + // cores. + for claimed_core_index in claimed_core_indices.iter_ones() { + if !resulting_cores.contains(&CoreIndex(claimed_core_index as u32)) { + gum::debug!( + target: LOG_TARGET, + ?resulting_cores, + ?claimed_core_indices, + vrf_modulo_cores = ?resulting_cores, + "Assignment claimed cores mismatch", + ); + return Err(InvalidAssignment(Reason::VRFModuloCoreIndexMismatch)) + } + } + + Ok(0) + }, + AssignmentCertKindV2::RelayVRFModulo { sample } => { + if *sample >= config.relay_vrf_modulo_samples { return Err(InvalidAssignment(Reason::SampleOutOfBounds)) } + // Enforce claimed candidates is 1. + if claimed_core_indices.count_ones() != 1 { + gum::warn!( + target: LOG_TARGET, + ?claimed_core_indices, + "`RelayVRFModulo` assignment must always claim 1 core", + ); + return Err(InvalidAssignment(Reason::InvalidArguments)) + } + let (vrf_in_out, _) = public .vrf_verify_extra( - relay_vrf_modulo_transcript(relay_vrf_story, sample), - &vrf_signature.output.0, - &vrf_signature.proof.0, - assigned_core_transcript(claimed_core_index), + relay_vrf_modulo_transcript_v1(relay_vrf_story, *sample), + &vrf_output.0, + &vrf_proof.0, + assigned_core_transcript(CoreIndex(first_claimed_core_index)), ) .map_err(|_| InvalidAssignment(Reason::VRFModuloOutputMismatch))?; + let core = relay_vrf_modulo_core(&vrf_in_out, config.n_cores); // ensure that the `vrf_in_out` actually gives us the claimed core. - if relay_vrf_modulo_core(&vrf_in_out, config.n_cores) == claimed_core_index { + if core.0 == first_claimed_core_index { Ok(0) } else { + gum::debug!( + target: LOG_TARGET, + ?core, + ?claimed_core_indices, + "Assignment claimed cores mismatch", + ); Err(InvalidAssignment(Reason::VRFModuloCoreIndexMismatch)) } }, - AssignmentCertKind::RelayVRFDelay { core_index } => { - if core_index != claimed_core_index { + AssignmentCertKindV2::RelayVRFDelay { core_index } => { + // Enforce claimed candidates is 1. + if claimed_core_indices.count_ones() != 1 { + gum::debug!( + target: LOG_TARGET, + ?claimed_core_indices, + "`RelayVRFDelay` assignment must always claim 1 core", + ); + return Err(InvalidAssignment(Reason::InvalidArguments)) + } + + if core_index.0 != first_claimed_core_index { return Err(InvalidAssignment(Reason::VRFDelayCoreIndexMismatch)) } let (vrf_in_out, _) = public .vrf_verify( - relay_vrf_delay_transcript(relay_vrf_story, core_index), - &vrf_signature.output.0, - &vrf_signature.proof.0, + relay_vrf_delay_transcript(relay_vrf_story, *core_index), + &vrf_output.0, + &vrf_proof.0, ) .map_err(|_| InvalidAssignment(Reason::VRFDelayOutputMismatch))?; @@ -550,6 +808,19 @@ fn is_in_backing_group( validator_groups.get(group).map_or(false, |g| g.contains(&validator)) } +/// Migration helpers. +impl From for OurAssignment { + fn from(value: crate::approval_db::v1::OurAssignment) -> Self { + Self { + cert: value.cert.into(), + tranche: value.tranche, + validator_index: value.validator_index, + // Whether the assignment has been triggered already. + triggered: value.triggered, + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -630,10 +901,11 @@ mod tests { ]), n_cores: 2, zeroth_delay_tranche_width: 10, - relay_vrf_modulo_samples: 3, + relay_vrf_modulo_samples: 10, n_delay_tranches: 40, }, vec![(c_a, CoreIndex(0), GroupIndex(1)), (c_b, CoreIndex(1), GroupIndex(0))], + false, ); // Note that alice is in group 0, which was the backing group for core 1. @@ -665,10 +937,11 @@ mod tests { ]), n_cores: 2, zeroth_delay_tranche_width: 10, - relay_vrf_modulo_samples: 3, + relay_vrf_modulo_samples: 10, n_delay_tranches: 40, }, vec![(c_a, CoreIndex(0), GroupIndex(0)), (c_b, CoreIndex(1), GroupIndex(1))], + false, ); assert_eq!(assignments.len(), 1); @@ -692,19 +965,21 @@ mod tests { validator_groups: Default::default(), n_cores: 0, zeroth_delay_tranche_width: 10, - relay_vrf_modulo_samples: 3, + relay_vrf_modulo_samples: 10, n_delay_tranches: 40, }, vec![], + false, ); assert!(assignments.is_empty()); } + #[derive(Debug)] struct MutatedAssignment { - core: CoreIndex, - cert: AssignmentCert, - group: GroupIndex, + cores: CoreBitfield, + cert: AssignmentCertV2, + groups: Vec, own_group: GroupIndex, val_index: ValidatorIndex, config: Config, @@ -729,12 +1004,12 @@ mod tests { validator_groups: basic_groups(n_validators, n_cores), n_cores: n_cores as u32, zeroth_delay_tranche_width: 10, - relay_vrf_modulo_samples: 3, + relay_vrf_modulo_samples: 15, n_delay_tranches: 40, }; let relay_vrf_story = RelayVRFStory([42u8; 32]); - let assignments = compute_assignments( + let mut assignments = compute_assignments( &keystore, relay_vrf_story.clone(), &config, @@ -747,19 +1022,42 @@ mod tests { ) }) .collect::>(), + false, ); + // Extend with v2 assignments as well + assignments.extend(compute_assignments( + &keystore, + relay_vrf_story.clone(), + &config, + (0..n_cores) + .map(|i| { + ( + CandidateHash(Hash::repeat_byte(i as u8)), + CoreIndex(i as u32), + group_for_core(i), + ) + }) + .collect::>(), + true, + )); + let mut counted = 0; for (core, assignment) in assignments { + let cores = match assignment.cert.kind.clone() { + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => core_bitfield, + AssignmentCertKindV2::RelayVRFModulo { sample: _ } => core.into(), + AssignmentCertKindV2::RelayVRFDelay { core_index } => core_index.into(), + }; + let mut mutated = MutatedAssignment { - core, - group: group_for_core(core.0 as _), + cores: cores.clone(), + groups: cores.iter_ones().map(|core| group_for_core(core)).collect(), cert: assignment.cert, own_group: GroupIndex(0), val_index: ValidatorIndex(0), config: config.clone(), }; - let expected = match f(&mut mutated) { None => continue, Some(e) => e, @@ -768,16 +1066,16 @@ mod tests { counted += 1; let is_good = check_assignment_cert( - mutated.core, + mutated.cores, mutated.val_index, &mutated.config, relay_vrf_story.clone(), &mutated.cert, - mutated.group, + mutated.groups, ) .is_ok(); - assert_eq!(expected, is_good) + assert_eq!(expected, is_good); } assert!(counted > 0); @@ -791,7 +1089,7 @@ mod tests { #[test] fn check_rejects_claimed_core_out_of_bounds() { check_mutated_assignments(200, 100, 25, |m| { - m.core.0 += 100; + m.cores = CoreIndex(100).into(); Some(false) }); } @@ -799,7 +1097,7 @@ mod tests { #[test] fn check_rejects_in_backing_group() { check_mutated_assignments(200, 100, 25, |m| { - m.group = m.own_group; + m.groups[0] = m.own_group; Some(false) }); } @@ -815,9 +1113,10 @@ mod tests { #[test] fn check_rejects_delay_bad_vrf() { check_mutated_assignments(40, 10, 8, |m| { + let vrf_signature = garbage_vrf_signature(); match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFDelay { .. } => { - m.cert.vrf = garbage_vrf_signature(); + AssignmentCertKindV2::RelayVRFDelay { .. } => { + m.cert.vrf = vrf_signature; Some(false) }, _ => None, // skip everything else. @@ -828,9 +1127,14 @@ mod tests { #[test] fn check_rejects_modulo_bad_vrf() { check_mutated_assignments(200, 100, 25, |m| { + let vrf_signature = garbage_vrf_signature(); match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFModulo { .. } => { - m.cert.vrf = garbage_vrf_signature(); + AssignmentCertKindV2::RelayVRFModulo { .. } => { + m.cert.vrf = vrf_signature; + Some(false) + }, + AssignmentCertKindV2::RelayVRFModuloCompact { .. } => { + m.cert.vrf = vrf_signature; Some(false) }, _ => None, // skip everything else. @@ -842,10 +1146,11 @@ mod tests { fn check_rejects_modulo_sample_out_of_bounds() { check_mutated_assignments(200, 100, 25, |m| { match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFModulo { sample } => { + AssignmentCertKindV2::RelayVRFModulo { sample } => { m.config.relay_vrf_modulo_samples = sample; Some(false) }, + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield: _ } => Some(true), _ => None, // skip everything else. } }); @@ -855,8 +1160,11 @@ mod tests { fn check_rejects_delay_claimed_core_wrong() { check_mutated_assignments(200, 100, 25, |m| { match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFDelay { .. } => { - m.core = CoreIndex((m.core.0 + 1) % 100); + AssignmentCertKindV2::RelayVRFDelay { .. } => { + // for core in &mut m.cores { + // core.0 = (core.0 + 1) % 100; + // } + m.cores = CoreIndex((m.cores.first_one().unwrap() + 1) as u32 % 100).into(); Some(false) }, _ => None, // skip everything else. @@ -868,8 +1176,10 @@ mod tests { fn check_rejects_modulo_core_wrong() { check_mutated_assignments(200, 100, 25, |m| { match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFModulo { .. } => { - m.core = CoreIndex((m.core.0 + 1) % 100); + AssignmentCertKindV2::RelayVRFModulo { .. } | + AssignmentCertKindV2::RelayVRFModuloCompact { .. } => { + m.cores = CoreIndex((m.cores.first_one().unwrap() + 1) as u32 % 100).into(); + Some(false) }, _ => None, // skip everything else. diff --git a/polkadot/node/core/approval-voting/src/import.rs b/polkadot/node/core/approval-voting/src/import.rs index c504ba71b3c2..4345380ed2f9 100644 --- a/polkadot/node/core/approval-voting/src/import.rs +++ b/polkadot/node/core/approval-voting/src/import.rs @@ -30,7 +30,10 @@ use polkadot_node_jaeger as jaeger; use polkadot_node_primitives::{ - approval::{self as approval_types, BlockApprovalMeta, RelayVRFStory}, + approval::{ + self as approval_types, + v1::{BlockApprovalMeta, RelayVRFStory}, + }, MAX_FINALITY_LAG, }; use polkadot_node_subsystem::{ @@ -53,7 +56,7 @@ use futures::{channel::oneshot, prelude::*}; use std::collections::HashMap; -use super::approval_db::v1; +use super::approval_db::v2; use crate::{ backend::{Backend, OverlayedBackend}, criteria::{AssignmentCriteria, OurAssignment}, @@ -92,7 +95,7 @@ enum ImportedBlockInfoError { FutureCancelled(&'static str, futures::channel::oneshot::Canceled), #[error(transparent)] - ApprovalError(approval_types::ApprovalError), + ApprovalError(approval_types::v1::ApprovalError), #[error("block is already finalized")] BlockAlreadyFinalized, @@ -216,7 +219,7 @@ async fn imported_block_info( .ok_or(ImportedBlockInfoError::SessionInfoUnavailable)?; let (assignments, slot, relay_vrf_story) = { - let unsafe_vrf = approval_types::babe_unsafe_vrf_info(&block_header); + let unsafe_vrf = approval_types::v1::babe_unsafe_vrf_info(&block_header); match unsafe_vrf { Some(unsafe_vrf) => { @@ -497,7 +500,7 @@ pub(crate) async fn handle_new_head( ctx.send_message(ChainSelectionMessage::Approved(block_hash)).await; } - let block_entry = v1::BlockEntry { + let block_entry = v2::BlockEntry { block_hash, parent_hash: block_header.parent_hash, block_number: block_header.number, @@ -510,6 +513,7 @@ pub(crate) async fn handle_new_head( .collect(), approved_bitfield, children: Vec::new(), + distributed_assignments: Default::default(), }; gum::trace!( @@ -588,11 +592,11 @@ pub(crate) async fn handle_new_head( #[cfg(test)] pub(crate) mod tests { use super::*; - use crate::{approval_db::v1::DbBackend, RuntimeInfo, RuntimeInfoConfig}; + use crate::{approval_db::v2::DbBackend, RuntimeInfo, RuntimeInfoConfig}; use ::test_helpers::{dummy_candidate_receipt, dummy_hash}; use assert_matches::assert_matches; use polkadot_node_primitives::{ - approval::{VrfSignature, VrfTranscript}, + approval::v1::{VrfSignature, VrfTranscript}, DISPUTE_WINDOW, }; use polkadot_node_subsystem::messages::{AllMessages, ApprovalVotingMessage}; @@ -608,7 +612,7 @@ pub(crate) mod tests { pub(crate) use sp_runtime::{Digest, DigestItem}; use std::{pin::Pin, sync::Arc}; - use crate::{approval_db::v1::Config as DatabaseConfig, criteria, BlockEntry}; + use crate::{approval_db::v2::Config as DatabaseConfig, criteria, BlockEntry}; const DATA_COL: u32 = 0; @@ -654,7 +658,7 @@ pub(crate) mod tests { fn compute_assignments( &self, _keystore: &LocalKeystore, - _relay_vrf_story: polkadot_node_primitives::approval::RelayVRFStory, + _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, _config: &criteria::Config, _leaving_cores: Vec<( CandidateHash, @@ -667,13 +671,14 @@ pub(crate) mod tests { fn check_assignment_cert( &self, - _claimed_core_index: polkadot_primitives::CoreIndex, + _claimed_core_bitfield: polkadot_node_primitives::approval::v2::CoreBitfield, _validator_index: polkadot_primitives::ValidatorIndex, _config: &criteria::Config, - _relay_vrf_story: polkadot_node_primitives::approval::RelayVRFStory, - _assignment: &polkadot_node_primitives::approval::AssignmentCert, - _backing_group: polkadot_primitives::GroupIndex, - ) -> Result { + _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, + _assignment: &polkadot_node_primitives::approval::v2::AssignmentCertV2, + _backing_groups: Vec, + ) -> Result + { Ok(0) } } @@ -1252,7 +1257,7 @@ pub(crate) mod tests { let (state, mut session_info_provider) = single_session_state(); overlay_db.write_block_entry( - v1::BlockEntry { + v2::BlockEntry { block_hash: parent_hash, parent_hash: Default::default(), block_number: 4, @@ -1262,6 +1267,7 @@ pub(crate) mod tests { candidates: Vec::new(), approved_bitfield: Default::default(), children: Vec::new(), + distributed_assignments: Default::default(), } .into(), ); @@ -1294,7 +1300,7 @@ pub(crate) mod tests { // the first candidate should be insta-approved // the second should not let entry: BlockEntry = - v1::load_block_entry(db_writer.as_ref(), &TEST_CONFIG, &hash) + v2::load_block_entry(db_writer.as_ref(), &TEST_CONFIG, &hash) .unwrap() .unwrap() .into(); diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index b29e47b4c435..beaca43ee528 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -25,7 +25,11 @@ use jaeger::{hash_to_trace_identifier, PerLeafSpan}; use polkadot_node_jaeger as jaeger; use polkadot_node_primitives::{ approval::{ - BlockApprovalMeta, DelayTranche, IndirectAssignmentCert, IndirectSignedApprovalVote, + v1::{BlockApprovalMeta, DelayTranche, IndirectSignedApprovalVote}, + v2::{ + AssignmentCertKindV2, BitfieldError, CandidateBitfield, CoreBitfield, + IndirectAssignmentCertV2, + }, }, ValidationResult, DISPUTE_WINDOW, }; @@ -75,12 +79,13 @@ use std::{ }; use approval_checking::RequiredTranches; +use bitvec::{order::Lsb0, vec::BitVec}; use criteria::{AssignmentCriteria, RealAssignmentCriteria}; use persisted_entries::{ApprovalEntry, BlockEntry, CandidateEntry}; use time::{slot_number_to_tick, Clock, ClockExt, SystemClock, Tick}; mod approval_checking; -mod approval_db; +pub mod approval_db; mod backend; mod criteria; mod import; @@ -89,8 +94,9 @@ mod persisted_entries; mod time; use crate::{ - approval_db::v1::{Config as DatabaseConfig, DbBackend}, + approval_db::v2::{Config as DatabaseConfig, DbBackend}, backend::{Backend, OverlayedBackend}, + criteria::InvalidAssignmentReason, }; #[cfg(test)] @@ -109,7 +115,7 @@ const APPROVAL_CACHE_SIZE: NonZeroUsize = match NonZeroUsize::new(1024) { const TICK_TOO_FAR_IN_FUTURE: Tick = 20; // 10 seconds. const APPROVAL_DELAY: Tick = 2; -const LOG_TARGET: &str = "parachain::approval-voting"; +pub(crate) const LOG_TARGET: &str = "parachain::approval-voting"; /// Configuration for the approval voting subsystem #[derive(Debug, Clone)] @@ -379,8 +385,8 @@ impl ApprovalVotingSubsystem { /// The operation is not allowed for blocks older than the last finalized one. pub fn revert_to(&self, hash: Hash) -> Result<(), SubsystemError> { let config = - approval_db::v1::Config { col_approval_data: self.db_config.col_approval_data }; - let mut backend = approval_db::v1::DbBackend::new(self.db.clone(), config); + approval_db::v2::Config { col_approval_data: self.db_config.col_approval_data }; + let mut backend = approval_db::v2::DbBackend::new(self.db.clone(), config); let mut overlay = OverlayedBackend::new(&backend); ops::revert_to(&mut overlay, hash)?; @@ -741,14 +747,15 @@ enum Action { tick: Tick, }, LaunchApproval { + claimed_candidate_indices: CandidateBitfield, candidate_hash: CandidateHash, - indirect_cert: IndirectAssignmentCert, + indirect_cert: IndirectAssignmentCertV2, assignment_tranche: DelayTranche, relay_block_hash: Hash, - candidate_index: CandidateIndex, session: SessionIndex, candidate: CandidateReceipt, backing_group: GroupIndex, + distribute_assignment: bool, }, NoteApprovedInChainSelection(Hash), IssueApproval(CandidateHash, ApprovalVoteRequest), @@ -963,14 +970,15 @@ async fn handle_actions( actions_iter = next_actions.into_iter(); }, Action::LaunchApproval { + claimed_candidate_indices, candidate_hash, indirect_cert, assignment_tranche, relay_block_hash, - candidate_index, session, candidate, backing_group, + distribute_assignment, } => { // Don't launch approval work if the node is syncing. if let Mode::Syncing(_) = *mode { @@ -991,10 +999,12 @@ async fn handle_actions( launch_approval_span.add_string_tag("block-hash", format!("{:?}", block_hash)); let validator_index = indirect_cert.validator; - ctx.send_unbounded_message(ApprovalDistributionMessage::DistributeAssignment( - indirect_cert, - candidate_index, - )); + if distribute_assignment { + ctx.send_unbounded_message(ApprovalDistributionMessage::DistributeAssignment( + indirect_cert, + claimed_candidate_indices, + )); + } match approvals_cache.get(&candidate_hash) { Some(ApprovalOutcome::Approved) => { @@ -1061,6 +1071,49 @@ async fn handle_actions( Ok(conclude) } +fn cores_to_candidate_indices( + core_indices: &CoreBitfield, + block_entry: &BlockEntry, +) -> Result { + let mut candidate_indices = Vec::new(); + + // Map from core index to candidate index. + for claimed_core_index in core_indices.iter_ones() { + if let Some(candidate_index) = block_entry + .candidates() + .iter() + .position(|(core_index, _)| core_index.0 == claimed_core_index as u32) + { + candidate_indices.push(candidate_index as CandidateIndex); + } + } + + CandidateBitfield::try_from(candidate_indices) +} + +// Returns the claimed core bitfield from the assignment cert, the candidate hash and a +// `BlockEntry`. Can fail only for VRF Delay assignments for which we cannot find the candidate hash +// in the block entry which indicates a bug or corrupted storage. +fn get_assignment_core_indices( + assignment: &AssignmentCertKindV2, + candidate_hash: &CandidateHash, + block_entry: &BlockEntry, +) -> Option { + match &assignment { + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => + Some(core_bitfield.clone()), + AssignmentCertKindV2::RelayVRFModulo { sample: _ } => block_entry + .candidates() + .iter() + .find(|(_core_index, h)| candidate_hash == h) + .map(|(core_index, _candidate_hash)| { + CoreBitfield::try_from(vec![*core_index]).expect("Not an empty vec; qed") + }), + AssignmentCertKindV2::RelayVRFDelay { core_index } => + Some(CoreBitfield::try_from(vec![*core_index]).expect("Not an empty vec; qed")), + } +} + fn distribution_messages_for_activation( db: &OverlayedBackend<'_, impl Backend>, state: &State, @@ -1125,33 +1178,95 @@ fn distribution_messages_for_activation( match approval_entry.local_statements() { (None, None) | (None, Some(_)) => {}, // second is impossible case. (Some(assignment), None) => { - messages.push(ApprovalDistributionMessage::DistributeAssignment( - IndirectAssignmentCert { - block_hash, - validator: assignment.validator_index(), - cert: assignment.cert().clone(), - }, - i as _, - )); + if let Some(claimed_core_indices) = get_assignment_core_indices( + &assignment.cert().kind, + &candidate_hash, + &block_entry, + ) { + match cores_to_candidate_indices( + &claimed_core_indices, + &block_entry, + ) { + Ok(bitfield) => messages.push( + ApprovalDistributionMessage::DistributeAssignment( + IndirectAssignmentCertV2 { + block_hash, + validator: assignment.validator_index(), + cert: assignment.cert().clone(), + }, + bitfield, + ), + ), + Err(err) => { + // Should never happen. If we fail here it means the + // assignment is null (no cores claimed). + gum::warn!( + target: LOG_TARGET, + ?block_hash, + ?candidate_hash, + ?err, + "Failed to create assignment bitfield", + ); + }, + } + } else { + gum::warn!( + target: LOG_TARGET, + ?block_hash, + ?candidate_hash, + "Cannot get assignment claimed core indices", + ); + } }, (Some(assignment), Some(approval_sig)) => { - messages.push(ApprovalDistributionMessage::DistributeAssignment( - IndirectAssignmentCert { - block_hash, - validator: assignment.validator_index(), - cert: assignment.cert().clone(), - }, - i as _, - )); + if let Some(claimed_core_indices) = get_assignment_core_indices( + &assignment.cert().kind, + &candidate_hash, + &block_entry, + ) { + match cores_to_candidate_indices( + &claimed_core_indices, + &block_entry, + ) { + Ok(bitfield) => messages.push( + ApprovalDistributionMessage::DistributeAssignment( + IndirectAssignmentCertV2 { + block_hash, + validator: assignment.validator_index(), + cert: assignment.cert().clone(), + }, + bitfield, + ), + ), + Err(err) => { + gum::warn!( + target: LOG_TARGET, + ?block_hash, + ?candidate_hash, + ?err, + "Failed to create assignment bitfield", + ); + // If we didn't send assignment, we don't send approval. + continue + }, + } - messages.push(ApprovalDistributionMessage::DistributeApproval( - IndirectSignedApprovalVote { - block_hash, - candidate_index: i as _, - validator: assignment.validator_index(), - signature: approval_sig, - }, - )) + messages.push(ApprovalDistributionMessage::DistributeApproval( + IndirectSignedApprovalVote { + block_hash, + candidate_index: i as _, + validator: assignment.validator_index(), + signature: approval_sig, + }, + )); + } else { + gum::warn!( + target: LOG_TARGET, + ?block_hash, + ?candidate_hash, + "Cannot get assignment claimed core indices", + ); + } }, } }, @@ -1271,14 +1386,14 @@ async fn handle_from_overseer( vec![Action::Conclude] }, FromOrchestra::Communication { msg } => match msg { - ApprovalVotingMessage::CheckAndImportAssignment(a, claimed_core, res) => { + ApprovalVotingMessage::CheckAndImportAssignment(a, claimed_cores, res) => { let (check_outcome, actions) = check_and_import_assignment( ctx.sender(), state, db, session_info_provider, a, - claimed_core, + claimed_cores, ) .await?; let _ = res.send(check_outcome); @@ -1448,7 +1563,6 @@ async fn handle_approved_ancestor( let mut span = span .child("handle-approved-ancestor") .with_stage(jaeger::Stage::ApprovalChecking); - use bitvec::{order::Lsb0, vec::BitVec}; let mut all_approved_max = None; @@ -1787,8 +1901,8 @@ async fn check_and_import_assignment( state: &State, db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, - assignment: IndirectAssignmentCert, - candidate_index: CandidateIndex, + assignment: IndirectAssignmentCertV2, + candidate_indices: CandidateBitfield, ) -> SubsystemResult<(AssignmentCheckResult, Vec)> where Sender: SubsystemSender, @@ -1801,9 +1915,12 @@ where .map(|span| span.child("check-and-import-assignment")) .unwrap_or_else(|| jaeger::Span::new(assignment.block_hash, "check-and-import-assignment")) .with_relay_parent(assignment.block_hash) - .with_uint_tag("candidate-index", candidate_index as u64) .with_stage(jaeger::Stage::ApprovalChecking); + for candidate_index in candidate_indices.iter_ones() { + check_and_import_assignment_span.add_uint_tag("candidate-index", candidate_index as u64); + } + let block_entry = match db.load_block_entry(&assignment.block_hash)? { Some(b) => b, None => @@ -1833,39 +1950,64 @@ where )), }; - let (claimed_core_index, assigned_candidate_hash) = - match block_entry.candidate(candidate_index as usize) { - Some((c, h)) => (*c, *h), + let n_cores = session_info.n_cores as usize; + + // Early check the candidate bitfield and core bitfields lengths < `n_cores`. + // Core bitfield length is checked later in `check_assignment_cert`. + if candidate_indices.len() > n_cores { + gum::debug!( + target: LOG_TARGET, + validator = assignment.validator.0, + n_cores, + candidate_bitfield_len = ?candidate_indices.len(), + "Oversized bitfield", + ); + + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidBitfield( + candidate_indices.len(), + )), + Vec::new(), + )) + } + + // The Compact VRF modulo assignment cert has multiple core assignments. + let mut backing_groups = Vec::new(); + let mut claimed_core_indices = Vec::new(); + let mut assigned_candidate_hashes = Vec::new(); + + for candidate_index in candidate_indices.iter_ones() { + let (claimed_core_index, assigned_candidate_hash) = + match block_entry.candidate(candidate_index) { + Some((c, h)) => (*c, *h), + None => + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidateIndex( + candidate_index as _, + )), + Vec::new(), + )), // no candidate at core. + }; + + let mut candidate_entry = match db.load_candidate_entry(&assigned_candidate_hash)? { + Some(c) => c, None => return Ok(( - AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidateIndex( - candidate_index, + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidate( + candidate_index as _, + assigned_candidate_hash, )), Vec::new(), )), // no candidate at core. }; - check_and_import_assignment_span - .add_string_tag("candidate-hash", format!("{:?}", assigned_candidate_hash)); - check_and_import_assignment_span.add_string_tag( - "traceID", - format!("{:?}", jaeger::hash_to_trace_identifier(assigned_candidate_hash.0)), - ); - - let mut candidate_entry = match db.load_candidate_entry(&assigned_candidate_hash)? { - Some(c) => c, - None => - return Ok(( - AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidate( - candidate_index, - assigned_candidate_hash, - )), - Vec::new(), - )), - }; + check_and_import_assignment_span + .add_string_tag("candidate-hash", format!("{:?}", assigned_candidate_hash)); + check_and_import_assignment_span.add_string_tag( + "traceID", + format!("{:?}", jaeger::hash_to_trace_identifier(assigned_candidate_hash.0)), + ); - let res = { - // import the assignment. let approval_entry = match candidate_entry.approval_entry_mut(&assignment.block_hash) { Some(a) => a, None => @@ -1878,79 +2020,144 @@ where )), }; - let res = state.assignment_criteria.check_assignment_cert( - claimed_core_index, - assignment.validator, - &criteria::Config::from(session_info), - block_entry.relay_vrf_story(), - &assignment.cert, - approval_entry.backing_group(), - ); + backing_groups.push(approval_entry.backing_group()); + claimed_core_indices.push(claimed_core_index); + assigned_candidate_hashes.push(assigned_candidate_hash); + } - let tranche = match res { - Err(crate::criteria::InvalidAssignment(reason)) => - return Ok(( - AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCert( - assignment.validator, - format!("{:?}", reason), - )), - Vec::new(), + // Error on null assignments. + if claimed_core_indices.is_empty() { + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCert( + assignment.validator, + format!("{:?}", InvalidAssignmentReason::NullAssignment), + )), + Vec::new(), + )) + } + + // Check the assignment certificate. + let res = state.assignment_criteria.check_assignment_cert( + claimed_core_indices + .clone() + .try_into() + .expect("Checked for null assignment above; qed"), + assignment.validator, + &criteria::Config::from(session_info), + block_entry.relay_vrf_story(), + &assignment.cert, + backing_groups, + ); + + let tranche = match res { + Err(crate::criteria::InvalidAssignment(reason)) => + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCert( + assignment.validator, + format!("{:?}", reason), )), - Ok(tranche) => { - let current_tranche = - state.clock.tranche_now(state.slot_duration_millis, block_entry.slot()); + Vec::new(), + )), + Ok(tranche) => { + let current_tranche = + state.clock.tranche_now(state.slot_duration_millis, block_entry.slot()); - let too_far_in_future = current_tranche + TICK_TOO_FAR_IN_FUTURE as DelayTranche; + let too_far_in_future = current_tranche + TICK_TOO_FAR_IN_FUTURE as DelayTranche; - if tranche >= too_far_in_future { - return Ok((AssignmentCheckResult::TooFarInFuture, Vec::new())) - } + if tranche >= too_far_in_future { + return Ok((AssignmentCheckResult::TooFarInFuture, Vec::new())) + } - tranche - }, - }; + tranche + }, + }; - check_and_import_assignment_span.add_uint_tag("tranche", tranche as u64); + let mut actions = Vec::new(); + let res = { + let mut is_duplicate = true; + // Import the assignments for all cores in the cert. + for (assigned_candidate_hash, candidate_index) in + assigned_candidate_hashes.iter().zip(candidate_indices.iter_ones()) + { + let mut candidate_entry = match db.load_candidate_entry(&assigned_candidate_hash)? { + Some(c) => c, + None => + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidate( + candidate_index as _, + *assigned_candidate_hash, + )), + Vec::new(), + )), + }; - let is_duplicate = approval_entry.is_assigned(assignment.validator); - approval_entry.import_assignment(tranche, assignment.validator, tick_now); + let approval_entry = match candidate_entry.approval_entry_mut(&assignment.block_hash) { + Some(a) => a, + None => + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::Internal( + assignment.block_hash, + *assigned_candidate_hash, + )), + Vec::new(), + )), + }; + is_duplicate &= approval_entry.is_assigned(assignment.validator); + approval_entry.import_assignment(tranche, assignment.validator, tick_now); + check_and_import_assignment_span.add_uint_tag("tranche", tranche as u64); + + // We've imported a new assignment, so we need to schedule a wake-up for when that might + // no-show. + if let Some((approval_entry, status)) = state + .approval_status(sender, session_info_provider, &block_entry, &candidate_entry) + .await + { + actions.extend(schedule_wakeup_action( + approval_entry, + block_entry.block_hash(), + block_entry.block_number(), + *assigned_candidate_hash, + status.block_tick, + tick_now, + status.required_tranches, + )); + } + + // We also write the candidate entry as it now contains the new candidate. + db.write_candidate_entry(candidate_entry.into()); + } + // Since we don't account for tranche in distribution message fingerprinting, some + // validators can be assigned to the same core (VRF modulo vs VRF delay). These can be + // safely ignored ignored. However, if an assignment is for multiple cores (these are only + // tranche0), we cannot ignore it, because it would mean ignoring other non duplicate + // assignments. if is_duplicate { AssignmentCheckResult::AcceptedDuplicate + } else if candidate_indices.count_ones() > 1 { + gum::trace!( + target: LOG_TARGET, + validator = assignment.validator.0, + candidate_hashes = ?assigned_candidate_hashes, + assigned_cores = ?claimed_core_indices, + ?tranche, + "Imported assignments for multiple cores.", + ); + + AssignmentCheckResult::Accepted } else { gum::trace!( target: LOG_TARGET, validator = assignment.validator.0, - candidate_hash = ?assigned_candidate_hash, - para_id = ?candidate_entry.candidate_receipt().descriptor.para_id, - "Imported assignment.", + candidate_hashes = ?assigned_candidate_hashes, + assigned_cores = ?claimed_core_indices, + "Imported assignment for a single core.", ); AssignmentCheckResult::Accepted } }; - let mut actions = Vec::new(); - - // We've imported a new approval, so we need to schedule a wake-up for when that might no-show. - if let Some((approval_entry, status)) = state - .approval_status(sender, session_info_provider, &block_entry, &candidate_entry) - .await - { - actions.extend(schedule_wakeup_action( - approval_entry, - block_entry.block_hash(), - block_entry.block_number(), - assigned_candidate_hash, - status.block_tick, - tick_now, - status.required_tranches, - )); - } - - // We also write the candidate entry as it now contains the new candidate. - db.write_candidate_entry(candidate_entry.into()); - Ok((res, actions)) } @@ -2324,7 +2531,7 @@ async fn process_wakeup( let candidate_entry = db.load_candidate_entry(&candidate_hash)?; // If either is not present, we have nothing to wakeup. Might have lost a race with finality - let (block_entry, mut candidate_entry) = match (block_entry, candidate_entry) { + let (mut block_entry, mut candidate_entry) = match (block_entry, candidate_entry) { (Some(b), Some(c)) => (b, c), _ => return Ok(Vec::new()), }; @@ -2404,31 +2611,58 @@ async fn process_wakeup( if let Some((cert, val_index, tranche)) = maybe_cert { let indirect_cert = - IndirectAssignmentCert { block_hash: relay_block, validator: val_index, cert }; + IndirectAssignmentCertV2 { block_hash: relay_block, validator: val_index, cert }; + + gum::trace!( + target: LOG_TARGET, + ?candidate_hash, + para_id = ?candidate_receipt.descriptor.para_id, + block_hash = ?relay_block, + "Launching approval work.", + ); - let index_in_candidate = - block_entry.candidates().iter().position(|(_, h)| &candidate_hash == h); + if let Some(claimed_core_indices) = + get_assignment_core_indices(&indirect_cert.cert.kind, &candidate_hash, &block_entry) + { + match cores_to_candidate_indices(&claimed_core_indices, &block_entry) { + Ok(claimed_candidate_indices) => { + // Ensure we distribute multiple core assignments just once. + let distribute_assignment = if claimed_candidate_indices.count_ones() > 1 { + !block_entry.mark_assignment_distributed(claimed_candidate_indices.clone()) + } else { + true + }; + db.write_block_entry(block_entry.clone()); - if let Some(i) = index_in_candidate { - gum::trace!( + actions.push(Action::LaunchApproval { + claimed_candidate_indices, + candidate_hash, + indirect_cert, + assignment_tranche: tranche, + relay_block_hash: relay_block, + session: block_entry.session(), + candidate: candidate_receipt, + backing_group, + distribute_assignment, + }); + }, + Err(err) => { + // Never happens, it should only happen if no cores are claimed, which is a bug. + gum::warn!( + target: LOG_TARGET, + block_hash = ?relay_block, + ?err, + "Failed to create assignment bitfield" + ); + }, + }; + } else { + gum::warn!( target: LOG_TARGET, - ?candidate_hash, - para_id = ?candidate_receipt.descriptor.para_id, block_hash = ?relay_block, - "Launching approval work.", + ?candidate_hash, + "Cannot get assignment claimed core indices", ); - - // sanity: should always be present. - actions.push(Action::LaunchApproval { - candidate_hash, - indirect_cert, - assignment_tranche: tranche, - relay_block_hash: relay_block, - candidate_index: i as _, - session: block_entry.session(), - candidate: candidate_receipt, - backing_group, - }); } } // Although we checked approval earlier in this function, diff --git a/polkadot/node/core/approval-voting/src/ops.rs b/polkadot/node/core/approval-voting/src/ops.rs index 6f57b2f80e8a..a6f0ecf9d1f0 100644 --- a/polkadot/node/core/approval-voting/src/ops.rs +++ b/polkadot/node/core/approval-voting/src/ops.rs @@ -25,7 +25,7 @@ use polkadot_primitives::{BlockNumber, CandidateHash, CandidateReceipt, GroupInd use std::collections::{hash_map::Entry, BTreeMap, HashMap}; use super::{ - approval_db::v1::{OurAssignment, StoredBlockRange}, + approval_db::v2::{OurAssignment, StoredBlockRange}, backend::{Backend, OverlayedBackend}, persisted_entries::{ApprovalEntry, BlockEntry, CandidateEntry}, LOG_TARGET, diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index 9b6592220275..155b2f9c4e02 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -20,16 +20,21 @@ //! Within that context, things are plain-old-data. Within this module, //! data and logic are intertwined. -use polkadot_node_primitives::approval::{AssignmentCert, DelayTranche, RelayVRFStory}; +use polkadot_node_primitives::approval::{ + v1::{DelayTranche, RelayVRFStory}, + v2::{AssignmentCertV2, CandidateBitfield}, +}; use polkadot_primitives::{ BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, SessionIndex, ValidatorIndex, ValidatorSignature, }; use sp_consensus_slots::Slot; -use bitvec::{order::Lsb0 as BitOrderLsb0, slice::BitSlice, vec::BitVec}; +use bitvec::{order::Lsb0 as BitOrderLsb0, slice::BitSlice}; use std::collections::BTreeMap; +use crate::approval_db::v2::Bitfield; + use super::{criteria::OurAssignment, time::Tick}; /// Metadata regarding a specific tranche of assignments for a specific candidate. @@ -53,8 +58,8 @@ impl TrancheEntry { } } -impl From for TrancheEntry { - fn from(entry: crate::approval_db::v1::TrancheEntry) -> Self { +impl From for TrancheEntry { + fn from(entry: crate::approval_db::v2::TrancheEntry) -> Self { TrancheEntry { tranche: entry.tranche, assignments: entry.assignments.into_iter().map(|(v, t)| (v, t.into())).collect(), @@ -62,7 +67,7 @@ impl From for TrancheEntry { } } -impl From for crate::approval_db::v1::TrancheEntry { +impl From for crate::approval_db::v2::TrancheEntry { fn from(entry: TrancheEntry) -> Self { Self { tranche: entry.tranche, @@ -80,7 +85,7 @@ pub struct ApprovalEntry { our_assignment: Option, our_approval_sig: Option, // `n_validators` bits. - assignments: BitVec, + assigned_validators: Bitfield, approved: bool, } @@ -92,10 +97,17 @@ impl ApprovalEntry { our_assignment: Option, our_approval_sig: Option, // `n_validators` bits. - assignments: BitVec, + assigned_validators: Bitfield, approved: bool, ) -> Self { - Self { tranches, backing_group, our_assignment, our_approval_sig, assignments, approved } + Self { + tranches, + backing_group, + our_assignment, + our_approval_sig, + assigned_validators, + approved, + } } // Access our assignment for this approval entry. @@ -107,7 +119,7 @@ impl ApprovalEntry { pub fn trigger_our_assignment( &mut self, tick_now: Tick, - ) -> Option<(AssignmentCert, ValidatorIndex, DelayTranche)> { + ) -> Option<(AssignmentCertV2, ValidatorIndex, DelayTranche)> { let our = self.our_assignment.as_mut().and_then(|a| { if a.triggered() { return None @@ -131,7 +143,10 @@ impl ApprovalEntry { /// Whether a validator is already assigned. pub fn is_assigned(&self, validator_index: ValidatorIndex) -> bool { - self.assignments.get(validator_index.0 as usize).map(|b| *b).unwrap_or(false) + self.assigned_validators + .get(validator_index.0 as usize) + .map(|b| *b) + .unwrap_or(false) } /// Import an assignment. No-op if already assigned on the same tranche. @@ -158,14 +173,14 @@ impl ApprovalEntry { }; self.tranches[idx].assignments.push((validator_index, tick_now)); - self.assignments.set(validator_index.0 as _, true); + self.assigned_validators.set(validator_index.0 as _, true); } // Produce a bitvec indicating the assignments of all validators up to and // including `tranche`. - pub fn assignments_up_to(&self, tranche: DelayTranche) -> BitVec { + pub fn assignments_up_to(&self, tranche: DelayTranche) -> Bitfield { self.tranches.iter().take_while(|e| e.tranche <= tranche).fold( - bitvec::bitvec![u8, BitOrderLsb0; 0; self.assignments.len()], + bitvec::bitvec![u8, BitOrderLsb0; 0; self.assigned_validators.len()], |mut a, e| { for &(v, _) in &e.assignments { a.set(v.0 as _, true); @@ -193,12 +208,12 @@ impl ApprovalEntry { /// Get the number of validators in this approval entry. pub fn n_validators(&self) -> usize { - self.assignments.len() + self.assigned_validators.len() } /// Get the number of assignments by validators, including the local validator. pub fn n_assignments(&self) -> usize { - self.assignments.count_ones() + self.assigned_validators.count_ones() } /// Get the backing group index of the approval entry. @@ -219,27 +234,27 @@ impl ApprovalEntry { } } -impl From for ApprovalEntry { - fn from(entry: crate::approval_db::v1::ApprovalEntry) -> Self { +impl From for ApprovalEntry { + fn from(entry: crate::approval_db::v2::ApprovalEntry) -> Self { ApprovalEntry { tranches: entry.tranches.into_iter().map(Into::into).collect(), backing_group: entry.backing_group, our_assignment: entry.our_assignment.map(Into::into), our_approval_sig: entry.our_approval_sig.map(Into::into), - assignments: entry.assignments, + assigned_validators: entry.assigned_validators, approved: entry.approved, } } } -impl From for crate::approval_db::v1::ApprovalEntry { +impl From for crate::approval_db::v2::ApprovalEntry { fn from(entry: ApprovalEntry) -> Self { Self { tranches: entry.tranches.into_iter().map(Into::into).collect(), backing_group: entry.backing_group, our_assignment: entry.our_assignment.map(Into::into), our_approval_sig: entry.our_approval_sig.map(Into::into), - assignments: entry.assignments, + assigned_validators: entry.assigned_validators, approved: entry.approved, } } @@ -253,7 +268,7 @@ pub struct CandidateEntry { // Assignments are based on blocks, so we need to track assignments separately // based on the block we are looking at. pub block_assignments: BTreeMap, - pub approvals: BitVec, + pub approvals: Bitfield, } impl CandidateEntry { @@ -290,8 +305,8 @@ impl CandidateEntry { } } -impl From for CandidateEntry { - fn from(entry: crate::approval_db::v1::CandidateEntry) -> Self { +impl From for CandidateEntry { + fn from(entry: crate::approval_db::v2::CandidateEntry) -> Self { CandidateEntry { candidate: entry.candidate, session: entry.session, @@ -305,7 +320,7 @@ impl From for CandidateEntry { } } -impl From for crate::approval_db::v1::CandidateEntry { +impl From for crate::approval_db::v2::CandidateEntry { fn from(entry: CandidateEntry) -> Self { Self { candidate: entry.candidate, @@ -336,8 +351,12 @@ pub struct BlockEntry { // A bitfield where the i'th bit corresponds to the i'th candidate in `candidates`. // The i'th bit is `true` iff the candidate has been approved in the context of this // block. The block can be considered approved if the bitfield has all bits set to `true`. - pub approved_bitfield: BitVec, + pub approved_bitfield: Bitfield, pub children: Vec, + // A list of assignments for which wea already distributed the assignment. + // We use this to ensure we don't distribute multiple core assignments twice as we track + // individual wakeups for each core. + distributed_assignments: Bitfield, } impl BlockEntry { @@ -412,6 +431,39 @@ impl BlockEntry { pub fn parent_hash(&self) -> Hash { self.parent_hash } + + /// Mark distributed assignment for many candidate indices. + /// Returns `true` if an assignment was already distributed for the `candidates`. + pub fn mark_assignment_distributed(&mut self, candidates: CandidateBitfield) -> bool { + let bitfield = candidates.into_inner(); + let total_one_bits = self.distributed_assignments.count_ones(); + + let new_len = std::cmp::max(self.distributed_assignments.len(), bitfield.len()); + self.distributed_assignments.resize(new_len, false); + self.distributed_assignments |= bitfield; + + // If the an operation did not change our current bitfied, we return true. + let distributed = total_one_bits == self.distributed_assignments.count_ones(); + + distributed + } +} + +impl From for BlockEntry { + fn from(entry: crate::approval_db::v2::BlockEntry) -> Self { + BlockEntry { + block_hash: entry.block_hash, + parent_hash: entry.parent_hash, + block_number: entry.block_number, + session: entry.session, + slot: entry.slot, + relay_vrf_story: RelayVRFStory(entry.relay_vrf_story), + candidates: entry.candidates, + approved_bitfield: entry.approved_bitfield, + children: entry.children, + distributed_assignments: entry.distributed_assignments, + } + } } impl From for BlockEntry { @@ -426,11 +478,12 @@ impl From for BlockEntry { candidates: entry.candidates, approved_bitfield: entry.approved_bitfield, children: entry.children, + distributed_assignments: Default::default(), } } } -impl From for crate::approval_db::v1::BlockEntry { +impl From for crate::approval_db::v2::BlockEntry { fn from(entry: BlockEntry) -> Self { Self { block_hash: entry.block_hash, @@ -442,6 +495,36 @@ impl From for crate::approval_db::v1::BlockEntry { candidates: entry.candidates, approved_bitfield: entry.approved_bitfield, children: entry.children, + distributed_assignments: entry.distributed_assignments, + } + } +} + +/// Migration helpers. +impl From for CandidateEntry { + fn from(value: crate::approval_db::v1::CandidateEntry) -> Self { + Self { + approvals: value.approvals, + block_assignments: value + .block_assignments + .into_iter() + .map(|(h, ae)| (h, ae.into())) + .collect(), + candidate: value.candidate, + session: value.session, + } + } +} + +impl From for ApprovalEntry { + fn from(value: crate::approval_db::v1::ApprovalEntry) -> Self { + ApprovalEntry { + tranches: value.tranches.into_iter().map(|tranche| tranche.into()).collect(), + backing_group: value.backing_group, + our_assignment: value.our_assignment.map(|assignment| assignment.into()), + our_approval_sig: value.our_approval_sig, + assigned_validators: value.assignments, + approved: value.approved, } } } diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index f58e60c6a487..c2ef109ad4ca 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -15,10 +15,14 @@ // along with Polkadot. If not, see . use super::*; +use crate::backend::V1ReadBackend; use polkadot_node_primitives::{ approval::{ - AssignmentCert, AssignmentCertKind, DelayTranche, VrfOutput, VrfProof, VrfSignature, - RELAY_VRF_MODULO_CONTEXT, + v1::{ + AssignmentCert, AssignmentCertKind, DelayTranche, VrfOutput, VrfProof, VrfSignature, + RELAY_VRF_MODULO_CONTEXT, + }, + v2::{AssignmentCertKindV2, AssignmentCertV2}, }, AvailableData, BlockData, PoV, }; @@ -51,7 +55,7 @@ use std::{ }; use super::{ - approval_db::v1::StoredBlockRange, + approval_db::v2::StoredBlockRange, backend::BackendWriteOp, import::tests::{ garbage_vrf_signature, AllowedSlots, BabeEpoch, BabeEpochConfiguration, @@ -111,7 +115,7 @@ fn make_sync_oracle(val: bool) -> (Box, TestSyncOracleHan #[cfg(test)] pub mod test_constants { - use crate::approval_db::v1::Config as DatabaseConfig; + use crate::approval_db::v2::Config as DatabaseConfig; const DATA_COL: u32 = 0; pub(crate) const NUM_COLUMNS: u32 = 1; @@ -161,7 +165,7 @@ impl Clock for MockClock { // This mock clock allows us to manipulate the time and // be notified when wakeups have been triggered. -#[derive(Default)] +#[derive(Default, Debug)] struct MockClockInner { tick: Tick, wakeups: Vec<(Tick, oneshot::Sender<()>)>, @@ -231,7 +235,7 @@ where fn compute_assignments( &self, _keystore: &LocalKeystore, - _relay_vrf_story: polkadot_node_primitives::approval::RelayVRFStory, + _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, _config: &criteria::Config, _leaving_cores: Vec<( CandidateHash, @@ -244,13 +248,13 @@ where fn check_assignment_cert( &self, - _claimed_core_index: polkadot_primitives::CoreIndex, + _claimed_core_bitfield: polkadot_node_primitives::approval::v2::CoreBitfield, validator_index: ValidatorIndex, _config: &criteria::Config, - _relay_vrf_story: polkadot_node_primitives::approval::RelayVRFStory, - _assignment: &polkadot_node_primitives::approval::AssignmentCert, - _backing_group: polkadot_primitives::GroupIndex, - ) -> Result { + _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, + _assignment: &polkadot_node_primitives::approval::v2::AssignmentCertV2, + _backing_groups: Vec, + ) -> Result { self.1(validator_index) } } @@ -271,6 +275,18 @@ struct TestStoreInner { candidate_entries: HashMap, } +impl V1ReadBackend for TestStoreInner { + fn load_candidate_entry_v1( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult> { + self.load_candidate_entry(candidate_hash) + } + fn load_block_entry_v1(&self, block_hash: &Hash) -> SubsystemResult> { + self.load_block_entry(block_hash) + } +} + impl Backend for TestStoreInner { fn load_block_entry(&self, block_hash: &Hash) -> SubsystemResult> { Ok(self.block_entries.get(block_hash).cloned()) @@ -342,6 +358,18 @@ pub struct TestStore { store: Arc>, } +impl V1ReadBackend for TestStore { + fn load_candidate_entry_v1( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult> { + self.load_candidate_entry(candidate_hash) + } + fn load_block_entry_v1(&self, block_hash: &Hash) -> SubsystemResult> { + self.load_block_entry(block_hash) + } +} + impl Backend for TestStore { fn load_block_entry(&self, block_hash: &Hash) -> SubsystemResult> { let store = self.store.lock(); @@ -391,6 +419,17 @@ fn garbage_assignment_cert(kind: AssignmentCertKind) -> AssignmentCert { AssignmentCert { kind, vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) } } } +fn garbage_assignment_cert_v2(kind: AssignmentCertKindV2) -> AssignmentCertV2 { + let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); + let msg = b"test-garbage"; + let mut prng = rand_core::OsRng; + let keypair = schnorrkel::Keypair::generate_with(&mut prng); + let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); + let out = inout.to_output(); + + AssignmentCertV2 { kind, vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) } } +} + fn sign_approval( key: Sr25519Keyring, candidate_hash: CandidateHash, @@ -467,6 +506,12 @@ fn test_harness>( config: HarnessConfig, test: impl FnOnce(TestHarness) -> T, ) { + let _ = env_logger::builder() + .is_test(true) + .filter(Some("polkadot_node_core_approval_voting"), log::LevelFilter::Trace) + .filter(Some(LOG_TARGET), log::LevelFilter::Trace) + .try_init(); + let HarnessConfig { sync_oracle, sync_oracle_handle, clock, backend, assignment_criteria } = config; @@ -616,12 +661,13 @@ async fn check_and_import_assignment( overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportAssignment( - IndirectAssignmentCert { + IndirectAssignmentCertV2 { block_hash, validator, - cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }), + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), }, - candidate_index, + candidate_index.into(), tx, ), }, @@ -630,6 +676,38 @@ async fn check_and_import_assignment( rx } +async fn check_and_import_assignment_v2( + overseer: &mut VirtualOverseer, + block_hash: Hash, + core_indices: Vec, + validator: ValidatorIndex, +) -> oneshot::Receiver { + let (tx, rx) = oneshot::channel(); + overseer_send( + overseer, + FromOrchestra::Communication { + msg: ApprovalVotingMessage::CheckAndImportAssignment( + IndirectAssignmentCertV2 { + block_hash, + validator, + cert: garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: core_indices + .clone() + .into_iter() + .map(|c| CoreIndex(c)) + .collect::>() + .try_into() + .unwrap(), + }), + }, + core_indices.try_into().unwrap(), + tx, + ), + }, + ) + .await; + rx +} struct BlockConfig { slot: Slot, candidates: Option>, @@ -742,7 +820,7 @@ fn session_info(keys: &[Sr25519Keyring]) -> SessionInfo { vec![ValidatorIndex(0)], vec![ValidatorIndex(1)], ]), - n_cores: keys.len() as _, + n_cores: 10, needed_approvals: 2, zeroth_delay_tranche_width: 5, relay_vrf_modulo_samples: 3, @@ -1059,14 +1137,15 @@ fn blank_subsystem_act_on_bad_block() { &mut virtual_overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportAssignment( - IndirectAssignmentCert { + IndirectAssignmentCertV2 { block_hash: bad_block_hash, validator: 0u32.into(), cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0, - }), + }) + .into(), }, - 0u32, + 0u32.into(), tx, ), }, @@ -1322,9 +1401,22 @@ fn subsystem_accepts_duplicate_assignment() { } ); - let block_hash = Hash::repeat_byte(0x01); - let candidate_index = 0; let validator = ValidatorIndex(0); + let candidate_index = 0; + let block_hash = Hash::repeat_byte(0x01); + + let candidate_receipt1 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(1_u32); + receipt + }; + let candidate_receipt2 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(2_u32); + receipt + }; + let candidate_index1 = 0; + let candidate_index2 = 1; // Add block hash 00. ChainBuilder::new() @@ -1332,21 +1424,30 @@ fn subsystem_accepts_duplicate_assignment() { block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(1), + candidates: Some(vec![ + (candidate_receipt1, CoreIndex(0), GroupIndex(1)), + (candidate_receipt2, CoreIndex(1), GroupIndex(1)), + ]), + session_info: None, + }, ) .build(&mut virtual_overseer) .await; - let rx = check_and_import_assignment( + // Initial assignment. + let rx = check_and_import_assignment_v2( &mut virtual_overseer, block_hash, - candidate_index, + vec![candidate_index1, candidate_index2], validator, ) .await; assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted)); + // Test with single assigned core. let rx = check_and_import_assignment( &mut virtual_overseer, block_hash, @@ -1357,6 +1458,18 @@ fn subsystem_accepts_duplicate_assignment() { assert_eq!(rx.await, Ok(AssignmentCheckResult::AcceptedDuplicate)); + // Test with multiple assigned cores. This cannot happen in practice, as tranche0 + // assignments are sent first, but we should still ensure correct behavior. + let rx = check_and_import_assignment_v2( + &mut virtual_overseer, + block_hash, + vec![candidate_index1, candidate_index2], + validator, + ) + .await; + + assert_eq!(rx.await, Ok(AssignmentCheckResult::AcceptedDuplicate)); + virtual_overseer }); } @@ -1407,6 +1520,63 @@ fn subsystem_rejects_assignment_with_unknown_candidate() { }); } +#[test] +fn subsystem_rejects_oversized_bitfields() { + test_harness(HarnessConfig::default(), |test_harness| async move { + let TestHarness { mut virtual_overseer, sync_oracle_handle: _sync_oracle_handle, .. } = + test_harness; + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + let candidate_index = 10; + let validator = ValidatorIndex(0); + + ChainBuilder::new() + .add_block( + block_hash, + ChainBuilder::GENESIS_HASH, + 1, + BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + ) + .build(&mut virtual_overseer) + .await; + + let rx = check_and_import_assignment( + &mut virtual_overseer, + block_hash, + candidate_index, + validator, + ) + .await; + + assert_eq!( + rx.await, + Ok(AssignmentCheckResult::Bad(AssignmentCheckError::InvalidBitfield( + candidate_index as usize + 1 + ))), + ); + + let rx = check_and_import_assignment_v2( + &mut virtual_overseer, + block_hash, + vec![1, 2, 10, 50], + validator, + ) + .await; + + assert_eq!( + rx.await, + Ok(AssignmentCheckResult::Bad(AssignmentCheckError::InvalidBitfield(51))), + ); + virtual_overseer + }); +} + #[test] fn subsystem_accepts_and_imports_approval_after_assignment() { test_harness(HarnessConfig::default(), |test_harness| async move { @@ -1727,14 +1897,15 @@ fn linear_import_act_on_leaf() { &mut virtual_overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportAssignment( - IndirectAssignmentCert { + IndirectAssignmentCertV2 { block_hash: head, validator: 0u32.into(), cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0, - }), + }) + .into(), }, - 0u32, + 0u32.into(), tx, ), }, @@ -1797,14 +1968,15 @@ fn forkful_import_at_same_height_act_on_leaf() { &mut virtual_overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportAssignment( - IndirectAssignmentCert { + IndirectAssignmentCertV2 { block_hash: head, validator: 0u32.into(), cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0, - }), + }) + .into(), }, - 0u32, + 0u32.into(), tx, ), }, @@ -2248,8 +2420,24 @@ fn subsystem_validate_approvals_cache() { let mut assignments = HashMap::new(); let _ = assignments.insert( CoreIndex(0), - approval_db::v1::OurAssignment { - cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: vec![CoreIndex(0), CoreIndex(1), CoreIndex(2)] + .try_into() + .unwrap(), + }), tranche: 0, validator_index: ValidatorIndex(0), triggered: false, @@ -2346,6 +2534,137 @@ fn subsystem_validate_approvals_cache() { }); } +#[test] +fn subsystem_doesnt_distribute_duplicate_compact_assignments() { + let assignment_criteria = Box::new(MockAssignmentCriteria( + || { + let mut assignments = HashMap::new(); + let cert = garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: vec![CoreIndex(0), CoreIndex(1)].try_into().unwrap(), + }); + + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: cert.clone(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let _ = assignments.insert( + CoreIndex(1), + approval_db::v2::OurAssignment { + cert, + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + assignments + }, + |_| Ok(0), + )); + + let config = HarnessConfigBuilder::default().assignment_criteria(assignment_criteria).build(); + let store = config.backend(); + + test_harness(config, |test_harness| async move { + let TestHarness { + mut virtual_overseer, + sync_oracle_handle: _sync_oracle_handle, + clock, + .. + } = test_harness; + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + + let candidate_receipt1 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(1_u32); + receipt + }; + let candidate_receipt2 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(2_u32); + receipt + }; + let candidate_index1 = 0; + let candidate_index2 = 1; + + // Add block hash 00. + ChainBuilder::new() + .add_block( + block_hash, + ChainBuilder::GENESIS_HASH, + 1, + BlockConfig { + slot: Slot::from(0), + candidates: Some(vec![ + (candidate_receipt1.clone(), CoreIndex(0), GroupIndex(1)), + (candidate_receipt2.clone(), CoreIndex(1), GroupIndex(1)), + ]), + session_info: None, + }, + ) + .build(&mut virtual_overseer) + .await; + + // Activate the wakeup present above, and sleep to allow process_wakeups to execute.. + assert_eq!(Some(2), clock.inner.lock().next_wakeup()); + gum::trace!("clock \n{:?}\n", clock.inner.lock()); + + clock.inner.lock().wakeup_all(100); + + assert_eq!(clock.inner.lock().wakeups.len(), 0); + + futures_timer::Delay::new(Duration::from_millis(100)).await; + + // Assignment is distributed only once from `approval-voting` + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + c_indices, + )) => { + assert_eq!(c_indices, vec![candidate_index1, candidate_index2].try_into().unwrap()); + } + ); + + // Candidate 1 + recover_available_data(&mut virtual_overseer).await; + fetch_validation_code(&mut virtual_overseer).await; + + // Candidate 2 + recover_available_data(&mut virtual_overseer).await; + fetch_validation_code(&mut virtual_overseer).await; + + // Check if assignment was triggered for candidate 1. + let candidate_entry = + store.load_candidate_entry(&candidate_receipt1.hash()).unwrap().unwrap(); + let our_assignment = + candidate_entry.approval_entry(&block_hash).unwrap().our_assignment().unwrap(); + assert!(our_assignment.triggered()); + + // Check if assignment was triggered for candidate 2. + let candidate_entry = + store.load_candidate_entry(&candidate_receipt2.hash()).unwrap().unwrap(); + let our_assignment = + candidate_entry.approval_entry(&block_hash).unwrap().our_assignment().unwrap(); + assert!(our_assignment.triggered()); + virtual_overseer + }); +} + /// Ensure that when two assignments are imported, only one triggers the Approval Checking work async fn handle_double_assignment_import( virtual_overseer: &mut VirtualOverseer, @@ -2355,9 +2674,9 @@ async fn handle_double_assignment_import( overseer_recv(virtual_overseer).await, AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( _, - c_index, + c_indices, )) => { - assert_eq!(candidate_index, c_index); + assert_eq!(Into::::into(candidate_index), c_indices); } ); @@ -2370,7 +2689,7 @@ async fn handle_double_assignment_import( _, c_index )) => { - assert_eq!(candidate_index, c_index); + assert_eq!(Into::::into(candidate_index), c_index); } ); @@ -2391,7 +2710,6 @@ async fn handle_double_assignment_import( overseer_recv(virtual_overseer).await, AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(_)) ); - // Assert that there are no more messages being sent by the subsystem assert!(overseer_recv(virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); } @@ -2466,8 +2784,9 @@ where let mut assignments = HashMap::new(); let _ = assignments.insert( CoreIndex(0), - approval_db::v1::OurAssignment { - cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), tranche: our_assigned_tranche, validator_index: ValidatorIndex(0), triggered: false, @@ -2601,14 +2920,12 @@ async fn step_until_done(clock: &MockClock) { futures_timer::Delay::new(Duration::from_millis(200)).await; let mut clock = clock.inner.lock(); if let Some(tick) = clock.next_wakeup() { - println!("TICK: {:?}", tick); relevant_ticks.push(tick); clock.set_tick(tick); } else { break } } - println!("relevant_ticks: {:?}", relevant_ticks); } #[test] diff --git a/polkadot/node/core/approval-voting/src/time.rs b/polkadot/node/core/approval-voting/src/time.rs index 34132dc22b23..a45866402c82 100644 --- a/polkadot/node/core/approval-voting/src/time.rs +++ b/polkadot/node/core/approval-voting/src/time.rs @@ -17,7 +17,7 @@ //! Time utilities for approval voting. use futures::prelude::*; -use polkadot_node_primitives::approval::DelayTranche; +use polkadot_node_primitives::approval::v1::DelayTranche; use sp_consensus_slots::Slot; use std::{ pin::Pin, diff --git a/polkadot/node/core/dispute-coordinator/src/import.rs b/polkadot/node/core/dispute-coordinator/src/import.rs index 0da3723ebf22..6222aab1cb10 100644 --- a/polkadot/node/core/dispute-coordinator/src/import.rs +++ b/polkadot/node/core/dispute-coordinator/src/import.rs @@ -109,7 +109,7 @@ pub enum OwnVoteState { } impl OwnVoteState { - fn new(votes: &CandidateVotes, env: &CandidateEnvironment<'_>) -> Self { + fn new(votes: &CandidateVotes, env: &CandidateEnvironment) -> Self { let controlled_indices = env.controlled_indices(); if controlled_indices.is_empty() { return Self::CannotVote diff --git a/polkadot/node/network/approval-distribution/Cargo.toml b/polkadot/node/network/approval-distribution/Cargo.toml index 9638c53d318b..15602a9796b5 100644 --- a/polkadot/node/network/approval-distribution/Cargo.toml +++ b/polkadot/node/network/approval-distribution/Cargo.toml @@ -14,10 +14,12 @@ polkadot-node-subsystem-util = { path = "../../subsystem-util" } polkadot-primitives = { path = "../../../primitives" } polkadot-node-jaeger = { path = "../../jaeger" } rand = "0.8" +itertools = "0.10.5" futures = "0.3.21" futures-timer = "3.0.2" gum = { package = "tracing-gum", path = "../../gum" } +bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } [dev-dependencies] sp-authority-discovery = { path = "../../../../substrate/primitives/authority-discovery" } diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index ac525ea6faf3..746a4b4dab5c 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -20,17 +20,23 @@ #![warn(missing_docs)] +use self::metrics::Metrics; use futures::{channel::oneshot, select, FutureExt as _}; +use itertools::Itertools; +use net_protocol::peer_set::{ProtocolVersion, ValidationVersion}; use polkadot_node_jaeger as jaeger; use polkadot_node_network_protocol::{ - self as net_protocol, + self as net_protocol, filter_by_peer_version, grid_topology::{RandomRouting, RequiredRouting, SessionGridTopologies, SessionGridTopology}, - peer_set::{ValidationVersion, MAX_NOTIFICATION_SIZE}, + peer_set::MAX_NOTIFICATION_SIZE, v1 as protocol_v1, vstaging as protocol_vstaging, PeerId, UnifiedReputationChange as Rep, - Versioned, VersionedValidationProtocol, View, + Versioned, View, }; use polkadot_node_primitives::approval::{ - AssignmentCert, BlockApprovalMeta, IndirectAssignmentCert, IndirectSignedApprovalVote, + v1::{ + AssignmentCertKind, BlockApprovalMeta, IndirectAssignmentCert, IndirectSignedApprovalVote, + }, + v2::{AsBitIndex, AssignmentCertKindV2, CandidateBitfield, IndirectAssignmentCertV2}, }; use polkadot_node_subsystem::{ messages::{ @@ -49,8 +55,6 @@ use std::{ time::Duration, }; -use self::metrics::Metrics; - mod metrics; #[cfg(test)] @@ -64,11 +68,15 @@ const COST_DUPLICATE_MESSAGE: Rep = Rep::CostMinorRepeated("Peer sent identical const COST_ASSIGNMENT_TOO_FAR_IN_THE_FUTURE: Rep = Rep::CostMinor("The vote was valid but too far in the future"); const COST_INVALID_MESSAGE: Rep = Rep::CostMajor("The vote was bad"); +const COST_OVERSIZED_BITFIELD: Rep = Rep::CostMajor("Oversized certificate or candidate bitfield"); const BENEFIT_VALID_MESSAGE: Rep = Rep::BenefitMinor("Peer sent a valid message"); const BENEFIT_VALID_MESSAGE_FIRST: Rep = Rep::BenefitMinorFirst("Valid message with new information"); +// Maximum valid size for the `CandidateBitfield` in the assignment messages. +const MAX_BITFIELD_SIZE: usize = 500; + /// The Approval Distribution subsystem. pub struct ApprovalDistribution { metrics: Metrics, @@ -97,7 +105,144 @@ impl RecentlyOutdated { } } -// In case the original gtid topology mechanisms don't work on their own, we need to trade bandwidth +// Contains topology routing information for assignments and approvals. +struct ApprovalRouting { + required_routing: RequiredRouting, + local: bool, + random_routing: RandomRouting, +} + +// This struct is responsible for tracking the full state of an assignment and grid routing +// information. +struct ApprovalEntry { + // The assignment certificate. + assignment: IndirectAssignmentCertV2, + // The candidates claimed by the certificate. A mapping between bit index and candidate index. + candidates: CandidateBitfield, + // The approval signatures for each `CandidateIndex` claimed by the assignment certificate. + approvals: HashMap, + // The validator index of the assignment signer. + validator_index: ValidatorIndex, + // Information required for gossiping to other peers using the grid topology. + routing_info: ApprovalRouting, +} + +#[derive(Debug)] +enum ApprovalEntryError { + InvalidValidatorIndex, + CandidateIndexOutOfBounds, + InvalidCandidateIndex, + DuplicateApproval, +} + +impl ApprovalEntry { + pub fn new( + assignment: IndirectAssignmentCertV2, + candidates: CandidateBitfield, + routing_info: ApprovalRouting, + ) -> ApprovalEntry { + Self { + validator_index: assignment.validator, + assignment, + approvals: HashMap::with_capacity(candidates.len()), + candidates, + routing_info, + } + } + + // Create a `MessageSubject` to reference the assignment. + pub fn create_assignment_knowledge(&self, block_hash: Hash) -> (MessageSubject, MessageKind) { + ( + MessageSubject(block_hash, self.candidates.clone(), self.validator_index), + MessageKind::Assignment, + ) + } + + // Create a `MessageSubject` to reference the approval. + pub fn create_approval_knowledge( + &self, + block_hash: Hash, + candidate_index: CandidateIndex, + ) -> (MessageSubject, MessageKind) { + ( + MessageSubject(block_hash, candidate_index.into(), self.validator_index), + MessageKind::Approval, + ) + } + + // Updates routing information and returns the previous information if any. + pub fn routing_info_mut(&mut self) -> &mut ApprovalRouting { + &mut self.routing_info + } + + // Get the routing information. + pub fn routing_info(&self) -> &ApprovalRouting { + &self.routing_info + } + + // Update routing information. + pub fn update_required_routing(&mut self, required_routing: RequiredRouting) { + self.routing_info.required_routing = required_routing; + } + + // Records a new approval. Returns false if the claimed candidate is not found or we already + // have received the approval. + pub fn note_approval( + &mut self, + approval: IndirectSignedApprovalVote, + ) -> Result<(), ApprovalEntryError> { + // First do some sanity checks: + // - check validator index matches + // - check claimed candidate + // - check for duplicate approval + if self.validator_index != approval.validator { + return Err(ApprovalEntryError::InvalidValidatorIndex) + } + + if self.candidates.len() <= approval.candidate_index as usize { + return Err(ApprovalEntryError::CandidateIndexOutOfBounds) + } + + if !self.candidates.bit_at(approval.candidate_index.as_bit_index()) { + return Err(ApprovalEntryError::InvalidCandidateIndex) + } + + if self.approvals.contains_key(&approval.candidate_index) { + return Err(ApprovalEntryError::DuplicateApproval) + } + + self.approvals.insert(approval.candidate_index, approval); + Ok(()) + } + + // Get the assignment certiticate and claimed candidates. + pub fn assignment(&self) -> (IndirectAssignmentCertV2, CandidateBitfield) { + (self.assignment.clone(), self.candidates.clone()) + } + + // Get all approvals for all candidates claimed by the assignment. + pub fn approvals(&self) -> Vec { + self.approvals.values().cloned().collect::>() + } + + // Get the approval for a specific candidate index. + pub fn approval(&self, candidate_index: CandidateIndex) -> Option { + self.approvals.get(&candidate_index).cloned() + } + + // Get validator index. + pub fn validator_index(&self) -> ValidatorIndex { + self.validator_index + } +} + +// We keep track of each peer view and protocol version using this struct. +struct PeerEntry { + pub view: View, + pub version: ProtocolVersion, +} + +// In case the original grid topology mechanisms don't work on their own, we need to trade bandwidth // for protocol liveliness by introducing aggression. // // Aggression has 3 levels: @@ -117,7 +262,6 @@ impl RecentlyOutdated { // not to all blocks older than the threshold. Most likely, a few assignments struggle to // be propagated in a single block and this holds up all of its descendants blocks. // Accordingly, we only step on the gas for the block which is most obviously holding up finality. - /// Aggression configuration representation #[derive(Clone)] struct AggressionConfig { @@ -160,15 +304,6 @@ enum Resend { No, } -/// Data stored on a per-peer basis. -#[derive(Debug)] -struct PeerData { - /// The peer's view. - view: View, - /// The peer's protocol version. - version: ValidationVersion, -} - /// The [`State`] struct is responsible for tracking the overall state of the subsystem. /// /// It tracks metadata about our view of the unfinalized chain, @@ -189,7 +324,7 @@ struct State { pending_known: HashMap>, /// Peer data is partially stored here, and partially inline within the [`BlockEntry`]s - peer_data: HashMap, + peer_views: HashMap, /// Keeps a topology for various different sessions. topologies: SessionGridTopologies, @@ -197,12 +332,12 @@ struct State { /// Tracks recently finalized blocks. recent_outdated_blocks: RecentlyOutdated, - /// Config for aggression. - aggression_config: AggressionConfig, - - /// `HashMap` from active leaves to spans + /// HashMap from active leaves to spans spans: HashMap, + /// Aggression configuration. + aggression_config: AggressionConfig, + /// Current approval checking finality lag. approval_checking_lag: BlockNumber, @@ -216,8 +351,11 @@ enum MessageKind { Approval, } +// Utility structure to identify assignments and approvals for specific candidates. +// Assignments can span multiple candidates, while approvals refer to only one candidate. +// #[derive(Debug, Clone, Hash, PartialEq, Eq)] -struct MessageSubject(Hash, CandidateIndex, ValidatorIndex); +struct MessageSubject(Hash, pub CandidateBitfield, ValidatorIndex); #[derive(Debug, Clone, Default)] struct Knowledge { @@ -238,9 +376,11 @@ impl Knowledge { } fn insert(&mut self, message: MessageSubject, kind: MessageKind) -> bool { - match self.known_messages.entry(message) { + let mut success = match self.known_messages.entry(message.clone()) { hash_map::Entry::Vacant(vacant) => { vacant.insert(kind); + // If there are multiple candidates assigned in the message, create + // separate entries for each one. true }, hash_map::Entry::Occupied(mut occupied) => match (*occupied.get(), kind) { @@ -252,7 +392,25 @@ impl Knowledge { true }, }, + }; + + // In case of succesful insertion of multiple candidate assignments create additional + // entries for each assigned candidate. This fakes knowledge of individual assignments, but + // we need to share the same `MessageSubject` with the followup approval candidate index. + if kind == MessageKind::Assignment && success && message.1.count_ones() > 1 { + for candidate_index in message.1.iter_ones() { + success = success && + self.insert( + MessageSubject( + message.0, + vec![candidate_index as u32].try_into().expect("Non-empty vec; qed"), + message.2, + ), + kind, + ); + } } + success } } @@ -286,47 +444,97 @@ struct BlockEntry { candidates: Vec, /// The session index of this block. session: SessionIndex, + /// Approval entries for whole block. These also contain all approvals in the case of multiple + /// candidates being claimed by assignments. + approval_entries: HashMap<(ValidatorIndex, CandidateBitfield), ApprovalEntry>, } -#[derive(Debug)] -enum ApprovalState { - Assigned(AssignmentCert), - Approved(AssignmentCert, ValidatorSignature), -} +impl BlockEntry { + // Returns the peer which currently know this block. + pub fn known_by(&self) -> Vec { + self.known_by.keys().cloned().collect::>() + } -impl ApprovalState { - fn assignment_cert(&self) -> &AssignmentCert { - match *self { - ApprovalState::Assigned(ref cert) => cert, - ApprovalState::Approved(ref cert, _) => cert, + pub fn insert_approval_entry(&mut self, entry: ApprovalEntry) -> &mut ApprovalEntry { + // First map one entry per candidate to the same key we will use in `approval_entries`. + // Key is (Validator_index, CandidateBitfield) that links the `ApprovalEntry` to the (K,V) + // entry in `candidate_entry.messages`. + for claimed_candidate_index in entry.candidates.iter_ones() { + match self.candidates.get_mut(claimed_candidate_index) { + Some(candidate_entry) => { + candidate_entry + .messages + .entry(entry.validator_index()) + .or_insert(entry.candidates.clone()); + }, + None => { + // This should never happen, but if it happens, it means the subsystem is + // broken. + gum::warn!( + target: LOG_TARGET, + hash = ?entry.assignment.block_hash, + ?claimed_candidate_index, + "Missing candidate entry on `import_and_circulate_assignment`", + ); + }, + }; } + + self.approval_entries + .entry((entry.validator_index, entry.candidates.clone())) + .or_insert(entry) } - fn approval_signature(&self) -> Option { - match *self { - ApprovalState::Assigned(_) => None, - ApprovalState::Approved(_, ref sig) => Some(sig.clone()), - } + // Returns a mutable reference of `ApprovalEntry` for `candidate_index` from validator + // `validator_index`. + pub fn approval_entry( + &mut self, + candidate_index: CandidateIndex, + validator_index: ValidatorIndex, + ) -> Option<&mut ApprovalEntry> { + self.candidates + .get(candidate_index as usize) + .map_or(None, |candidate_entry| candidate_entry.messages.get(&validator_index)) + .map_or(None, |candidate_indices| { + self.approval_entries.get_mut(&(validator_index, candidate_indices.clone())) + }) } -} -// routing state bundled with messages for the candidate. Corresponding assignments -// and approvals are stored together and should be routed in the same way, with -// assignments preceding approvals in all cases. -#[derive(Debug)] -struct MessageState { - required_routing: RequiredRouting, - local: bool, - random_routing: RandomRouting, - approval_state: ApprovalState, + // Get all approval entries for a given candidate. + pub fn approval_entries(&self, candidate_index: CandidateIndex) -> Vec<&ApprovalEntry> { + // Get the keys for fetching `ApprovalEntry` from `self.approval_entries`, + let approval_entry_keys = self + .candidates + .get(candidate_index as usize) + .map(|candidate_entry| &candidate_entry.messages); + + if let Some(approval_entry_keys) = approval_entry_keys { + // Ensure no duplicates. + let approval_entry_keys = approval_entry_keys.iter().unique().collect::>(); + + let mut entries = Vec::new(); + for (validator_index, candidate_indices) in approval_entry_keys { + if let Some(entry) = + self.approval_entries.get(&(*validator_index, candidate_indices.clone())) + { + entries.push(entry); + } + } + entries + } else { + vec![] + } + } } -/// Information about candidates in the context of a particular block they are included in. -/// In other words, multiple `CandidateEntry`s may exist for the same candidate, -/// if it is included by multiple blocks - this is likely the case when there are forks. +// Information about candidates in the context of a particular block they are included in. +// In other words, multiple `CandidateEntry`s may exist for the same candidate, +// if it is included by multiple blocks - this is likely the case when there are forks. #[derive(Debug, Default)] struct CandidateEntry { - messages: HashMap, + // The value represents part of the lookup key in `approval_entries` to fetch the assignment + // and existing votes. + messages: HashMap, } #[derive(Debug, Clone, PartialEq)] @@ -345,7 +553,7 @@ impl MessageSource { } enum PendingMessage { - Assignment(IndirectAssignmentCert, CandidateIndex), + Assignment(IndirectAssignmentCertV2, CandidateBitfield), Approval(IndirectSignedApprovalVote), } @@ -362,27 +570,13 @@ impl State { NetworkBridgeEvent::PeerConnected(peer_id, role, version, _) => { // insert a blank view if none already present gum::trace!(target: LOG_TARGET, ?peer_id, ?role, "Peer connected"); - let version = match ValidationVersion::try_from(version).ok() { - Some(v) => v, - None => { - // sanity: network bridge is supposed to detect this already. - gum::error!( - target: LOG_TARGET, - ?peer_id, - ?version, - "Unsupported protocol version" - ); - return - }, - }; - - self.peer_data + self.peer_views .entry(peer_id) - .or_insert_with(|| PeerData { version, view: Default::default() }); + .or_insert(PeerEntry { view: Default::default(), version }); }, NetworkBridgeEvent::PeerDisconnected(peer_id) => { gum::trace!(target: LOG_TARGET, ?peer_id, "Peer disconnected"); - self.peer_data.remove(&peer_id); + self.peer_views.remove(&peer_id); self.blocks.iter_mut().for_each(|(_hash, entry)| { entry.known_by.remove(&peer_id); }) @@ -419,8 +613,8 @@ impl State { live }); }, - NetworkBridgeEvent::PeerMessage(peer_id, msg) => { - self.process_incoming_peer_message(ctx, metrics, peer_id, msg, rng).await; + NetworkBridgeEvent::PeerMessage(peer_id, message) => { + self.process_incoming_peer_message(ctx, metrics, peer_id, message, rng).await; }, NetworkBridgeEvent::UpdatedAuthorityIds { .. } => { // The approval-distribution subsystem doesn't deal with `AuthorityDiscoveryId`s. @@ -459,6 +653,7 @@ impl State { knowledge: Knowledge::default(), candidates, session: meta.session, + approval_entries: HashMap::new(), }); self.topologies.inc_session_refs(meta.session); @@ -481,18 +676,17 @@ impl State { { let sender = ctx.sender(); - for (peer_id, data) in self.peer_data.iter() { - let intersection = data.view.iter().filter(|h| new_hashes.contains(h)); - let view_intersection = - View::new(intersection.cloned(), data.view.finalized_number); + for (peer_id, PeerEntry { view, version }) in self.peer_views.iter() { + let intersection = view.iter().filter(|h| new_hashes.contains(h)); + let view_intersection = View::new(intersection.cloned(), view.finalized_number); Self::unify_with_peer( sender, metrics, &mut self.blocks, &self.topologies, - self.peer_data.len(), + self.peer_views.len(), *peer_id, - data.version, + *version, view_intersection, rng, ) @@ -530,13 +724,13 @@ impl State { for (peer_id, message) in to_import { match message { - PendingMessage::Assignment(assignment, claimed_index) => { + PendingMessage::Assignment(assignment, claimed_indices) => { self.import_and_circulate_assignment( ctx, metrics, MessageSource::Peer(peer_id), assignment, - claimed_index, + claimed_indices, rng, ) .await; @@ -575,33 +769,78 @@ impl State { adjust_required_routing_and_propagate( ctx, - &self.peer_data, &mut self.blocks, &self.topologies, |block_entry| block_entry.session == session, |required_routing, local, validator_index| { - if *required_routing == RequiredRouting::PendingTopology { - *required_routing = topology + if required_routing == &RequiredRouting::PendingTopology { + topology .local_grid_neighbors() - .required_routing_by_index(*validator_index, local); + .required_routing_by_index(*validator_index, local) + } else { + *required_routing } }, + &self.peer_views, ) .await; } + async fn process_incoming_assignments( + &mut self, + ctx: &mut Context, + metrics: &Metrics, + peer_id: PeerId, + assignments: Vec<(IndirectAssignmentCertV2, CandidateBitfield)>, + rng: &mut R, + ) where + R: CryptoRng + Rng, + { + for (assignment, claimed_indices) in assignments { + if let Some(pending) = self.pending_known.get_mut(&assignment.block_hash) { + let block_hash = &assignment.block_hash; + let validator_index = assignment.validator; + + gum::trace!( + target: LOG_TARGET, + %peer_id, + ?block_hash, + ?claimed_indices, + ?validator_index, + "Pending assignment", + ); + + pending.push((peer_id, PendingMessage::Assignment(assignment, claimed_indices))); + + continue + } + + self.import_and_circulate_assignment( + ctx, + metrics, + MessageSource::Peer(peer_id), + assignment, + claimed_indices, + rng, + ) + .await; + } + } + async fn process_incoming_peer_message( &mut self, ctx: &mut Context, metrics: &Metrics, peer_id: PeerId, - msg: net_protocol::ApprovalDistributionMessage, + msg: Versioned< + protocol_v1::ApprovalDistributionMessage, + protocol_vstaging::ApprovalDistributionMessage, + >, rng: &mut R, ) where R: CryptoRng + Rng, { match msg { - Versioned::V1(protocol_v1::ApprovalDistributionMessage::Assignments(assignments)) | Versioned::VStaging(protocol_vstaging::ApprovalDistributionMessage::Assignments( assignments, )) => { @@ -611,42 +850,42 @@ impl State { num = assignments.len(), "Processing assignments from a peer", ); - for (assignment, claimed_index) in assignments.into_iter() { - if let Some(pending) = self.pending_known.get_mut(&assignment.block_hash) { - let message_subject = MessageSubject( - assignment.block_hash, - claimed_index, - assignment.validator, - ); - - gum::trace!( - target: LOG_TARGET, - %peer_id, - ?message_subject, - "Pending assignment", - ); + let sanitized_assignments = + self.sanitize_v2_assignments(peer_id, ctx.sender(), assignments).await; - pending - .push((peer_id, PendingMessage::Assignment(assignment, claimed_index))); + self.process_incoming_assignments( + ctx, + metrics, + peer_id, + sanitized_assignments, + rng, + ) + .await; + }, + Versioned::V1(protocol_v1::ApprovalDistributionMessage::Assignments(assignments)) => { + gum::trace!( + target: LOG_TARGET, + peer_id = %peer_id, + num = assignments.len(), + "Processing assignments from a peer", + ); - continue - } + let sanitized_assignments = + self.sanitize_v1_assignments(peer_id, ctx.sender(), assignments).await; - self.import_and_circulate_assignment( - ctx, - metrics, - MessageSource::Peer(peer_id), - assignment, - claimed_index, - rng, - ) - .await; - } + self.process_incoming_assignments( + ctx, + metrics, + peer_id, + sanitized_assignments, + rng, + ) + .await; }, - Versioned::V1(protocol_v1::ApprovalDistributionMessage::Approvals(approvals)) | Versioned::VStaging(protocol_vstaging::ApprovalDistributionMessage::Approvals( approvals, - )) => { + )) | + Versioned::V1(protocol_v1::ApprovalDistributionMessage::Approvals(approvals)) => { gum::trace!( target: LOG_TARGET, peer_id = %peer_id, @@ -655,17 +894,17 @@ impl State { ); for approval_vote in approvals.into_iter() { if let Some(pending) = self.pending_known.get_mut(&approval_vote.block_hash) { - let message_subject = MessageSubject( - approval_vote.block_hash, - approval_vote.candidate_index, - approval_vote.validator, - ); + let block_hash = approval_vote.block_hash; + let candidate_index = approval_vote.candidate_index; + let validator_index = approval_vote.validator; gum::trace!( target: LOG_TARGET, %peer_id, - ?message_subject, - "Pending approval", + ?block_hash, + ?candidate_index, + ?validator_index, + "Pending assignment", ); pending.push((peer_id, PendingMessage::Approval(approval_vote))); @@ -699,14 +938,23 @@ impl State { { gum::trace!(target: LOG_TARGET, ?view, "Peer view change"); let finalized_number = view.finalized_number; - let (peer_protocol_version, old_finalized_number) = match self - .peer_data - .get_mut(&peer_id) - .map(|d| (d.version, std::mem::replace(&mut d.view, view.clone()))) - { - Some((v, view)) => (v, view.finalized_number), - None => return, // unknown peer - }; + + let (old_view, protocol_version) = + if let Some(peer_entry) = self.peer_views.get_mut(&peer_id) { + (Some(std::mem::replace(&mut peer_entry.view, view.clone())), peer_entry.version) + } else { + // This shouldn't happen, but if it does we assume protocol version 1. + gum::warn!( + target: LOG_TARGET, + ?peer_id, + ?view, + "Peer view change for missing `peer_entry`" + ); + + (None, ValidationVersion::V1.into()) + }; + + let old_finalized_number = old_view.map(|v| v.finalized_number).unwrap_or(0); // we want to prune every block known_by peer up to (including) view.finalized_number let blocks = &mut self.blocks; @@ -731,9 +979,9 @@ impl State { metrics, &mut self.blocks, &self.topologies, - self.peer_data.len(), + self.peer_views.len(), peer_id, - peer_protocol_version, + protocol_version, view, rng, ) @@ -774,8 +1022,8 @@ impl State { ctx: &mut Context, metrics: &Metrics, source: MessageSource, - assignment: IndirectAssignmentCert, - claimed_candidate_index: CandidateIndex, + assignment: IndirectAssignmentCertV2, + claimed_candidate_indices: CandidateBitfield, rng: &mut R, ) where R: CryptoRng + Rng, @@ -823,9 +1071,11 @@ impl State { }, }; - // compute metadata on the assignment. - let message_subject = MessageSubject(block_hash, claimed_candidate_index, validator_index); - let message_kind = MessageKind::Assignment; + // Compute metadata on the assignment. + let (message_subject, message_kind) = ( + MessageSubject(block_hash, claimed_candidate_indices.clone(), validator_index), + MessageKind::Assignment, + ); if let Some(peer_id) = source.peer_id() { // check if our knowledge of the peer already contains this assignment @@ -841,6 +1091,7 @@ impl State { ?message_subject, "Duplicate assignment", ); + modify_reputation( &mut self.reputation, ctx.sender(), @@ -848,6 +1099,15 @@ impl State { COST_DUPLICATE_MESSAGE, ) .await; + } else { + gum::trace!( + target: LOG_TARGET, + ?peer_id, + hash = ?block_hash, + ?validator_index, + ?message_subject, + "We sent the message to the peer while peer was sending it to us. Known race condition.", + ); } return } @@ -889,7 +1149,7 @@ impl State { ctx.send_message(ApprovalVotingMessage::CheckAndImportAssignment( assignment.clone(), - claimed_candidate_index, + claimed_candidate_indices.clone(), tx, )) .await; @@ -920,7 +1180,7 @@ impl State { BENEFIT_VALID_MESSAGE_FIRST, ) .await; - entry.knowledge.known_messages.insert(message_subject.clone(), message_kind); + entry.knowledge.insert(message_subject.clone(), message_kind); if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { peer_knowledge.received.insert(message_subject.clone(), message_kind); } @@ -994,7 +1254,7 @@ impl State { // Invariant: to our knowledge, none of the peers except for the `source` know about the // assignment. - metrics.on_assignment_imported(); + metrics.on_assignment_imported(&assignment.cert.kind); let topology = self.topologies.get_topology(entry.session); let local = source == MessageSource::Local; @@ -1003,28 +1263,14 @@ impl State { t.local_grid_neighbors().required_routing_by_index(validator_index, local) }); - let message_state = match entry.candidates.get_mut(claimed_candidate_index as usize) { - Some(candidate_entry) => { - // set the approval state for validator_index to Assigned - // unless the approval state is set already - candidate_entry.messages.entry(validator_index).or_insert_with(|| MessageState { - required_routing, - local, - random_routing: Default::default(), - approval_state: ApprovalState::Assigned(assignment.cert.clone()), - }) - }, - None => { - gum::warn!( - target: LOG_TARGET, - hash = ?block_hash, - ?claimed_candidate_index, - "Expected a candidate entry on import_and_circulate_assignment", - ); + // All the peers that know the relay chain block. + let peers_to_filter = entry.known_by(); - return - }, - }; + let approval_entry = entry.insert_approval_entry(ApprovalEntry::new( + assignment.clone(), + claimed_candidate_indices.clone(), + ApprovalRouting { required_routing, local, random_routing: Default::default() }, + )); // Dispatch the message to all peers in the routing set which // know the block. @@ -1032,83 +1278,65 @@ impl State { // If the topology isn't known yet (race with networking subsystems) // then messages will be sent when we get it. - let assignments = vec![(assignment, claimed_candidate_index)]; - let n_peers_total = self.peer_data.len(); + let assignments = vec![(assignment, claimed_candidate_indices.clone())]; + let n_peers_total = self.peer_views.len(); let source_peer = source.peer_id(); - let mut peer_filter = move |peer| { - if Some(peer) == source_peer.as_ref() { - return false + // Peers that we will send the assignment to. + let mut peers = Vec::new(); + + // Filter destination peers + for peer in peers_to_filter.into_iter() { + if Some(peer) == source_peer { + continue } if let Some(true) = topology .as_ref() - .map(|t| t.local_grid_neighbors().route_to_peer(required_routing, peer)) + .map(|t| t.local_grid_neighbors().route_to_peer(required_routing, &peer)) { - return true + peers.push(peer); + continue } // Note: at this point, we haven't received the message from any peers // other than the source peer, and we just got it, so we haven't sent it // to any peers either. - let route_random = message_state.random_routing.sample(n_peers_total, rng); + let route_random = + approval_entry.routing_info().random_routing.sample(n_peers_total, rng); if route_random { - message_state.random_routing.inc_sent(); - } - - route_random - }; - - let (v1_peers, vstaging_peers) = { - let peer_data = &self.peer_data; - let peers = entry - .known_by - .keys() - .filter_map(|p| peer_data.get_key_value(p)) - .filter(|(p, _)| peer_filter(p)) - .map(|(p, peer_data)| (*p, peer_data.version)) - .collect::>(); - - // Add the metadata of the assignment to the knowledge of each peer. - for (peer, _) in peers.iter() { - // we already filtered peers above, so this should always be Some - if let Some(peer_knowledge) = entry.known_by.get_mut(peer) { - peer_knowledge.sent.insert(message_subject.clone(), message_kind); - } + approval_entry.routing_info_mut().random_routing.inc_sent(); + peers.push(peer); } + } - if !peers.is_empty() { - gum::trace!( - target: LOG_TARGET, - ?block_hash, - ?claimed_candidate_index, - local = source.peer_id().is_none(), - num_peers = peers.len(), - "Sending an assignment to peers", - ); + // Add the metadata of the assignment to the knowledge of each peer. + for peer in peers.iter() { + // we already filtered peers above, so this should always be Some + if let Some(peer_knowledge) = entry.known_by.get_mut(peer) { + peer_knowledge.sent.insert(message_subject.clone(), message_kind); } + } - let v1_peers = filter_peers_by_version(&peers, ValidationVersion::V1); - let vstaging_peers = filter_peers_by_version(&peers, ValidationVersion::VStaging); + if !peers.is_empty() { + gum::trace!( + target: LOG_TARGET, + ?block_hash, + ?claimed_candidate_indices, + local = source.peer_id().is_none(), + num_peers = peers.len(), + "Sending an assignment to peers", + ); - (v1_peers, vstaging_peers) - }; + let peers = peers + .iter() + .filter_map(|peer_id| { + self.peer_views.get(peer_id).map(|peer_entry| (*peer_id, peer_entry.version)) + }) + .collect::>(); - if !v1_peers.is_empty() { - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - v1_peers, - versioned_assignments_packet(ValidationVersion::V1, assignments.clone()), - )) - .await; - } - - if !vstaging_peers.is_empty() { - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - vstaging_peers, - versioned_assignments_packet(ValidationVersion::VStaging, assignments.clone()), - )) - .await; + send_assignments_batched(ctx.sender(), assignments, &peers).await; } } @@ -1143,6 +1371,14 @@ impl State { _ => { if let Some(peer_id) = source.peer_id() { if !self.recent_outdated_blocks.is_recent_outdated(&block_hash) { + gum::debug!( + target: LOG_TARGET, + ?peer_id, + ?block_hash, + ?validator_index, + ?candidate_index, + "Approval from a peer is out of view", + ); modify_reputation( &mut self.reputation, ctx.sender(), @@ -1157,7 +1393,7 @@ impl State { }; // compute metadata on the assignment. - let message_subject = MessageSubject(block_hash, candidate_index, validator_index); + let message_subject = MessageSubject(block_hash, candidate_index.into(), validator_index); let message_kind = MessageKind::Approval; if let Some(peer_id) = source.peer_id() { @@ -1307,71 +1543,48 @@ impl State { } } - // Invariant: to our knowledge, none of the peers except for the `source` know about the - // approval. - metrics.on_approval_imported(); - - let required_routing = match entry.candidates.get_mut(candidate_index as usize) { - Some(candidate_entry) => { - // set the approval state for validator_index to Approved - // it should be in assigned state already - match candidate_entry.messages.remove(&validator_index) { - Some(MessageState { - approval_state: ApprovalState::Assigned(cert), - required_routing, - local, - random_routing, - }) => { - candidate_entry.messages.insert( - validator_index, - MessageState { - approval_state: ApprovalState::Approved( - cert, - vote.signature.clone(), - ), - required_routing, - local, - random_routing, - }, - ); - - required_routing - }, - Some(_) => { - unreachable!( - "we only insert it after the metadata, checked the metadata above; qed" - ); - }, - None => { - // this would indicate a bug in approval-voting - gum::warn!( - target: LOG_TARGET, - hash = ?block_hash, - ?candidate_index, - ?validator_index, - "Importing an approval we don't have an assignment for", - ); + let required_routing = match entry.approval_entry(candidate_index, validator_index) { + Some(approval_entry) => { + // Invariant: to our knowledge, none of the peers except for the `source` know about + // the approval. + metrics.on_approval_imported(); + + if let Err(err) = approval_entry.note_approval(vote.clone()) { + // this would indicate a bug in approval-voting: + // - validator index mismatch + // - candidate index mismatch + // - duplicate approval + gum::warn!( + target: LOG_TARGET, + hash = ?block_hash, + ?candidate_index, + ?validator_index, + ?err, + "Possible bug: Vote import failed", + ); - return - }, + return } + + approval_entry.routing_info().required_routing }, None => { + let peer_id = source.peer_id(); + // This indicates a bug in approval-distribution, since we check the knowledge at + // the begining of the function. gum::warn!( target: LOG_TARGET, - hash = ?block_hash, - ?candidate_index, - ?validator_index, - "Expected a candidate entry on import_and_circulate_approval", + ?peer_id, + ?message_subject, + "Unknown approval assignment", ); - + // No rep change as this is caused by an issue return }, }; // Dispatch a ApprovalDistributionV1Message::Approval(vote) // to all peers required by the topology, with the exception of the source peer. - let topology = self.topologies.get_topology(entry.session); let source_peer = source.peer_id(); @@ -1395,57 +1608,56 @@ impl State { in_topology || knowledge.sent.contains(message_subject, MessageKind::Assignment) }; - let (v1_peers, vstaging_peers) = { - let peer_data = &self.peer_data; - let peers = entry - .known_by - .iter() - .filter_map(|(p, k)| peer_data.get(&p).map(|pd| (p, k, pd.version))) - .filter(|(p, k, _)| peer_filter(p, k)) - .map(|(p, _, v)| (*p, v)) - .collect::>(); - - // Add the metadata of the assignment to the knowledge of each peer. - for (peer, _) in peers.iter() { - // we already filtered peers above, so this should always be Some - if let Some(peer_knowledge) = entry.known_by.get_mut(peer) { - peer_knowledge.sent.insert(message_subject.clone(), message_kind); - } + let peers = entry + .known_by + .iter() + .filter(|(p, k)| peer_filter(p, k)) + .filter_map(|(p, _)| self.peer_views.get(p).map(|entry| (*p, entry.version))) + .collect::>(); + + // Add the metadata of the assignment to the knowledge of each peer. + for peer in peers.iter() { + // we already filtered peers above, so this should always be Some + if let Some(entry) = entry.known_by.get_mut(&peer.0) { + entry.sent.insert(message_subject.clone(), message_kind); } + } - if !peers.is_empty() { - gum::trace!( - target: LOG_TARGET, - ?block_hash, - ?candidate_index, - local = source.peer_id().is_none(), - num_peers = peers.len(), - "Sending an approval to peers", - ); - } - - let v1_peers = filter_peers_by_version(&peers, ValidationVersion::V1); - let vstaging_peers = filter_peers_by_version(&peers, ValidationVersion::VStaging); - - (v1_peers, vstaging_peers) - }; + if !peers.is_empty() { + let approvals = vec![vote]; + gum::trace!( + target: LOG_TARGET, + ?block_hash, + ?candidate_index, + local = source.peer_id().is_none(), + num_peers = peers.len(), + "Sending an approval to peers", + ); - let approvals = vec![vote]; + let v1_peers = filter_by_peer_version(&peers, ValidationVersion::V1.into()); + let v2_peers = filter_by_peer_version(&peers, ValidationVersion::VStaging.into()); - if !v1_peers.is_empty() { - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - v1_peers, - versioned_approvals_packet(ValidationVersion::V1, approvals.clone()), - )) - .await; - } + if !v1_peers.is_empty() { + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + v1_peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Approvals(approvals.clone()), + )), + )) + .await; + } - if !vstaging_peers.is_empty() { - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - vstaging_peers, - versioned_approvals_packet(ValidationVersion::VStaging, approvals), - )) - .await; + if !v2_peers.is_empty() { + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + v2_peers, + Versioned::VStaging( + protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals), + ), + ), + )) + .await; + } } } @@ -1476,25 +1688,12 @@ impl State { Some(e) => e, }; - let candidate_entry = match block_entry.candidates.get(index as usize) { - None => { - gum::debug!( - target: LOG_TARGET, - ?hash, - ?index, - "`get_approval_signatures`: could not find candidate entry for given hash and index!" - ); - continue - }, - Some(e) => e, - }; - let sigs = - candidate_entry.messages.iter().filter_map(|(validator_index, message_state)| { - match &message_state.approval_state { - ApprovalState::Approved(_, sig) => Some((*validator_index, sig.clone())), - ApprovalState::Assigned(_) => None, - } - }); + let sigs = block_entry + .approval_entries(index) + .into_iter() + .filter_map(|approval_entry| approval_entry.approval(index)) + .map(|approval| (approval.validator, approval.signature)) + .collect::>(); all_sigs.extend(sigs); } all_sigs @@ -1507,7 +1706,7 @@ impl State { topologies: &SessionGridTopologies, total_peers: usize, peer_id: PeerId, - peer_protocol_version: ValidationVersion, + protocol_version: ProtocolVersion, view: View, rng: &mut (impl CryptoRng + Rng), ) { @@ -1520,6 +1719,8 @@ impl State { let view_finalized_number = view.finalized_number; for head in view.into_iter() { let mut block = head; + + // Walk the chain back to last finalized block of the peer view. loop { let entry = match entries.get_mut(&block) { Some(entry) if entry.number > view_finalized_number => entry, @@ -1534,19 +1735,16 @@ impl State { } let peer_knowledge = entry.known_by.entry(peer_id).or_default(); - let topology = topologies.get_topology(entry.session); - // Iterate all messages in all candidates. - for (candidate_index, validator, message_state) in - entry.candidates.iter_mut().enumerate().flat_map(|(c_i, c)| { - c.messages.iter_mut().map(move |(k, v)| (c_i as _, k, v)) - }) { + // We want to iterate the `approval_entries` of the block entry as these contain all + // assignments that also link all approval votes. + for approval_entry in entry.approval_entries.values_mut() { // Propagate the message to all peers in the required routing set OR // randomly sample peers. { - let random_routing = &mut message_state.random_routing; - let required_routing = message_state.required_routing; + let required_routing = approval_entry.routing_info().required_routing; + let random_routing = &mut approval_entry.routing_info_mut().random_routing; let rng = &mut *rng; let mut peer_filter = move |peer_id| { let in_topology = topology.as_ref().map_or(false, |t| { @@ -1567,39 +1765,24 @@ impl State { } } - let message_subject = MessageSubject(block, candidate_index, *validator); - - let assignment_message = ( - IndirectAssignmentCert { - block_hash: block, - validator: *validator, - cert: message_state.approval_state.assignment_cert().clone(), - }, - candidate_index, - ); - - let approval_message = - message_state.approval_state.approval_signature().map(|signature| { - IndirectSignedApprovalVote { - block_hash: block, - validator: *validator, - candidate_index, - signature, - } - }); + let assignment_message = approval_entry.assignment(); + let approval_messages = approval_entry.approvals(); + let (assignment_knowledge, message_kind) = + approval_entry.create_assignment_knowledge(block); - if !peer_knowledge.contains(&message_subject, MessageKind::Assignment) { - peer_knowledge - .sent - .insert(message_subject.clone(), MessageKind::Assignment); + // Only send stuff a peer doesn't know in the context of a relay chain block. + if !peer_knowledge.contains(&assignment_knowledge, message_kind) { + peer_knowledge.sent.insert(assignment_knowledge, message_kind); assignments_to_send.push(assignment_message); } - if let Some(approval_message) = approval_message { - if !peer_knowledge.contains(&message_subject, MessageKind::Approval) { - peer_knowledge - .sent - .insert(message_subject.clone(), MessageKind::Approval); + // Filter approval votes. + for approval_message in approval_messages { + let (approval_knowledge, message_kind) = approval_entry + .create_approval_knowledge(block, approval_message.candidate_index); + + if !peer_knowledge.contains(&approval_knowledge, message_kind) { + peer_knowledge.sent.insert(approval_knowledge, message_kind); approvals_to_send.push(approval_message); } } @@ -1613,23 +1796,30 @@ impl State { gum::trace!( target: LOG_TARGET, ?peer_id, + ?protocol_version, num = assignments_to_send.len(), "Sending assignments to unified peer", ); - send_assignments_batched(sender, assignments_to_send, peer_id, peer_protocol_version) - .await; + send_assignments_batched( + sender, + assignments_to_send, + &vec![(peer_id, protocol_version)], + ) + .await; } if !approvals_to_send.is_empty() { gum::trace!( target: LOG_TARGET, ?peer_id, + ?protocol_version, num = approvals_to_send.len(), "Sending approvals to unified peer", ); - send_approvals_batched(sender, approvals_to_send, peer_id, peer_protocol_version).await; + send_approvals_batched(sender, approvals_to_send, &vec![(peer_id, protocol_version)]) + .await; } } @@ -1665,7 +1855,6 @@ impl State { adjust_required_routing_and_propagate( ctx, - &self.peer_data, &mut self.blocks, &self.topologies, |block_entry| { @@ -1687,13 +1876,13 @@ impl State { false } }, - |_, _, _| {}, + |required_routing, _, _| *required_routing, + &self.peer_views, ) .await; adjust_required_routing_and_propagate( ctx, - &self.peer_data, &mut self.blocks, &self.topologies, |block_entry| { @@ -1712,30 +1901,110 @@ impl State { lag = ?self.approval_checking_lag, "Encountered old block pending gossip topology", ); - return + return *required_routing } + let mut new_required_routing = *required_routing; + if config.l1_threshold.as_ref().map_or(false, |t| &self.approval_checking_lag >= t) { // Message originator sends to everyone. - if local && *required_routing != RequiredRouting::All { + if local && new_required_routing != RequiredRouting::All { metrics.on_aggression_l1(); - *required_routing = RequiredRouting::All; + new_required_routing = RequiredRouting::All; } } if config.l2_threshold.as_ref().map_or(false, |t| &self.approval_checking_lag >= t) { // Message originator sends to everyone. Everyone else sends to XY. - if !local && *required_routing != RequiredRouting::GridXY { + if !local && new_required_routing != RequiredRouting::GridXY { metrics.on_aggression_l2(); - *required_routing = RequiredRouting::GridXY; + new_required_routing = RequiredRouting::GridXY; } } + new_required_routing }, + &self.peer_views, ) .await; } + + // Filter out invalid candidate index and certificate core bitfields. + // For each invalid assignment we also punish the peer. + async fn sanitize_v1_assignments( + &mut self, + peer_id: PeerId, + sender: &mut impl overseer::ApprovalDistributionSenderTrait, + assignments: Vec<(IndirectAssignmentCert, CandidateIndex)>, + ) -> Vec<(IndirectAssignmentCertV2, CandidateBitfield)> { + let mut sanitized_assignments = Vec::new(); + for (cert, candidate_index) in assignments.into_iter() { + let cert_bitfield_bits = match cert.cert.kind { + AssignmentCertKind::RelayVRFDelay { core_index } => core_index.0 as usize + 1, + // We don't want to run the VRF yet, but the output is always bounded by `n_cores`. + // We assume `candidate_bitfield` length for the core bitfield and we just check + // against `MAX_BITFIELD_SIZE` later. + AssignmentCertKind::RelayVRFModulo { .. } => candidate_index as usize + 1, + }; + + let candidate_bitfield_bits = candidate_index as usize + 1; + + // Ensure bitfields length under hard limit. + if cert_bitfield_bits > MAX_BITFIELD_SIZE || candidate_bitfield_bits > MAX_BITFIELD_SIZE + { + // Punish the peer for the invalid message. + modify_reputation(&mut self.reputation, sender, peer_id, COST_OVERSIZED_BITFIELD) + .await; + } else { + sanitized_assignments.push((cert.into(), candidate_index.into())) + } + } + + sanitized_assignments + } + + // Filter out oversized candidate and certificate core bitfields. + // For each invalid assignment we also punish the peer. + async fn sanitize_v2_assignments( + &mut self, + peer_id: PeerId, + sender: &mut impl overseer::ApprovalDistributionSenderTrait, + assignments: Vec<(IndirectAssignmentCertV2, CandidateBitfield)>, + ) -> Vec<(IndirectAssignmentCertV2, CandidateBitfield)> { + let mut sanitized_assignments = Vec::new(); + for (cert, candidate_bitfield) in assignments.into_iter() { + let cert_bitfield_bits = match &cert.cert.kind { + AssignmentCertKindV2::RelayVRFDelay { core_index } => core_index.0 as usize + 1, + // We don't want to run the VRF yet, but the output is always bounded by `n_cores`. + // We assume `candidate_bitfield` length for the core bitfield and we just check + // against `MAX_BITFIELD_SIZE` later. + AssignmentCertKindV2::RelayVRFModulo { .. } => candidate_bitfield.len(), + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => + core_bitfield.len(), + }; + + let candidate_bitfield_bits = candidate_bitfield.len(); + + // Our bitfield has `Lsb0`. + let msb = candidate_bitfield_bits - 1; + + // Ensure bitfields length under hard limit. + if cert_bitfield_bits > MAX_BITFIELD_SIZE + || candidate_bitfield_bits > MAX_BITFIELD_SIZE + // Ensure minimum bitfield size - MSB needs to be one. + || !candidate_bitfield.bit_at(msb.as_bit_index()) + { + // Punish the peer for the invalid message. + modify_reputation(&mut self.reputation, sender, peer_id, COST_OVERSIZED_BITFIELD) + .await; + } else { + sanitized_assignments.push((cert, candidate_bitfield)) + } + } + + sanitized_assignments + } } // This adjusts the required routing of messages in blocks that pass the block filter @@ -1753,14 +2022,14 @@ impl State { #[overseer::contextbounds(ApprovalDistribution, prefix = self::overseer)] async fn adjust_required_routing_and_propagate( ctx: &mut Context, - peer_data: &HashMap, blocks: &mut HashMap, topologies: &SessionGridTopologies, block_filter: BlockFilter, routing_modifier: RoutingModifier, + peer_views: &HashMap, ) where BlockFilter: Fn(&mut BlockEntry) -> bool, - RoutingModifier: Fn(&mut RequiredRouting, bool, &ValidatorIndex), + RoutingModifier: Fn(&RequiredRouting, bool, &ValidatorIndex) -> RequiredRouting, { let mut peer_assignments = HashMap::new(); let mut peer_approvals = HashMap::new(); @@ -1772,64 +2041,55 @@ async fn adjust_required_routing_and_propagate t, + None => continue, + }; - if message_state.required_routing.is_empty() { - continue - } + // We just need to iterate the `approval_entries` of the block entry as these contain all + // assignments that also link all approval votes. + for approval_entry in block_entry.approval_entries.values_mut() { + let new_required_routing = routing_modifier( + &approval_entry.routing_info().required_routing, + approval_entry.routing_info().local, + &approval_entry.validator_index(), + ); - let topology = match topologies.get_topology(block_entry.session) { - Some(t) => t, - None => continue, - }; + approval_entry.update_required_routing(new_required_routing); - // Propagate the message to all peers in the required routing set. - let message_subject = MessageSubject(*block_hash, candidate_index, *validator); + if approval_entry.routing_info().required_routing.is_empty() { + continue + } - let assignment_message = ( - IndirectAssignmentCert { - block_hash: *block_hash, - validator: *validator, - cert: message_state.approval_state.assignment_cert().clone(), - }, - candidate_index, - ); - let approval_message = - message_state.approval_state.approval_signature().map(|signature| { - IndirectSignedApprovalVote { - block_hash: *block_hash, - validator: *validator, - candidate_index, - signature, - } - }); + let assignment_message = approval_entry.assignment(); + let approval_messages = approval_entry.approvals(); + let (assignment_knowledge, message_kind) = + approval_entry.create_assignment_knowledge(*block_hash); for (peer, peer_knowledge) in &mut block_entry.known_by { if !topology .local_grid_neighbors() - .route_to_peer(message_state.required_routing, peer) + .route_to_peer(approval_entry.routing_info().required_routing, peer) { continue } - if !peer_knowledge.contains(&message_subject, MessageKind::Assignment) { - peer_knowledge.sent.insert(message_subject.clone(), MessageKind::Assignment); + // Only send stuff a peer doesn't know in the context of a relay chain block. + if !peer_knowledge.contains(&assignment_knowledge, message_kind) { + peer_knowledge.sent.insert(assignment_knowledge.clone(), message_kind); peer_assignments .entry(*peer) .or_insert_with(Vec::new) .push(assignment_message.clone()); } - if let Some(approval_message) = approval_message.as_ref() { - if !peer_knowledge.contains(&message_subject, MessageKind::Approval) { - peer_knowledge.sent.insert(message_subject.clone(), MessageKind::Approval); + // Filter approval votes. + for approval_message in &approval_messages { + let (approval_knowledge, message_kind) = approval_entry + .create_approval_knowledge(*block_hash, approval_message.candidate_index); + + if !peer_knowledge.contains(&approval_knowledge, message_kind) { + peer_knowledge.sent.insert(approval_knowledge, message_kind); peer_approvals .entry(*peer) .or_insert_with(Vec::new) @@ -1841,24 +2101,32 @@ async fn adjust_required_routing_and_propagate continue, - Some(v) => v, - }; - - send_assignments_batched(ctx.sender(), assignments_packet, peer, peer_protocol_version) + if let Some(peer_view) = peer_views.get(&peer) { + send_assignments_batched( + ctx.sender(), + assignments_packet, + &vec![(peer, peer_view.version)], + ) .await; + } else { + // This should never happen. + gum::warn!(target: LOG_TARGET, ?peer, "Unknown protocol version for peer",); + } } for (peer, approvals_packet) in peer_approvals { - let peer_protocol_version = match peer_data.get(&peer).map(|pd| pd.version) { - None => continue, - Some(v) => v, - }; - - send_approvals_batched(ctx.sender(), approvals_packet, peer, peer_protocol_version).await; + if let Some(peer_view) = peer_views.get(&peer) { + send_approvals_batched( + ctx.sender(), + approvals_packet, + &vec![(peer, peer_view.version)], + ) + .await; + } else { + // This should never happen. + gum::warn!(target: LOG_TARGET, ?peer, "Unknown protocol version for peer",); + } } } @@ -1960,12 +2228,21 @@ impl ApprovalDistribution { ApprovalDistributionMessage::NewBlocks(metas) => { state.handle_new_blocks(ctx, metrics, metas, rng).await; }, - ApprovalDistributionMessage::DistributeAssignment(cert, candidate_index) => { + ApprovalDistributionMessage::DistributeAssignment(cert, candidate_indices) => { + let _span = state + .spans + .get(&cert.block_hash) + .map(|span| span.child("import-and-distribute-assignment")) + .unwrap_or_else(|| jaeger::Span::new(&cert.block_hash, "distribute-assignment")) + .with_string_tag("block-hash", format!("{:?}", cert.block_hash)) + .with_stage(jaeger::Stage::ApprovalDistribution); + gum::debug!( target: LOG_TARGET, - "Distributing our assignment on candidate (block={}, index={})", - cert.block_hash, - candidate_index, + ?candidate_indices, + block_hash = ?cert.block_hash, + assignment_kind = ?cert.cert.kind, + "Distributing our assignment on candidates", ); state @@ -1974,7 +2251,7 @@ impl ApprovalDistribution { &metrics, MessageSource::Local, cert, - candidate_index, + candidate_indices, rng, ) .await; @@ -2008,49 +2285,6 @@ impl ApprovalDistribution { } } -fn versioned_approvals_packet( - version: ValidationVersion, - approvals: Vec, -) -> VersionedValidationProtocol { - match version { - ValidationVersion::V1 => - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(approvals), - )), - ValidationVersion::VStaging => - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals), - )), - } -} - -fn versioned_assignments_packet( - version: ValidationVersion, - assignments: Vec<(IndirectAssignmentCert, CandidateIndex)>, -) -> VersionedValidationProtocol { - match version { - ValidationVersion::V1 => - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(assignments), - )), - ValidationVersion::VStaging => - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments), - )), - } -} - -fn filter_peers_by_version( - peers: &[(PeerId, ValidationVersion)], - version: ValidationVersion, -) -> Vec { - peers - .iter() - .filter(|(_, v)| v == &version) - .map(|(peer_id, _)| *peer_id) - .collect() -} - #[overseer::subsystem(ApprovalDistribution, error=SubsystemError, prefix=self::overseer)] impl ApprovalDistribution { fn start(self, ctx: Context) -> SpawnedSubsystem { @@ -2075,7 +2309,7 @@ const fn ensure_size_not_zero(size: usize) -> usize { /// the protocol configuration. pub const MAX_ASSIGNMENT_BATCH_SIZE: usize = ensure_size_not_zero( MAX_NOTIFICATION_SIZE as usize / - std::mem::size_of::<(IndirectAssignmentCert, CandidateIndex)>() / + std::mem::size_of::<(IndirectAssignmentCertV2, CandidateIndex)>() / 3, ); @@ -2084,6 +2318,54 @@ pub const MAX_APPROVAL_BATCH_SIZE: usize = ensure_size_not_zero( MAX_NOTIFICATION_SIZE as usize / std::mem::size_of::() / 3, ); +// Low level helper for sending assignments. +async fn send_assignments_batched_inner( + sender: &mut impl overseer::ApprovalDistributionSenderTrait, + batch: impl IntoIterator, + peers: &[PeerId], + peer_version: ValidationVersion, +) { + let peers = peers.into_iter().cloned().collect::>(); + if peer_version == ValidationVersion::VStaging { + sender + .send_message(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments( + batch.into_iter().collect(), + ), + )), + )) + .await; + } else { + // Create a batch of v1 assignments from v2 assignments that are compatible with v1. + // `IndirectAssignmentCertV2` -> `IndirectAssignmentCert` + let batch = batch + .into_iter() + .filter_map(|(cert, candidates)| { + cert.try_into().ok().map(|cert| { + ( + cert, + // First 1 bit index is the candidate index. + candidates + .first_one() + .map(|index| index as CandidateIndex) + .expect("Assignment was checked for not being empty; qed"), + ) + }) + }) + .collect(); + sender + .send_message(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(batch), + )), + )) + .await; + } +} + /// Send assignments while honoring the `max_notification_size` of the protocol. /// /// Splitting the messages into multiple notifications allows more granular processing at the @@ -2091,37 +2373,81 @@ pub const MAX_APPROVAL_BATCH_SIZE: usize = ensure_size_not_zero( /// of assignments and can `select!` other tasks. pub(crate) async fn send_assignments_batched( sender: &mut impl overseer::ApprovalDistributionSenderTrait, - assignments: Vec<(IndirectAssignmentCert, CandidateIndex)>, - peer: PeerId, - protocol_version: ValidationVersion, + v2_assignments: impl IntoIterator + Clone, + peers: &[(PeerId, ProtocolVersion)], ) { - let mut batches = assignments.into_iter().peekable(); + let v1_peers = filter_by_peer_version(peers, ValidationVersion::V1.into()); + let v2_peers = filter_by_peer_version(peers, ValidationVersion::VStaging.into()); + + if !v1_peers.is_empty() { + // Older peers(v1) do not understand `AssignmentsV2` messages, so we have to filter these + // out. + let v1_assignments = v2_assignments + .clone() + .into_iter() + .filter(|(_, candidates)| candidates.count_ones() == 1); + + let mut v1_batches = v1_assignments.peekable(); + + while v1_batches.peek().is_some() { + let batch: Vec<_> = v1_batches.by_ref().take(MAX_ASSIGNMENT_BATCH_SIZE).collect(); + send_assignments_batched_inner(sender, batch, &v1_peers, ValidationVersion::V1).await; + } + } - while batches.peek().is_some() { - let batch: Vec<_> = batches.by_ref().take(MAX_ASSIGNMENT_BATCH_SIZE).collect(); - let versioned = versioned_assignments_packet(protocol_version, batch); + if !v2_peers.is_empty() { + let mut v2_batches = v2_assignments.into_iter().peekable(); - sender - .send_message(NetworkBridgeTxMessage::SendValidationMessage(vec![peer], versioned)) - .await; + while v2_batches.peek().is_some() { + let batch = v2_batches.by_ref().take(MAX_ASSIGNMENT_BATCH_SIZE).collect::>(); + send_assignments_batched_inner(sender, batch, &v2_peers, ValidationVersion::VStaging) + .await; + } } } -/// Send approvals while honoring the `max_notification_size` of the protocol. +/// Send approvals while honoring the `max_notification_size` of the protocol and peer version. pub(crate) async fn send_approvals_batched( sender: &mut impl overseer::ApprovalDistributionSenderTrait, - approvals: Vec, - peer: PeerId, - protocol_version: ValidationVersion, + approvals: impl IntoIterator + Clone, + peers: &[(PeerId, ProtocolVersion)], ) { - let mut batches = approvals.into_iter().peekable(); - - while batches.peek().is_some() { - let batch: Vec<_> = batches.by_ref().take(MAX_APPROVAL_BATCH_SIZE).collect(); - let versioned = versioned_approvals_packet(protocol_version, batch); + let v1_peers = filter_by_peer_version(peers, ValidationVersion::V1.into()); + let v2_peers = filter_by_peer_version(peers, ValidationVersion::VStaging.into()); + + if !v1_peers.is_empty() { + let mut batches = approvals.clone().into_iter().peekable(); + + while batches.peek().is_some() { + let batch: Vec<_> = batches.by_ref().take(MAX_APPROVAL_BATCH_SIZE).collect(); + + sender + .send_message(NetworkBridgeTxMessage::SendValidationMessage( + v1_peers.clone(), + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Approvals(batch), + )), + )) + .await; + } + } - sender - .send_message(NetworkBridgeTxMessage::SendValidationMessage(vec![peer], versioned)) - .await; + if !v2_peers.is_empty() { + let mut batches = approvals.into_iter().peekable(); + + while batches.peek().is_some() { + let batch: Vec<_> = batches.by_ref().take(MAX_APPROVAL_BATCH_SIZE).collect(); + + sender + .send_message(NetworkBridgeTxMessage::SendValidationMessage( + v2_peers.clone(), + Versioned::VStaging( + protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(batch), + ), + ), + )) + .await; + } } } diff --git a/polkadot/node/network/approval-distribution/src/metrics.rs b/polkadot/node/network/approval-distribution/src/metrics.rs index 896866ce099a..6864259e6fdb 100644 --- a/polkadot/node/network/approval-distribution/src/metrics.rs +++ b/polkadot/node/network/approval-distribution/src/metrics.rs @@ -15,6 +15,7 @@ // along with Polkadot. If not, see . use polkadot_node_metrics::metrics::{prometheus, Metrics as MetricsTrait}; +use polkadot_node_primitives::approval::v2::AssignmentCertKindV2; /// Approval Distribution metrics. #[derive(Default, Clone)] @@ -22,21 +23,34 @@ pub struct Metrics(Option); #[derive(Clone)] struct MetricsInner { - assignments_imported_total: prometheus::Counter, + assignments_imported_total: prometheus::CounterVec, approvals_imported_total: prometheus::Counter, unified_with_peer_total: prometheus::Counter, aggression_l1_messages_total: prometheus::Counter, aggression_l2_messages_total: prometheus::Counter, - time_unify_with_peer: prometheus::Histogram, time_import_pending_now_known: prometheus::Histogram, time_awaiting_approval_voting: prometheus::Histogram, } +trait AsLabel { + fn as_label(&self) -> &str; +} + +impl AsLabel for &AssignmentCertKindV2 { + fn as_label(&self) -> &str { + match self { + AssignmentCertKindV2::RelayVRFDelay { .. } => "VRF Delay", + AssignmentCertKindV2::RelayVRFModulo { .. } => "VRF Modulo", + AssignmentCertKindV2::RelayVRFModuloCompact { .. } => "VRF Modulo Compact", + } + } +} + impl Metrics { - pub(crate) fn on_assignment_imported(&self) { + pub(crate) fn on_assignment_imported(&self, kind: &AssignmentCertKindV2) { if let Some(metrics) = &self.0 { - metrics.assignments_imported_total.inc(); + metrics.assignments_imported_total.with_label_values(&[kind.as_label()]).inc(); } } @@ -89,9 +103,12 @@ impl MetricsTrait for Metrics { fn try_register(registry: &prometheus::Registry) -> Result { let metrics = MetricsInner { assignments_imported_total: prometheus::register( - prometheus::Counter::new( - "polkadot_parachain_assignments_imported_total", - "Number of valid assignments imported locally or from other peers.", + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_assignments_imported_total", + "Number of valid assignments imported locally or from other peers.", + ), + &["kind"], )?, registry, )?, @@ -124,10 +141,16 @@ impl MetricsTrait for Metrics { registry, )?, time_unify_with_peer: prometheus::register( - prometheus::Histogram::with_opts(prometheus::HistogramOpts::new( - "polkadot_parachain_time_unify_with_peer", - "Time spent within fn `unify_with_peer`.", - ).buckets(vec![0.000625, 0.00125,0.0025, 0.005, 0.0075, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,]))?, + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "polkadot_parachain_time_unify_with_peer", + "Time spent within fn `unify_with_peer`.", + ) + .buckets(vec![ + 0.000625, 0.00125, 0.0025, 0.005, 0.0075, 0.01, 0.025, 0.05, 0.1, 0.25, + 0.5, 1.0, 2.5, 5.0, 10.0, + ]), + )?, registry, )?, time_import_pending_now_known: prometheus::register( diff --git a/polkadot/node/network/approval-distribution/src/tests.rs b/polkadot/node/network/approval-distribution/src/tests.rs index 1e9ae7b62007..f0c3c4f8ba64 100644 --- a/polkadot/node/network/approval-distribution/src/tests.rs +++ b/polkadot/node/network/approval-distribution/src/tests.rs @@ -24,20 +24,26 @@ use polkadot_node_network_protocol::{ view, ObservedRole, }; use polkadot_node_primitives::approval::{ - AssignmentCertKind, VrfOutput, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT, + v1::{ + AssignmentCert, AssignmentCertKind, IndirectAssignmentCert, VrfOutput, VrfProof, + VrfSignature, + }, + v2::{ + AssignmentCertKindV2, AssignmentCertV2, CoreBitfield, IndirectAssignmentCertV2, + RELAY_VRF_MODULO_CONTEXT, + }, }; use polkadot_node_subsystem::messages::{ network_bridge_event, AllMessages, ApprovalCheckError, ReportPeerMessage, }; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::{reputation::add_reputation, TimeoutExt as _}; -use polkadot_primitives::{AuthorityDiscoveryId, BlakeTwo256, HashT}; +use polkadot_primitives::{AuthorityDiscoveryId, BlakeTwo256, CoreIndex, HashT}; use polkadot_primitives_test_helpers::dummy_signature; use rand::SeedableRng; use sp_authority_discovery::AuthorityPair as AuthorityDiscoveryPair; use sp_core::crypto::Pair as PairT; use std::time::Duration; - type VirtualOverseer = test_helpers::TestSubsystemContextHandle; fn test_harness>( @@ -219,15 +225,15 @@ async fn setup_gossip_topology( async fn setup_peer_with_view( virtual_overseer: &mut VirtualOverseer, peer_id: &PeerId, - validation_version: ValidationVersion, view: View, + version: ValidationVersion, ) { overseer_send( virtual_overseer, ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerConnected( *peer_id, ObservedRole::Full, - validation_version.into(), + version.into(), None, )), ) @@ -244,12 +250,28 @@ async fn setup_peer_with_view( async fn send_message_from_peer( virtual_overseer: &mut VirtualOverseer, peer_id: &PeerId, - msg: net_protocol::ApprovalDistributionMessage, + msg: protocol_v1::ApprovalDistributionMessage, +) { + overseer_send( + virtual_overseer, + ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage( + *peer_id, + Versioned::V1(msg), + )), + ) + .await; +} + +async fn send_message_from_peer_v2( + virtual_overseer: &mut VirtualOverseer, + peer_id: &PeerId, + msg: protocol_vstaging::ApprovalDistributionMessage, ) { overseer_send( virtual_overseer, ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage( - *peer_id, msg, + *peer_id, + Versioned::VStaging(msg), )), ) .await; @@ -273,6 +295,28 @@ fn fake_assignment_cert(block_hash: Hash, validator: ValidatorIndex) -> Indirect } } +fn fake_assignment_cert_v2( + block_hash: Hash, + validator: ValidatorIndex, + core_bitfield: CoreBitfield, +) -> IndirectAssignmentCertV2 { + let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); + let msg = b"WhenParachains?"; + let mut prng = rand_core::OsRng; + let keypair = schnorrkel::Keypair::generate_with(&mut prng); + let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); + let out = inout.to_output(); + + IndirectAssignmentCertV2 { + block_hash, + validator, + cert: AssignmentCertV2 { + kind: AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield }, + vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) }, + }, + } +} + async fn expect_reputation_change( virtual_overseer: &mut VirtualOverseer, peer_id: &PeerId, @@ -331,9 +375,9 @@ fn try_import_the_same_assignment() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; // setup peers - setup_peer_with_view(overseer, &peer_a, ValidationVersion::V1, view![]).await; - setup_peer_with_view(overseer, &peer_b, ValidationVersion::V1, view![hash]).await; - setup_peer_with_view(overseer, &peer_c, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -353,7 +397,7 @@ fn try_import_the_same_assignment() { let assignments = vec![(cert.clone(), 0u32)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, &peer_a, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer_a, msg).await; expect_reputation_change(overseer, &peer_a, COST_UNEXPECTED_MESSAGE).await; @@ -362,10 +406,11 @@ fn try_import_the_same_assignment() { overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( assignment, - 0u32, + claimed_indices, tx, )) => { - assert_eq!(assignment, cert); + assert_eq!(claimed_indices, 0u32.into()); + assert_eq!(assignment, cert.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -385,12 +430,104 @@ fn try_import_the_same_assignment() { } ); - // setup new peer - setup_peer_with_view(overseer, &peer_d, ValidationVersion::V1, view![]).await; + // setup new peer with V2 + setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::VStaging).await; // send the same assignment from peer_d let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments); - send_message_from_peer(overseer, &peer_d, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer_d, msg).await; + + expect_reputation_change(overseer, &peer_d, COST_UNEXPECTED_MESSAGE).await; + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE).await; + + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }); +} + +/// Just like `try_import_the_same_assignment` but use `VRFModuloCompact` assignments for multiple +/// cores. +#[test] +fn try_import_the_same_assignment_v2() { + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + let peer_c = PeerId::random(); + let peer_d = PeerId::random(); + let parent_hash = Hash::repeat_byte(0xFF); + let hash = Hash::repeat_byte(0xAA); + + let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // setup peers + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::VStaging).await; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 2, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // send the assignment related to `hash` + let validator_index = ValidatorIndex(0); + let cores = vec![1, 2, 3, 4]; + let core_bitfield: CoreBitfield = cores + .iter() + .map(|index| CoreIndex(*index)) + .collect::>() + .try_into() + .unwrap(); + + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfield.clone()); + let assignments = vec![(cert.clone(), cores.clone().try_into().unwrap())]; + + let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer_v2(overseer, &peer_a, msg).await; + + expect_reputation_change(overseer, &peer_a, COST_UNEXPECTED_MESSAGE).await; + + // send an `Accept` message from the Approval Voting subsystem + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + assignment, + claimed_indices, + tx, + )) => { + assert_eq!(claimed_indices, cores.try_into().unwrap()); + assert_eq!(assignment, cert.into()); + tx.send(AssignmentCheckResult::Accepted).unwrap(); + } + ); + + expect_reputation_change(overseer, &peer_a, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 2); + assert_eq!(assignments.len(), 1); + } + ); + + // setup new peer + setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::VStaging).await; + + // send the same assignment from peer_d + let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments); + send_message_from_peer_v2(overseer, &peer_d, msg).await; expect_reputation_change(overseer, &peer_d, COST_UNEXPECTED_MESSAGE).await; expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE).await; @@ -413,7 +550,7 @@ fn delay_reputation_change() { let overseer = &mut virtual_overseer; // Setup peers - setup_peer_with_view(overseer, &peer, ValidationVersion::V1, view![]).await; + setup_peer_with_view(overseer, &peer, view![], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -433,17 +570,18 @@ fn delay_reputation_change() { let assignments = vec![(cert.clone(), 0u32)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, &peer, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer, msg).await; // send an `Accept` message from the Approval Voting subsystem assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( assignment, - 0u32, + claimed_candidates, tx, )) => { - assert_eq!(assignment, cert); + assert_eq!(assignment.cert, cert.cert.into()); + assert_eq!(claimed_candidates, vec![0u32].try_into().unwrap()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -474,7 +612,7 @@ fn spam_attack_results_in_negative_reputation_change() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; let peer = &peer_a; - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![]).await; + setup_peer_with_view(overseer, peer, view![], ValidationVersion::V1).await; // new block `hash_b` with 20 candidates let candidates_count = 20; @@ -501,7 +639,7 @@ fn spam_attack_results_in_negative_reputation_change() { .collect(); let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, peer, Versioned::V1(msg.clone())).await; + send_message_from_peer(overseer, peer, msg.clone()).await; for i in 0..candidates_count { expect_reputation_change(overseer, peer, COST_UNEXPECTED_MESSAGE).await; @@ -513,8 +651,8 @@ fn spam_attack_results_in_negative_reputation_change() { claimed_candidate_index, tx, )) => { - assert_eq!(assignment, assignments[i].0); - assert_eq!(claimed_candidate_index, assignments[i].1); + assert_eq!(assignment, assignments[i].0.clone().into()); + assert_eq!(claimed_candidate_index, assignments[i].1.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -533,7 +671,7 @@ fn spam_attack_results_in_negative_reputation_change() { .await; // send the assignments again - send_message_from_peer(overseer, peer, Versioned::V1(msg.clone())).await; + send_message_from_peer(overseer, peer, msg.clone()).await; // each of them will incur `COST_UNEXPECTED_MESSAGE`, not only the first one for _ in 0..candidates_count { @@ -558,7 +696,7 @@ fn peer_sending_us_the_same_we_just_sent_them_is_ok() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; let peer = &peer_a; - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![]).await; + setup_peer_with_view(overseer, peer, view![], ValidationVersion::V1).await; // new block `hash` with 1 candidates let meta = BlockApprovalMeta { @@ -578,7 +716,10 @@ fn peer_sending_us_the_same_we_just_sent_them_is_ok() { let cert = fake_assignment_cert(hash, validator_index); overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -610,12 +751,12 @@ fn peer_sending_us_the_same_we_just_sent_them_is_ok() { // the peer could send us it as well let assignments = vec![(cert, candidate_index)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments); - send_message_from_peer(overseer, peer, Versioned::V1(msg.clone())).await; + send_message_from_peer(overseer, peer, msg.clone()).await; assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "we should not punish the peer"); // send the assignments again - send_message_from_peer(overseer, peer, Versioned::V1(msg)).await; + send_message_from_peer(overseer, peer, msg).await; // now we should expect_reputation_change(overseer, peer, COST_DUPLICATE_MESSAGE).await; @@ -633,10 +774,10 @@ fn import_approval_happy_path() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; - // setup peers - setup_peer_with_view(overseer, &peer_a, ValidationVersion::V1, view![]).await; - setup_peer_with_view(overseer, &peer_b, ValidationVersion::V1, view![hash]).await; - setup_peer_with_view(overseer, &peer_c, ValidationVersion::V1, view![hash]).await; + // setup peers with V1 and V2 protocol versions + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -656,10 +797,14 @@ fn import_approval_happy_path() { let cert = fake_assignment_cert(hash, validator_index); overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert, candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; + // 1 peer is v1 assert_matches!( overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( @@ -668,7 +813,21 @@ fn import_approval_happy_path() { protocol_v1::ApprovalDistributionMessage::Assignments(assignments) )) )) => { - assert_eq!(peers.len(), 2); + assert_eq!(peers.len(), 1); + assert_eq!(assignments.len(), 1); + } + ); + + // 1 peer is v2 + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 1); assert_eq!(assignments.len(), 1); } ); @@ -681,7 +840,7 @@ fn import_approval_happy_path() { signature: dummy_signature(), }; let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_b, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -722,8 +881,8 @@ fn import_approval_bad() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; // setup peers - setup_peer_with_view(overseer, &peer_a, ValidationVersion::V1, view![]).await; - setup_peer_with_view(overseer, &peer_b, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -749,14 +908,14 @@ fn import_approval_bad() { signature: dummy_signature(), }; let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_b, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer_b, msg).await; expect_reputation_change(overseer, &peer_b, COST_UNEXPECTED_MESSAGE).await; // now import an assignment from peer_b let assignments = vec![(cert.clone(), candidate_index)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments); - send_message_from_peer(overseer, &peer_b, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -765,8 +924,8 @@ fn import_approval_bad() { i, tx, )) => { - assert_eq!(assignment, cert); - assert_eq!(i, candidate_index); + assert_eq!(assignment, cert.into()); + assert_eq!(i, candidate_index.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -775,7 +934,7 @@ fn import_approval_bad() { // and try again let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_b, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -911,12 +1070,20 @@ fn update_peer_view() { let cert_a = fake_assignment_cert(hash_a, ValidatorIndex(0)); let cert_b = fake_assignment_cert(hash_b, ValidatorIndex(0)); - overseer_send(overseer, ApprovalDistributionMessage::DistributeAssignment(cert_a, 0)).await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment(cert_a.into(), 0.into()), + ) + .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeAssignment(cert_b, 0)).await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment(cert_b.into(), 0.into()), + ) + .await; // connect a peer - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash_a]).await; + setup_peer_with_view(overseer, peer, view![hash_a], ValidationVersion::V1).await; // we should send relevant assignments to the peer assert_matches!( @@ -934,7 +1101,7 @@ fn update_peer_view() { virtual_overseer }); - assert_eq!(state.peer_data.get(peer).map(|data| data.view.finalized_number), Some(0)); + assert_eq!(state.peer_views.get(peer).map(|v| v.view.finalized_number), Some(0)); assert_eq!( state .blocks @@ -965,7 +1132,7 @@ fn update_peer_view() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert_c.clone(), 0), + ApprovalDistributionMessage::DistributeAssignment(cert_c.clone().into(), 0.into()), ) .await; @@ -986,7 +1153,7 @@ fn update_peer_view() { virtual_overseer }); - assert_eq!(state.peer_data.get(peer).map(|data| data.view.finalized_number), Some(2)); + assert_eq!(state.peer_views.get(peer).map(|v| v.view.finalized_number), Some(2)); assert_eq!( state .blocks @@ -1016,10 +1183,7 @@ fn update_peer_view() { virtual_overseer }); - assert_eq!( - state.peer_data.get(peer).map(|data| data.view.finalized_number), - Some(finalized_number) - ); + assert_eq!(state.peer_views.get(peer).map(|v| v.view.finalized_number), Some(finalized_number)); assert!(state.blocks.get(&hash_c).unwrap().known_by.get(peer).is_none()); } @@ -1034,7 +1198,7 @@ fn import_remotely_then_locally() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; // setup the peer - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -1054,7 +1218,7 @@ fn import_remotely_then_locally() { let cert = fake_assignment_cert(hash, validator_index); let assignments = vec![(cert.clone(), candidate_index)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, peer, Versioned::V1(msg)).await; + send_message_from_peer(overseer, peer, msg).await; // send an `Accept` message from the Approval Voting subsystem assert_matches!( @@ -1064,8 +1228,8 @@ fn import_remotely_then_locally() { i, tx, )) => { - assert_eq!(assignment, cert); - assert_eq!(i, candidate_index); + assert_eq!(assignment, cert.clone().into()); + assert_eq!(i, candidate_index.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -1075,7 +1239,10 @@ fn import_remotely_then_locally() { // import the same assignment locally overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert, candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -1089,7 +1256,7 @@ fn import_remotely_then_locally() { signature: dummy_signature(), }; let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, peer, Versioned::V1(msg)).await; + send_message_from_peer(overseer, peer, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1147,7 +1314,10 @@ fn sends_assignments_even_when_state_is_approved() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -1155,7 +1325,7 @@ fn sends_assignments_even_when_state_is_approved() { .await; // connect the peer. - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; let assignments = vec![(cert.clone(), candidate_index)]; let approvals = vec![approval.clone()]; @@ -1191,6 +1361,112 @@ fn sends_assignments_even_when_state_is_approved() { }); } +/// Same as `sends_assignments_even_when_state_is_approved_v2` but with `VRFModuloCompact` +/// assignemnts. +#[test] +fn sends_assignments_even_when_state_is_approved_v2() { + let peer_a = PeerId::random(); + let parent_hash = Hash::repeat_byte(0xFF); + let hash = Hash::repeat_byte(0xAA); + let peer = &peer_a; + + let _ = test_harness(State::default(), |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 4], + slot: 1.into(), + session: 1, + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + let validator_index = ValidatorIndex(0); + let cores = vec![0, 1, 2, 3]; + let candidate_bitfield: CandidateBitfield = cores.clone().try_into().unwrap(); + + let core_bitfield: CoreBitfield = cores + .iter() + .map(|index| CoreIndex(*index)) + .collect::>() + .try_into() + .unwrap(); + + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfield.clone()); + + // Assumes candidate index == core index. + let approvals = cores + .iter() + .map(|core| IndirectSignedApprovalVote { + block_hash: hash, + candidate_index: *core, + validator: validator_index, + signature: dummy_signature(), + }) + .collect::>(); + + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_bitfield.clone(), + ), + ) + .await; + + for approval in &approvals { + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone()), + ) + .await; + } + + // connect the peer. + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::VStaging).await; + + let assignments = vec![(cert.clone(), candidate_bitfield.clone())]; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(sent_assignments) + )) + )) => { + assert_eq!(peers, vec![*peer]); + assert_eq!(sent_assignments, assignments); + } + ); + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(sent_approvals) + )) + )) => { + // Construct a hashmaps of approvals for comparison. Approval distribution reorders messages because they are kept in a + // hashmap as well. + let sent_approvals = sent_approvals.into_iter().map(|approval| (approval.candidate_index, approval)).collect::>(); + let approvals = approvals.into_iter().map(|approval| (approval.candidate_index, approval)).collect::>(); + + assert_eq!(peers, vec![*peer]); + assert_eq!(sent_approvals, approvals); + } + ); + + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }); +} + /// /// /// 1. Receive remote peer view update with an unknown head @@ -1219,7 +1495,7 @@ fn race_condition_in_local_vs_remote_view_update() { }; // This will send a peer view that is ahead of our view - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash_b]).await; + setup_peer_with_view(overseer, peer, view![hash_b], ValidationVersion::V1).await; // Send our view update to include a new head overseer_send( @@ -1240,7 +1516,7 @@ fn race_condition_in_local_vs_remote_view_update() { .collect(); let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, peer, Versioned::V1(msg.clone())).await; + send_message_from_peer(overseer, peer, msg.clone()).await; // This will handle pending messages being processed let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); @@ -1257,8 +1533,8 @@ fn race_condition_in_local_vs_remote_view_update() { claimed_candidate_index, tx, )) => { - assert_eq!(assignment, assignments[i].0); - assert_eq!(claimed_candidate_index, assignments[i].1); + assert_eq!(assignment, assignments[i].0.clone().into()); + assert_eq!(claimed_candidate_index, assignments[i].1.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -1283,7 +1559,7 @@ fn propagates_locally_generated_assignment_to_both_dimensions() { // Connect all peers. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // Set up a gossip topology. @@ -1325,7 +1601,10 @@ fn propagates_locally_generated_assignment_to_both_dimensions() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -1388,7 +1667,7 @@ fn propagates_assignments_along_unshared_dimension() { // Connect all peers. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // Set up a gossip topology. @@ -1424,7 +1703,7 @@ fn propagates_assignments_along_unshared_dimension() { // Issuer of the message is important, not the peer we receive from. // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peers[99].0, msg).await; assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( @@ -1473,7 +1752,7 @@ fn propagates_assignments_along_unshared_dimension() { // Issuer of the message is important, not the peer we receive from. // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peers[99].0, msg).await; assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( @@ -1530,7 +1809,7 @@ fn propagates_to_required_after_connect() { // Connect all peers except omitted. for (i, (peer, _)) in peers.iter().enumerate() { if !omitted.contains(&i) { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } } @@ -1573,7 +1852,10 @@ fn propagates_to_required_after_connect() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -1619,7 +1901,7 @@ fn propagates_to_required_after_connect() { ); for i in omitted.iter().copied() { - setup_peer_with_view(overseer, &peers[i].0, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, &peers[i].0, view![hash], ValidationVersion::V1).await; assert_matches!( overseer_recv(overseer).await, @@ -1668,7 +1950,7 @@ fn sends_to_more_peers_after_getting_topology() { // Connect all peers except omitted. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // new block `hash_a` with 1 candidates @@ -1698,7 +1980,10 @@ fn sends_to_more_peers_after_getting_topology() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -1820,7 +2105,7 @@ fn originator_aggression_l1() { // Connect all peers except omitted. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // new block `hash_a` with 1 candidates @@ -1857,7 +2142,10 @@ fn originator_aggression_l1() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -1979,7 +2267,7 @@ fn non_originator_aggression_l1() { // Connect all peers except omitted. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // new block `hash_a` with 1 candidates @@ -2008,12 +2296,12 @@ fn non_originator_aggression_l1() { ) .await; - let assignments = vec![(cert.clone(), candidate_index)]; + let assignments = vec![(cert.clone().into(), candidate_index)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); // Issuer of the message is important, not the peer we receive from. // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peers[99].0, msg).await; assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( @@ -2084,7 +2372,7 @@ fn non_originator_aggression_l2() { // Connect all peers except omitted. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // new block `hash_a` with 1 candidates @@ -2118,7 +2406,7 @@ fn non_originator_aggression_l2() { // Issuer of the message is important, not the peer we receive from. // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peers[99].0, msg).await; assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( @@ -2249,7 +2537,7 @@ fn resends_messages_periodically() { // Connect all peers. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // Set up a gossip topology. @@ -2284,7 +2572,7 @@ fn resends_messages_periodically() { // Issuer of the message is important, not the peer we receive from. // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peers[99].0, msg).await; assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( @@ -2375,126 +2663,6 @@ fn resends_messages_periodically() { }); } -/// Tests that peers correctly receive versioned messages. -#[test] -fn import_versioned_approval() { - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - let peer_c = PeerId::random(); - let parent_hash = Hash::repeat_byte(0xFF); - let hash = Hash::repeat_byte(0xAA); - - let state = state_without_reputation_delay(); - let _ = test_harness(state, |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // All peers are aware of relay parent. - setup_peer_with_view(overseer, &peer_a, ValidationVersion::VStaging, view![hash]).await; - setup_peer_with_view(overseer, &peer_b, ValidationVersion::V1, view![hash]).await; - setup_peer_with_view(overseer, &peer_c, ValidationVersion::VStaging, view![hash]).await; - - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - // import an assignment related to `hash` locally - let validator_index = ValidatorIndex(0); - let candidate_index = 0u32; - let cert = fake_assignment_cert(hash, validator_index); - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment(cert, candidate_index), - ) - .await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert_eq!(peers, vec![peer_b]); - assert_eq!(assignments.len(), 1); - } - ); - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert_eq!(peers.len(), 2); - assert!(peers.contains(&peer_a)); - assert!(peers.contains(&peer_c)); - - assert_eq!(assignments.len(), 1); - } - ); - - // send the an approval from peer_a - let approval = IndirectSignedApprovalVote { - block_hash: hash, - candidate_index, - validator: validator_index, - signature: dummy_signature(), - }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_a, Versioned::VStaging(msg)).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( - vote, - tx, - )) => { - assert_eq!(vote, approval); - tx.send(ApprovalCheckResult::Accepted).unwrap(); - } - ); - - expect_reputation_change(overseer, &peer_a, BENEFIT_VALID_MESSAGE_FIRST).await; - - // Peers b and c receive versioned approval messages. - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(approvals) - )) - )) => { - assert_eq!(peers, vec![peer_b]); - assert_eq!(approvals.len(), 1); - } - ); - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) - )) - )) => { - assert_eq!(peers, vec![peer_c]); - assert_eq!(approvals.len(), 1); - } - ); - virtual_overseer - }); -} - fn batch_test_round(message_count: usize) { use polkadot_node_subsystem::SubsystemContext; let pool = sp_core::testing::TaskExecutor::new(); @@ -2512,7 +2680,9 @@ fn batch_test_round(message_count: usize) { let validators = 0..message_count; let assignments: Vec<_> = validators .clone() - .map(|index| (fake_assignment_cert(Hash::zero(), ValidatorIndex(index as u32)), 0)) + .map(|index| { + (fake_assignment_cert(Hash::zero(), ValidatorIndex(index as u32)).into(), 0.into()) + }) .collect(); let approvals: Vec<_> = validators @@ -2525,9 +2695,18 @@ fn batch_test_round(message_count: usize) { .collect(); let peer = PeerId::random(); - send_assignments_batched(&mut sender, assignments.clone(), peer, ValidationVersion::V1) - .await; - send_approvals_batched(&mut sender, approvals.clone(), peer, ValidationVersion::V1).await; + send_assignments_batched( + &mut sender, + assignments.clone(), + &vec![(peer, ValidationVersion::V1.into())], + ) + .await; + send_approvals_batched( + &mut sender, + approvals.clone(), + &vec![(peer, ValidationVersion::V1.into())], + ) + .await; // Check expected assignments batches. for assignment_index in (0..assignments.len()).step_by(super::MAX_ASSIGNMENT_BATCH_SIZE) { @@ -2549,7 +2728,7 @@ fn batch_test_round(message_count: usize) { assert_eq!(peers.len(), 1); for (message_index, assignment) in sent_assignments.iter().enumerate() { - assert_eq!(assignment.0, assignments[assignment_index + message_index].0); + assert_eq!(assignment.0, assignments[assignment_index + message_index].0.clone().try_into().unwrap()); assert_eq!(assignment.1, 0); } } diff --git a/polkadot/node/network/bitfield-distribution/src/lib.rs b/polkadot/node/network/bitfield-distribution/src/lib.rs index c85d874bc4db..55243edd3a5e 100644 --- a/polkadot/node/network/bitfield-distribution/src/lib.rs +++ b/polkadot/node/network/bitfield-distribution/src/lib.rs @@ -131,9 +131,9 @@ pub struct PeerData { /// Data used to track information of peers and relay parents the /// overseer ordered us to work on. -#[derive(Default, Debug)] +#[derive(Default)] struct ProtocolState { - /// Track all active peers and their views + /// Track all active peer views and protocol versions /// to determine what is relevant to them. peer_data: HashMap, @@ -775,9 +775,11 @@ async fn handle_network_msg( handle_peer_view_change(ctx, state, new_peer, old_view, rng).await; } }, - NetworkBridgeEvent::PeerViewChange(peerid, new_view) => { - gum::trace!(target: LOG_TARGET, ?peerid, ?new_view, "Peer view change"); - handle_peer_view_change(ctx, state, peerid, new_view, rng).await; + NetworkBridgeEvent::PeerViewChange(peer_id, new_view) => { + gum::trace!(target: LOG_TARGET, ?peer_id, ?new_view, "Peer view change"); + if state.peer_data.get(&peer_id).is_some() { + handle_peer_view_change(ctx, state, peer_id, new_view, rng).await; + } }, NetworkBridgeEvent::OurViewChange(new_view) => { gum::trace!(target: LOG_TARGET, ?new_view, "Our view change"); diff --git a/polkadot/node/network/bridge/src/network.rs b/polkadot/node/network/bridge/src/network.rs index 4f21212dcb64..4d2d1fba4891 100644 --- a/polkadot/node/network/bridge/src/network.rs +++ b/polkadot/node/network/bridge/src/network.rs @@ -28,23 +28,109 @@ use sc_network::{ }; use polkadot_node_network_protocol::{ - peer_set::{PeerSet, PeerSetProtocolNames, ProtocolVersion}, + peer_set::{ + CollationVersion, PeerSet, PeerSetProtocolNames, ProtocolVersion, ValidationVersion, + }, request_response::{OutgoingRequest, Recipient, ReqProtocolNames, Requests}, - PeerId, + v1 as protocol_v1, vstaging as protocol_vstaging, PeerId, }; use polkadot_primitives::{AuthorityDiscoveryId, Block, Hash}; -use crate::validator_discovery::AuthorityDiscovery; +use crate::{metrics::Metrics, validator_discovery::AuthorityDiscovery, WireMessage}; // network bridge network abstraction log target const LOG_TARGET: &'static str = "parachain::network-bridge-net"; -/// Send a message to the network. +// Helper function to send a validation v1 message to a list of peers. +// Messages are always sent via the main protocol, even legacy protocol messages. +pub(crate) fn send_validation_message_v1( + net: &mut impl Network, + peers: Vec, + peerset_protocol_names: &PeerSetProtocolNames, + message: WireMessage, + metrics: &Metrics, +) { + gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation v1 message to peers",); + + send_message( + net, + peers, + PeerSet::Validation, + ValidationVersion::V1.into(), + peerset_protocol_names, + message, + metrics, + ); +} + +// Helper function to send a validation v2 message to a list of peers. +// Messages are always sent via the main protocol, even legacy protocol messages. +pub(crate) fn send_validation_message_v2( + net: &mut impl Network, + peers: Vec, + peerset_protocol_names: &PeerSetProtocolNames, + message: WireMessage, + metrics: &Metrics, +) { + gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation v2 message to peers",); + + send_message( + net, + peers, + PeerSet::Validation, + ValidationVersion::VStaging.into(), + peerset_protocol_names, + message, + metrics, + ); +} + +// Helper function to send a collation v1 message to a list of peers. +// Messages are always sent via the main protocol, even legacy protocol messages. +pub(crate) fn send_collation_message_v1( + net: &mut impl Network, + peers: Vec, + peerset_protocol_names: &PeerSetProtocolNames, + message: WireMessage, + metrics: &Metrics, +) { + send_message( + net, + peers, + PeerSet::Collation, + CollationVersion::V1.into(), + peerset_protocol_names, + message, + metrics, + ); +} + +// Helper function to send a collation v2 message to a list of peers. +// Messages are always sent via the main protocol, even legacy protocol messages. +pub(crate) fn send_collation_message_v2( + net: &mut impl Network, + peers: Vec, + peerset_protocol_names: &PeerSetProtocolNames, + message: WireMessage, + metrics: &Metrics, +) { + send_message( + net, + peers, + PeerSet::Collation, + CollationVersion::VStaging.into(), + peerset_protocol_names, + message, + metrics, + ); +} + +/// Lower level function that sends a message to the network using the main protocol version. /// /// This function is only used internally by the network-bridge, which is responsible to only send /// messages that are compatible with the passed peer set, as that is currently not enforced by /// this function. These are messages of type `WireMessage` parameterized on the matching type. -pub(crate) fn send_message( +fn send_message( net: &mut impl Network, mut peers: Vec, peer_set: PeerSet, @@ -64,6 +150,17 @@ pub(crate) fn send_message( encoded }; + // optimization: generate the protocol name once. + let protocol_name = protocol_names.get_name(peer_set, version); + gum::trace!( + target: LOG_TARGET, + ?peers, + ?version, + ?protocol_name, + ?message, + "Sending message to peers", + ); + // optimization: avoid cloning the message for the last peer in the // list. The message payload can be quite large. If the underlying // network used `Bytes` this would not be necessary. diff --git a/polkadot/node/network/bridge/src/rx/mod.rs b/polkadot/node/network/bridge/src/rx/mod.rs index 002919c5b0e5..15ed6b48745d 100644 --- a/polkadot/node/network/bridge/src/rx/mod.rs +++ b/polkadot/node/network/bridge/src/rx/mod.rs @@ -64,9 +64,11 @@ use super::validator_discovery; /// Actual interfacing to the network based on the `Network` trait. /// /// Defines the `Network` trait with an implementation for an `Arc`. -use crate::network::{send_message, Network}; - -use crate::network::get_peer_id_by_authority_id; +use crate::network::{ + send_collation_message_v1, send_collation_message_v2, send_validation_message_v1, + send_validation_message_v2, Network, +}; +use crate::{network::get_peer_id_by_authority_id, WireMessage}; use super::metrics::Metrics; @@ -250,22 +252,18 @@ where match ValidationVersion::try_from(version) .expect("try_get_protocol has already checked version is known; qed") { - ValidationVersion::V1 => send_message( + ValidationVersion::V1 => send_validation_message_v1( &mut network_service, vec![peer], - PeerSet::Validation, - version, &peerset_protocol_names, WireMessage::::ViewUpdate( local_view, ), &metrics, ), - ValidationVersion::VStaging => send_message( + ValidationVersion::VStaging => send_validation_message_v2( &mut network_service, vec![peer], - PeerSet::Validation, - version, &peerset_protocol_names, WireMessage::::ViewUpdate( local_view, @@ -292,22 +290,18 @@ where match CollationVersion::try_from(version) .expect("try_get_protocol has already checked version is known; qed") { - CollationVersion::V1 => send_message( + CollationVersion::V1 => send_collation_message_v1( &mut network_service, vec![peer], - PeerSet::Collation, - version, &peerset_protocol_names, WireMessage::::ViewUpdate( local_view, ), &metrics, ), - CollationVersion::VStaging => send_message( + CollationVersion::VStaging => send_collation_message_v2( &mut network_service, vec![peer], - PeerSet::Collation, - version, &peerset_protocol_names, WireMessage::::ViewUpdate( local_view, @@ -384,8 +378,16 @@ where .filter_map(|(protocol, msg_bytes)| { // version doesn't matter because we always receive on the 'correct' // protocol name, not the negotiated fallback. - let (peer_set, _version) = + let (peer_set, version) = peerset_protocol_names.try_get_protocol(protocol)?; + gum::trace!( + target: LOG_TARGET, + ?peer_set, + ?protocol, + ?version, + "Received notification" + ); + if peer_set == PeerSet::Validation { if expected_versions[PeerSet::Validation].is_none() { return Some(Err(UNCONNECTED_PEERSET_COST)) @@ -832,7 +834,7 @@ fn update_our_view( metrics, ); - send_validation_message_vstaging( + send_validation_message_v2( net, vstaging_validation_peers, peerset_protocol_names, @@ -840,7 +842,7 @@ fn update_our_view( metrics, ); - send_collation_message_vstaging( + send_collation_message_v2( net, vstaging_collation_peers, peerset_protocol_names, @@ -917,78 +919,6 @@ fn handle_peer_messages>( (outgoing_events, reports) } -fn send_validation_message_v1( - net: &mut impl Network, - peers: Vec, - peerset_protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Validation, - ValidationVersion::V1.into(), - peerset_protocol_names, - message, - metrics, - ); -} - -fn send_collation_message_v1( - net: &mut impl Network, - peers: Vec, - peerset_protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Collation, - CollationVersion::V1.into(), - peerset_protocol_names, - message, - metrics, - ); -} - -fn send_validation_message_vstaging( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Validation, - ValidationVersion::VStaging.into(), - protocol_names, - message, - metrics, - ); -} - -fn send_collation_message_vstaging( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Collation, - CollationVersion::VStaging.into(), - protocol_names, - message, - metrics, - ); -} - async fn dispatch_validation_event_to_all( event: NetworkBridgeEvent, ctx: &mut impl overseer::NetworkBridgeRxSenderTrait, diff --git a/polkadot/node/network/bridge/src/tx/mod.rs b/polkadot/node/network/bridge/src/tx/mod.rs index e0ca633547f4..7bd7f1e25c06 100644 --- a/polkadot/node/network/bridge/src/tx/mod.rs +++ b/polkadot/node/network/bridge/src/tx/mod.rs @@ -18,9 +18,7 @@ use super::*; use polkadot_node_network_protocol::{ - peer_set::{CollationVersion, PeerSet, PeerSetProtocolNames, ValidationVersion}, - request_response::ReqProtocolNames, - v1 as protocol_v1, vstaging as protocol_vstaging, PeerId, Versioned, + peer_set::PeerSetProtocolNames, request_response::ReqProtocolNames, Versioned, }; use polkadot_node_subsystem::{ @@ -40,7 +38,10 @@ use crate::validator_discovery; /// Actual interfacing to the network based on the `Network` trait. /// /// Defines the `Network` trait with an implementation for an `Arc`. -use crate::network::{send_message, Network}; +use crate::network::{ + send_collation_message_v1, send_collation_message_v2, send_validation_message_v1, + send_validation_message_v2, Network, +}; use crate::metrics::Metrics; @@ -186,6 +187,7 @@ where gum::trace!( target: LOG_TARGET, action = "SendValidationMessages", + ?msg, num_messages = 1usize, ); @@ -197,7 +199,7 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), - Versioned::VStaging(msg) => send_validation_message_vstaging( + Versioned::VStaging(msg) => send_validation_message_v2( &mut network_service, peers, peerset_protocol_names, @@ -211,6 +213,7 @@ where target: LOG_TARGET, action = "SendValidationMessages", num_messages = %msgs.len(), + ?msgs, ); for (peers, msg) in msgs { @@ -222,7 +225,7 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), - Versioned::VStaging(msg) => send_validation_message_vstaging( + Versioned::VStaging(msg) => send_validation_message_v2( &mut network_service, peers, peerset_protocol_names, @@ -247,7 +250,7 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), - Versioned::VStaging(msg) => send_collation_message_vstaging( + Versioned::VStaging(msg) => send_collation_message_v2( &mut network_service, peers, peerset_protocol_names, @@ -272,7 +275,7 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), - Versioned::VStaging(msg) => send_collation_message_vstaging( + Versioned::VStaging(msg) => send_collation_message_v2( &mut network_service, peers, peerset_protocol_names, @@ -373,75 +376,3 @@ where Ok(()) } - -fn send_validation_message_v1( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Validation, - ValidationVersion::V1.into(), - protocol_names, - message, - metrics, - ); -} - -fn send_collation_message_v1( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Collation, - CollationVersion::V1.into(), - protocol_names, - message, - metrics, - ); -} - -fn send_validation_message_vstaging( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Validation, - ValidationVersion::VStaging.into(), - protocol_names, - message, - metrics, - ); -} - -fn send_collation_message_vstaging( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Collation, - CollationVersion::VStaging.into(), - protocol_names, - message, - metrics, - ); -} diff --git a/polkadot/node/network/bridge/src/tx/tests.rs b/polkadot/node/network/bridge/src/tx/tests.rs index 21cd134c54f2..6e28a4032183 100644 --- a/polkadot/node/network/bridge/src/tx/tests.rs +++ b/polkadot/node/network/bridge/src/tx/tests.rs @@ -25,9 +25,9 @@ use std::collections::HashSet; use sc_network::{Event as NetworkEvent, IfDisconnected, ProtocolName, ReputationChange}; use polkadot_node_network_protocol::{ - peer_set::PeerSetProtocolNames, + peer_set::{PeerSetProtocolNames, ValidationVersion}, request_response::{outgoing::Requests, ReqProtocolNames}, - ObservedRole, Versioned, + v1 as protocol_v1, vstaging as protocol_vstaging, ObservedRole, Versioned, }; use polkadot_node_subsystem::{FromOrchestra, OverseerSignal}; use polkadot_node_subsystem_test_helpers::TestSubsystemContextHandle; @@ -356,7 +356,6 @@ fn network_protocol_versioning_send() { } // send a validation protocol message. - { let approval_distribution_message = protocol_vstaging::ApprovalDistributionMessage::Approvals(Vec::new()); diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index 1bed2c12fe20..ba9b9a7f4900 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -313,7 +313,6 @@ macro_rules! impl_versioned_full_protocol_from { } }; } - /// Implement `TryFrom` for one versioned enum variant into the inner type. /// `$m_ty::$variant(inner) -> Ok(inner)` macro_rules! impl_versioned_try_from { @@ -441,7 +440,7 @@ pub mod v1 { }; use polkadot_node_primitives::{ - approval::{IndirectAssignmentCert, IndirectSignedApprovalVote}, + approval::v1::{IndirectAssignmentCert, IndirectSignedApprovalVote}, UncheckedSignedFullStatement, }; @@ -595,12 +594,15 @@ pub mod vstaging { use parity_scale_codec::{Decode, Encode}; use polkadot_primitives::vstaging::{ - CandidateHash, CandidateIndex, CollatorId, CollatorSignature, GroupIndex, Hash, - Id as ParaId, UncheckedSignedAvailabilityBitfield, UncheckedSignedStatement, + CandidateHash, CollatorId, CollatorSignature, GroupIndex, Hash, Id as ParaId, + UncheckedSignedAvailabilityBitfield, UncheckedSignedStatement, }; use polkadot_node_primitives::{ - approval::{IndirectAssignmentCert, IndirectSignedApprovalVote}, + approval::{ + v1::IndirectSignedApprovalVote, + v2::{CandidateBitfield, IndirectAssignmentCertV2}, + }, UncheckedSignedFullStatement, }; @@ -769,10 +771,13 @@ pub mod vstaging { #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] pub enum ApprovalDistributionMessage { /// Assignments for candidates in recent, unfinalized blocks. + /// We use a bitfield to reference claimed candidates, where the bit index is equal to + /// candidate index. /// /// Actually checking the assignment may yield a different result. + /// TODO: Look at getting rid of bitfield in the future. #[codec(index = 0)] - Assignments(Vec<(IndirectAssignmentCert, CandidateIndex)>), + Assignments(Vec<(IndirectAssignmentCertV2, CandidateBitfield)>), /// Approvals for candidates in some recent, unfinalized block. #[codec(index = 1)] Approvals(Vec), @@ -841,3 +846,11 @@ pub mod vstaging { payload } } + +/// Returns the subset of `peers` with the specified `version`. +pub fn filter_by_peer_version( + peers: &[(PeerId, peer_set::ProtocolVersion)], + version: peer_set::ProtocolVersion, +) -> Vec { + peers.iter().filter(|(_, v)| v == &version).map(|(p, _)| *p).collect::>() +} diff --git a/polkadot/node/primitives/Cargo.toml b/polkadot/node/primitives/Cargo.toml index c7c323985510..170626b9cf41 100644 --- a/polkadot/node/primitives/Cargo.toml +++ b/polkadot/node/primitives/Cargo.toml @@ -21,6 +21,7 @@ polkadot-parachain = { path = "../../parachain", default-features = false } schnorrkel = "0.9.1" thiserror = "1.0.31" serde = { version = "1.0.163", features = ["derive"] } +bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } [target.'cfg(not(target_os = "unknown"))'.dependencies] zstd = { version = "0.11.2", default-features = false } diff --git a/polkadot/node/primitives/src/approval.rs b/polkadot/node/primitives/src/approval.rs index 01f45a207874..bfa1f1aa47d2 100644 --- a/polkadot/node/primitives/src/approval.rs +++ b/polkadot/node/primitives/src/approval.rs @@ -16,190 +16,515 @@ //! Types relevant for approval. -pub use sp_consensus_babe::{Randomness, Slot, VrfOutput, VrfProof, VrfSignature, VrfTranscript}; - -use parity_scale_codec::{Decode, Encode}; -use polkadot_primitives::{ - BlockNumber, CandidateHash, CandidateIndex, CoreIndex, Hash, Header, SessionIndex, - ValidatorIndex, ValidatorSignature, -}; -use sp_application_crypto::ByteArray; -use sp_consensus_babe as babe_primitives; - -/// Validators assigning to check a particular candidate are split up into tranches. -/// Earlier tranches of validators check first, with later tranches serving as backup. -pub type DelayTranche = u32; - -/// A static context used to compute the Relay VRF story based on the -/// VRF output included in the header-chain. -pub const RELAY_VRF_STORY_CONTEXT: &[u8] = b"A&V RC-VRF"; - -/// A static context used for all relay-vrf-modulo VRFs. -pub const RELAY_VRF_MODULO_CONTEXT: &[u8] = b"A&V MOD"; - -/// A static context used for all relay-vrf-modulo VRFs. -pub const RELAY_VRF_DELAY_CONTEXT: &[u8] = b"A&V DELAY"; - -/// A static context used for transcripts indicating assigned availability core. -pub const ASSIGNED_CORE_CONTEXT: &[u8] = b"A&V ASSIGNED"; - -/// A static context associated with producing randomness for a core. -pub const CORE_RANDOMNESS_CONTEXT: &[u8] = b"A&V CORE"; - -/// A static context associated with producing randomness for a tranche. -pub const TRANCHE_RANDOMNESS_CONTEXT: &[u8] = b"A&V TRANCHE"; - -/// random bytes derived from the VRF submitted within the block by the -/// block author as a credential and used as input to approval assignment criteria. -#[derive(Debug, Clone, Encode, Decode, PartialEq)] -pub struct RelayVRFStory(pub [u8; 32]); - -/// Different kinds of input data or criteria that can prove a validator's assignment -/// to check a particular parachain. -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -pub enum AssignmentCertKind { - /// An assignment story based on the VRF that authorized the relay-chain block where the - /// candidate was included combined with a sample number. +/// A list of primitives introduced in v1. +pub mod v1 { + use sp_consensus_babe as babe_primitives; + pub use sp_consensus_babe::{ + Randomness, Slot, VrfOutput, VrfProof, VrfSignature, VrfTranscript, + }; + + use parity_scale_codec::{Decode, Encode}; + use polkadot_primitives::{ + BlockNumber, CandidateHash, CandidateIndex, CoreIndex, Hash, Header, SessionIndex, + ValidatorIndex, ValidatorSignature, + }; + use sp_application_crypto::ByteArray; + + /// Validators assigning to check a particular candidate are split up into tranches. + /// Earlier tranches of validators check first, with later tranches serving as backup. + pub type DelayTranche = u32; + + /// A static context used to compute the Relay VRF story based on the + /// VRF output included in the header-chain. + pub const RELAY_VRF_STORY_CONTEXT: &[u8] = b"A&V RC-VRF"; + + /// A static context used for all relay-vrf-modulo VRFs. + pub const RELAY_VRF_MODULO_CONTEXT: &[u8] = b"A&V MOD"; + + /// A static context used for all relay-vrf-modulo VRFs. + pub const RELAY_VRF_DELAY_CONTEXT: &[u8] = b"A&V DELAY"; + + /// A static context used for transcripts indicating assigned availability core. + pub const ASSIGNED_CORE_CONTEXT: &[u8] = b"A&V ASSIGNED"; + + /// A static context associated with producing randomness for a core. + pub const CORE_RANDOMNESS_CONTEXT: &[u8] = b"A&V CORE"; + + /// A static context associated with producing randomness for a tranche. + pub const TRANCHE_RANDOMNESS_CONTEXT: &[u8] = b"A&V TRANCHE"; + + /// random bytes derived from the VRF submitted within the block by the + /// block author as a credential and used as input to approval assignment criteria. + #[derive(Debug, Clone, Encode, Decode, PartialEq)] + pub struct RelayVRFStory(pub [u8; 32]); + + /// Different kinds of input data or criteria that can prove a validator's assignment + /// to check a particular parachain. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum AssignmentCertKind { + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with a sample number. + /// + /// The context used to produce bytes is [`RELAY_VRF_MODULO_CONTEXT`] + RelayVRFModulo { + /// The sample number used in this cert. + sample: u32, + }, + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with the index of a particular core. + /// + /// The context is [`RELAY_VRF_DELAY_CONTEXT`] + RelayVRFDelay { + /// The core index chosen in this cert. + core_index: CoreIndex, + }, + } + + /// A certification of assignment. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct AssignmentCert { + /// The criterion which is claimed to be met by this cert. + pub kind: AssignmentCertKind, + /// The VRF signature showing the criterion is met. + pub vrf: VrfSignature, + } + + /// An assignment criterion which refers to the candidate under which the assignment is + /// relevant by block hash. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct IndirectAssignmentCert { + /// A block hash where the candidate appears. + pub block_hash: Hash, + /// The validator index. + pub validator: ValidatorIndex, + /// The cert itself. + pub cert: AssignmentCert, + } + + /// A signed approval vote which references the candidate indirectly via the block. /// - /// The context used to produce bytes is [`RELAY_VRF_MODULO_CONTEXT`] - RelayVRFModulo { - /// The sample number used in this cert. - sample: u32, - }, - /// An assignment story based on the VRF that authorized the relay-chain block where the - /// candidate was included combined with the index of a particular core. + /// In practice, we have a look-up from block hash and candidate index to candidate hash, + /// so this can be transformed into a `SignedApprovalVote`. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct IndirectSignedApprovalVote { + /// A block hash where the candidate appears. + pub block_hash: Hash, + /// The index of the candidate in the list of candidates fully included as-of the block. + pub candidate_index: CandidateIndex, + /// The validator index. + pub validator: ValidatorIndex, + /// The signature by the validator. + pub signature: ValidatorSignature, + } + + /// Metadata about a block which is now live in the approval protocol. + #[derive(Debug)] + pub struct BlockApprovalMeta { + /// The hash of the block. + pub hash: Hash, + /// The number of the block. + pub number: BlockNumber, + /// The hash of the parent block. + pub parent_hash: Hash, + /// The candidates included by the block. + /// Note that these are not the same as the candidates that appear within the block body. + pub candidates: Vec, + /// The consensus slot of the block. + pub slot: Slot, + /// The session of the block. + pub session: SessionIndex, + } + + /// Errors that can occur during the approvals protocol. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum ApprovalError { + #[error("Schnorrkel signature error")] + SchnorrkelSignature(schnorrkel::errors::SignatureError), + #[error("Authority index {0} out of bounds")] + AuthorityOutOfBounds(usize), + } + + /// An unsafe VRF output. Provide BABE Epoch info to create a `RelayVRFStory`. + pub struct UnsafeVRFOutput { + vrf_output: VrfOutput, + slot: Slot, + authority_index: u32, + } + + impl UnsafeVRFOutput { + /// Get the slot. + pub fn slot(&self) -> Slot { + self.slot + } + + /// Compute the randomness associated with this VRF output. + pub fn compute_randomness( + self, + authorities: &[(babe_primitives::AuthorityId, babe_primitives::BabeAuthorityWeight)], + randomness: &babe_primitives::Randomness, + epoch_index: u64, + ) -> Result { + let author = match authorities.get(self.authority_index as usize) { + None => return Err(ApprovalError::AuthorityOutOfBounds(self.authority_index as _)), + Some(x) => &x.0, + }; + + let pubkey = schnorrkel::PublicKey::from_bytes(author.as_slice()) + .map_err(ApprovalError::SchnorrkelSignature)?; + + let transcript = + sp_consensus_babe::make_vrf_transcript(randomness, self.slot, epoch_index); + + let inout = self + .vrf_output + .0 + .attach_input_hash(&pubkey, transcript.0) + .map_err(ApprovalError::SchnorrkelSignature)?; + Ok(RelayVRFStory(inout.make_bytes(super::v1::RELAY_VRF_STORY_CONTEXT))) + } + } + + /// Extract the slot number and relay VRF from a header. /// - /// The context is [`RELAY_VRF_DELAY_CONTEXT`] - RelayVRFDelay { - /// The core index chosen in this cert. - core_index: CoreIndex, - }, -} + /// This fails if either there is no BABE `PreRuntime` digest or + /// the digest has type `SecondaryPlain`, which Substrate nodes do + /// not produce or accept anymore. + pub fn babe_unsafe_vrf_info(header: &Header) -> Option { + use babe_primitives::digests::CompatibleDigestItem; -/// A certification of assignment. -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -pub struct AssignmentCert { - /// The criterion which is claimed to be met by this cert. - pub kind: AssignmentCertKind, - /// The VRF signature showing the criterion is met. - pub vrf: VrfSignature, -} + for digest in &header.digest.logs { + if let Some(pre) = digest.as_babe_pre_digest() { + let slot = pre.slot(); + let authority_index = pre.authority_index(); -/// An assignment criterion which refers to the candidate under which the assignment is -/// relevant by block hash. -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -pub struct IndirectAssignmentCert { - /// A block hash where the candidate appears. - pub block_hash: Hash, - /// The validator index. - pub validator: ValidatorIndex, - /// The cert itself. - pub cert: AssignmentCert, -} + return pre.vrf_signature().map(|sig| UnsafeVRFOutput { + vrf_output: sig.output.clone(), + slot, + authority_index, + }) + } + } -/// A signed approval vote which references the candidate indirectly via the block. -/// -/// In practice, we have a look-up from block hash and candidate index to candidate hash, -/// so this can be transformed into a `SignedApprovalVote`. -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -pub struct IndirectSignedApprovalVote { - /// A block hash where the candidate appears. - pub block_hash: Hash, - /// The index of the candidate in the list of candidates fully included as-of the block. - pub candidate_index: CandidateIndex, - /// The validator index. - pub validator: ValidatorIndex, - /// The signature by the validator. - pub signature: ValidatorSignature, + None + } } -/// Metadata about a block which is now live in the approval protocol. -#[derive(Debug)] -pub struct BlockApprovalMeta { - /// The hash of the block. - pub hash: Hash, - /// The number of the block. - pub number: BlockNumber, - /// The hash of the parent block. - pub parent_hash: Hash, - /// The candidates included by the block. - /// Note that these are not the same as the candidates that appear within the block body. - pub candidates: Vec, - /// The consensus slot of the block. - pub slot: Slot, - /// The session of the block. - pub session: SessionIndex, -} +/// A list of primitives introduced by v2. +pub mod v2 { + use parity_scale_codec::{Decode, Encode}; + pub use sp_consensus_babe::{ + Randomness, Slot, VrfOutput, VrfProof, VrfSignature, VrfTranscript, + }; + use std::ops::BitOr; -/// Errors that can occur during the approvals protocol. -#[derive(Debug, thiserror::Error)] -#[allow(missing_docs)] -pub enum ApprovalError { - #[error("Schnorrkel signature error")] - SchnorrkelSignature(schnorrkel::errors::SignatureError), - #[error("Authority index {0} out of bounds")] - AuthorityOutOfBounds(usize), -} + use bitvec::{prelude::Lsb0, vec::BitVec}; + use polkadot_primitives::{CandidateIndex, CoreIndex, Hash, ValidatorIndex}; -/// An unsafe VRF output. Provide BABE Epoch info to create a `RelayVRFStory`. -pub struct UnsafeVRFOutput { - vrf_output: VrfOutput, - slot: Slot, - authority_index: u32, -} + /// A static context associated with producing randomness for a core. + pub const CORE_RANDOMNESS_CONTEXT: &[u8] = b"A&V CORE v2"; + /// A static context associated with producing randomness for v2 multi-core assignments. + pub const ASSIGNED_CORE_CONTEXT: &[u8] = b"A&V ASSIGNED v2"; + /// A static context used for all relay-vrf-modulo VRFs for v2 multi-core assignments. + pub const RELAY_VRF_MODULO_CONTEXT: &[u8] = b"A&V MOD v2"; + /// A read-only bitvec wrapper + #[derive(Clone, Debug, Encode, Decode, Hash, PartialEq, Eq)] + pub struct Bitfield(BitVec, std::marker::PhantomData); + + /// A `read-only`, `non-zero` bitfield. + /// Each 1 bit identifies a candidate by the bitfield bit index. + pub type CandidateBitfield = Bitfield; + /// A bitfield of core assignments. + pub type CoreBitfield = Bitfield; + + /// Errors that can occur when creating and manipulating bitfields. + #[derive(Debug)] + pub enum BitfieldError { + /// All bits are zero. + NullAssignment, + } + + /// A bit index in `Bitfield`. + #[cfg_attr(test, derive(PartialEq, Clone))] + pub struct BitIndex(pub usize); + + /// Helper trait to convert primitives to `BitIndex`. + pub trait AsBitIndex { + /// Returns the index of the corresponding bit in `Bitfield`. + fn as_bit_index(&self) -> BitIndex; + } + + impl Bitfield { + /// Returns the bit value at specified `index`. If `index` is greater than bitfield size, + /// returns `false`. + pub fn bit_at(&self, index: BitIndex) -> bool { + if self.0.len() <= index.0 { + false + } else { + self.0[index.0] + } + } + + /// Returns number of bits. + pub fn len(&self) -> usize { + self.0.len() + } + + /// Returns the number of 1 bits. + pub fn count_ones(&self) -> usize { + self.0.count_ones() + } + + /// Returns the index of the first 1 bit. + pub fn first_one(&self) -> Option { + self.0.first_one() + } + + /// Returns an iterator over inner bits. + pub fn iter_ones(&self) -> bitvec::slice::IterOnes { + self.0.iter_ones() + } -impl UnsafeVRFOutput { - /// Get the slot. - pub fn slot(&self) -> Slot { - self.slot + /// For testing purpose, we want a inner mutable ref. + #[cfg(test)] + pub fn inner_mut(&mut self) -> &mut BitVec { + &mut self.0 + } + + /// Returns the inner bitfield and consumes `self`. + pub fn into_inner(self) -> BitVec { + self.0 + } } - /// Compute the randomness associated with this VRF output. - pub fn compute_randomness( - self, - authorities: &[(babe_primitives::AuthorityId, babe_primitives::BabeAuthorityWeight)], - randomness: &babe_primitives::Randomness, - epoch_index: u64, - ) -> Result { - let author = match authorities.get(self.authority_index as usize) { - None => return Err(ApprovalError::AuthorityOutOfBounds(self.authority_index as _)), - Some(x) => &x.0, - }; + impl AsBitIndex for CandidateIndex { + fn as_bit_index(&self) -> BitIndex { + BitIndex(*self as usize) + } + } - let pubkey = schnorrkel::PublicKey::from_bytes(author.as_slice()) - .map_err(ApprovalError::SchnorrkelSignature)?; + impl AsBitIndex for CoreIndex { + fn as_bit_index(&self) -> BitIndex { + BitIndex(self.0 as usize) + } + } - let transcript = sp_consensus_babe::make_vrf_transcript(randomness, self.slot, epoch_index); + impl AsBitIndex for usize { + fn as_bit_index(&self) -> BitIndex { + BitIndex(*self) + } + } - let inout = self - .vrf_output - .0 - .attach_input_hash(&pubkey, transcript.0) - .map_err(ApprovalError::SchnorrkelSignature)?; - Ok(RelayVRFStory(inout.make_bytes(RELAY_VRF_STORY_CONTEXT))) + impl From for Bitfield + where + T: AsBitIndex, + { + fn from(value: T) -> Self { + Self( + { + let mut bv = bitvec::bitvec![u8, Lsb0; 0; value.as_bit_index().0 + 1]; + bv.set(value.as_bit_index().0, true); + bv + }, + Default::default(), + ) + } + } + + impl TryFrom> for Bitfield + where + T: Into>, + { + type Error = BitfieldError; + + fn try_from(mut value: Vec) -> Result { + if value.is_empty() { + return Err(BitfieldError::NullAssignment) + } + + let initial_bitfield = + value.pop().expect("Just checked above it's not empty; qed").into(); + + Ok(Self( + value.into_iter().fold(initial_bitfield.0, |initial_bitfield, element| { + let mut bitfield: Bitfield = element.into(); + bitfield + .0 + .resize(std::cmp::max(initial_bitfield.len(), bitfield.0.len()), false); + bitfield.0.bitor(initial_bitfield) + }), + Default::default(), + )) + } + } + + /// Certificate is changed compared to `AssignmentCertKind`: + /// - introduced RelayVRFModuloCompact + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum AssignmentCertKindV2 { + /// Multiple assignment stories based on the VRF that authorized the relay-chain block + /// where the candidates were included. + /// + /// The context is [`v2::RELAY_VRF_MODULO_CONTEXT`] + #[codec(index = 0)] + RelayVRFModuloCompact { + /// A bitfield representing the core indices claimed by this assignment. + core_bitfield: CoreBitfield, + }, + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with the index of a particular core. + /// + /// The context is [`v2::RELAY_VRF_DELAY_CONTEXT`] + #[codec(index = 1)] + RelayVRFDelay { + /// The core index chosen in this cert. + core_index: CoreIndex, + }, + /// Deprectated assignment. Soon to be removed. + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with a sample number. + /// + /// The context used to produce bytes is [`v1::RELAY_VRF_MODULO_CONTEXT`] + #[codec(index = 2)] + RelayVRFModulo { + /// The sample number used in this cert. + sample: u32, + }, + } + + /// A certification of assignment. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct AssignmentCertV2 { + /// The criterion which is claimed to be met by this cert. + pub kind: AssignmentCertKindV2, + /// The VRF showing the criterion is met. + pub vrf: VrfSignature, + } + + impl From for AssignmentCertV2 { + fn from(cert: super::v1::AssignmentCert) -> Self { + Self { + kind: match cert.kind { + super::v1::AssignmentCertKind::RelayVRFDelay { core_index } => + AssignmentCertKindV2::RelayVRFDelay { core_index }, + super::v1::AssignmentCertKind::RelayVRFModulo { sample } => + AssignmentCertKindV2::RelayVRFModulo { sample }, + }, + vrf: cert.vrf, + } + } } -} -/// Extract the slot number and relay VRF from a header. -/// -/// This fails if either there is no BABE `PreRuntime` digest or -/// the digest has type `SecondaryPlain`, which Substrate nodes do -/// not produce or accept anymore. -pub fn babe_unsafe_vrf_info(header: &Header) -> Option { - use babe_primitives::digests::CompatibleDigestItem; - - for digest in &header.digest.logs { - if let Some(pre) = digest.as_babe_pre_digest() { - let slot = pre.slot(); - let authority_index = pre.authority_index(); - - return pre.vrf_signature().map(|sig| UnsafeVRFOutput { - vrf_output: sig.output.clone(), - slot, - authority_index, + /// Errors that can occur when trying to convert to/from assignment v1/v2 + #[derive(Debug)] + pub enum AssignmentConversionError { + /// Assignment certificate is not supported in v1. + CertificateNotSupported, + } + + impl TryFrom for super::v1::AssignmentCert { + type Error = AssignmentConversionError; + fn try_from(cert: AssignmentCertV2) -> Result { + Ok(Self { + kind: match cert.kind { + AssignmentCertKindV2::RelayVRFDelay { core_index } => + super::v1::AssignmentCertKind::RelayVRFDelay { core_index }, + AssignmentCertKindV2::RelayVRFModulo { sample } => + super::v1::AssignmentCertKind::RelayVRFModulo { sample }, + // Not supported + _ => return Err(AssignmentConversionError::CertificateNotSupported), + }, + vrf: cert.vrf, }) } } - None + /// An assignment criterion which refers to the candidate under which the assignment is + /// relevant by block hash. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct IndirectAssignmentCertV2 { + /// A block hash where the candidate appears. + pub block_hash: Hash, + /// The validator index. + pub validator: ValidatorIndex, + /// The cert itself. + pub cert: AssignmentCertV2, + } + + impl From for IndirectAssignmentCertV2 { + fn from(indirect_cert: super::v1::IndirectAssignmentCert) -> Self { + Self { + block_hash: indirect_cert.block_hash, + validator: indirect_cert.validator, + cert: indirect_cert.cert.into(), + } + } + } + + impl TryFrom for super::v1::IndirectAssignmentCert { + type Error = AssignmentConversionError; + fn try_from( + indirect_cert: IndirectAssignmentCertV2, + ) -> Result { + Ok(Self { + block_hash: indirect_cert.block_hash, + validator: indirect_cert.validator, + cert: indirect_cert.cert.try_into()?, + }) + } + } +} + +#[cfg(test)] +mod test { + use super::v2::{BitIndex, Bitfield}; + + use polkadot_primitives::{CandidateIndex, CoreIndex}; + + #[test] + fn test_assignment_bitfield_from_vec() { + let candidate_indices = vec![1u32, 7, 3, 10, 45, 8, 200, 2]; + let max_index = *candidate_indices.iter().max().unwrap(); + let bitfield = Bitfield::try_from(candidate_indices.clone()).unwrap(); + let candidate_indices = + candidate_indices.into_iter().map(|i| BitIndex(i as usize)).collect::>(); + + // Test 1 bits. + for index in candidate_indices.clone() { + assert!(bitfield.bit_at(index)); + } + + // Test 0 bits. + for index in 0..max_index { + if candidate_indices.contains(&BitIndex(index as usize)) { + continue + } + assert!(!bitfield.bit_at(BitIndex(index as usize))); + } + } + + #[test] + fn test_assignment_bitfield_invariant_msb() { + let core_indices = vec![CoreIndex(1), CoreIndex(3), CoreIndex(10), CoreIndex(20)]; + let mut bitfield = Bitfield::try_from(core_indices.clone()).unwrap(); + assert!(bitfield.inner_mut().pop().unwrap()); + + for i in 0..1024 { + assert!(Bitfield::try_from(CoreIndex(i)).unwrap().inner_mut().pop().unwrap()); + assert!(Bitfield::try_from(i).unwrap().inner_mut().pop().unwrap()); + } + } + + #[test] + fn test_assignment_bitfield_basic() { + let bitfield = Bitfield::try_from(CoreIndex(0)).unwrap(); + assert!(bitfield.bit_at(BitIndex(0))); + assert!(!bitfield.bit_at(BitIndex(1))); + assert_eq!(bitfield.len(), 1); + + let mut bitfield = Bitfield::try_from(20 as CandidateIndex).unwrap(); + assert!(bitfield.bit_at(BitIndex(20))); + assert_eq!(bitfield.inner_mut().count_ones(), 1); + assert_eq!(bitfield.len(), 21); + } } diff --git a/polkadot/node/service/src/parachains_db/mod.rs b/polkadot/node/service/src/parachains_db/mod.rs index 519afbe0ccd1..92f3f167f22f 100644 --- a/polkadot/node/service/src/parachains_db/mod.rs +++ b/polkadot/node/service/src/parachains_db/mod.rs @@ -41,7 +41,15 @@ pub(crate) mod columns { pub const COL_SESSION_WINDOW_DATA: u32 = 5; } + // Version 4 only changed structures in approval voting, so we can re-export the v4 definitions. pub mod v3 { + pub use super::v4::{ + COL_APPROVAL_DATA, COL_AVAILABILITY_DATA, COL_AVAILABILITY_META, + COL_CHAIN_SELECTION_DATA, COL_DISPUTE_COORDINATOR_DATA, NUM_COLUMNS, ORDERED_COL, + }; + } + + pub mod v4 { pub const NUM_COLUMNS: u32 = 5; pub const COL_AVAILABILITY_DATA: u32 = 0; pub const COL_AVAILABILITY_META: u32 = 1; @@ -73,14 +81,14 @@ pub struct ColumnsConfig { /// The real columns used by the parachains DB. #[cfg(any(test, feature = "full-node"))] pub const REAL_COLUMNS: ColumnsConfig = ColumnsConfig { - col_availability_data: columns::v3::COL_AVAILABILITY_DATA, - col_availability_meta: columns::v3::COL_AVAILABILITY_META, - col_approval_data: columns::v3::COL_APPROVAL_DATA, - col_chain_selection_data: columns::v3::COL_CHAIN_SELECTION_DATA, - col_dispute_coordinator_data: columns::v3::COL_DISPUTE_COORDINATOR_DATA, + col_availability_data: columns::v4::COL_AVAILABILITY_DATA, + col_availability_meta: columns::v4::COL_AVAILABILITY_META, + col_approval_data: columns::v4::COL_APPROVAL_DATA, + col_chain_selection_data: columns::v4::COL_CHAIN_SELECTION_DATA, + col_dispute_coordinator_data: columns::v4::COL_DISPUTE_COORDINATOR_DATA, }; -#[derive(PartialEq)] +#[derive(PartialEq, Copy, Clone)] pub(crate) enum DatabaseKind { ParityDB, RocksDB, @@ -125,28 +133,28 @@ pub fn open_creating_rocksdb( let path = root.join("parachains").join("db"); - let mut db_config = DatabaseConfig::with_columns(columns::v3::NUM_COLUMNS); + let mut db_config = DatabaseConfig::with_columns(columns::v4::NUM_COLUMNS); let _ = db_config .memory_budget - .insert(columns::v3::COL_AVAILABILITY_DATA, cache_sizes.availability_data); + .insert(columns::v4::COL_AVAILABILITY_DATA, cache_sizes.availability_data); let _ = db_config .memory_budget - .insert(columns::v3::COL_AVAILABILITY_META, cache_sizes.availability_meta); + .insert(columns::v4::COL_AVAILABILITY_META, cache_sizes.availability_meta); let _ = db_config .memory_budget - .insert(columns::v3::COL_APPROVAL_DATA, cache_sizes.approval_data); + .insert(columns::v4::COL_APPROVAL_DATA, cache_sizes.approval_data); let path_str = path .to_str() .ok_or_else(|| other_io_error(format!("Bad database path: {:?}", path)))?; std::fs::create_dir_all(&path_str)?; - upgrade::try_upgrade_db(&path, DatabaseKind::RocksDB)?; + upgrade::try_upgrade_db(&path, DatabaseKind::RocksDB, upgrade::CURRENT_VERSION)?; let db = Database::open(&db_config, &path_str)?; let db = polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new( db, - columns::v3::ORDERED_COL, + columns::v4::ORDERED_COL, ); Ok(Arc::new(db)) @@ -164,14 +172,14 @@ pub fn open_creating_paritydb( .ok_or_else(|| other_io_error(format!("Bad database path: {:?}", path)))?; std::fs::create_dir_all(&path_str)?; - upgrade::try_upgrade_db(&path, DatabaseKind::ParityDB)?; + upgrade::try_upgrade_db(&path, DatabaseKind::ParityDB, upgrade::CURRENT_VERSION)?; let db = parity_db::Db::open_or_create(&upgrade::paritydb_version_3_config(&path)) .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?; let db = polkadot_node_subsystem_util::database::paritydb_impl::DbAdapter::new( db, - columns::v3::ORDERED_COL, + columns::v4::ORDERED_COL, ); Ok(Arc::new(db)) } diff --git a/polkadot/node/service/src/parachains_db/upgrade.rs b/polkadot/node/service/src/parachains_db/upgrade.rs index 54ef97afd71c..b99f885176b0 100644 --- a/polkadot/node/service/src/parachains_db/upgrade.rs +++ b/polkadot/node/service/src/parachains_db/upgrade.rs @@ -22,13 +22,17 @@ use std::{ str::FromStr, }; +use polkadot_node_core_approval_voting::approval_db::v2::{ + migration_helpers::v1_to_v2, Config as ApprovalDbConfig, +}; type Version = u32; /// Version file name. const VERSION_FILE_NAME: &'static str = "parachain_db_version"; /// Current db version. -const CURRENT_VERSION: Version = 3; +/// Version 4 changes approval db format for `OurAssignment`. +pub(crate) const CURRENT_VERSION: Version = 4; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -38,6 +42,10 @@ pub enum Error { CorruptedVersionFile, #[error("Parachains DB has a future version (expected {current:?}, found {got:?})")] FutureVersion { current: Version, got: Version }, + #[error("Parachain DB migration failed")] + MigrationFailed, + #[error("Parachain DB migration would take forever")] + MigrationLoop, } impl From for io::Error { @@ -49,10 +57,42 @@ impl From for io::Error { } } -/// Try upgrading parachain's database to the current version. -pub(crate) fn try_upgrade_db(db_path: &Path, db_kind: DatabaseKind) -> Result<(), Error> { +/// Try upgrading parachain's database to a target version. +pub(crate) fn try_upgrade_db( + db_path: &Path, + db_kind: DatabaseKind, + target_version: Version, +) -> Result<(), Error> { + // Ensure we don't loop forever below befcause of a bug. + const MAX_MIGRATIONS: u32 = 30; + + #[cfg(test)] + remove_file_lock(&db_path); + + // Loop migrations until we reach the target version. + for _ in 0..MAX_MIGRATIONS { + let version = try_upgrade_db_to_next_version(db_path, db_kind)?; + + #[cfg(test)] + remove_file_lock(&db_path); + + if version == target_version { + return Ok(()) + } + } + + Err(Error::MigrationLoop) +} + +/// Try upgrading parachain's database to the next version. +/// If successfull, it returns the current version. +pub(crate) fn try_upgrade_db_to_next_version( + db_path: &Path, + db_kind: DatabaseKind, +) -> Result { let is_empty = db_path.read_dir().map_or(true, |mut d| d.next().is_none()); - if !is_empty { + + let new_version = if !is_empty { match get_db_version(db_path)? { // 0 -> 1 migration Some(0) => migrate_from_version_0_to_1(db_path, db_kind)?, @@ -60,21 +100,26 @@ pub(crate) fn try_upgrade_db(db_path: &Path, db_kind: DatabaseKind) -> Result<() Some(1) => migrate_from_version_1_to_2(db_path, db_kind)?, // 2 -> 3 migration Some(2) => migrate_from_version_2_to_3(db_path, db_kind)?, + // 3 -> 4 migration + Some(3) => migrate_from_version_3_to_4(db_path, db_kind)?, // Already at current version, do nothing. - Some(CURRENT_VERSION) => (), + Some(CURRENT_VERSION) => CURRENT_VERSION, // This is an arbitrary future version, we don't handle it. Some(v) => return Err(Error::FutureVersion { current: CURRENT_VERSION, got: v }), // No version file. For `RocksDB` we dont need to do anything. - None if db_kind == DatabaseKind::RocksDB => (), + None if db_kind == DatabaseKind::RocksDB => CURRENT_VERSION, // No version file. `ParityDB` did not previously have a version defined. // We handle this as a `0 -> 1` migration. None if db_kind == DatabaseKind::ParityDB => migrate_from_version_0_to_1(db_path, db_kind)?, None => unreachable!(), } - } + } else { + CURRENT_VERSION + }; - update_version(db_path) + update_version(db_path, new_version)?; + Ok(new_version) } /// Reads current database version from the file at given path. @@ -91,9 +136,9 @@ fn get_db_version(path: &Path) -> Result, Error> { /// Writes current database version to the file. /// Creates a new file if the version file does not exist yet. -fn update_version(path: &Path) -> Result<(), Error> { +fn update_version(path: &Path, new_version: Version) -> Result<(), Error> { fs::create_dir_all(path)?; - fs::write(version_file_path(path), CURRENT_VERSION.to_string()).map_err(Into::into) + fs::write(version_file_path(path), new_version.to_string()).map_err(Into::into) } /// Returns the version file path. @@ -103,7 +148,7 @@ fn version_file_path(path: &Path) -> PathBuf { file_path } -fn migrate_from_version_0_to_1(path: &Path, db_kind: DatabaseKind) -> Result<(), Error> { +fn migrate_from_version_0_to_1(path: &Path, db_kind: DatabaseKind) -> Result { gum::info!(target: LOG_TARGET, "Migrating parachains db from version 0 to version 1 ..."); match db_kind { @@ -116,7 +161,7 @@ fn migrate_from_version_0_to_1(path: &Path, db_kind: DatabaseKind) -> Result<(), }) } -fn migrate_from_version_1_to_2(path: &Path, db_kind: DatabaseKind) -> Result<(), Error> { +fn migrate_from_version_1_to_2(path: &Path, db_kind: DatabaseKind) -> Result { gum::info!(target: LOG_TARGET, "Migrating parachains db from version 1 to version 2 ..."); match db_kind { @@ -129,7 +174,48 @@ fn migrate_from_version_1_to_2(path: &Path, db_kind: DatabaseKind) -> Result<(), }) } -fn migrate_from_version_2_to_3(path: &Path, db_kind: DatabaseKind) -> Result<(), Error> { +// Migrade approval voting database. `OurAssignment` has been changed to support the v2 assignments. +// As these are backwards compatible, we'll convert the old entries in the new format. +fn migrate_from_version_3_to_4(path: &Path, db_kind: DatabaseKind) -> Result { + gum::info!(target: LOG_TARGET, "Migrating parachains db from version 3 to version 4 ..."); + use polkadot_node_subsystem_util::database::{ + kvdb_impl::DbAdapter as RocksDbAdapter, paritydb_impl::DbAdapter as ParityDbAdapter, + }; + use std::sync::Arc; + + let approval_db_config = + ApprovalDbConfig { col_approval_data: super::REAL_COLUMNS.col_approval_data }; + + let _result = match db_kind { + DatabaseKind::ParityDB => { + let db = ParityDbAdapter::new( + parity_db::Db::open(&paritydb_version_3_config(path)) + .map_err(|e| other_io_error(format!("Error opening db {:?}", e)))?, + super::columns::v3::ORDERED_COL, + ); + + v1_to_v2(Arc::new(db), approval_db_config).map_err(|_| Error::MigrationFailed)?; + }, + DatabaseKind::RocksDB => { + let db_path = path + .to_str() + .ok_or_else(|| super::other_io_error("Invalid database path".into()))?; + let db_cfg = + kvdb_rocksdb::DatabaseConfig::with_columns(super::columns::v3::NUM_COLUMNS); + let db = RocksDbAdapter::new( + kvdb_rocksdb::Database::open(&db_cfg, db_path)?, + &super::columns::v3::ORDERED_COL, + ); + + v1_to_v2(Arc::new(db), approval_db_config).map_err(|_| Error::MigrationFailed)?; + }, + }; + + gum::info!(target: LOG_TARGET, "Migration complete! "); + Ok(CURRENT_VERSION) +} + +fn migrate_from_version_2_to_3(path: &Path, db_kind: DatabaseKind) -> Result { gum::info!(target: LOG_TARGET, "Migrating parachains db from version 2 to version 3 ..."); match db_kind { DatabaseKind::ParityDB => paritydb_migrate_from_version_2_to_3(path), @@ -143,7 +229,7 @@ fn migrate_from_version_2_to_3(path: &Path, db_kind: DatabaseKind) -> Result<(), /// Migration from version 0 to version 1: /// * the number of columns has changed from 3 to 5; -fn rocksdb_migrate_from_version_0_to_1(path: &Path) -> Result<(), Error> { +fn rocksdb_migrate_from_version_0_to_1(path: &Path) -> Result { use kvdb_rocksdb::{Database, DatabaseConfig}; let db_path = path @@ -155,12 +241,12 @@ fn rocksdb_migrate_from_version_0_to_1(path: &Path) -> Result<(), Error> { db.add_column()?; db.add_column()?; - Ok(()) + Ok(1) } /// Migration from version 1 to version 2: /// * the number of columns has changed from 5 to 6; -fn rocksdb_migrate_from_version_1_to_2(path: &Path) -> Result<(), Error> { +fn rocksdb_migrate_from_version_1_to_2(path: &Path) -> Result { use kvdb_rocksdb::{Database, DatabaseConfig}; let db_path = path @@ -171,10 +257,10 @@ fn rocksdb_migrate_from_version_1_to_2(path: &Path) -> Result<(), Error> { db.add_column()?; - Ok(()) + Ok(2) } -fn rocksdb_migrate_from_version_2_to_3(path: &Path) -> Result<(), Error> { +fn rocksdb_migrate_from_version_2_to_3(path: &Path) -> Result { use kvdb_rocksdb::{Database, DatabaseConfig}; let db_path = path @@ -185,7 +271,7 @@ fn rocksdb_migrate_from_version_2_to_3(path: &Path) -> Result<(), Error> { db.remove_last_column()?; - Ok(()) + Ok(3) } // This currently clears columns which had their configs altered between versions. @@ -249,7 +335,7 @@ fn paritydb_fix_columns( pub(crate) fn paritydb_version_1_config(path: &Path) -> parity_db::Options { let mut options = parity_db::Options::with_columns(&path, super::columns::v1::NUM_COLUMNS as u8); - for i in columns::v3::ORDERED_COL { + for i in columns::v4::ORDERED_COL { options.columns[*i as usize].btree_index = true; } @@ -260,7 +346,7 @@ pub(crate) fn paritydb_version_1_config(path: &Path) -> parity_db::Options { pub(crate) fn paritydb_version_2_config(path: &Path) -> parity_db::Options { let mut options = parity_db::Options::with_columns(&path, super::columns::v2::NUM_COLUMNS as u8); - for i in columns::v3::ORDERED_COL { + for i in columns::v4::ORDERED_COL { options.columns[*i as usize].btree_index = true; } @@ -278,48 +364,160 @@ pub(crate) fn paritydb_version_3_config(path: &Path) -> parity_db::Options { options } +/// Database configuration for version 0. This is useful just for testing. +#[cfg(test)] +pub(crate) fn paritydb_version_0_config(path: &Path) -> parity_db::Options { + let mut options = + parity_db::Options::with_columns(&path, super::columns::v0::NUM_COLUMNS as u8); + options.columns[super::columns::v4::COL_AVAILABILITY_META as usize].btree_index = true; + + options +} + /// Migration from version 0 to version 1. /// Cases covered: /// - upgrading from v0.9.23 or earlier -> the `dispute coordinator column` was changed /// - upgrading from v0.9.24+ -> this is a no op assuming the DB has been manually fixed as per /// release notes -fn paritydb_migrate_from_version_0_to_1(path: &Path) -> Result<(), Error> { +fn paritydb_migrate_from_version_0_to_1(path: &Path) -> Result { // Delete the `dispute coordinator` column if needed (if column configuration is changed). paritydb_fix_columns( path, paritydb_version_1_config(path), - vec![super::columns::v3::COL_DISPUTE_COORDINATOR_DATA], + vec![super::columns::v4::COL_DISPUTE_COORDINATOR_DATA], )?; - Ok(()) + Ok(1) } /// Migration from version 1 to version 2: /// - add a new column for session information storage -fn paritydb_migrate_from_version_1_to_2(path: &Path) -> Result<(), Error> { +fn paritydb_migrate_from_version_1_to_2(path: &Path) -> Result { let mut options = paritydb_version_1_config(path); // Adds the session info column. parity_db::Db::add_column(&mut options, Default::default()) .map_err(|e| other_io_error(format!("Error adding column {:?}", e)))?; - Ok(()) + Ok(2) } /// Migration from version 2 to version 3: /// - drop the column used by `RollingSessionWindow` -fn paritydb_migrate_from_version_2_to_3(path: &Path) -> Result<(), Error> { +fn paritydb_migrate_from_version_2_to_3(path: &Path) -> Result { parity_db::Db::drop_last_column(&mut paritydb_version_2_config(path)) .map_err(|e| other_io_error(format!("Error removing COL_SESSION_WINDOW_DATA {:?}", e)))?; - Ok(()) + Ok(3) +} + +/// Remove the lock file. If file is locked, it will wait up to 1s. +#[cfg(test)] +pub fn remove_file_lock(path: &std::path::Path) { + use std::{io::ErrorKind, thread::sleep, time::Duration}; + + let mut lock_path = std::path::PathBuf::from(path); + lock_path.push("lock"); + + for _ in 0..10 { + let result = std::fs::remove_file(lock_path.as_path()); + match result { + Err(error) => match error.kind() { + ErrorKind::WouldBlock => { + sleep(Duration::from_millis(100)); + continue + }, + _ => return, + }, + Ok(_) => {}, + } + } + + unreachable!("Database is locked, waited 1s for lock file: {:?}", lock_path); } #[cfg(test)] mod tests { use super::{ - columns::{v2::COL_SESSION_WINDOW_DATA, v3::*}, + columns::{v2::COL_SESSION_WINDOW_DATA, v4::*}, *, }; + use polkadot_node_core_approval_voting::approval_db::v2::migration_helpers::v1_to_v2_fill_test_data; + + #[test] + fn test_paritydb_migrate_0_to_1() { + use parity_db::Db; + + let db_dir = tempfile::tempdir().unwrap(); + let path = db_dir.path(); + { + let db = Db::open_or_create(&paritydb_version_0_config(&path)).unwrap(); + + db.commit(vec![( + COL_AVAILABILITY_META as u8, + b"5678".to_vec(), + Some(b"somevalue".to_vec()), + )]) + .unwrap(); + } + + try_upgrade_db(&path, DatabaseKind::ParityDB, 1).unwrap(); + + let db = Db::open(&paritydb_version_1_config(&path)).unwrap(); + assert_eq!( + db.get(COL_AVAILABILITY_META as u8, b"5678").unwrap(), + Some("somevalue".as_bytes().to_vec()) + ); + } + + #[test] + fn test_paritydb_migrate_1_to_2() { + use parity_db::Db; + + let db_dir = tempfile::tempdir().unwrap(); + let path = db_dir.path(); + + // We need to properly set db version for upgrade to work. + fs::write(version_file_path(path), "1").expect("Failed to write DB version"); + + { + let db = Db::open_or_create(&paritydb_version_1_config(&path)).unwrap(); + + // Write some dummy data + db.commit(vec![( + COL_DISPUTE_COORDINATOR_DATA as u8, + b"1234".to_vec(), + Some(b"somevalue".to_vec()), + )]) + .unwrap(); + + assert_eq!(db.num_columns(), columns::v1::NUM_COLUMNS as u8); + } + + try_upgrade_db(&path, DatabaseKind::ParityDB, 2).unwrap(); + + let db = Db::open(&paritydb_version_2_config(&path)).unwrap(); + + assert_eq!(db.num_columns(), columns::v2::NUM_COLUMNS as u8); + + assert_eq!( + db.get(COL_DISPUTE_COORDINATOR_DATA as u8, b"1234").unwrap(), + Some("somevalue".as_bytes().to_vec()) + ); + + // Test we can write the new column. + db.commit(vec![( + COL_SESSION_WINDOW_DATA as u8, + b"1337".to_vec(), + Some(b"0xdeadb00b".to_vec()), + )]) + .unwrap(); + + // Read back data from new column. + assert_eq!( + db.get(COL_SESSION_WINDOW_DATA as u8, b"1337").unwrap(), + Some("0xdeadb00b".as_bytes().to_vec()) + ); + } #[test] fn test_rocksdb_migrate_1_to_2() { @@ -338,7 +536,7 @@ mod tests { // We need to properly set db version for upgrade to work. fs::write(version_file_path(db_dir.path()), "1").expect("Failed to write DB version"); { - let db = DbAdapter::new(db, columns::v3::ORDERED_COL); + let db = DbAdapter::new(db, columns::v4::ORDERED_COL); db.write(DBTransaction { ops: vec![DBOp::Insert { col: COL_DISPUTE_COORDINATOR_DATA, @@ -349,14 +547,14 @@ mod tests { .unwrap(); } - try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB).unwrap(); + try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB, 2).unwrap(); let db_cfg = DatabaseConfig::with_columns(super::columns::v2::NUM_COLUMNS); let db = Database::open(&db_cfg, db_path).unwrap(); assert_eq!(db.num_columns(), super::columns::v2::NUM_COLUMNS); - let db = DbAdapter::new(db, columns::v3::ORDERED_COL); + let db = DbAdapter::new(db, columns::v4::ORDERED_COL); assert_eq!( db.get(COL_DISPUTE_COORDINATOR_DATA, b"1234").unwrap(), @@ -380,6 +578,108 @@ mod tests { ); } + #[test] + fn test_migrate_3_to_4() { + use kvdb_rocksdb::{Database, DatabaseConfig}; + use polkadot_node_core_approval_voting::approval_db::v2::migration_helpers::v1_to_v2_sanity_check; + use polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter; + + let db_dir = tempfile::tempdir().unwrap(); + let db_path = db_dir.path().to_str().unwrap(); + let db_cfg: DatabaseConfig = DatabaseConfig::with_columns(super::columns::v3::NUM_COLUMNS); + + let approval_cfg = ApprovalDbConfig { + col_approval_data: crate::parachains_db::REAL_COLUMNS.col_approval_data, + }; + + // We need to properly set db version for upgrade to work. + fs::write(version_file_path(db_dir.path()), "3").expect("Failed to write DB version"); + let expected_candidates = { + let db = Database::open(&db_cfg, db_path.clone()).unwrap(); + assert_eq!(db.num_columns(), super::columns::v3::NUM_COLUMNS as u32); + let db = DbAdapter::new(db, columns::v3::ORDERED_COL); + // Fill the approval voting column with test data. + v1_to_v2_fill_test_data(std::sync::Arc::new(db), approval_cfg).unwrap() + }; + + try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB, 4).unwrap(); + + let db_cfg = DatabaseConfig::with_columns(super::columns::v4::NUM_COLUMNS); + let db = Database::open(&db_cfg, db_path).unwrap(); + let db = DbAdapter::new(db, columns::v4::ORDERED_COL); + + v1_to_v2_sanity_check(std::sync::Arc::new(db), approval_cfg, expected_candidates).unwrap(); + } + + #[test] + fn test_rocksdb_migrate_0_to_4() { + use kvdb_rocksdb::{Database, DatabaseConfig}; + + let db_dir = tempfile::tempdir().unwrap(); + let db_path = db_dir.path().to_str().unwrap(); + + fs::write(version_file_path(db_dir.path()), "0").expect("Failed to write DB version"); + try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB, 4).unwrap(); + + let db_cfg = DatabaseConfig::with_columns(super::columns::v4::NUM_COLUMNS); + let db = Database::open(&db_cfg, db_path).unwrap(); + + assert_eq!(db.num_columns(), columns::v4::NUM_COLUMNS); + } + + #[test] + fn test_paritydb_migrate_0_to_4() { + use parity_db::Db; + + let db_dir = tempfile::tempdir().unwrap(); + let path = db_dir.path(); + + // We need to properly set db version for upgrade to work. + fs::write(version_file_path(path), "0").expect("Failed to write DB version"); + + { + let db = Db::open_or_create(&paritydb_version_0_config(&path)).unwrap(); + assert_eq!(db.num_columns(), columns::v0::NUM_COLUMNS as u8); + } + + try_upgrade_db(&path, DatabaseKind::ParityDB, 4).unwrap(); + + let db = Db::open(&paritydb_version_3_config(&path)).unwrap(); + assert_eq!(db.num_columns(), columns::v4::NUM_COLUMNS as u8); + } + + #[test] + fn test_paritydb_migrate_2_to_3() { + use parity_db::Db; + + let db_dir = tempfile::tempdir().unwrap(); + let path = db_dir.path(); + let test_key = b"1337"; + + // We need to properly set db version for upgrade to work. + fs::write(version_file_path(path), "2").expect("Failed to write DB version"); + + { + let db = Db::open_or_create(&paritydb_version_2_config(&path)).unwrap(); + + // Write some dummy data + db.commit(vec![( + COL_SESSION_WINDOW_DATA as u8, + test_key.to_vec(), + Some(b"0xdeadb00b".to_vec()), + )]) + .unwrap(); + + assert_eq!(db.num_columns(), columns::v2::NUM_COLUMNS as u8); + } + + try_upgrade_db(&path, DatabaseKind::ParityDB, 3).unwrap(); + + let db = Db::open(&paritydb_version_3_config(&path)).unwrap(); + + assert_eq!(db.num_columns(), columns::v3::NUM_COLUMNS as u8); + } + #[test] fn test_rocksdb_migrate_2_to_3() { use kvdb_rocksdb::{Database, DatabaseConfig}; @@ -387,6 +687,7 @@ mod tests { let db_dir = tempfile::tempdir().unwrap(); let db_path = db_dir.path().to_str().unwrap(); let db_cfg = DatabaseConfig::with_columns(super::columns::v2::NUM_COLUMNS); + { let db = Database::open(&db_cfg, db_path).unwrap(); assert_eq!(db.num_columns(), super::columns::v2::NUM_COLUMNS as u32); @@ -395,7 +696,7 @@ mod tests { // We need to properly set db version for upgrade to work. fs::write(version_file_path(db_dir.path()), "2").expect("Failed to write DB version"); - try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB).unwrap(); + try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB, 3).unwrap(); let db_cfg = DatabaseConfig::with_columns(super::columns::v3::NUM_COLUMNS); let db = Database::open(&db_cfg, db_path).unwrap(); diff --git a/polkadot/node/service/src/tests.rs b/polkadot/node/service/src/tests.rs index 86119662d9bc..26c8083185d8 100644 --- a/polkadot/node/service/src/tests.rs +++ b/polkadot/node/service/src/tests.rs @@ -17,7 +17,7 @@ use super::{relay_chain_selection::*, *}; use futures::channel::oneshot::Receiver; -use polkadot_node_primitives::approval::VrfSignature; +use polkadot_node_primitives::approval::v2::VrfSignature; use polkadot_node_subsystem::messages::{AllMessages, BlockDescription}; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::TimeoutExt; diff --git a/polkadot/node/subsystem-types/Cargo.toml b/polkadot/node/subsystem-types/Cargo.toml index 4f2d753b7993..51337c2cbeb3 100644 --- a/polkadot/node/subsystem-types/Cargo.toml +++ b/polkadot/node/subsystem-types/Cargo.toml @@ -24,3 +24,4 @@ smallvec = "1.8.0" substrate-prometheus-endpoint = { path = "../../../substrate/utils/prometheus" } thiserror = "1.0.31" async-trait = "0.1.57" +bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index 8adc39eed56d..f561a0e28512 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -32,7 +32,10 @@ use polkadot_node_network_protocol::{ self as net_protocol, peer_set::PeerSet, request_response::Requests, PeerId, }; use polkadot_node_primitives::{ - approval::{BlockApprovalMeta, IndirectAssignmentCert, IndirectSignedApprovalVote}, + approval::{ + v1::{BlockApprovalMeta, IndirectSignedApprovalVote}, + v2::{CandidateBitfield, IndirectAssignmentCertV2}, + }, AvailableData, BabeEpoch, BlockWeight, CandidateVotes, CollationGenerationConfig, CollationSecondedSignal, DisputeMessage, DisputeStatus, ErasureChunk, PoV, SignedDisputeStatement, SignedFullStatement, SignedFullStatementWithPVD, SubmitCollationParams, @@ -833,6 +836,8 @@ pub enum AssignmentCheckError { InvalidCert(ValidatorIndex, String), #[error("Internal state mismatch: {0:?}, {1:?}")] Internal(Hash, CandidateHash), + #[error("Oversized candidate or core bitfield >= {0}")] + InvalidBitfield(usize), } /// The result type of [`ApprovalVotingMessage::CheckAndImportApproval`] request. @@ -898,8 +903,8 @@ pub enum ApprovalVotingMessage { /// Check if the assignment is valid and can be accepted by our view of the protocol. /// Should not be sent unless the block hash is known. CheckAndImportAssignment( - IndirectAssignmentCert, - CandidateIndex, + IndirectAssignmentCertV2, + CandidateBitfield, oneshot::Sender, ), /// Check if the approval vote is valid and can be accepted by our view of the @@ -934,7 +939,7 @@ pub enum ApprovalDistributionMessage { NewBlocks(Vec), /// Distribute an assignment cert from the local validator. The cert is assumed /// to be valid, relevant, and for the given relay-parent and validator index. - DistributeAssignment(IndirectAssignmentCert, CandidateIndex), + DistributeAssignment(IndirectAssignmentCertV2, CandidateBitfield), /// Distribute an approval vote for the local validator. The approval vote is assumed to be /// valid, relevant, and the corresponding approval already issued. /// If not, the subsystem is free to drop the message. diff --git a/polkadot/node/subsystem-util/src/reputation.rs b/polkadot/node/subsystem-util/src/reputation.rs index 89e3eb64df9b..35746dd5fef3 100644 --- a/polkadot/node/subsystem-util/src/reputation.rs +++ b/polkadot/node/subsystem-util/src/reputation.rs @@ -25,6 +25,7 @@ use std::{collections::HashMap, time::Duration}; /// Default delay for sending reputation changes pub const REPUTATION_CHANGE_INTERVAL: Duration = Duration::from_secs(30); +const LOG_TARGET: &'static str = "parachain::reputation-aggregator"; type BatchReputationChange = HashMap; @@ -75,6 +76,10 @@ impl ReputationAggregator { peer_id: PeerId, rep: UnifiedReputationChange, ) { + if rep.cost_or_benefit() < 0 { + gum::debug!(target: LOG_TARGET, peer = ?peer_id, ?rep, "Modify reputation"); + } + if (self.send_immediately_if)(rep) { self.single_send(sender, peer_id, rep).await; } else { diff --git a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md index 88744e50cf79..375b8f1f12b3 100644 --- a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md +++ b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md @@ -49,22 +49,27 @@ struct TrancheEntry { assignments: Vec<(ValidatorIndex, Tick)>, } -struct OurAssignment { - cert: AssignmentCert, - tranche: DelayTranche, - validator_index: ValidatorIndex, - triggered: bool, +pub struct OurAssignment { + /// Our assignment certificate. + cert: AssignmentCertV2, + /// The tranche for which the assignment refers to. + tranche: DelayTranche, + /// Our validator index for the session in which the candidates were included. + validator_index: ValidatorIndex, + /// Whether the assignment has been triggered already. + triggered: bool, } -struct ApprovalEntry { - tranches: Vec, // sorted ascending by tranche number. - backing_group: GroupIndex, - our_assignment: Option, - our_approval_sig: Option, - assignments: Bitfield, // n_validators bits - approved: bool, +pub struct ApprovalEntry { + tranches: Vec, // sorted ascending by tranche number. + backing_group: GroupIndex, + our_assignment: Option, + our_approval_sig: Option, + assigned_validators: Bitfield, // `n_validators` bits. + approved: bool, } + struct CandidateEntry { candidate: CandidateReceipt, session: SessionIndex, @@ -200,6 +205,8 @@ On receiving a `ApprovalVotingMessage::CheckAndImportAssignment` message, we che * Determine the claimed core index by looking up the candidate with given index in `block_entry.candidates`. Return `AssignmentCheckResult::Bad` if missing. * Check the assignment cert * If the cert kind is `RelayVRFModulo`, then the certificate is valid as long as `sample < session_info.relay_vrf_samples` and the VRF is valid for the validator's key with the input `block_entry.relay_vrf_story ++ sample.encode()` as described with [the approvals protocol section](../../protocol-approval.md#assignment-criteria). We set `core_index = vrf.make_bytes().to_u32() % session_info.n_cores`. If the `BlockEntry` causes inclusion of a candidate at `core_index`, then this is a valid assignment for the candidate at `core_index` and has delay tranche 0. Otherwise, it can be ignored. + * If the cert kind is `RelayVRFModuloCompact`, then the certificate is valid as long as the VRF is valid for the validator's key with the input `block_entry.relay_vrf_story ++ relay_vrf_samples.encode()` as described with [the approvals protocol section](../../protocol-approval.md#assignment-criteria). We enforce that all `core_bitfield` indices are included in the set of the core indices sampled from the VRF Output. The assignment is considered a valid tranche0 assignment for all claimed candidates if all `core_bitfield` indices match the core indices where the claimed candidates were included at. + * If the cert kind is `RelayVRFDelay`, then we check if the VRF is valid for the validator's key with the input `block_entry.relay_vrf_story ++ cert.core_index.encode()` as described in [the approvals protocol section](../../protocol-approval.md#assignment-criteria). The cert can be ignored if the block did not cause inclusion of a candidate on that core index. Otherwise, this is a valid assignment for the included candidate. The delay tranche for the assignment is determined by reducing `(vrf.make_bytes().to_u64() % (session_info.n_delay_tranches + session_info.zeroth_delay_tranche_width)).saturating_sub(session_info.zeroth_delay_tranche_width)`. * We also check that the core index derived by the output is covered by the `VRFProof` by means of an auxiliary signature. * If the delay tranche is too far in the future, return `AssignmentCheckResult::TooFarInFuture`. diff --git a/polkadot/roadmap/implementers-guide/src/protocol-approval.md b/polkadot/roadmap/implementers-guide/src/protocol-approval.md index 693822ce0797..aa513c16292d 100644 --- a/polkadot/roadmap/implementers-guide/src/protocol-approval.md +++ b/polkadot/roadmap/implementers-guide/src/protocol-approval.md @@ -98,12 +98,14 @@ We want checkers for candidate equivocations that lie outside our preferred rela Assignment criteria compute actual assignments using stories and the validators' secret approval assignment key. Assignment criteria output a `Position` consisting of both a `ParaId` to be checked, as well as a precedence `DelayTranche` for when the assignment becomes valid. -Assignment criteria come in three flavors, `RelayVRFModulo`, `RelayVRFDelay` and `RelayEquivocation`. Among these, both `RelayVRFModulo` and `RelayVRFDelay` run a VRF whose input is the output of a `RelayVRFStory`, while `RelayEquivocation` runs a VRF whose input is the output of a `RelayEquivocationStory`. +Assignment criteria come in four flavors, `RelayVRFModuloCompact`, `RelayVRFDelay`, `RelayEquivocation` and the deprecated `RelayVRFModulo`. Among these, `RelayVRFModulo`, `RelayVRFModuloCompact` and `RelayVRFDelay` run a VRF whose input is the output of a `RelayVRFStory`, while `RelayEquivocation` runs a VRF whose input is the output of a `RelayEquivocationStory`. Among these, we have two distinct VRF output computations: `RelayVRFModulo` runs several distinct samples whose VRF input is the `RelayVRFStory` and the sample number. It computes the VRF output with `schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Core", reduces this number modulo the number of availability cores, and outputs the candidate just declared available by, and included by aka leaving, that availability core. We drop any samples that return no candidate because no candidate was leaving the sampled availability core in this relay chain block. We choose three samples initially, but we could make polkadot more secure and efficient by increasing this to four or five, and reducing the backing checks accordingly. All successful `RelayVRFModulo` samples are assigned delay tranche zero. +`RelayVRFModuloCompact` runs a single samples whose VRF input is the `RelayVRFStory` and the sample count. Similar to `RelayVRFModulo` introduces multiple core assignments for tranche zero. It computes the VRF output with `schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Core v2" and samples up to 160 bytes of the output as an array of `u32`. Then reduces each `u32` modulo the number of availability cores, and outputs up to `relay_vrf_modulo_samples` availability core indices. + There is no sampling process for `RelayVRFDelay` and `RelayEquivocation`. We instead run them on specific candidates and they compute a delay from their VRF output. `RelayVRFDelay` runs for all candidates included under, aka declared available by, a relay chain block, and inputs the associated VRF output via `RelayVRFStory`. `RelayEquivocation` runs only on candidate block equivocations, and inputs their block hashes via the `RelayEquivocation` story. `RelayVRFDelay` and `RelayEquivocation` both compute their output with `schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Tranche" and reduce the result modulo `num_delay_tranches + zeroth_delay_tranche_width`, and consolidate results 0 through `zeroth_delay_tranche_width` to be 0. In this way, they ensure the zeroth delay tranche has `zeroth_delay_tranche_width+1` times as many assignments as any other tranche. @@ -114,9 +116,9 @@ As future work (or TODO?), we should merge assignment notices with the same dela We track all validators' announced approval assignments for each candidate associated to each relay chain block, which tells us which validators were assigned to which candidates. -We permit at most one assignment per candidate per story per validator, so one validator could be assigned under both the `RelayVRFDelay` and `RelayEquivocation` criteria, but not under both `RelayVRFModulo` and `RelayVRFDelay` criteria, since those both use the same story. We permit only one approval vote per candidate per validator, which counts for any applicable criteria. +We permit at most one assignment per candidate per story per validator, so one validator could be assigned under both the `RelayVRFDelay` and `RelayEquivocation` criteria, but not under both `RelayVRFModulo/RelayVRFModuloCompact` and `RelayVRFDelay` criteria, since those both use the same story. We permit only one approval vote per candidate per validator, which counts for any applicable criteria. -We announce, and start checking for, our own assignments when the delay of their tranche is reached, but only if the tracker says the assignee candidate requires more approval checkers. We never announce an assignment we believe unnecessary because early announcements gives an adversary information. All delay tranche zero assignments always get announced, which includes all `RelayVRFModulo` assignments. +We announce, and start checking for, our own assignments when the delay of their tranche is reached, but only if the tracker says the assignee candidate requires more approval checkers. We never announce an assignment we believe unnecessary because early announcements gives an adversary information. All delay tranche zero assignments always get announced, which includes all `RelayVRFModulo` and `RelayVRFModuloCompact` assignments. In other words, if some candidate `C` needs more approval checkers by the time we reach round `t` then any validators with an assignment to `C` in delay tranche `t` gossip their send assignment notice for `C`, and begin reconstruction and validation for 'C. If however `C` reached enough assignments, then validators with later assignments skip announcing their assignments. @@ -164,7 +166,7 @@ We need the chain to win in this case, but doing this requires imposing an annoy ## Parameters -We prefer doing approval checkers assignments under `RelayVRFModulo` as opposed to `RelayVRFDelay` because `RelayVRFModulo` avoids giving individual checkers too many assignments and tranche zero assignments benefit security the most. We suggest assigning at least 16 checkers under `RelayVRFModulo` although assignment levels have never been properly analyzed. +We prefer doing approval checkers assignments under `RelayVRFModulo` or `RelayVRFModuloCompact` as opposed to `RelayVRFDelay` because `RelayVRFModulo` avoids giving individual checkers too many assignments and tranche zero assignments benefit security the most. We suggest assigning at least 16 checkers under `RelayVRFModulo` or `RelayVRFModuloCompact` although assignment levels have never been properly analyzed. Our delay criteria `RelayVRFDelay` and `RelayEquivocation` both have two primary paramaters, expected checkers per tranche and the zeroth delay tranche width. diff --git a/polkadot/roadmap/implementers-guide/src/types/approval.md b/polkadot/roadmap/implementers-guide/src/types/approval.md index b58e0a8187e1..2cc9b34cc700 100644 --- a/polkadot/roadmap/implementers-guide/src/types/approval.md +++ b/polkadot/roadmap/implementers-guide/src/types/approval.md @@ -20,6 +20,35 @@ enum AssignmentCertKind { } } +enum AssignmentCertKindV2 { + /// Multiple assignment stories based on the VRF that authorized the relay-chain block where the + /// candidates were included. + /// + /// The context is [`v2::RELAY_VRF_MODULO_CONTEXT`] + RelayVRFModuloCompact { + /// A bitfield representing the core indices claimed by this assignment. + core_bitfield: CoreBitfield, + }, + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with the index of a particular core. + /// + /// The context is [`v2::RELAY_VRF_DELAY_CONTEXT`] + RelayVRFDelay { + /// The core index chosen in this cert. + core_index: CoreIndex, + }, + /// Deprectated assignment. Soon to be removed. + /// + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with a sample number. + /// + /// The context used to produce bytes is [`v1::RELAY_VRF_MODULO_CONTEXT`] + RelayVRFModulo { + /// The sample number used in this cert. + sample: u32, + }, +} + struct AssignmentCert { // The criterion which is claimed to be met by this cert. kind: AssignmentCertKind, diff --git a/polkadot/roadmap/implementers-guide/src/types/runtime.md b/polkadot/roadmap/implementers-guide/src/types/runtime.md index 79da899bd35e..4b97409f8df3 100644 --- a/polkadot/roadmap/implementers-guide/src/types/runtime.md +++ b/polkadot/roadmap/implementers-guide/src/types/runtime.md @@ -55,7 +55,7 @@ struct HostConfiguration { pub zeroth_delay_tranche_width: u32, /// The number of validators needed to approve a block. pub needed_approvals: u32, - /// The number of samples to do of the RelayVRFModulo approval assignment criterion. + /// The number of samples to use in `RelayVRFModulo` or `RelayVRFModuloCompact` approval assignment criterions. pub relay_vrf_modulo_samples: u32, /// Total number of individual messages allowed in the parachain -> relay-chain message queue. pub max_upward_queue_count: u32, diff --git a/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml b/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml index 62e081d1de0e..1c8df44cbaa7 100644 --- a/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml +++ b/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml @@ -160,6 +160,34 @@ zombienet-tests-parachains-disputes-past-session: tags: - zombienet-polkadot-integration-test +zombienet-tests-parachains-max-tranche0-approvals: + stage: zombienet + image: "${ZOMBIENET_IMAGE}" + extends: + - .kubernetes-env + - .zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + variables: + RUN_IN_CONTAINER: "1" + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0005-parachains-max-tranche0.zndsl" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test zombienet-test-parachains-upgrade-smoke-test: stage: zombienet @@ -182,7 +210,7 @@ zombienet-test-parachains-upgrade-smoke-test: - echo "${GH_DIR}" - export DEBUG=zombie,zombie::network-node - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export COL_IMAGE="docker.io/parity/polkadot-collator:latest" # Use cumulus lastest image + - export COL_IMAGE="docker.io/parity/polkadot-parachain:latest" # Use cumulus lastest image script: - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh --github-remote-dir="${GH_DIR}" diff --git a/polkadot/zombienet_tests/functional/0005-parachains-max-tranche0.toml b/polkadot/zombienet_tests/functional/0005-parachains-max-tranche0.toml new file mode 100644 index 000000000000..ab7fa0195d13 --- /dev/null +++ b/polkadot/zombienet_tests/functional/0005-parachains-max-tranche0.toml @@ -0,0 +1,40 @@ +[settings] +timeout = 1000 +bootnode = true + +[relaychain.genesis.runtime.runtime_genesis_config.configuration.config] + max_validators_per_core = 1 + needed_approvals = 7 + relay_vrf_modulo_samples = 5 + +[relaychain] +default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" +chain = "rococo-local" +chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default-bootnode" +default_command = "polkadot" + +[relaychain.default_resources] +limits = { memory = "4G", cpu = "2" } +requests = { memory = "2G", cpu = "1" } + + [[relaychain.node_groups]] + name = "some-validator" + count = 8 + args = ["-lparachain=debug,runtime=debug"] + +{% for id in range(2000,2005) %} +[[parachains]] +id = {{id}} +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size={{10000*(id-1999)}} --pvf-complexity={{id - 1999}}" + [parachains.collator] + image = "{{COL_IMAGE}}" + name = "collator" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size={{10000*(id-1999)}}", "--parachain-id={{id}}", "--pvf-complexity={{id - 1999}}"] +{% endfor %} + +[types.Header] +number = "u64" +parent_hash = "Hash" +post_state = "Hash" diff --git a/polkadot/zombienet_tests/functional/0005-parachains-max-tranche0.zndsl b/polkadot/zombienet_tests/functional/0005-parachains-max-tranche0.zndsl new file mode 100644 index 000000000000..48a56f60c9bc --- /dev/null +++ b/polkadot/zombienet_tests/functional/0005-parachains-max-tranche0.zndsl @@ -0,0 +1,27 @@ +Description: Test if parachains make progress with most of approvals being tranch0 +Network: ./0005-parachains-max-tranche0.toml +Creds: config + +# Check authority status. +some-validator-0: reports node_roles is 4 +some-validator-1: reports node_roles is 4 +some-validator-3: reports node_roles is 4 +some-validator-4: reports node_roles is 4 +some-validator-5: reports node_roles is 4 +some-validator-6: reports node_roles is 4 +some-validator-7: reports node_roles is 4 + +some-validator-0: parachain 2000 block height is at least 5 within 180 seconds +some-validator-1: parachain 2001 block height is at least 5 within 180 seconds +some-validator-2: parachain 2002 block height is at least 5 within 180 seconds +some-validator-3: parachain 2003 block height is at least 5 within 180 seconds +some-validator-4: parachain 2004 block height is at least 5 within 180 seconds + +some-validator-0: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-1: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-2: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-3: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-4: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-5: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-6: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-7: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 diff --git a/polkadot/zombienet_tests/smoke/0002-parachains-upgrade-smoke-test.toml b/polkadot/zombienet_tests/smoke/0002-parachains-upgrade-smoke-test.toml index 0becb408550a..88b789f37fa1 100644 --- a/polkadot/zombienet_tests/smoke/0002-parachains-upgrade-smoke-test.toml +++ b/polkadot/zombienet_tests/smoke/0002-parachains-upgrade-smoke-test.toml @@ -31,7 +31,7 @@ cumulus_based = true [parachains.collator] name = "collator01" image = "{{COL_IMAGE}}" - command = "polkadot-collator" + command = "polkadot-parachain" [[parachains.collator.env]] name = "RUST_LOG" From d04c18239763fb50f29a49ddd789b7ed242a98d2 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 25 Aug 2023 19:15:34 +0300 Subject: [PATCH 002/192] cargo lock Signed-off-by: Andrei Sandu --- Cargo.lock | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 4b0028a64513..2d75973de493 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11617,9 +11617,11 @@ name = "polkadot-approval-distribution" version = "1.0.0" dependencies = [ "assert_matches", + "bitvec", "env_logger 0.9.3", "futures", "futures-timer", + "itertools 0.10.5", "log", "polkadot-node-jaeger", "polkadot-node-metrics", @@ -11926,10 +11928,13 @@ dependencies = [ "async-trait", "bitvec", "derive_more", + "env_logger 0.9.3", "futures", "futures-timer", + "itertools 0.10.5", "kvdb", "kvdb-memorydb", + "log", "lru 0.11.0", "merlin 2.0.1", "parity-scale-codec", @@ -12398,6 +12403,7 @@ dependencies = [ name = "polkadot-node-primitives" version = "1.0.0" dependencies = [ + "bitvec", "bounded-vec", "futures", "parity-scale-codec", @@ -12448,6 +12454,7 @@ name = "polkadot-node-subsystem-types" version = "1.0.0" dependencies = [ "async-trait", + "bitvec", "derive_more", "futures", "orchestra", From 341c7af51f91f981831b79629a9141b61a875e1c Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 25 Aug 2023 17:53:49 +0300 Subject: [PATCH 003/192] Approve multiple candidates with a single signature The pr migrates: - https://github.com/paritytech/polkadot/pull/7554 Signed-off-by: Alexandru Gheorghe --- .../src/blockchain_rpc_client.rs | 6 +- .../approval-voting/src/approval_checking.rs | 67 +- .../approval-voting/src/approval_db/v2/mod.rs | 34 +- .../src/approval_db/v2/tests.rs | 1 + .../node/core/approval-voting/src/criteria.rs | 2 +- .../node/core/approval-voting/src/import.rs | 3 + polkadot/node/core/approval-voting/src/lib.rs | 801 ++++++++++++++---- .../approval-voting/src/persisted_entries.rs | 133 ++- .../node/core/approval-voting/src/tests.rs | 577 ++++++++++++- .../node/core/approval-voting/src/time.rs | 165 +++- .../core/dispute-coordinator/src/import.rs | 33 +- .../dispute-coordinator/src/initialized.rs | 4 +- .../node/core/dispute-coordinator/src/lib.rs | 2 +- .../core/dispute-coordinator/src/tests.rs | 11 +- .../src/disputes/prioritized_selection/mod.rs | 2 +- polkadot/node/core/pvf/src/host.rs | 3 +- polkadot/node/core/runtime-api/src/cache.rs | 33 +- polkadot/node/core/runtime-api/src/lib.rs | 7 + polkadot/node/core/runtime-api/src/tests.rs | 17 +- .../network/approval-distribution/src/lib.rs | 576 ++++++++----- .../approval-distribution/src/metrics.rs | 160 ++++ .../approval-distribution/src/tests.rs | 211 +++-- .../network/availability-recovery/src/lib.rs | 3 +- .../network/protocol/src/grid_topology.rs | 21 +- polkadot/node/network/protocol/src/lib.rs | 7 +- polkadot/node/primitives/src/approval.rs | 57 +- .../node/primitives/src/disputes/message.rs | 2 +- polkadot/node/primitives/src/disputes/mod.rs | 23 +- polkadot/node/service/src/fake_runtime_api.rs | 13 +- polkadot/node/subsystem-types/src/messages.rs | 30 +- .../subsystem-types/src/runtime_client.rs | 21 +- polkadot/primitives/src/runtime_api.rs | 12 +- polkadot/primitives/src/v5/mod.rs | 73 +- polkadot/primitives/src/vstaging/mod.rs | 20 + .../src/node/approval/approval-voting.md | 35 +- .../src/protocol-approval.md | 6 + polkadot/runtime/kusama/src/lib.rs | 13 +- polkadot/runtime/parachains/src/builder.rs | 2 +- .../runtime/parachains/src/configuration.rs | 28 +- .../src/configuration/migration/v7.rs | 1 + .../src/configuration/migration/v8.rs | 5 +- .../parachains/src/configuration/tests.rs | 1 + polkadot/runtime/parachains/src/disputes.rs | 26 +- .../src/runtime_api_impl/vstaging.rs | 7 + polkadot/runtime/polkadot/src/lib.rs | 13 +- polkadot/runtime/rococo/src/lib.rs | 12 +- polkadot/runtime/westend/src/lib.rs | 13 +- polkadot/scripts/ci/gitlab/pipeline/build.yml | 2 +- .../scripts/ci/gitlab/pipeline/zombienet.yml | 30 + .../functional/0001-parachains-pvf.zndsl | 2 + .../functional/0002-parachains-disputes.toml | 4 + .../0006-approval-voting-coalescing.toml | 195 +++++ .../0006-approval-voting-coalescing.zndsl | 51 ++ 53 files changed, 2941 insertions(+), 635 deletions(-) create mode 100644 polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml create mode 100644 polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.zndsl diff --git a/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs b/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs index fc4d803002cb..0ff3de0b296d 100644 --- a/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs +++ b/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs @@ -23,7 +23,7 @@ use polkadot_core_primitives::{Block, BlockNumber, Hash, Header}; use polkadot_overseer::RuntimeApiSubsystemClient; use polkadot_primitives::{ slashing, - vstaging::{AsyncBackingParams, BackingState}, + vstaging::{ApprovalVotingParams, AsyncBackingParams, BackingState}, }; use sc_authority_discovery::{AuthorityDiscovery, Error as AuthorityDiscoveryError}; use sp_api::{ApiError, RuntimeApiInfo}; @@ -349,6 +349,10 @@ impl RuntimeApiSubsystemClient for BlockChainRpcClient { ) -> Result, ApiError> { Ok(self.rpc_client.parachain_host_staging_para_backing_state(at, para_id).await?) } + /// Approval voting configuration parameters + async fn approval_voting_params(&self, _at: Hash) -> Result { + Ok(ApprovalVotingParams { max_approval_coalesce_count: 1 }) + } } #[async_trait::async_trait] diff --git a/polkadot/node/core/approval-voting/src/approval_checking.rs b/polkadot/node/core/approval-voting/src/approval_checking.rs index 5d24ff164193..2f0ea4160ae1 100644 --- a/polkadot/node/core/approval-voting/src/approval_checking.rs +++ b/polkadot/node/core/approval-voting/src/approval_checking.rs @@ -64,7 +64,7 @@ pub enum RequiredTranches { } /// The result of a check. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq)] pub enum Check { /// The candidate is unapproved. Unapproved, @@ -372,19 +372,22 @@ pub fn tranches_to_approve( block_tick: Tick, no_show_duration: Tick, needed_approvals: usize, -) -> RequiredTranches { +) -> (RequiredTranches, usize) { let tick_now = tranche_now as Tick + block_tick; let n_validators = approval_entry.n_validators(); - let initial_state = State { - assignments: 0, - depth: 0, - covered: 0, - covering: needed_approvals, - uncovered: 0, - next_no_show: None, - last_assignment_tick: None, - }; + let initial_state = ( + State { + assignments: 0, + depth: 0, + covered: 0, + covering: needed_approvals, + uncovered: 0, + next_no_show: None, + last_assignment_tick: None, + }, + 0usize, + ); // The `ApprovalEntry` doesn't have any data for empty tranches. We still want to iterate over // these empty tranches, so we create an iterator to fill the gaps. @@ -395,7 +398,7 @@ pub fn tranches_to_approve( tranches_with_gaps_filled .scan(Some(initial_state), |state, (tranche, assignments)| { // The `Option` here is used for early exit. - let s = state.take()?; + let (s, prev_no_shows) = state.take()?; let clock_drift = s.clock_drift(no_show_duration); let drifted_tick_now = tick_now.saturating_sub(clock_drift); @@ -444,11 +447,11 @@ pub fn tranches_to_approve( RequiredTranches::Pending { .. } => { // Pending results are only interesting when they are the last result of the iterator // i.e. we never achieve a satisfactory level of assignment. - Some(s) + Some((s, no_shows + prev_no_shows)) } }; - Some(output) + Some((output, no_shows + prev_no_shows)) }) .last() .expect("the underlying iterator is infinite, starts at 0, and never exits early before tranche 1; qed") @@ -675,7 +678,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Exact { needed: 1, tolerated_missing: 0, @@ -715,7 +719,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 2, next_no_show: Some(block_tick + no_show_duration), @@ -759,7 +764,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 11, next_no_show: None, @@ -807,7 +813,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 1, next_no_show: None, @@ -826,7 +833,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 1, next_no_show: None, @@ -879,7 +887,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Exact { needed: 1, tolerated_missing: 0, @@ -898,7 +907,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Exact { needed: 2, tolerated_missing: 1, @@ -917,7 +927,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 2, next_no_show: None, @@ -970,7 +981,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Exact { needed: 2, tolerated_missing: 1, @@ -992,7 +1004,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 2, next_no_show: None, @@ -1013,7 +1026,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Exact { needed: 3, tolerated_missing: 2, @@ -1068,7 +1082,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 10, next_no_show: None, diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs index 66df6ee8f653..88650a539130 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs @@ -17,12 +17,15 @@ //! Version 2 of the DB schema. use parity_scale_codec::{Decode, Encode}; -use polkadot_node_primitives::approval::{v1::DelayTranche, v2::AssignmentCertV2}; +use polkadot_node_primitives::approval::{ + v1::DelayTranche, + v2::{AssignmentCertV2, CandidateBitfield}, +}; use polkadot_node_subsystem::{SubsystemError, SubsystemResult}; use polkadot_node_subsystem_util::database::{DBTransaction, Database}; use polkadot_primitives::{ - BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, SessionIndex, - ValidatorIndex, ValidatorSignature, + BlockNumber, CandidateHash, CandidateIndex, CandidateReceipt, CoreIndex, GroupIndex, Hash, + SessionIndex, ValidatorIndex, ValidatorSignature, }; use sp_consensus_slots::Slot; @@ -197,6 +200,16 @@ pub struct TrancheEntry { pub assignments: Vec<(ValidatorIndex, Tick)>, } +/// Metadata about our approval signature +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct OurApproval { + /// The signature for the candidates hashes pointed by indices. + pub signature: ValidatorSignature, + /// The indices of the candidates signed in this approval, an empty value means only + /// the candidate referred by this approval entry was signed. + pub signed_candidates_indices: Option, +} + /// Metadata regarding approval of a particular candidate within the context of some /// particular block. #[derive(Encode, Decode, Debug, Clone, PartialEq)] @@ -204,7 +217,7 @@ pub struct ApprovalEntry { pub tranches: Vec, pub backing_group: GroupIndex, pub our_assignment: Option, - pub our_approval_sig: Option, + pub our_approval_sig: Option, // `n_validators` bits. pub assigned_validators: Bitfield, pub approved: bool, @@ -241,12 +254,25 @@ pub struct BlockEntry { // block. The block can be considered approved if the bitfield has all bits set to `true`. pub approved_bitfield: Bitfield, pub children: Vec, + // A list of candidates that has been approved, but we didn't not sign and + // advertise the vote yet. + pub candidates_pending_signature: BTreeMap, // Assignments we already distributed. A 1 bit means the candidate index for which // we already have sent out an assignment. We need this to avoid distributing // multiple core assignments more than once. pub distributed_assignments: Bitfield, } +#[derive(Encode, Decode, Debug, Clone, PartialEq)] + +/// Context needed for creating an approval signature for a given candidate. +pub struct CandidateSigningContext { + /// The candidate hash, to be included in the signature. + pub candidate_hash: CandidateHash, + /// The latest tick we have to create and send the approval. + pub send_no_later_than_tick: Tick, +} + impl From for Tick { fn from(tick: crate::Tick) -> Tick { Tick(tick) diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs index 50a5a924ca8d..31b66d88577f 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs @@ -56,6 +56,7 @@ fn make_block_entry( approved_bitfield: make_bitvec(candidates.len()), candidates, children: Vec::new(), + candidates_pending_signature: Default::default(), distributed_assignments: Default::default(), } } diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 1f751e2bf140..db74302f0225 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -278,7 +278,7 @@ impl AssignmentCriteria for RealAssignmentCriteria { config: &Config, leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>, ) -> HashMap { - compute_assignments(keystore, relay_vrf_story, config, leaving_cores, false) + compute_assignments(keystore, relay_vrf_story, config, leaving_cores, true) } fn check_assignment_cert( diff --git a/polkadot/node/core/approval-voting/src/import.rs b/polkadot/node/core/approval-voting/src/import.rs index 4345380ed2f9..019a74a51a74 100644 --- a/polkadot/node/core/approval-voting/src/import.rs +++ b/polkadot/node/core/approval-voting/src/import.rs @@ -513,6 +513,7 @@ pub(crate) async fn handle_new_head( .collect(), approved_bitfield, children: Vec::new(), + candidates_pending_signature: Default::default(), distributed_assignments: Default::default(), }; @@ -639,6 +640,7 @@ pub(crate) mod tests { clock: Box::new(MockClock::default()), assignment_criteria: Box::new(MockAssignmentCriteria), spans: HashMap::new(), + approval_voting_params_cache: None, } } @@ -1267,6 +1269,7 @@ pub(crate) mod tests { candidates: Vec::new(), approved_bitfield: Default::default(), children: Vec::new(), + candidates_pending_signature: Default::default(), distributed_assignments: Default::default(), } .into(), diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index beaca43ee528..69c5603624d3 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -21,14 +21,16 @@ //! of others. It uses this information to determine when candidates and blocks have //! been sufficiently approved to finalize. +use itertools::Itertools; use jaeger::{hash_to_trace_identifier, PerLeafSpan}; +use lru::LruCache; use polkadot_node_jaeger as jaeger; use polkadot_node_primitives::{ approval::{ - v1::{BlockApprovalMeta, DelayTranche, IndirectSignedApprovalVote}, + v1::{BlockApprovalMeta, DelayTranche}, v2::{ AssignmentCertKindV2, BitfieldError, CandidateBitfield, CoreBitfield, - IndirectAssignmentCertV2, + IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2, }, }, ValidationResult, DISPUTE_WINDOW, @@ -53,9 +55,10 @@ use polkadot_node_subsystem_util::{ TimeoutExt, }; use polkadot_primitives::{ - ApprovalVote, BlockNumber, CandidateHash, CandidateIndex, CandidateReceipt, DisputeStatement, - GroupIndex, Hash, PvfExecTimeoutKind, SessionIndex, SessionInfo, ValidDisputeStatementKind, - ValidatorId, ValidatorIndex, ValidatorPair, ValidatorSignature, + vstaging::{ApprovalVoteMultipleCandidates, ApprovalVotingParams}, + BlockNumber, CandidateHash, CandidateIndex, CandidateReceipt, DisputeStatement, GroupIndex, + Hash, PvfExecTimeoutKind, SessionIndex, SessionInfo, ValidDisputeStatementKind, ValidatorId, + ValidatorIndex, ValidatorPair, ValidatorSignature, }; use sc_keystore::LocalKeystore; use sp_application_crypto::Pair; @@ -67,9 +70,11 @@ use futures::{ future::{BoxFuture, RemoteHandle}, prelude::*, stream::FuturesUnordered, + StreamExt, }; use std::{ + cmp::min, collections::{ btree_map::Entry as BTMEntry, hash_map::Entry as HMEntry, BTreeMap, HashMap, HashSet, }, @@ -82,7 +87,7 @@ use approval_checking::RequiredTranches; use bitvec::{order::Lsb0, vec::BitVec}; use criteria::{AssignmentCriteria, RealAssignmentCriteria}; use persisted_entries::{ApprovalEntry, BlockEntry, CandidateEntry}; -use time::{slot_number_to_tick, Clock, ClockExt, SystemClock, Tick}; +use time::{slot_number_to_tick, Clock, ClockExt, DelayedApprovalTimer, SystemClock, Tick}; mod approval_checking; pub mod approval_db; @@ -94,9 +99,11 @@ mod persisted_entries; mod time; use crate::{ + approval_checking::Check, approval_db::v2::{Config as DatabaseConfig, DbBackend}, backend::{Backend, OverlayedBackend}, criteria::InvalidAssignmentReason, + persisted_entries::OurApproval, }; #[cfg(test)] @@ -117,6 +124,16 @@ const TICK_TOO_FAR_IN_FUTURE: Tick = 20; // 10 seconds. const APPROVAL_DELAY: Tick = 2; pub(crate) const LOG_TARGET: &str = "parachain::approval-voting"; +// The max number of ticks we delay sending the approval after we are ready to issue the approval +const MAX_APPROVAL_COALESCE_WAIT_TICKS: Tick = 12; + +// The maximum approval params we cache locally in the subsytem, so that we don't have +// to do the back and forth to the runtime subsystem api. +const APPROVAL_PARAMS_CACHE_SIZE: NonZeroUsize = match NonZeroUsize::new(128) { + Some(cap) => cap, + None => panic!("Approval params cache size must be non-zero."), +}; + /// Configuration for the approval voting subsystem #[derive(Debug, Clone)] pub struct Config { @@ -152,6 +169,9 @@ pub struct ApprovalVotingSubsystem { db: Arc, mode: Mode, metrics: Metrics, + // Store approval-voting params, so that we don't to always go to RuntimeAPI + // to ask this information which requires a task context switch. + approval_voting_params_cache: Option>, } #[derive(Clone)] @@ -160,7 +180,13 @@ struct MetricsInner { assignments_produced: prometheus::Histogram, approvals_produced_total: prometheus::CounterVec, no_shows_total: prometheus::Counter, + // The difference from `no_shows_total` is that this counts all observed no-shows at any + // moment in time. While `no_shows_total` catches that the no-shows at the moment the candidate + // is approved, approvals might arrive late and `no_shows_total` wouldn't catch that number. + observed_no_shows: prometheus::Counter, + approved_by_one_third: prometheus::Counter, wakeups_triggered_total: prometheus::Counter, + coalesced_approvals_buckets: prometheus::Histogram, candidate_approval_time_ticks: prometheus::Histogram, block_approval_time_ticks: prometheus::Histogram, time_db_transaction: prometheus::Histogram, @@ -186,6 +212,16 @@ impl Metrics { } } + fn on_approval_coalesce(&self, num_coalesced: u32) { + if let Some(metrics) = &self.0 { + // Count how many candidates we covered with this coalesced approvals, + // so that the heat-map really gives a good understanding of the scales. + for _ in 0..num_coalesced { + metrics.coalesced_approvals_buckets.observe(num_coalesced as f64) + } + } + } + fn on_approval_stale(&self) { if let Some(metrics) = &self.0 { metrics.approvals_produced_total.with_label_values(&["stale"]).inc() @@ -222,6 +258,18 @@ impl Metrics { } } + fn on_observed_no_shows(&self, n: usize) { + if let Some(metrics) = &self.0 { + metrics.observed_no_shows.inc_by(n as u64); + } + } + + fn on_approved_by_one_third(&self) { + if let Some(metrics) = &self.0 { + metrics.approved_by_one_third.inc(); + } + } + fn on_wakeup(&self) { if let Some(metrics) = &self.0 { metrics.wakeups_triggered_total.inc(); @@ -299,6 +347,13 @@ impl metrics::Metrics for Metrics { )?, registry, )?, + observed_no_shows: prometheus::register( + prometheus::Counter::new( + "polkadot_parachain_approvals_observed_no_shows_total", + "Number of observed no shows at any moment in time", + )?, + registry, + )?, wakeups_triggered_total: prometheus::register( prometheus::Counter::new( "polkadot_parachain_approvals_wakeups_total", @@ -315,6 +370,22 @@ impl metrics::Metrics for Metrics { )?, registry, )?, + coalesced_approvals_buckets: prometheus::register( + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "polkadot_parachain_approvals_coalesced_approvals_buckets", + "Number of coalesced approvals.", + ).buckets(vec![1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 9.5]), + )?, + registry, + )?, + approved_by_one_third: prometheus::register( + prometheus::Counter::new( + "polkadot_parachain_approved_by_one_third", + "Number of candidates where more than one third had to vote ", + )?, + registry, + )?, block_approval_time_ticks: prometheus::register( prometheus::Histogram::with_opts( prometheus::HistogramOpts::new( @@ -370,6 +441,24 @@ impl ApprovalVotingSubsystem { keystore: Arc, sync_oracle: Box, metrics: Metrics, + ) -> Self { + Self::with_config_and_cache( + config, + db, + keystore, + sync_oracle, + metrics, + Some(LruCache::new(APPROVAL_PARAMS_CACHE_SIZE)), + ) + } + + pub fn with_config_and_cache( + config: Config, + db: Arc, + keystore: Arc, + sync_oracle: Box, + metrics: Metrics, + approval_voting_params_cache: Option>, ) -> Self { ApprovalVotingSubsystem { keystore, @@ -378,6 +467,7 @@ impl ApprovalVotingSubsystem { db_config: DatabaseConfig { col_approval_data: config.col_approval_data }, mode: Mode::Syncing(sync_oracle), metrics, + approval_voting_params_cache, } } @@ -561,6 +651,7 @@ struct ApprovalStatus { required_tranches: RequiredTranches, tranche_now: DelayTranche, block_tick: Tick, + last_no_shows: usize, } #[derive(Copy, Clone)] @@ -682,6 +773,9 @@ struct State { clock: Box, assignment_criteria: Box, spans: HashMap, + // Store approval-voting params, so that we don't to always go to RuntimeAPI + // to ask this information which requires a task context switch. + approval_voting_params_cache: Option>, } #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] @@ -720,7 +814,7 @@ impl State { ); if let Some(approval_entry) = candidate_entry.approval_entry(&block_hash) { - let required_tranches = approval_checking::tranches_to_approve( + let (required_tranches, last_no_shows) = approval_checking::tranches_to_approve( approval_entry, candidate_entry.approvals(), tranche_now, @@ -729,13 +823,56 @@ impl State { session_info.needed_approvals as _, ); - let status = ApprovalStatus { required_tranches, block_tick, tranche_now }; + let status = + ApprovalStatus { required_tranches, block_tick, tranche_now, last_no_shows }; Some((approval_entry, status)) } else { None } } + + // Returns the approval voting from the RuntimeApi. + // To avoid crossing the subsystem boundary every-time we are caching locally the values. + #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] + async fn get_approval_voting_params_or_default( + &mut self, + _ctx: &mut Context, + block_hash: Hash, + ) -> ApprovalVotingParams { + if let Some(params) = self + .approval_voting_params_cache + .as_mut() + .and_then(|cache| cache.get(&block_hash)) + { + *params + } else { + // let (s_tx, s_rx) = oneshot::channel(); + + // ctx.send_message(RuntimeApiMessage::Request( + // block_hash, + // RuntimeApiRequest::ApprovalVotingParams(s_tx), + // )) + // .await; + + // match s_rx.await { + // Ok(Ok(params)) => { + // self.approval_voting_params_cache + // .as_mut() + // .map(|cache| cache.put(block_hash, params)); + // params + // }, + // _ => { + // gum::error!( + // target: LOG_TARGET, + // "Could not request approval voting params from runtime using defaults" + // ); + // TODO: Uncomment this after versi test + ApprovalVotingParams { max_approval_coalesce_count: 6 } + // }, + // } + } + } } #[derive(Debug, Clone)] @@ -784,6 +921,7 @@ where clock, assignment_criteria, spans: HashMap::new(), + approval_voting_params_cache: subsystem.approval_voting_params_cache.take(), }; // `None` on start-up. Gets initialized/updated on leaf update @@ -794,6 +932,7 @@ where let mut wakeups = Wakeups::default(); let mut currently_checking_set = CurrentlyCheckingSet::default(); let mut approvals_cache = lru::LruCache::new(APPROVAL_CACHE_SIZE); + let mut delayed_approvals_timers = DelayedApprovalTimer::default(); let mut last_finalized_height: Option = { let (tx, rx) = oneshot::channel(); @@ -871,18 +1010,50 @@ where } actions + }, + (block_hash, validator_index) = delayed_approvals_timers.select_next_some() => { + gum::debug!( + target: LOG_TARGET, + "Sign approval for multiple candidates", + ); + + let approval_params = state.get_approval_voting_params_or_default(&mut ctx, block_hash).await; + + match maybe_create_signature( + &mut overlayed_db, + &mut session_info_provider, + approval_params, + &state, &mut ctx, + block_hash, + validator_index, + &subsystem.metrics, + ).await { + Ok(Some(next_wakeup)) => { + delayed_approvals_timers.maybe_arm_timer(next_wakeup, state.clock.as_ref(), block_hash, validator_index); + }, + Ok(None) => {} + Err(err) => { + gum::error!( + target: LOG_TARGET, + ?err, + "Failed to create signature", + ); + } + } + vec![] } }; if handle_actions( &mut ctx, - &state, + &mut state, &mut overlayed_db, &mut session_info_provider, &subsystem.metrics, &mut wakeups, &mut currently_checking_set, &mut approvals_cache, + &mut delayed_approvals_timers, &mut subsystem.mode, actions, ) @@ -923,13 +1094,14 @@ where #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn handle_actions( ctx: &mut Context, - state: &State, + state: &mut State, overlayed_db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, metrics: &Metrics, wakeups: &mut Wakeups, currently_checking_set: &mut CurrentlyCheckingSet, approvals_cache: &mut lru::LruCache, + delayed_approvals_timers: &mut DelayedApprovalTimer, mode: &mut Mode, actions: Vec, ) -> SubsystemResult { @@ -959,6 +1131,7 @@ async fn handle_actions( session_info_provider, metrics, candidate_hash, + delayed_approvals_timers, approval_request, ) .await? @@ -1058,7 +1231,11 @@ async fn handle_actions( Action::BecomeActive => { *mode = Mode::Active; - let messages = distribution_messages_for_activation(overlayed_db, state)?; + let messages = distribution_messages_for_activation( + overlayed_db, + state, + delayed_approvals_timers, + )?; ctx.send_messages(messages.into_iter()).await; }, @@ -1084,7 +1261,7 @@ fn cores_to_candidate_indices( .iter() .position(|(core_index, _)| core_index.0 == claimed_core_index as u32) { - candidate_indices.push(candidate_index as CandidateIndex); + candidate_indices.push(candidate_index as _); } } @@ -1117,6 +1294,7 @@ fn get_assignment_core_indices( fn distribution_messages_for_activation( db: &OverlayedBackend<'_, impl Backend>, state: &State, + delayed_approvals_timers: &mut DelayedApprovalTimer, ) -> SubsystemResult> { let all_blocks: Vec = db.load_all_blocks()?; @@ -1155,7 +1333,7 @@ fn distribution_messages_for_activation( slot: block_entry.slot(), session: block_entry.session(), }); - + let mut signatures_queued = HashSet::new(); for (i, (_, candidate_hash)) in block_entry.candidates().iter().enumerate() { let _candidate_span = distribution_message_span.child("candidate").with_candidate(*candidate_hash); @@ -1183,6 +1361,15 @@ fn distribution_messages_for_activation( &candidate_hash, &block_entry, ) { + if block_entry.has_candidates_pending_signature() { + delayed_approvals_timers.maybe_arm_timer( + state.clock.tick_now(), + state.clock.as_ref(), + block_entry.block_hash(), + assignment.validator_index(), + ) + } + match cores_to_candidate_indices( &claimed_core_indices, &block_entry, @@ -1250,15 +1437,19 @@ fn distribution_messages_for_activation( continue }, } - - messages.push(ApprovalDistributionMessage::DistributeApproval( - IndirectSignedApprovalVote { - block_hash, - candidate_index: i as _, - validator: assignment.validator_index(), - signature: approval_sig, - }, - )); + let candidate_indices = approval_sig + .signed_candidates_indices + .unwrap_or((i as CandidateIndex).into()); + if signatures_queued.insert(candidate_indices.clone()) { + messages.push(ApprovalDistributionMessage::DistributeApproval( + IndirectSignedApprovalVoteV2 { + block_hash, + candidate_indices, + validator: assignment.validator_index(), + signature: approval_sig.signature, + }, + )) + }; } else { gum::warn!( target: LOG_TARGET, @@ -1464,7 +1655,7 @@ async fn get_approval_signatures_for_candidate( ctx: &mut Context, db: &OverlayedBackend<'_, impl Backend>, candidate_hash: CandidateHash, - tx: oneshot::Sender>, + tx: oneshot::Sender, ValidatorSignature)>>, ) -> SubsystemResult<()> { let send_votes = |votes| { if let Err(_) = tx.send(votes) { @@ -1490,6 +1681,11 @@ async fn get_approval_signatures_for_candidate( let relay_hashes = entry.block_assignments.keys(); let mut candidate_indices = HashSet::new(); + let mut candidate_indices_to_candidate_hashes: HashMap< + Hash, + HashMap, + > = HashMap::new(); + // Retrieve `CoreIndices`/`CandidateIndices` as required by approval-distribution: for hash in relay_hashes { let entry = match db.load_block_entry(hash)? { @@ -1507,8 +1703,11 @@ async fn get_approval_signatures_for_candidate( for (candidate_index, (_core_index, c_hash)) in entry.candidates().iter().enumerate() { if c_hash == &candidate_hash { candidate_indices.insert((*hash, candidate_index as u32)); - break } + candidate_indices_to_candidate_hashes + .entry(*hash) + .or_default() + .insert(candidate_index as _, *c_hash); } } @@ -1533,7 +1732,24 @@ async fn get_approval_signatures_for_candidate( target: LOG_TARGET, "Request for approval signatures got cancelled by `approval-distribution`." ), - Some(Ok(votes)) => send_votes(votes), + Some(Ok(votes)) => { + let votes = votes + .into_iter() + .map(|(validator_index, (hash, signed_candidates_indices, signature))| { + let candidates_hashes = + candidate_indices_to_candidate_hashes.get(&hash).expect("Can't fail because it is the same hash we sent to approval-distribution; qed"); + let signed_candidates_hashes: Vec = + signed_candidates_indices + .into_iter() + .map(|candidate_index| { + *(candidates_hashes.get(&candidate_index).expect("Can't fail because we already checked the signature was valid, so we should be able to find the hash; qed")) + }) + .collect(); + (validator_index, (signed_candidates_hashes, signature)) + }) + .collect(); + send_votes(votes) + }, } }; @@ -2167,7 +2383,7 @@ async fn check_and_import_approval( db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, metrics: &Metrics, - approval: IndirectSignedApprovalVote, + approval: IndirectSignedApprovalVoteV2, with_response: impl FnOnce(ApprovalCheckResult) -> T, ) -> SubsystemResult<(Vec, T)> where @@ -2179,13 +2395,12 @@ where return Ok((Vec::new(), t)) }}; } - let mut span = state .spans .get(&approval.block_hash) .map(|span| span.child("check-and-import-approval")) .unwrap_or_else(|| jaeger::Span::new(approval.block_hash, "check-and-import-approval")) - .with_uint_tag("candidate-index", approval.candidate_index as u64) + .with_string_fmt_debug_tag("candidate-index", approval.candidate_indices.clone()) .with_relay_parent(approval.block_hash) .with_stage(jaeger::Stage::ApprovalChecking); @@ -2198,105 +2413,157 @@ where }, }; - let session_info = match get_session_info( - session_info_provider, - sender, - approval.block_hash, - block_entry.session(), - ) - .await - { - Some(s) => s, - None => { - respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::UnknownSessionIndex( - block_entry.session() - ),)) - }, - }; + let approved_candidates_info: Result, ApprovalCheckError> = + approval + .candidate_indices + .iter_ones() + .map(|candidate_index| { + block_entry + .candidate(candidate_index) + .ok_or(ApprovalCheckError::InvalidCandidateIndex(candidate_index as _)) + .map(|candidate| (candidate_index as _, candidate.1)) + }) + .collect(); - let approved_candidate_hash = match block_entry.candidate(approval.candidate_index as usize) { - Some((_, h)) => *h, - None => respond_early!(ApprovalCheckResult::Bad( - ApprovalCheckError::InvalidCandidateIndex(approval.candidate_index), - )), + let approved_candidates_info = match approved_candidates_info { + Ok(approved_candidates_info) => approved_candidates_info, + Err(err) => { + respond_early!(ApprovalCheckResult::Bad(err)) + }, }; - span.add_string_tag("candidate-hash", format!("{:?}", approved_candidate_hash)); + span.add_string_tag("candidate-hashes", format!("{:?}", approved_candidates_info)); span.add_string_tag( - "traceID", - format!("{:?}", hash_to_trace_identifier(approved_candidate_hash.0)), + "traceIDs", + format!( + "{:?}", + approved_candidates_info + .iter() + .map(|(_, approved_candidate_hash)| hash_to_trace_identifier( + approved_candidate_hash.0 + )) + .collect_vec() + ), ); - let pubkey = match session_info.validators.get(approval.validator) { - Some(k) => k, - None => respond_early!(ApprovalCheckResult::Bad( - ApprovalCheckError::InvalidValidatorIndex(approval.validator), - )), - }; + { + let session_info = match get_session_info( + session_info_provider, + sender, + approval.block_hash, + block_entry.session(), + ) + .await + { + Some(s) => s, + None => { + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::UnknownSessionIndex( + block_entry.session() + ),)) + }, + }; - // Signature check: - match DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking).check_signature( - &pubkey, - approved_candidate_hash, - block_entry.session(), - &approval.signature, - ) { - Err(_) => respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::InvalidSignature( - approval.validator - ),)), - Ok(()) => {}, - }; + let pubkey = match session_info.validators.get(approval.validator) { + Some(k) => k, + None => respond_early!(ApprovalCheckResult::Bad( + ApprovalCheckError::InvalidValidatorIndex(approval.validator), + )), + }; - let candidate_entry = match db.load_candidate_entry(&approved_candidate_hash)? { - Some(c) => c, - None => { - respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::InvalidCandidate( - approval.candidate_index, - approved_candidate_hash - ),)) - }, - }; + gum::trace!( + target: LOG_TARGET, + "Received approval for num_candidates {:}", + approval.candidate_indices.count_ones() + ); - // Don't accept approvals until assignment. - match candidate_entry.approval_entry(&approval.block_hash) { - None => { - respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::Internal( - approval.block_hash, - approved_candidate_hash - ),)) - }, - Some(e) if !e.is_assigned(approval.validator) => { - respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::NoAssignment( - approval.validator - ),)) - }, - _ => {}, + let candidate_hashes: Vec = + approved_candidates_info.iter().map(|candidate| candidate.1).collect(); + // Signature check: + match DisputeStatement::Valid( + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(candidate_hashes.clone()), + ) + .check_signature( + &pubkey, + *candidate_hashes.first().expect("Checked above this is not empty; qed"), + block_entry.session(), + &approval.signature, + ) { + Err(_) => { + gum::error!( + target: LOG_TARGET, + "Error while checking signature {:}", + approval.candidate_indices.count_ones() + ); + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::InvalidSignature( + approval.validator + ),)) + }, + Ok(()) => {}, + }; } - // importing the approval can be heavy as it may trigger acceptance for a series of blocks. - let t = with_response(ApprovalCheckResult::Accepted); + let mut actions = Vec::new(); + for (approval_candidate_index, approved_candidate_hash) in approved_candidates_info { + let block_entry = match db.load_block_entry(&approval.block_hash)? { + Some(b) => b, + None => { + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::UnknownBlock( + approval.block_hash + ),)) + }, + }; - gum::trace!( - target: LOG_TARGET, - validator_index = approval.validator.0, - validator = ?pubkey, - candidate_hash = ?approved_candidate_hash, - para_id = ?candidate_entry.candidate_receipt().descriptor.para_id, - "Importing approval vote", - ); + let candidate_entry = match db.load_candidate_entry(&approved_candidate_hash)? { + Some(c) => c, + None => { + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::InvalidCandidate( + approval_candidate_index, + approved_candidate_hash + ),)) + }, + }; - let actions = advance_approval_state( - sender, - state, - db, - session_info_provider, - &metrics, - block_entry, - approved_candidate_hash, - candidate_entry, - ApprovalStateTransition::RemoteApproval(approval.validator), - ) - .await; + // Don't accept approvals until assignment. + match candidate_entry.approval_entry(&approval.block_hash) { + None => { + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::Internal( + approval.block_hash, + approved_candidate_hash + ),)) + }, + Some(e) if !e.is_assigned(approval.validator) => { + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::NoAssignment( + approval.validator + ),)) + }, + _ => {}, + } + + gum::debug!( + target: LOG_TARGET, + validator_index = approval.validator.0, + candidate_hash = ?approved_candidate_hash, + para_id = ?candidate_entry.candidate_receipt().descriptor.para_id, + "Importing approval vote", + ); + + let new_actions = advance_approval_state( + sender, + state, + db, + session_info_provider, + &metrics, + block_entry, + approved_candidate_hash, + candidate_entry, + ApprovalStateTransition::RemoteApproval(approval.validator), + ) + .await; + actions.extend(new_actions); + } + + // importing the approval can be heavy as it may trigger acceptance for a series of blocks. + let t = with_response(ApprovalCheckResult::Accepted); Ok((actions, t)) } @@ -2304,7 +2571,7 @@ where #[derive(Debug)] enum ApprovalStateTransition { RemoteApproval(ValidatorIndex), - LocalApproval(ValidatorIndex, ValidatorSignature), + LocalApproval(ValidatorIndex), WakeupProcessed, } @@ -2312,7 +2579,7 @@ impl ApprovalStateTransition { fn validator_index(&self) -> Option { match *self { ApprovalStateTransition::RemoteApproval(v) | - ApprovalStateTransition::LocalApproval(v, _) => Some(v), + ApprovalStateTransition::LocalApproval(v) => Some(v), ApprovalStateTransition::WakeupProcessed => None, } } @@ -2320,7 +2587,7 @@ impl ApprovalStateTransition { fn is_local_approval(&self) -> bool { match *self { ApprovalStateTransition::RemoteApproval(_) => false, - ApprovalStateTransition::LocalApproval(_, _) => true, + ApprovalStateTransition::LocalApproval(_) => true, ApprovalStateTransition::WakeupProcessed => false, } } @@ -2387,7 +2654,16 @@ where // assignment tick of `now - APPROVAL_DELAY` - that is, that // all counted assignments are at least `APPROVAL_DELAY` ticks old. let is_approved = check.is_approved(tick_now.saturating_sub(APPROVAL_DELAY)); - + if status.last_no_shows != 0 { + metrics.on_observed_no_shows(status.last_no_shows); + gum::debug!( + target: LOG_TARGET, + ?candidate_hash, + ?block_hash, + last_no_shows = ?status.last_no_shows, + "Observed no_shows", + ); + } if is_approved { gum::trace!( target: LOG_TARGET, @@ -2405,6 +2681,12 @@ where if no_shows != 0 { metrics.on_no_shows(no_shows); } + if check == Check::ApprovedOneThird { + // No-shows are not counted when more than one third of validators approve a + // candidate, so count candidates where more than one third of validators had to + // approve it, this is indicative of something breaking. + metrics.on_approved_by_one_third() + } metrics.on_candidate_approved(status.tranche_now as _); @@ -2413,6 +2695,10 @@ where actions.push(Action::NoteApprovedInChainSelection(block_hash)); } + db.write_block_entry(block_entry.into()); + } else if transition.is_local_approval() { + // Local approvals always update the block_entry, so we need to flush it to + // the database. db.write_block_entry(block_entry.into()); } @@ -2441,10 +2727,6 @@ where approval_entry.mark_approved(); } - if let ApprovalStateTransition::LocalApproval(_, ref sig) = transition { - approval_entry.import_approval_sig(sig.clone()); - } - actions.extend(schedule_wakeup_action( &approval_entry, block_hash, @@ -2569,7 +2851,7 @@ async fn process_wakeup( None => return Ok(Vec::new()), }; - let tranches_to_approve = approval_checking::tranches_to_approve( + let (tranches_to_approve, _last_no_shows) = approval_checking::tranches_to_approve( &approval_entry, candidate_entry.approvals(), tranche_now, @@ -2903,11 +3185,12 @@ async fn launch_approval( #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn issue_approval( ctx: &mut Context, - state: &State, + state: &mut State, db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, metrics: &Metrics, candidate_hash: CandidateHash, + delayed_approvals_timers: &mut DelayedApprovalTimer, ApprovalVoteRequest { validator_index, block_hash }: ApprovalVoteRequest, ) -> SubsystemResult> { let mut issue_approval_span = state @@ -2921,7 +3204,7 @@ async fn issue_approval( .with_validator_index(validator_index) .with_stage(jaeger::Stage::ApprovalChecking); - let block_entry = match db.load_block_entry(&block_hash)? { + let mut block_entry = match db.load_block_entry(&block_hash)? { Some(b) => b, None => { // not a cause for alarm - just lost a race with pruning, most likely. @@ -2947,21 +3230,6 @@ async fn issue_approval( }; issue_approval_span.add_int_tag("candidate_index", candidate_index as i64); - let session_info = match get_session_info( - session_info_provider, - ctx.sender(), - block_entry.parent_hash(), - block_entry.session(), - ) - .await - { - Some(s) => s, - None => { - metrics.on_approval_error(); - return Ok(Vec::new()) - }, - }; - let candidate_hash = match block_entry.candidate(candidate_index as usize) { Some((_, h)) => *h, None => { @@ -2992,10 +3260,150 @@ async fn issue_approval( }, }; + let session_info = match get_session_info( + session_info_provider, + ctx.sender(), + block_entry.parent_hash(), + block_entry.session(), + ) + .await + { + Some(s) => s, + None => return Ok(Vec::new()), + }; + + if block_entry + .defer_candidate_signature( + candidate_index as _, + candidate_hash, + compute_delayed_approval_sending_tick(state, &block_entry, session_info), + ) + .is_some() + { + gum::error!( + target: LOG_TARGET, + ?candidate_hash, + ?block_hash, + validator_index = validator_index.0, + "Possible bug, we shouldn't have to defer a candidate more than once", + ); + } + + gum::info!( + target: LOG_TARGET, + ?candidate_hash, + ?block_hash, + validator_index = validator_index.0, + "Ready to issue approval vote", + ); + + let approval_params = state.get_approval_voting_params_or_default(ctx, block_hash).await; + + let actions = advance_approval_state( + ctx.sender(), + state, + db, + session_info_provider, + metrics, + block_entry, + candidate_hash, + candidate_entry, + ApprovalStateTransition::LocalApproval(validator_index as _), + ) + .await; + + if let Some(next_wakeup) = maybe_create_signature( + db, + session_info_provider, + approval_params, + state, + ctx, + block_hash, + validator_index, + metrics, + ) + .await? + { + delayed_approvals_timers.maybe_arm_timer( + next_wakeup, + state.clock.as_ref(), + block_hash, + validator_index, + ); + } + Ok(actions) +} + +// Create signature for the approved candidates pending signatures +#[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] +async fn maybe_create_signature( + db: &mut OverlayedBackend<'_, impl Backend>, + session_info_provider: &mut RuntimeInfo, + approval_params: ApprovalVotingParams, + state: &State, + ctx: &mut Context, + block_hash: Hash, + validator_index: ValidatorIndex, + metrics: &Metrics, +) -> SubsystemResult> { + let mut block_entry = match db.load_block_entry(&block_hash)? { + Some(b) => b, + None => { + // not a cause for alarm - just lost a race with pruning, most likely. + metrics.on_approval_stale(); + gum::debug!( + target: LOG_TARGET, + "Could not find block that needs signature {:}", block_hash + ); + return Ok(None) + }, + }; + + gum::trace!( + target: LOG_TARGET, + "Candidates pending signatures {:}", block_entry.num_candidates_pending_signature() + ); + + let oldest_candidate_to_sign = match block_entry.longest_waiting_candidate_signature() { + Some(candidate) => candidate, + // No cached candidates, nothing to do here, this just means the timer fired, + // but the signatures were already sent because we gathered more than + // max_approval_coalesce_count. + None => return Ok(None), + }; + + let tick_now = state.clock.tick_now(); + + if oldest_candidate_to_sign.send_no_later_than_tick > tick_now && + (block_entry.num_candidates_pending_signature() as u32) < + approval_params.max_approval_coalesce_count + { + return Ok(Some(oldest_candidate_to_sign.send_no_later_than_tick)) + } + + let session_info = match get_session_info( + session_info_provider, + ctx.sender(), + block_entry.parent_hash(), + block_entry.session(), + ) + .await + { + Some(s) => s, + None => { + metrics.on_approval_error(); + gum::error!( + target: LOG_TARGET, + "Could not retrieve the session" + ); + return Ok(None) + }, + }; + let validator_pubkey = match session_info.validators.get(validator_index) { Some(p) => p, None => { - gum::warn!( + gum::error!( target: LOG_TARGET, "Validator index {} out of bounds in session {}", validator_index.0, @@ -3003,72 +3411,93 @@ async fn issue_approval( ); metrics.on_approval_error(); - return Ok(Vec::new()) + return Ok(None) }, }; - let session = block_entry.session(); - let sig = match sign_approval(&state.keystore, &validator_pubkey, candidate_hash, session) { + let candidate_hashes = block_entry.candidate_hashes_pending_signature(); + + let signature = match sign_approval( + &state.keystore, + &validator_pubkey, + candidate_hashes.clone(), + block_entry.session(), + ) { Some(sig) => sig, None => { - gum::warn!( + gum::error!( target: LOG_TARGET, validator_index = ?validator_index, - session, + session = ?block_entry.session(), "Could not issue approval signature. Assignment key present but not validator key?", ); metrics.on_approval_error(); - return Ok(Vec::new()) + return Ok(None) }, }; - gum::trace!( - target: LOG_TARGET, - ?candidate_hash, - ?block_hash, - validator_index = validator_index.0, - "Issuing approval vote", - ); + let candidate_entries = candidate_hashes + .iter() + .map(|candidate_hash| db.load_candidate_entry(candidate_hash)) + .collect::>>>()?; - let actions = advance_approval_state( - ctx.sender(), - state, - db, - session_info_provider, - metrics, - block_entry, - candidate_hash, - candidate_entry, - ApprovalStateTransition::LocalApproval(validator_index as _, sig.clone()), - ) - .await; + let candidate_indices = block_entry.candidate_indices_pending_signature(); + + for candidate_entry in candidate_entries { + let mut candidate_entry = candidate_entry + .expect("Candidate was scheduled to be signed entry in db should exist; qed"); + let approval_entry = candidate_entry + .approval_entry_mut(&block_entry.block_hash()) + .expect("Candidate was scheduled to be signed entry in db should exist; qed"); + approval_entry.import_approval_sig(OurApproval { + signature: signature.clone(), + signed_candidates_indices: Some( + candidate_indices + .clone() + .try_into() + .expect("Fails only of array empty, it can't be, qed"), + ), + }); + db.write_candidate_entry(candidate_entry); + } + + metrics.on_approval_coalesce(candidate_indices.len() as u32); metrics.on_approval_produced(); - // dispatch to approval distribution. ctx.send_unbounded_message(ApprovalDistributionMessage::DistributeApproval( - IndirectSignedApprovalVote { - block_hash, - candidate_index: candidate_index as _, + IndirectSignedApprovalVoteV2 { + block_hash: block_entry.block_hash(), + candidate_indices: candidate_indices + .try_into() + .expect("Fails only of array empty, it can't be, qed"), validator: validator_index, - signature: sig, + signature, }, )); - Ok(actions) + gum::trace!( + target: LOG_TARGET, + ?block_hash, + signed_candidates = ?block_entry.num_candidates_pending_signature(), + "Issue approval votes", + ); + block_entry.issued_approval(); + db.write_block_entry(block_entry.into()); + Ok(None) } // Sign an approval vote. Fails if the key isn't present in the store. fn sign_approval( keystore: &LocalKeystore, public: &ValidatorId, - candidate_hash: CandidateHash, + candidate_hashes: Vec, session_index: SessionIndex, ) -> Option { let key = keystore.key_pair::(public).ok().flatten()?; - let payload = ApprovalVote(candidate_hash).signing_payload(session_index); + let payload = ApprovalVoteMultipleCandidates(&candidate_hashes).signing_payload(session_index); Some(key.sign(&payload[..])) } @@ -3098,3 +3527,27 @@ fn issue_local_invalid_statement( false, )); } + +// Computes what is the latest tick we can send an approval +fn compute_delayed_approval_sending_tick( + state: &State, + block_entry: &BlockEntry, + session_info: &SessionInfo, +) -> Tick { + let current_block_tick = slot_number_to_tick(state.slot_duration_millis, block_entry.slot()); + + let no_show_duration_ticks = slot_number_to_tick( + state.slot_duration_millis, + Slot::from(u64::from(session_info.no_show_slots)), + ); + let tick_now = state.clock.tick_now(); + + min( + tick_now + MAX_APPROVAL_COALESCE_WAIT_TICKS as Tick, + // We don't want to accidentally cause, no-shows so if we are past + // the 2 thirds of the no show time, force the sending of the + // approval immediately. + // TODO: TBD the right value here, so that we don't accidentally create no-shows. + current_block_tick + (no_show_duration_ticks * 2) / 3, + ) +} diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index 155b2f9c4e02..e441c23be122 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -25,8 +25,8 @@ use polkadot_node_primitives::approval::{ v2::{AssignmentCertV2, CandidateBitfield}, }; use polkadot_primitives::{ - BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, SessionIndex, - ValidatorIndex, ValidatorSignature, + BlockNumber, CandidateHash, CandidateIndex, CandidateReceipt, CoreIndex, GroupIndex, Hash, + SessionIndex, ValidatorIndex, ValidatorSignature, }; use sp_consensus_slots::Slot; @@ -76,6 +76,39 @@ impl From for crate::approval_db::v2::TrancheEntry { } } +impl From for OurApproval { + fn from(approval: crate::approval_db::v2::OurApproval) -> Self { + Self { + signature: approval.signature, + signed_candidates_indices: approval.signed_candidates_indices, + } + } +} +impl From for crate::approval_db::v2::OurApproval { + fn from(approval: OurApproval) -> Self { + Self { + signature: approval.signature, + signed_candidates_indices: approval.signed_candidates_indices, + } + } +} + +impl From for OurApproval { + fn from(value: ValidatorSignature) -> Self { + Self { signature: value, signed_candidates_indices: Default::default() } + } +} + +/// Metadata about our approval signature +#[derive(Debug, Clone, PartialEq)] +pub struct OurApproval { + /// The signature for the candidates hashes pointed by indices. + pub signature: ValidatorSignature, + /// The indices of the candidates signed in this approval, an empty value means only + /// the candidate referred by this approval entry was signed. + pub signed_candidates_indices: Option, +} + /// Metadata regarding approval of a particular candidate within the context of some /// particular block. #[derive(Debug, Clone, PartialEq)] @@ -83,7 +116,7 @@ pub struct ApprovalEntry { tranches: Vec, backing_group: GroupIndex, our_assignment: Option, - our_approval_sig: Option, + our_approval_sig: Option, // `n_validators` bits. assigned_validators: Bitfield, approved: bool, @@ -95,7 +128,7 @@ impl ApprovalEntry { tranches: Vec, backing_group: GroupIndex, our_assignment: Option, - our_approval_sig: Option, + our_approval_sig: Option, // `n_validators` bits. assigned_validators: Bitfield, approved: bool, @@ -137,7 +170,7 @@ impl ApprovalEntry { } /// Import our local approval vote signature for this candidate. - pub fn import_approval_sig(&mut self, approval_sig: ValidatorSignature) { + pub fn import_approval_sig(&mut self, approval_sig: OurApproval) { self.our_approval_sig = Some(approval_sig); } @@ -224,7 +257,7 @@ impl ApprovalEntry { /// Get the assignment cert & approval signature. /// /// The approval signature will only be `Some` if the assignment is too. - pub fn local_statements(&self) -> (Option, Option) { + pub fn local_statements(&self) -> (Option, Option) { let approval_sig = self.our_approval_sig.clone(); if let Some(our_assignment) = self.our_assignment.as_ref().filter(|a| a.triggered()) { (Some(our_assignment.clone()), approval_sig) @@ -353,12 +386,21 @@ pub struct BlockEntry { // block. The block can be considered approved if the bitfield has all bits set to `true`. pub approved_bitfield: Bitfield, pub children: Vec, + // A list of candidates that has been approved, but we didn't not sign and + // advertise the vote yet. + candidates_pending_signature: BTreeMap, // A list of assignments for which wea already distributed the assignment. // We use this to ensure we don't distribute multiple core assignments twice as we track // individual wakeups for each core. distributed_assignments: Bitfield, } +#[derive(Debug, Clone, PartialEq)] +pub struct CandidateSigningContext { + pub candidate_hash: CandidateHash, + pub send_no_later_than_tick: Tick, +} + impl BlockEntry { /// Mark a candidate as fully approved in the bitfield. pub fn mark_approved_by_hash(&mut self, candidate_hash: &CandidateHash) { @@ -447,6 +489,54 @@ impl BlockEntry { distributed } + + /// Defer signing and issuing an approval for a candidate no later than the specified tick + pub fn defer_candidate_signature( + &mut self, + candidate_index: CandidateIndex, + candidate_hash: CandidateHash, + send_no_later_than_tick: Tick, + ) -> Option { + self.candidates_pending_signature.insert( + candidate_index, + CandidateSigningContext { candidate_hash, send_no_later_than_tick }, + ) + } + + /// Returns the number of candidates waiting for an approval to be issued. + pub fn num_candidates_pending_signature(&self) -> usize { + self.candidates_pending_signature.len() + } + + /// Return if we have candidates waiting for signature to be issued + pub fn has_candidates_pending_signature(&self) -> bool { + !self.candidates_pending_signature.is_empty() + } + + /// Candidate hashes for candidates pending signatures + pub fn candidate_hashes_pending_signature(&self) -> Vec { + self.candidates_pending_signature + .values() + .map(|unsigned_approval| unsigned_approval.candidate_hash) + .collect() + } + + /// Candidate indices for candidates pending signature + pub fn candidate_indices_pending_signature(&self) -> Vec { + self.candidates_pending_signature.keys().map(|val| *val).collect() + } + + /// Returns the candidate that has been longest in the queue. + pub fn longest_waiting_candidate_signature(&self) -> Option<&CandidateSigningContext> { + self.candidates_pending_signature + .values() + .min_by(|a, b| a.send_no_later_than_tick.cmp(&b.send_no_later_than_tick)) + } + + /// Signals the approval was issued for the candidates pending signature + pub fn issued_approval(&mut self) { + self.candidates_pending_signature.clear(); + } } impl From for BlockEntry { @@ -461,6 +551,11 @@ impl From for BlockEntry { candidates: entry.candidates, approved_bitfield: entry.approved_bitfield, children: entry.children, + candidates_pending_signature: entry + .candidates_pending_signature + .into_iter() + .map(|(candidate_index, signing_context)| (candidate_index, signing_context.into())) + .collect(), distributed_assignments: entry.distributed_assignments, } } @@ -479,6 +574,7 @@ impl From for BlockEntry { approved_bitfield: entry.approved_bitfield, children: entry.children, distributed_assignments: Default::default(), + candidates_pending_signature: Default::default(), } } } @@ -495,11 +591,34 @@ impl From for crate::approval_db::v2::BlockEntry { candidates: entry.candidates, approved_bitfield: entry.approved_bitfield, children: entry.children, + candidates_pending_signature: entry + .candidates_pending_signature + .into_iter() + .map(|(candidate_index, signing_context)| (candidate_index, signing_context.into())) + .collect(), distributed_assignments: entry.distributed_assignments, } } } +impl From for CandidateSigningContext { + fn from(signing_context: crate::approval_db::v2::CandidateSigningContext) -> Self { + Self { + candidate_hash: signing_context.candidate_hash, + send_no_later_than_tick: signing_context.send_no_later_than_tick.into(), + } + } +} + +impl From for crate::approval_db::v2::CandidateSigningContext { + fn from(signing_context: CandidateSigningContext) -> Self { + Self { + candidate_hash: signing_context.candidate_hash, + send_no_later_than_tick: signing_context.send_no_later_than_tick.into(), + } + } +} + /// Migration helpers. impl From for CandidateEntry { fn from(value: crate::approval_db::v1::CandidateEntry) -> Self { @@ -522,7 +641,7 @@ impl From for ApprovalEntry { tranches: value.tranches.into_iter().map(|tranche| tranche.into()).collect(), backing_group: value.backing_group, our_assignment: value.our_assignment.map(|assignment| assignment.into()), - our_approval_sig: value.our_approval_sig, + our_approval_sig: value.our_approval_sig.map(Into::into), assigned_validators: value.assignments, approved: value.approved, } diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index c2ef109ad4ca..b8ae3003efd8 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -36,8 +36,8 @@ use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::TimeoutExt; use polkadot_overseer::HeadSupportsParachains; use polkadot_primitives::{ - CandidateCommitments, CandidateEvent, CoreIndex, GroupIndex, Header, Id as ParaId, IndexedVec, - ValidationCode, ValidatorSignature, + ApprovalVote, CandidateCommitments, CandidateEvent, CoreIndex, GroupIndex, Header, + Id as ParaId, IndexedVec, ValidationCode, ValidatorSignature, }; use std::time::Duration; @@ -438,6 +438,15 @@ fn sign_approval( key.sign(&ApprovalVote(candidate_hash).signing_payload(session_index)).into() } +fn sign_approval_multiple_candidates( + key: Sr25519Keyring, + candidate_hashes: Vec, + session_index: SessionIndex, +) -> ValidatorSignature { + key.sign(&ApprovalVoteMultipleCandidates(&candidate_hashes).signing_payload(session_index)) + .into() +} + type VirtualOverseer = test_helpers::TestSubsystemContextHandle; #[derive(Default)] @@ -530,7 +539,7 @@ fn test_harness>( let subsystem = run( context, - ApprovalVotingSubsystem::with_config( + ApprovalVotingSubsystem::with_config_and_cache( Config { col_approval_data: test_constants::TEST_CONFIG.col_approval_data, slot_duration_millis: SLOT_DURATION_MILLIS, @@ -539,6 +548,7 @@ fn test_harness>( Arc::new(keystore), sync_oracle, Metrics::default(), + None, ), clock.clone(), assignment_criteria, @@ -633,7 +643,12 @@ async fn check_and_import_approval( overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportApproval( - IndirectSignedApprovalVote { block_hash, candidate_index, validator, signature }, + IndirectSignedApprovalVoteV2 { + block_hash, + candidate_indices: candidate_index.into(), + validator, + signature, + }, tx, ), }, @@ -1989,6 +2004,91 @@ fn forkful_import_at_same_height_act_on_leaf() { }); } +#[test] +fn test_signing_a_single_candidate_is_backwards_compatible() { + let session_index = 1; + let block_hash = Hash::repeat_byte(0x01); + let candidate_descriptors = (1..10) + .into_iter() + .map(|val| make_candidate(ParaId::from(val as u32), &block_hash)) + .collect::>(); + + let candidate_hashes = candidate_descriptors + .iter() + .map(|candidate_descriptor| candidate_descriptor.hash()) + .collect_vec(); + + let first_descriptor = candidate_descriptors.first().expect("TODO"); + + let candidate_hash = first_descriptor.hash(); + + let sig_a = sign_approval(Sr25519Keyring::Alice, candidate_hash, session_index); + + let sig_b = sign_approval(Sr25519Keyring::Alice, candidate_hash, session_index); + + assert!(DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) + .check_signature( + &Sr25519Keyring::Alice.public().into(), + candidate_hash, + session_index, + &sig_a, + ) + .is_ok()); + + assert!(DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) + .check_signature( + &Sr25519Keyring::Alice.public().into(), + candidate_hash, + session_index, + &sig_b, + ) + .is_ok()); + + let sig_c = sign_approval_multiple_candidates( + Sr25519Keyring::Alice, + vec![candidate_hash], + session_index, + ); + + assert!(DisputeStatement::Valid( + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(vec![candidate_hash]) + ) + .check_signature(&Sr25519Keyring::Alice.public().into(), candidate_hash, session_index, &sig_c,) + .is_ok()); + + assert!(DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) + .check_signature( + &Sr25519Keyring::Alice.public().into(), + candidate_hash, + session_index, + &sig_c, + ) + .is_ok()); + + assert!(DisputeStatement::Valid( + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(vec![candidate_hash]) + ) + .check_signature(&Sr25519Keyring::Alice.public().into(), candidate_hash, session_index, &sig_a,) + .is_ok()); + + let sig_all = sign_approval_multiple_candidates( + Sr25519Keyring::Alice, + candidate_hashes.clone(), + session_index, + ); + + assert!(DisputeStatement::Valid( + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(candidate_hashes.clone()) + ) + .check_signature( + &Sr25519Keyring::Alice.public().into(), + *candidate_hashes.first().expect("test"), + session_index, + &sig_all, + ) + .is_ok()); +} + #[test] fn import_checked_approval_updates_entries_and_schedules() { let config = HarnessConfig::default(); @@ -2701,11 +2801,29 @@ async fn handle_double_assignment_import( } ); + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 1, + })); + } + ); + assert_matches!( overseer_recv(virtual_overseer).await, AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(_)) ); + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 1, + })); + } + ); + assert_matches!( overseer_recv(virtual_overseer).await, AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(_)) @@ -3440,3 +3558,454 @@ fn waits_until_approving_assignments_are_old_enough() { virtual_overseer }); } + +#[test] +fn test_approval_is_sent_on_max_approval_coalesce_count() { + let assignment_criteria = Box::new(MockAssignmentCriteria( + || { + let mut assignments = HashMap::new(); + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let assignments_cert = + garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: vec![CoreIndex(0), CoreIndex(1), CoreIndex(2)] + .try_into() + .unwrap(), + }); + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: assignments_cert.clone(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let _ = assignments.insert( + CoreIndex(1), + approval_db::v2::OurAssignment { + cert: assignments_cert.clone(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + assignments + }, + |_| Ok(0), + )); + + let config = HarnessConfigBuilder::default().assignment_criteria(assignment_criteria).build(); + let store = config.backend(); + + test_harness(config, |test_harness| async move { + let TestHarness { mut virtual_overseer, clock, sync_oracle_handle: _sync_oracle_handle } = + test_harness; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + + let candidate_commitments = CandidateCommitments::default(); + + let candidate_receipt1 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(1_u32); + receipt.commitments_hash = candidate_commitments.hash(); + receipt + }; + + let candidate_hash1 = candidate_receipt1.hash(); + + let candidate_receipt2 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(2_u32); + receipt.commitments_hash = candidate_commitments.hash(); + receipt + }; + + let slot = Slot::from(1); + let candidate_index1 = 0; + let candidate_index2 = 1; + + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Eve, + ]; + let session_info = SessionInfo { + validator_groups: IndexedVec::>::from(vec![ + vec![ValidatorIndex(0), ValidatorIndex(1)], + vec![ValidatorIndex(2)], + vec![ValidatorIndex(3), ValidatorIndex(4)], + ]), + ..session_info(&validators) + }; + + let candidates = Some(vec![ + (candidate_receipt1.clone(), CoreIndex(0), GroupIndex(0)), + (candidate_receipt2.clone(), CoreIndex(1), GroupIndex(1)), + ]); + ChainBuilder::new() + .add_block( + block_hash, + ChainBuilder::GENESIS_HASH, + 1, + BlockConfig { + slot, + candidates: candidates.clone(), + session_info: Some(session_info.clone()), + }, + ) + .build(&mut virtual_overseer) + .await; + + assert!(!clock.inner.lock().current_wakeup_is(1)); + clock.inner.lock().wakeup_all(1); + + assert!(clock.inner.lock().current_wakeup_is(slot_to_tick(slot))); + clock.inner.lock().wakeup_all(slot_to_tick(slot)); + + futures_timer::Delay::new(Duration::from_millis(200)).await; + + clock.inner.lock().wakeup_all(slot_to_tick(slot + 2)); + + assert_eq!(clock.inner.lock().wakeups.len(), 0); + + futures_timer::Delay::new(Duration::from_millis(200)).await; + + let candidate_entry = store.load_candidate_entry(&candidate_hash1).unwrap().unwrap(); + let our_assignment = + candidate_entry.approval_entry(&block_hash).unwrap().our_assignment().unwrap(); + assert!(our_assignment.triggered()); + + handle_approval_on_max_coalesce_count( + &mut virtual_overseer, + vec![candidate_index1, candidate_index2], + ) + .await; + + virtual_overseer + }); +} + +async fn handle_approval_on_max_coalesce_count( + virtual_overseer: &mut VirtualOverseer, + candidate_indicies: Vec, +) { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + c_indices, + )) => { + assert_eq!(TryInto::::try_into(candidate_indicies.clone()).unwrap(), c_indices); + } + ); + + for _ in &candidate_indicies { + recover_available_data(virtual_overseer).await; + fetch_validation_code(virtual_overseer).await; + } + + for _ in &candidate_indicies { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, timeout, tx)) if timeout == PvfExecTimeoutKind::Approval => { + tx.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) + .unwrap(); + } + ); + } + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 2, + })); + } + ); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 2, + })); + } + ); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(vote)) => { + assert_eq!(TryInto::::try_into(candidate_indicies).unwrap(), vote.candidate_indices); + } + ); + + // Assert that there are no more messages being sent by the subsystem + assert!(overseer_recv(virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); +} + +async fn handle_approval_on_max_wait_time( + virtual_overseer: &mut VirtualOverseer, + candidate_indicies: Vec, + clock: Box, +) { + const TICK_NOW_BEGIN: u64 = 1; + const MAX_COALESCE_COUNT: u32 = 3; + + clock.inner.lock().set_tick(TICK_NOW_BEGIN); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + c_indices, + )) => { + assert_eq!(TryInto::::try_into(candidate_indicies.clone()).unwrap(), c_indices); + } + ); + + for _ in &candidate_indicies { + recover_available_data(virtual_overseer).await; + fetch_validation_code(virtual_overseer).await; + } + + for _ in &candidate_indicies { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, timeout, tx)) if timeout == PvfExecTimeoutKind::Approval => { + tx.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) + .unwrap(); + } + ); + } + + // First time we fetch the configuration when we are ready to approve the first candidate + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: MAX_COALESCE_COUNT, + })); + } + ); + + // Second time we fetch the configuration when we are ready to approve the second candidate + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: MAX_COALESCE_COUNT, + })); + } + ); + + assert!(overseer_recv(virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); + + // Move the clock just before we should send the approval + clock + .inner + .lock() + .set_tick(MAX_APPROVAL_COALESCE_WAIT_TICKS as Tick + TICK_NOW_BEGIN - 1); + + assert!(overseer_recv(virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); + + // Move the clock tick, so we can trigger a force sending of the approvals + clock + .inner + .lock() + .set_tick(MAX_APPROVAL_COALESCE_WAIT_TICKS as Tick + TICK_NOW_BEGIN); + + // Third time we fetch the configuration when timer expires and we are ready to sent the approval + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 3, + })); + } + ); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(vote)) => { + assert_eq!(TryInto::::try_into(candidate_indicies).unwrap(), vote.candidate_indices); + } + ); + + // Assert that there are no more messages being sent by the subsystem + assert!(overseer_recv(virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); +} + +#[test] +fn test_approval_is_sent_on_max_approval_coalesce_wait() { + let assignment_criteria = Box::new(MockAssignmentCriteria( + || { + let mut assignments = HashMap::new(); + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let assignments_cert = + garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: vec![CoreIndex(0), CoreIndex(1), CoreIndex(2)] + .try_into() + .unwrap(), + }); + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: assignments_cert.clone(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let _ = assignments.insert( + CoreIndex(1), + approval_db::v2::OurAssignment { + cert: assignments_cert.clone(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + assignments + }, + |_| Ok(0), + )); + + let config = HarnessConfigBuilder::default().assignment_criteria(assignment_criteria).build(); + let store = config.backend(); + + test_harness(config, |test_harness| async move { + let TestHarness { mut virtual_overseer, clock, sync_oracle_handle: _sync_oracle_handle } = + test_harness; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + + let candidate_commitments = CandidateCommitments::default(); + + let candidate_receipt1 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(1_u32); + receipt.commitments_hash = candidate_commitments.hash(); + receipt + }; + + let candidate_hash1 = candidate_receipt1.hash(); + + let candidate_receipt2 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(2_u32); + receipt.commitments_hash = candidate_commitments.hash(); + receipt + }; + + let slot = Slot::from(1); + let candidate_index1 = 0; + let candidate_index2 = 1; + + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Eve, + ]; + let session_info = SessionInfo { + validator_groups: IndexedVec::>::from(vec![ + vec![ValidatorIndex(0), ValidatorIndex(1)], + vec![ValidatorIndex(2)], + vec![ValidatorIndex(3), ValidatorIndex(4)], + ]), + ..session_info(&validators) + }; + + let candidates = Some(vec![ + (candidate_receipt1.clone(), CoreIndex(0), GroupIndex(0)), + (candidate_receipt2.clone(), CoreIndex(1), GroupIndex(1)), + ]); + ChainBuilder::new() + .add_block( + block_hash, + ChainBuilder::GENESIS_HASH, + 1, + BlockConfig { + slot, + candidates: candidates.clone(), + session_info: Some(session_info.clone()), + }, + ) + .build(&mut virtual_overseer) + .await; + + assert!(!clock.inner.lock().current_wakeup_is(1)); + clock.inner.lock().wakeup_all(1); + + assert!(clock.inner.lock().current_wakeup_is(slot_to_tick(slot))); + clock.inner.lock().wakeup_all(slot_to_tick(slot)); + + futures_timer::Delay::new(Duration::from_millis(200)).await; + + clock.inner.lock().wakeup_all(slot_to_tick(slot + 2)); + + assert_eq!(clock.inner.lock().wakeups.len(), 0); + + futures_timer::Delay::new(Duration::from_millis(200)).await; + + let candidate_entry = store.load_candidate_entry(&candidate_hash1).unwrap().unwrap(); + let our_assignment = + candidate_entry.approval_entry(&block_hash).unwrap().our_assignment().unwrap(); + assert!(our_assignment.triggered()); + + handle_approval_on_max_wait_time( + &mut virtual_overseer, + vec![candidate_index1, candidate_index2], + clock, + ) + .await; + + virtual_overseer + }); +} diff --git a/polkadot/node/core/approval-voting/src/time.rs b/polkadot/node/core/approval-voting/src/time.rs index a45866402c82..61091f3c34cd 100644 --- a/polkadot/node/core/approval-voting/src/time.rs +++ b/polkadot/node/core/approval-voting/src/time.rs @@ -16,14 +16,23 @@ //! Time utilities for approval voting. -use futures::prelude::*; +use futures::{ + future::BoxFuture, + prelude::*, + stream::{FusedStream, FuturesUnordered}, + Stream, StreamExt, +}; + use polkadot_node_primitives::approval::v1::DelayTranche; use sp_consensus_slots::Slot; use std::{ + collections::HashSet, pin::Pin, + task::Poll, time::{Duration, SystemTime}, }; +use polkadot_primitives::{Hash, ValidatorIndex}; const TICK_DURATION_MILLIS: u64 = 500; /// A base unit of time, starting from the Unix epoch, split into half-second intervals. @@ -88,3 +97,157 @@ pub(crate) fn slot_number_to_tick(slot_duration_millis: u64, slot: Slot) -> Tick let ticks_per_slot = slot_duration_millis / TICK_DURATION_MILLIS; u64::from(slot) * ticks_per_slot } + +/// A list of delayed futures that gets triggered when the waiting time has expired and it is +/// time to sign the candidate. +/// We have a timer per relay-chain block. +#[derive(Default)] +pub struct DelayedApprovalTimer { + timers: FuturesUnordered>, + blocks: HashSet, +} + +impl DelayedApprovalTimer { + /// Starts a single timer per block hash + /// + /// Guarantees that if a timer already exits for the give block hash, + /// no additional timer is started. + pub(crate) fn maybe_arm_timer( + &mut self, + wait_untill: Tick, + clock: &dyn Clock, + block_hash: Hash, + validator_index: ValidatorIndex, + ) { + if self.blocks.insert(block_hash) { + let clock_wait = clock.wait(wait_untill); + self.timers.push(Box::pin(async move { + clock_wait.await; + (block_hash, validator_index) + })); + } + } +} + +impl Stream for DelayedApprovalTimer { + type Item = (Hash, ValidatorIndex); + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + let poll_result = self.timers.poll_next_unpin(cx); + match poll_result { + Poll::Ready(Some(result)) => { + self.blocks.remove(&result.0); + Poll::Ready(Some(result)) + }, + _ => poll_result, + } + } +} + +impl FusedStream for DelayedApprovalTimer { + fn is_terminated(&self) -> bool { + self.timers.is_terminated() + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use futures::{executor::block_on, FutureExt, StreamExt}; + use futures_timer::Delay; + use polkadot_primitives::{Hash, ValidatorIndex}; + + use crate::time::{Clock, SystemClock}; + + use super::DelayedApprovalTimer; + + #[test] + fn test_select_empty_timer() { + block_on(async move { + let mut timer = DelayedApprovalTimer::default(); + + for _ in 1..10 { + let result = futures::select!( + _ = timer.select_next_some() => { + 0 + } + // Only this arm should fire + _ = Delay::new(Duration::from_millis(100)).fuse() => { + 1 + } + ); + + assert_eq!(result, 1); + } + }); + } + + #[test] + fn test_timer_functionality() { + block_on(async move { + let mut timer = DelayedApprovalTimer::default(); + let test_hashes = + vec![Hash::repeat_byte(0x01), Hash::repeat_byte(0x02), Hash::repeat_byte(0x03)]; + for (index, hash) in test_hashes.iter().enumerate() { + timer.maybe_arm_timer( + SystemClock.tick_now() + index as u64, + &SystemClock, + *hash, + ValidatorIndex::from(2), + ); + timer.maybe_arm_timer( + SystemClock.tick_now() + index as u64, + &SystemClock, + *hash, + ValidatorIndex::from(2), + ); + } + let timeout_hash = Hash::repeat_byte(0x02); + for i in 0..test_hashes.len() * 2 { + let result = futures::select!( + (hash, _) = timer.select_next_some() => { + hash + } + // Timers should fire only once, so for the rest of the iterations we should timeout through here. + _ = Delay::new(Duration::from_secs(2)).fuse() => { + timeout_hash + } + ); + assert_eq!(test_hashes.get(i).cloned().unwrap_or(timeout_hash), result); + } + + // Now check timer can be restarted if already fired + for (index, hash) in test_hashes.iter().enumerate() { + timer.maybe_arm_timer( + SystemClock.tick_now() + index as u64, + &SystemClock, + *hash, + ValidatorIndex::from(2), + ); + timer.maybe_arm_timer( + SystemClock.tick_now() + index as u64, + &SystemClock, + *hash, + ValidatorIndex::from(2), + ); + } + + for i in 0..test_hashes.len() * 2 { + let result = futures::select!( + (hash, _) = timer.select_next_some() => { + hash + } + // Timers should fire only once, so for the rest of the iterations we should timeout through here. + _ = Delay::new(Duration::from_secs(2)).fuse() => { + timeout_hash + } + ); + assert_eq!(test_hashes.get(i).cloned().unwrap_or(timeout_hash), result); + } + }); + } +} diff --git a/polkadot/node/core/dispute-coordinator/src/import.rs b/polkadot/node/core/dispute-coordinator/src/import.rs index 6222aab1cb10..d2520e6f9a64 100644 --- a/polkadot/node/core/dispute-coordinator/src/import.rs +++ b/polkadot/node/core/dispute-coordinator/src/import.rs @@ -34,7 +34,7 @@ use polkadot_node_primitives::{ use polkadot_node_subsystem::overseer; use polkadot_node_subsystem_util::runtime::RuntimeInfo; use polkadot_primitives::{ - CandidateReceipt, DisputeStatement, Hash, IndexedVec, SessionIndex, SessionInfo, + CandidateHash, CandidateReceipt, DisputeStatement, Hash, IndexedVec, SessionIndex, SessionInfo, ValidDisputeStatementKind, ValidatorId, ValidatorIndex, ValidatorPair, ValidatorSignature, }; use sc_keystore::LocalKeystore; @@ -118,7 +118,9 @@ impl OwnVoteState { let our_valid_votes = controlled_indices .iter() .filter_map(|i| votes.valid.raw().get_key_value(i)) - .map(|(index, (kind, sig))| (*index, (DisputeStatement::Valid(*kind), sig.clone()))); + .map(|(index, (kind, sig))| { + (*index, (DisputeStatement::Valid(kind.clone()), sig.clone())) + }); let our_invalid_votes = controlled_indices .iter() .filter_map(|i| votes.invalid.get_key_value(i)) @@ -297,7 +299,7 @@ impl CandidateVoteState { DisputeStatement::Valid(valid_kind) => { let fresh = votes.valid.insert_vote( val_index, - *valid_kind, + valid_kind.clone(), statement.into_validator_signature(), ); if fresh { @@ -503,7 +505,7 @@ impl ImportResult { pub fn import_approval_votes( self, env: &CandidateEnvironment, - approval_votes: HashMap, + approval_votes: HashMap, ValidatorSignature)>, now: Timestamp, ) -> Self { let Self { @@ -517,19 +519,32 @@ impl ImportResult { let (mut votes, _) = new_state.into_old_state(); - for (index, sig) in approval_votes.into_iter() { + for (index, (candidate_hashes, sig)) in approval_votes.into_iter() { debug_assert!( { let pub_key = &env.session_info().validators.get(index).expect("indices are validated by approval-voting subsystem; qed"); - let candidate_hash = votes.candidate_receipt.hash(); let session_index = env.session_index(); - DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) - .check_signature(pub_key, candidate_hash, session_index, &sig) + candidate_hashes.contains(&votes.candidate_receipt.hash()) && DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(candidate_hashes.clone())) + .check_signature(pub_key, *candidate_hashes.first().expect("Valid votes have at least one candidate; qed"), session_index, &sig) .is_ok() }, "Signature check for imported approval votes failed! This is a serious bug. Session: {:?}, candidate hash: {:?}, validator index: {:?}", env.session_index(), votes.candidate_receipt.hash(), index ); - if votes.valid.insert_vote(index, ValidDisputeStatementKind::ApprovalChecking, sig) { + if votes.valid.insert_vote( + index, + // There is a hidden dependency here between approval-voting and this subsystem. + // We should be able to start emitting ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates only after: + // 1. Runtime have been upgraded to know about the new format. + // 2. All nodes have been upgraded to know about the new format. + // Once those two requirements have been met we should be able to increase max_approval_coalesce_count to values + // greater than 1. + if candidate_hashes.len() > 1 { + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(candidate_hashes) + } else { + ValidDisputeStatementKind::ApprovalChecking + }, + sig, + ) { imported_valid_votes += 1; imported_approval_votes += 1; } diff --git a/polkadot/node/core/dispute-coordinator/src/initialized.rs b/polkadot/node/core/dispute-coordinator/src/initialized.rs index c1d02ef976cb..7afc120d4161 100644 --- a/polkadot/node/core/dispute-coordinator/src/initialized.rs +++ b/polkadot/node/core/dispute-coordinator/src/initialized.rs @@ -633,7 +633,7 @@ impl Initialized { }; debug_assert!( SignedDisputeStatement::new_checked( - DisputeStatement::Valid(valid_statement_kind), + DisputeStatement::Valid(valid_statement_kind.clone()), candidate_hash, session, validator_public.clone(), @@ -647,7 +647,7 @@ impl Initialized { ); let signed_dispute_statement = SignedDisputeStatement::new_unchecked_from_trusted_source( - DisputeStatement::Valid(valid_statement_kind), + DisputeStatement::Valid(valid_statement_kind.clone()), candidate_hash, session, validator_public, diff --git a/polkadot/node/core/dispute-coordinator/src/lib.rs b/polkadot/node/core/dispute-coordinator/src/lib.rs index a2c500e08e28..b18cf8aa4aa9 100644 --- a/polkadot/node/core/dispute-coordinator/src/lib.rs +++ b/polkadot/node/core/dispute-coordinator/src/lib.rs @@ -575,7 +575,7 @@ pub fn make_dispute_message( .next() .ok_or(DisputeMessageCreationError::NoOppositeVote)?; let other_vote = SignedDisputeStatement::new_checked( - DisputeStatement::Valid(*statement_kind), + DisputeStatement::Valid(statement_kind.clone()), *our_vote.candidate_hash(), our_vote.session_index(), validators diff --git a/polkadot/node/core/dispute-coordinator/src/tests.rs b/polkadot/node/core/dispute-coordinator/src/tests.rs index 75eae8200dc6..552613c566b9 100644 --- a/polkadot/node/core/dispute-coordinator/src/tests.rs +++ b/polkadot/node/core/dispute-coordinator/src/tests.rs @@ -651,7 +651,7 @@ fn make_candidate_included_event(candidate_receipt: CandidateReceipt) -> Candida pub async fn handle_approval_vote_request( ctx_handle: &mut VirtualOverseer, expected_hash: &CandidateHash, - votes_to_send: HashMap, + votes_to_send: HashMap, ValidatorSignature)>, ) { assert_matches!( ctx_handle.recv().await, @@ -858,9 +858,12 @@ fn approval_vote_import_works() { .await; gum::trace!("After sending `ImportStatements`"); - let approval_votes = [(ValidatorIndex(4), approval_vote.into_validator_signature())] - .into_iter() - .collect(); + let approval_votes = [( + ValidatorIndex(4), + (vec![candidate_receipt1.hash()], approval_vote.into_validator_signature()), + )] + .into_iter() + .collect(); handle_approval_vote_request(&mut virtual_overseer, &candidate_hash1, approval_votes) .await; diff --git a/polkadot/node/core/provisioner/src/disputes/prioritized_selection/mod.rs b/polkadot/node/core/provisioner/src/disputes/prioritized_selection/mod.rs index 096b73d271a8..cb55ce39bc89 100644 --- a/polkadot/node/core/provisioner/src/disputes/prioritized_selection/mod.rs +++ b/polkadot/node/core/provisioner/src/disputes/prioritized_selection/mod.rs @@ -221,7 +221,7 @@ where votes.valid.retain(|validator_idx, (statement_kind, _)| { is_vote_worth_to_keep( validator_idx, - DisputeStatement::Valid(*statement_kind), + DisputeStatement::Valid(statement_kind.clone()), &onchain_state, ) }); diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs index 9f3b7e23fd89..aadc14775025 100644 --- a/polkadot/node/core/pvf/src/host.rs +++ b/polkadot/node/core/pvf/src/host.rs @@ -186,7 +186,8 @@ impl Config { prepare_workers_hard_max_num: 1, execute_worker_program_path, execute_worker_spawn_timeout: Duration::from_secs(3), - execute_workers_max_num: 2, + // TODO: cleanup increased for versi experimenting. + execute_workers_max_num: 4, } } } diff --git a/polkadot/node/core/runtime-api/src/cache.rs b/polkadot/node/core/runtime-api/src/cache.rs index 26aaf3fb6ec8..a0b2c7c0f2e4 100644 --- a/polkadot/node/core/runtime-api/src/cache.rs +++ b/polkadot/node/core/runtime-api/src/cache.rs @@ -20,12 +20,12 @@ use lru::LruCache; use sp_consensus_babe::Epoch; use polkadot_primitives::{ - vstaging, AuthorityDiscoveryId, BlockNumber, CandidateCommitments, CandidateEvent, - CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, - GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, - SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, - ValidatorSignature, + vstaging::{self, ApprovalVotingParams}, + AuthorityDiscoveryId, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, OccupiedCoreAssumption, + PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, + ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; /// For consistency we have the same capacity for all caches. We use 128 as we'll only need that @@ -64,6 +64,8 @@ pub(crate) struct RequestResultCache { LruCache<(Hash, ParaId, OccupiedCoreAssumption), Option>, version: LruCache, disputes: LruCache)>>, + approval_voting_params: LruCache, + unapplied_slashes: LruCache>, key_ownership_proof: @@ -100,7 +102,7 @@ impl Default for RequestResultCache { disputes: LruCache::new(DEFAULT_CACHE_CAP), unapplied_slashes: LruCache::new(DEFAULT_CACHE_CAP), key_ownership_proof: LruCache::new(DEFAULT_CACHE_CAP), - + approval_voting_params: LruCache::new(DEFAULT_CACHE_CAP), staging_para_backing_state: LruCache::new(DEFAULT_CACHE_CAP), staging_async_backing_params: LruCache::new(DEFAULT_CACHE_CAP), } @@ -399,6 +401,21 @@ impl RequestResultCache { self.disputes.put(relay_parent, value); } + pub(crate) fn approval_voting_params( + &mut self, + relay_parent: &Hash, + ) -> Option<&ApprovalVotingParams> { + self.approval_voting_params.get(relay_parent) + } + + pub(crate) fn cache_approval_voting_params( + &mut self, + relay_parent: Hash, + value: ApprovalVotingParams, + ) { + self.approval_voting_params.put(relay_parent, value); + } + pub(crate) fn unapplied_slashes( &mut self, relay_parent: &Hash, @@ -512,7 +529,7 @@ pub(crate) enum RequestResult { vstaging::slashing::OpaqueKeyOwnershipProof, Option<()>, ), - + ApprovalVotingParams(Hash, ApprovalVotingParams), StagingParaBackingState(Hash, ParaId, Option), StagingAsyncBackingParams(Hash, vstaging::AsyncBackingParams), } diff --git a/polkadot/node/core/runtime-api/src/lib.rs b/polkadot/node/core/runtime-api/src/lib.rs index 78531d41272b..1d97e4676c58 100644 --- a/polkadot/node/core/runtime-api/src/lib.rs +++ b/polkadot/node/core/runtime-api/src/lib.rs @@ -162,6 +162,8 @@ where KeyOwnershipProof(relay_parent, validator_id, key_ownership_proof) => self .requests_cache .cache_key_ownership_proof((relay_parent, validator_id), key_ownership_proof), + RequestResult::ApprovalVotingParams(relay_parent, params) => + self.requests_cache.cache_approval_voting_params(relay_parent, params), SubmitReportDisputeLost(_, _, _, _) => {}, StagingParaBackingState(relay_parent, para_id, constraints) => self @@ -294,6 +296,8 @@ where Request::SubmitReportDisputeLost(dispute_proof, key_ownership_proof, sender) }, ), + Request::ApprovalVotingParams(sender) => query!(approval_voting_params(), sender) + .map(|sender| Request::ApprovalVotingParams(sender)), Request::StagingParaBackingState(para, sender) => query!(staging_para_backing_state(para), sender) @@ -545,6 +549,9 @@ where ver = Request::KEY_OWNERSHIP_PROOF_RUNTIME_REQUIREMENT, sender ), + Request::ApprovalVotingParams(sender) => { + query!(ApprovalVotingParams, approval_voting_params(), ver = 6, sender) + }, Request::SubmitReportDisputeLost(dispute_proof, key_ownership_proof, sender) => query!( SubmitReportDisputeLost, submit_report_dispute_lost(dispute_proof, key_ownership_proof), diff --git a/polkadot/node/core/runtime-api/src/tests.rs b/polkadot/node/core/runtime-api/src/tests.rs index c3f8108312be..d13e6cede356 100644 --- a/polkadot/node/core/runtime-api/src/tests.rs +++ b/polkadot/node/core/runtime-api/src/tests.rs @@ -20,12 +20,12 @@ use polkadot_node_primitives::{BabeAllowedSlots, BabeEpoch, BabeEpochConfigurati use polkadot_node_subsystem::SpawnGlue; use polkadot_node_subsystem_test_helpers::make_subsystem_context; use polkadot_primitives::{ - vstaging, AuthorityDiscoveryId, BlockNumber, CandidateCommitments, CandidateEvent, - CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, - GroupRotationInfo, Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, - SessionIndex, SessionInfo, Slot, ValidationCode, ValidationCodeHash, ValidatorId, - ValidatorIndex, ValidatorSignature, + vstaging::{self, ApprovalVotingParams}, + AuthorityDiscoveryId, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, OccupiedCoreAssumption, + PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, + Slot, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; use sp_api::ApiError; use sp_core::testing::TaskExecutor; @@ -242,6 +242,11 @@ impl RuntimeApiSubsystemClient for MockSubsystemClient { todo!("Not required for tests") } + /// Approval voting configuration parameters + async fn approval_voting_params(&self, _: Hash) -> Result { + todo!("Not required for tests") + } + async fn current_epoch(&self, _: Hash) -> Result { Ok(self.babe_epoch.as_ref().unwrap().clone()) } diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 746a4b4dab5c..3b8624e9166d 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -36,7 +36,10 @@ use polkadot_node_primitives::approval::{ v1::{ AssignmentCertKind, BlockApprovalMeta, IndirectAssignmentCert, IndirectSignedApprovalVote, }, - v2::{AsBitIndex, AssignmentCertKindV2, CandidateBitfield, IndirectAssignmentCertV2}, + v2::{ + AsBitIndex, AssignmentCertKindV2, CandidateBitfield, IndirectAssignmentCertV2, + IndirectSignedApprovalVoteV2, + }, }; use polkadot_node_subsystem::{ messages::{ @@ -120,7 +123,7 @@ struct ApprovalEntry { // The candidates claimed by the certificate. A mapping between bit index and candidate index. candidates: CandidateBitfield, // The approval signatures for each `CandidateIndex` claimed by the assignment certificate. - approvals: HashMap, + approvals: HashMap, // The validator index of the assignment signer. validator_index: ValidatorIndex, // Information required for gossiping to other peers using the grid topology. @@ -189,7 +192,8 @@ impl ApprovalEntry { // have received the approval. pub fn note_approval( &mut self, - approval: IndirectSignedApprovalVote, + approval: IndirectSignedApprovalVoteV2, + candidate_index: CandidateIndex, ) -> Result<(), ApprovalEntryError> { // First do some sanity checks: // - check validator index matches @@ -199,19 +203,18 @@ impl ApprovalEntry { return Err(ApprovalEntryError::InvalidValidatorIndex) } - if self.candidates.len() <= approval.candidate_index as usize { + if self.candidates.len() <= candidate_index as usize { return Err(ApprovalEntryError::CandidateIndexOutOfBounds) } - - if !self.candidates.bit_at(approval.candidate_index.as_bit_index()) { + if !self.candidates.bit_at((candidate_index).as_bit_index()) { return Err(ApprovalEntryError::InvalidCandidateIndex) } - if self.approvals.contains_key(&approval.candidate_index) { + if self.approvals.contains_key(&candidate_index) { return Err(ApprovalEntryError::DuplicateApproval) } - self.approvals.insert(approval.candidate_index, approval); + self.approvals.insert(candidate_index, approval.clone()); Ok(()) } @@ -221,12 +224,15 @@ impl ApprovalEntry { } // Get all approvals for all candidates claimed by the assignment. - pub fn approvals(&self) -> Vec { + pub fn approvals(&self) -> Vec { self.approvals.values().cloned().collect::>() } // Get the approval for a specific candidate index. - pub fn approval(&self, candidate_index: CandidateIndex) -> Option { + pub fn approval( + &self, + candidate_index: CandidateIndex, + ) -> Option { self.approvals.get(&candidate_index).cloned() } @@ -554,7 +560,7 @@ impl MessageSource { enum PendingMessage { Assignment(IndirectAssignmentCertV2, CandidateBitfield), - Approval(IndirectSignedApprovalVote), + Approval(IndirectSignedApprovalVoteV2), } #[overseer::contextbounds(ApprovalDistribution, prefix = self::overseer)] @@ -827,6 +833,48 @@ impl State { } } + async fn process_incoming_approvals( + &mut self, + ctx: &mut Context, + metrics: &Metrics, + peer_id: PeerId, + approvals: Vec, + ) { + gum::trace!( + target: LOG_TARGET, + peer_id = %peer_id, + num = approvals.len(), + "Processing approvals from a peer", + ); + for approval_vote in approvals.into_iter() { + if let Some(pending) = self.pending_known.get_mut(&approval_vote.block_hash) { + let block_hash = approval_vote.block_hash; + let validator_index = approval_vote.validator; + + gum::trace!( + target: LOG_TARGET, + %peer_id, + ?block_hash, + ?validator_index, + "Pending assignment candidates {:?}", + approval_vote.candidate_indices, + ); + + pending.push((peer_id, PendingMessage::Approval(approval_vote))); + + continue + } + + self.import_and_circulate_approval( + ctx, + metrics, + MessageSource::Peer(peer_id), + approval_vote, + ) + .await; + } + } + async fn process_incoming_peer_message( &mut self, ctx: &mut Context, @@ -884,42 +932,17 @@ impl State { }, Versioned::VStaging(protocol_vstaging::ApprovalDistributionMessage::Approvals( approvals, - )) | + )) => { + self.process_incoming_approvals(ctx, metrics, peer_id, approvals).await; + }, Versioned::V1(protocol_v1::ApprovalDistributionMessage::Approvals(approvals)) => { - gum::trace!( - target: LOG_TARGET, - peer_id = %peer_id, - num = approvals.len(), - "Processing approvals from a peer", - ); - for approval_vote in approvals.into_iter() { - if let Some(pending) = self.pending_known.get_mut(&approval_vote.block_hash) { - let block_hash = approval_vote.block_hash; - let candidate_index = approval_vote.candidate_index; - let validator_index = approval_vote.validator; - - gum::trace!( - target: LOG_TARGET, - %peer_id, - ?block_hash, - ?candidate_index, - ?validator_index, - "Pending assignment", - ); - - pending.push((peer_id, PendingMessage::Approval(approval_vote))); - - continue - } - - self.import_and_circulate_approval( - ctx, - metrics, - MessageSource::Peer(peer_id), - approval_vote, - ) - .await; - } + self.process_incoming_approvals( + ctx, + metrics, + peer_id, + approvals.into_iter().map(|approval| approval.into()).collect::>(), + ) + .await; }, } } @@ -1065,8 +1088,11 @@ impl State { COST_UNEXPECTED_MESSAGE, ) .await; + gum::debug!(target: LOG_TARGET, "Received assignment for invalid block"); + metrics.on_assignment_recent_outdated(); } } + metrics.on_assignment_invalid_block(); return }, }; @@ -1099,6 +1125,7 @@ impl State { COST_DUPLICATE_MESSAGE, ) .await; + metrics.on_assignment_duplicate(); } else { gum::trace!( target: LOG_TARGET, @@ -1126,6 +1153,7 @@ impl State { COST_UNEXPECTED_MESSAGE, ) .await; + metrics.on_assignment_out_of_view(); }, } @@ -1142,6 +1170,7 @@ impl State { gum::trace!(target: LOG_TARGET, ?peer_id, ?message_subject, "Known assignment"); peer_knowledge.received.insert(message_subject, message_kind); } + metrics.on_assignment_good_known(); return } @@ -1198,6 +1227,8 @@ impl State { ?peer_id, "Got an `AcceptedDuplicate` assignment", ); + metrics.on_assignment_duplicatevoting(); + return }, AssignmentCheckResult::TooFarInFuture => { @@ -1214,6 +1245,8 @@ impl State { COST_ASSIGNMENT_TOO_FAR_IN_THE_FUTURE, ) .await; + metrics.on_assignment_far(); + return }, AssignmentCheckResult::Bad(error) => { @@ -1231,6 +1264,7 @@ impl State { COST_INVALID_MESSAGE, ) .await; + metrics.on_assignment_bad(); return }, } @@ -1299,6 +1333,9 @@ impl State { continue } + if !topology.map(|topology| topology.is_validator(&peer)).unwrap_or(false) { + continue + } // Note: at this point, we haven't received the message from any peers // other than the source peer, and we just got it, so we haven't sent it // to any peers either. @@ -1340,12 +1377,95 @@ impl State { } } + async fn check_approval_can_be_processed( + ctx: &mut Context, + message_subjects: &Vec, + message_kind: MessageKind, + entry: &mut BlockEntry, + reputation: &mut ReputationAggregator, + peer_id: PeerId, + metrics: &Metrics, + ) -> bool { + for message_subject in message_subjects { + if !entry.knowledge.contains(&message_subject, MessageKind::Assignment) { + gum::trace!( + target: LOG_TARGET, + ?peer_id, + ?message_subject, + "Unknown approval assignment", + ); + modify_reputation(reputation, ctx.sender(), peer_id, COST_UNEXPECTED_MESSAGE).await; + metrics.on_approval_unknown_assignment(); + return false + } + + // check if our knowledge of the peer already contains this approval + match entry.known_by.entry(peer_id) { + hash_map::Entry::Occupied(mut knowledge) => { + let peer_knowledge = knowledge.get_mut(); + if peer_knowledge.contains(&message_subject, message_kind) { + if !peer_knowledge.received.insert(message_subject.clone(), message_kind) { + gum::trace!( + target: LOG_TARGET, + ?peer_id, + ?message_subject, + "Duplicate approval", + ); + + modify_reputation( + reputation, + ctx.sender(), + peer_id, + COST_DUPLICATE_MESSAGE, + ) + .await; + metrics.on_approval_duplicate(); + } + return false + } + }, + hash_map::Entry::Vacant(_) => { + gum::debug!( + target: LOG_TARGET, + ?peer_id, + ?message_subject, + "Approval from a peer is out of view", + ); + modify_reputation(reputation, ctx.sender(), peer_id, COST_UNEXPECTED_MESSAGE) + .await; + metrics.on_approval_out_of_view(); + }, + } + } + + let good_known_approval = + message_subjects.iter().fold(false, |accumulator, message_subject| { + // if the approval is known to be valid, reward the peer + if entry.knowledge.contains(&message_subject, message_kind) { + if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { + peer_knowledge.received.insert(message_subject.clone(), message_kind); + } + // We already processed this approval no need to continue. + true + } else { + accumulator + } + }); + if good_known_approval { + gum::trace!(target: LOG_TARGET, ?peer_id, ?message_subjects, "Known approval"); + metrics.on_approval_good_known(); + modify_reputation(reputation, ctx.sender(), peer_id, BENEFIT_VALID_MESSAGE).await; + } + + !good_known_approval + } + async fn import_and_circulate_approval( &mut self, ctx: &mut Context, metrics: &Metrics, source: MessageSource, - vote: IndirectSignedApprovalVote, + vote: IndirectSignedApprovalVoteV2, ) { let _span = self .spans @@ -1364,10 +1484,21 @@ impl State { let block_hash = vote.block_hash; let validator_index = vote.validator; - let candidate_index = vote.candidate_index; - + let candidate_indices = &vote.candidate_indices; let entry = match self.blocks.get_mut(&block_hash) { - Some(entry) if entry.candidates.get(candidate_index as usize).is_some() => entry, + Some(entry) + if vote.candidate_indices.iter_ones().fold(true, |result, candidate_index| { + let approval_entry_exists = entry.candidates.get(candidate_index as usize).is_some(); + if !approval_entry_exists { + gum::debug!( + target: LOG_TARGET, ?block_hash, ?candidate_index, validator_index = ?vote.validator, candidate_indices = ?vote.candidate_indices, + peer_id = ?source.peer_id(), "Received approval before assignment" + ); + metrics.on_approval_entry_not_found(); + } + approval_entry_exists && result + }) => + entry, _ => { if let Some(peer_id) = source.peer_id() { if !self.recent_outdated_blocks.is_recent_outdated(&block_hash) { @@ -1376,7 +1507,7 @@ impl State { ?peer_id, ?block_hash, ?validator_index, - ?candidate_index, + ?candidate_indices, "Approval from a peer is out of view", ); modify_reputation( @@ -1386,6 +1517,10 @@ impl State { COST_UNEXPECTED_MESSAGE, ) .await; + metrics.on_approval_invalid_block(); + + } else { + metrics.on_approval_recent_outdated(); } } return @@ -1393,81 +1528,30 @@ impl State { }; // compute metadata on the assignment. - let message_subject = MessageSubject(block_hash, candidate_index.into(), validator_index); + let message_subjects = candidate_indices + .iter_ones() + .map(|candidate_index| { + MessageSubject( + block_hash, + (candidate_index as CandidateIndex).into(), + validator_index, + ) + }) + .collect_vec(); let message_kind = MessageKind::Approval; if let Some(peer_id) = source.peer_id() { - if !entry.knowledge.contains(&message_subject, MessageKind::Assignment) { - gum::debug!( - target: LOG_TARGET, - ?peer_id, - ?message_subject, - "Unknown approval assignment", - ); - modify_reputation( - &mut self.reputation, - ctx.sender(), - peer_id, - COST_UNEXPECTED_MESSAGE, - ) - .await; - return - } - - // check if our knowledge of the peer already contains this approval - match entry.known_by.entry(peer_id) { - hash_map::Entry::Occupied(mut knowledge) => { - let peer_knowledge = knowledge.get_mut(); - if peer_knowledge.contains(&message_subject, message_kind) { - if !peer_knowledge.received.insert(message_subject.clone(), message_kind) { - gum::debug!( - target: LOG_TARGET, - ?peer_id, - ?message_subject, - "Duplicate approval", - ); - - modify_reputation( - &mut self.reputation, - ctx.sender(), - peer_id, - COST_DUPLICATE_MESSAGE, - ) - .await; - } - return - } - }, - hash_map::Entry::Vacant(_) => { - gum::debug!( - target: LOG_TARGET, - ?peer_id, - ?message_subject, - "Approval from a peer is out of view", - ); - modify_reputation( - &mut self.reputation, - ctx.sender(), - peer_id, - COST_UNEXPECTED_MESSAGE, - ) - .await; - }, - } - - // if the approval is known to be valid, reward the peer - if entry.knowledge.contains(&message_subject, message_kind) { - gum::trace!(target: LOG_TARGET, ?peer_id, ?message_subject, "Known approval"); - modify_reputation( - &mut self.reputation, - ctx.sender(), - peer_id, - BENEFIT_VALID_MESSAGE, - ) - .await; - if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { - peer_knowledge.received.insert(message_subject.clone(), message_kind); - } + if !Self::check_approval_can_be_processed( + ctx, + &message_subjects, + message_kind, + entry, + &mut self.reputation, + peer_id, + metrics, + ) + .await + { return } @@ -1489,7 +1573,6 @@ impl State { gum::trace!( target: LOG_TARGET, ?peer_id, - ?message_subject, ?result, "Checked approval", ); @@ -1503,9 +1586,11 @@ impl State { ) .await; - entry.knowledge.insert(message_subject.clone(), message_kind); - if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { - peer_knowledge.received.insert(message_subject.clone(), message_kind); + for message_subject in &message_subjects { + entry.knowledge.insert(message_subject.clone(), message_kind); + if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { + peer_knowledge.received.insert(message_subject.clone(), message_kind); + } } }, ApprovalCheckResult::Bad(error) => { @@ -1522,73 +1607,80 @@ impl State { %error, "Got a bad approval from peer", ); + metrics.on_approval_bad(); return }, } } else { - if !entry.knowledge.insert(message_subject.clone(), message_kind) { - // if we already imported an approval, there is no need to distribute it again + let all_approvals_imported_already = + message_subjects.iter().fold(true, |result, message_subject| { + !entry.knowledge.insert(message_subject.clone(), message_kind) && result + }); + if all_approvals_imported_already { + // if we already imported all approvals, there is no need to distribute it again gum::warn!( target: LOG_TARGET, - ?message_subject, "Importing locally an already known approval", ); return } else { gum::debug!( target: LOG_TARGET, - ?message_subject, "Importing locally a new approval", ); } } - let required_routing = match entry.approval_entry(candidate_index, validator_index) { - Some(approval_entry) => { - // Invariant: to our knowledge, none of the peers except for the `source` know about - // the approval. - metrics.on_approval_imported(); - - if let Err(err) = approval_entry.note_approval(vote.clone()) { - // this would indicate a bug in approval-voting: - // - validator index mismatch - // - candidate index mismatch - // - duplicate approval - gum::warn!( - target: LOG_TARGET, - hash = ?block_hash, - ?candidate_index, - ?validator_index, - ?err, - "Possible bug: Vote import failed", - ); + let mut required_routing = RequiredRouting::None; - return - } - - approval_entry.routing_info().required_routing - }, - None => { + for candidate_index in candidate_indices.iter_ones() { + // The entry is created when assignment is imported, so we assume this exists. + let approval_entry = entry.approval_entry(candidate_index as _, validator_index); + if approval_entry.is_none() { let peer_id = source.peer_id(); // This indicates a bug in approval-distribution, since we check the knowledge at // the begining of the function. gum::warn!( target: LOG_TARGET, ?peer_id, - ?message_subject, "Unknown approval assignment", ); // No rep change as this is caused by an issue + metrics.on_approval_unexpected(); return - }, - }; + } + + let approval_entry = approval_entry.expect("Just checked above; qed"); + + if let Err(err) = approval_entry.note_approval(vote.clone(), candidate_index as _) { + // this would indicate a bug in approval-voting: + // - validator index mismatch + // - candidate index mismatch + // - duplicate approval + gum::warn!( + target: LOG_TARGET, + hash = ?block_hash, + ?candidate_index, + ?validator_index, + ?err, + "Possible bug: Vote import failed", + ); + metrics.on_approval_bug(); + return + } + required_routing = approval_entry.routing_info().required_routing; + } + + // Invariant: to our knowledge, none of the peers except for the `source` know about the + // approval. + metrics.on_approval_imported(); // Dispatch a ApprovalDistributionV1Message::Approval(vote) // to all peers required by the topology, with the exception of the source peer. let topology = self.topologies.get_topology(entry.session); let source_peer = source.peer_id(); - let message_subject = &message_subject; + let message_subjects_clone = message_subjects.clone(); let peer_filter = move |peer, knowledge: &PeerKnowledge| { if Some(peer) == source_peer.as_ref() { return false @@ -1605,7 +1697,10 @@ impl State { // 3. Any randomly selected peers have been sent the assignment already. let in_topology = topology .map_or(false, |t| t.local_grid_neighbors().route_to_peer(required_routing, peer)); - in_topology || knowledge.sent.contains(message_subject, MessageKind::Assignment) + in_topology || + message_subjects_clone.iter().fold(true, |result, message_subject| { + result && knowledge.sent.contains(message_subject, MessageKind::Assignment) + }) }; let peers = entry @@ -1619,7 +1714,9 @@ impl State { for peer in peers.iter() { // we already filtered peers above, so this should always be Some if let Some(entry) = entry.known_by.get_mut(&peer.0) { - entry.sent.insert(message_subject.clone(), message_kind); + for message_subject in &message_subjects { + entry.sent.insert(message_subject.clone(), message_kind); + } } } @@ -1628,7 +1725,6 @@ impl State { gum::trace!( target: LOG_TARGET, ?block_hash, - ?candidate_index, local = source.peer_id().is_none(), num_peers = peers.len(), "Sending an approval to peers", @@ -1641,10 +1737,22 @@ impl State { ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( v1_peers, Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(approvals.clone()), + protocol_v1::ApprovalDistributionMessage::Approvals( + approvals + .clone() + .into_iter() + .filter(|approval| approval.candidate_indices.count_ones() == 1) + .map(|approval| { + approval + .try_into() + .expect("We checked len() == 1 so it should not fail; qed") + }) + .collect::>(), + ), )), )) .await; + metrics.on_approval_sent_v1(); } if !v2_peers.is_empty() { @@ -1657,6 +1765,7 @@ impl State { ), )) .await; + metrics.on_approval_sent_v2(); } } } @@ -1665,7 +1774,7 @@ impl State { fn get_approval_signatures( &mut self, indices: HashSet<(Hash, CandidateIndex)>, - ) -> HashMap { + ) -> HashMap, ValidatorSignature)> { let mut all_sigs = HashMap::new(); for (hash, index) in indices { let _span = self @@ -1692,8 +1801,22 @@ impl State { .approval_entries(index) .into_iter() .filter_map(|approval_entry| approval_entry.approval(index)) - .map(|approval| (approval.validator, approval.signature)) - .collect::>(); + .map(|approval| { + ( + approval.validator, + ( + hash, + approval + .candidate_indices + .iter_ones() + .map(|val| val as CandidateIndex) + .collect_vec(), + approval.signature, + ), + ) + }) + .collect::, ValidatorSignature)>>( + ); all_sigs.extend(sigs); } all_sigs @@ -1714,7 +1837,7 @@ impl State { let _timer = metrics.time_unify_with_peer(); let mut assignments_to_send = Vec::new(); - let mut approvals_to_send = Vec::new(); + let mut approvals_to_send = HashMap::new(); let view_finalized_number = view.finalized_number; for head in view.into_iter() { @@ -1751,6 +1874,13 @@ impl State { t.local_grid_neighbors().route_to_peer(required_routing, peer_id) }); in_topology || { + if !topology + .map(|topology| topology.is_validator(peer_id)) + .unwrap_or(false) + { + return false + } + let route_random = random_routing.sample(total_peers, rng); if route_random { random_routing.inc_sent(); @@ -1778,12 +1908,51 @@ impl State { // Filter approval votes. for approval_message in approval_messages { - let (approval_knowledge, message_kind) = approval_entry - .create_approval_knowledge(block, approval_message.candidate_index); + let (should_forward_approval, candidates_covered_by_approvals) = + approval_message.candidate_indices.iter_ones().fold( + (true, Vec::new()), + |(should_forward_approval, mut new_covered_approvals), + approval_candidate_index| { + let (message_subject, message_kind) = approval_entry + .create_approval_knowledge( + block, + approval_candidate_index as _, + ); + // The assignments for all candidates signed in the approval + // should already have been sent to the peer, otherwise we can't + // send our approval and risk breaking our reputation. + let should_forward_approval = should_forward_approval && + peer_knowledge + .contains(&message_subject, MessageKind::Assignment); + if !peer_knowledge.contains(&message_subject, message_kind) { + new_covered_approvals.push((message_subject, message_kind)) + } + + (should_forward_approval, new_covered_approvals) + }, + ); - if !peer_knowledge.contains(&approval_knowledge, message_kind) { - peer_knowledge.sent.insert(approval_knowledge, message_kind); - approvals_to_send.push(approval_message); + if should_forward_approval { + if !approvals_to_send.contains_key(&( + approval_message.block_hash, + approval_message.validator, + approval_message.candidate_indices.clone(), + )) { + approvals_to_send.insert( + ( + approval_message.block_hash, + approval_message.validator, + approval_message.candidate_indices.clone(), + ), + approval_message, + ); + } + + candidates_covered_by_approvals.into_iter().for_each( + |(approval_knowledge, message_kind)| { + peer_knowledge.sent.insert(approval_knowledge, message_kind); + }, + ); } } } @@ -1818,8 +1987,12 @@ impl State { "Sending approvals to unified peer", ); - send_approvals_batched(sender, approvals_to_send, &vec![(peer_id, protocol_version)]) - .await; + send_approvals_batched( + sender, + approvals_to_send.into_values().collect_vec(), + &vec![(peer_id, protocol_version)], + ) + .await; } } @@ -1956,6 +2129,7 @@ impl State { // Punish the peer for the invalid message. modify_reputation(&mut self.reputation, sender, peer_id, COST_OVERSIZED_BITFIELD) .await; + gum::error!(target: LOG_TARGET, block_hash = ?cert.block_hash, ?candidate_index, validator_index = ?cert.validator, kind = ?cert.cert.kind, "Bad assignment v1"); } else { sanitized_assignments.push((cert.into(), candidate_index.into())) } @@ -1998,6 +2172,9 @@ impl State { // Punish the peer for the invalid message. modify_reputation(&mut self.reputation, sender, peer_id, COST_OVERSIZED_BITFIELD) .await; + for candidate_index in candidate_bitfield.iter_ones() { + gum::error!(target: LOG_TARGET, block_hash = ?cert.block_hash, ?candidate_index, validator_index = ?cert.validator, "Bad assignment v2"); + } } else { sanitized_assignments.push((cert, candidate_bitfield)) } @@ -2085,15 +2262,29 @@ async fn adjust_required_routing_and_propagate { gum::debug!( target: LOG_TARGET, - "Distributing our approval vote on candidate (block={}, index={})", + "Distributing our approval vote on candidate (block={}, index={:?})", vote.block_hash, - vote.candidate_index, + vote.candidate_indices, ); state @@ -2315,7 +2506,7 @@ pub const MAX_ASSIGNMENT_BATCH_SIZE: usize = ensure_size_not_zero( /// The maximum amount of approvals per batch is 33% of maximum allowed by protocol. pub const MAX_APPROVAL_BATCH_SIZE: usize = ensure_size_not_zero( - MAX_NOTIFICATION_SIZE as usize / std::mem::size_of::() / 3, + MAX_NOTIFICATION_SIZE as usize / std::mem::size_of::() / 3, ); // Low level helper for sending assignments. @@ -2409,14 +2600,19 @@ pub(crate) async fn send_assignments_batched( /// Send approvals while honoring the `max_notification_size` of the protocol and peer version. pub(crate) async fn send_approvals_batched( sender: &mut impl overseer::ApprovalDistributionSenderTrait, - approvals: impl IntoIterator + Clone, + approvals: impl IntoIterator + Clone, peers: &[(PeerId, ProtocolVersion)], ) { let v1_peers = filter_by_peer_version(peers, ValidationVersion::V1.into()); let v2_peers = filter_by_peer_version(peers, ValidationVersion::VStaging.into()); if !v1_peers.is_empty() { - let mut batches = approvals.clone().into_iter().peekable(); + let mut batches = approvals + .clone() + .into_iter() + .filter(|approval| approval.candidate_indices.count_ones() == 1) + .map(|val| val.try_into().expect("We checked conversion should succeed; qed")) + .peekable(); while batches.peek().is_some() { let batch: Vec<_> = batches.by_ref().take(MAX_APPROVAL_BATCH_SIZE).collect(); diff --git a/polkadot/node/network/approval-distribution/src/metrics.rs b/polkadot/node/network/approval-distribution/src/metrics.rs index 6864259e6fdb..094874820f26 100644 --- a/polkadot/node/network/approval-distribution/src/metrics.rs +++ b/polkadot/node/network/approval-distribution/src/metrics.rs @@ -31,6 +31,8 @@ struct MetricsInner { time_unify_with_peer: prometheus::Histogram, time_import_pending_now_known: prometheus::Histogram, time_awaiting_approval_voting: prometheus::Histogram, + assignments_received_result: prometheus::CounterVec, + approvals_received_result: prometheus::CounterVec, } trait AsLabel { @@ -78,6 +80,144 @@ impl Metrics { .map(|metrics| metrics.time_import_pending_now_known.start_timer()) } + pub fn on_approval_already_known(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["known"]).inc() + } + } + + pub fn on_approval_entry_not_found(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["noapprovalentry"]).inc() + } + } + + pub fn on_approval_recent_outdated(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["outdated"]).inc() + } + } + + pub fn on_approval_invalid_block(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["invalidblock"]).inc() + } + } + + pub fn on_approval_unknown_assignment(&self) { + if let Some(metrics) = &self.0 { + metrics + .approvals_received_result + .with_label_values(&["unknownassignment"]) + .inc() + } + } + + pub fn on_approval_duplicate(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["duplicate"]).inc() + } + } + + pub fn on_approval_out_of_view(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["outofview"]).inc() + } + } + + pub fn on_approval_good_known(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["goodknown"]).inc() + } + } + + pub fn on_approval_bad(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["bad"]).inc() + } + } + + pub fn on_approval_unexpected(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["unexpected"]).inc() + } + } + + pub fn on_approval_bug(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["bug"]).inc() + } + } + + pub fn on_approval_sent_v1(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["v1"]).inc() + } + } + + pub fn on_approval_sent_v2(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["v2"]).inc() + } + } + + pub fn on_assignment_already_known(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["known"]).inc() + } + } + + pub fn on_assignment_recent_outdated(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["outdated"]).inc() + } + } + + pub fn on_assignment_invalid_block(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["invalidblock"]).inc() + } + } + + pub fn on_assignment_duplicate(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["duplicate"]).inc() + } + } + + pub fn on_assignment_out_of_view(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["outofview"]).inc() + } + } + + pub fn on_assignment_good_known(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["goodknown"]).inc() + } + } + + pub fn on_assignment_bad(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["bad"]).inc() + } + } + + pub fn on_assignment_duplicatevoting(&self) { + if let Some(metrics) = &self.0 { + metrics + .assignments_received_result + .with_label_values(&["duplicatevoting"]) + .inc() + } + } + + pub fn on_assignment_far(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["far"]).inc() + } + } + pub(crate) fn time_awaiting_approval_voting( &self, ) -> Option { @@ -167,6 +307,26 @@ impl MetricsTrait for Metrics { ).buckets(vec![0.0001, 0.0004, 0.0016, 0.0064, 0.0256, 0.1024, 0.4096, 1.6384, 3.2768, 4.9152, 6.5536,]))?, registry, )?, + assignments_received_result: prometheus::register( + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_assignments_received_result", + "Result of a processed assignement", + ), + &["status"] + )?, + registry, + )?, + approvals_received_result: prometheus::register( + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_approvals_received_result", + "Result of a processed approval", + ), + &["status"] + )?, + registry, + )?, }; Ok(Metrics(Some(metrics))) } diff --git a/polkadot/node/network/approval-distribution/src/tests.rs b/polkadot/node/network/approval-distribution/src/tests.rs index f0c3c4f8ba64..b9fc5baa820c 100644 --- a/polkadot/node/network/approval-distribution/src/tests.rs +++ b/polkadot/node/network/approval-distribution/src/tests.rs @@ -133,14 +133,13 @@ fn make_gossip_topology( all_peers: &[(PeerId, AuthorityDiscoveryId)], neighbors_x: &[usize], neighbors_y: &[usize], + local_index: usize, ) -> network_bridge_event::NewGossipTopology { // This builds a grid topology which is a square matrix. // The local validator occupies the top left-hand corner. // The X peers occupy the same row and the Y peers occupy // the same column. - let local_index = 1; - assert_eq!( neighbors_x.len(), neighbors_y.len(), @@ -365,10 +364,11 @@ fn state_with_reputation_delay() -> State { /// the new peer sends us the same assignment #[test] fn try_import_the_same_assignment() { - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - let peer_c = PeerId::random(); - let peer_d = PeerId::random(); + let peers = make_peers_and_authority_ids(15); + let peer_a = peers.get(0).unwrap().0; + let peer_b = peers.get(1).unwrap().0; + let peer_c = peers.get(2).unwrap().0; + let peer_d = peers.get(4).unwrap().0; let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); @@ -379,6 +379,10 @@ fn try_import_the_same_assignment() { setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V1).await; setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V1).await; + // Set up a gossip topology, where a, b, c and d are topology neighboors to the node under + // testing. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0, 1], &[2, 4], 3)).await; + // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { hash, @@ -449,10 +453,11 @@ fn try_import_the_same_assignment() { /// cores. #[test] fn try_import_the_same_assignment_v2() { - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - let peer_c = PeerId::random(); - let peer_d = PeerId::random(); + let peers = make_peers_and_authority_ids(15); + let peer_a = peers.get(0).unwrap().0; + let peer_b = peers.get(1).unwrap().0; + let peer_c = peers.get(2).unwrap().0; + let peer_d = peers.get(4).unwrap().0; let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); @@ -463,6 +468,10 @@ fn try_import_the_same_assignment_v2() { setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::VStaging).await; + // Set up a gossip topology, where a, b, c and d are topology neighboors to the node under + // testing. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0, 1], &[2, 4], 3)).await; + // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { hash, @@ -690,14 +699,19 @@ fn spam_attack_results_in_negative_reputation_change() { #[test] fn peer_sending_us_the_same_we_just_sent_them_is_ok() { let parent_hash = Hash::repeat_byte(0xFF); - let peer_a = PeerId::random(); let hash = Hash::repeat_byte(0xAA); + let peers = make_peers_and_authority_ids(8); + let peer_a = peers.first().unwrap().0; + let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; let peer = &peer_a; setup_peer_with_view(overseer, peer, view![], ValidationVersion::V1).await; + // Setup a topology where peer_a is neigboor to current node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0], &[2], 1)).await; + // new block `hash` with 1 candidates let meta = BlockApprovalMeta { hash, @@ -766,9 +780,11 @@ fn peer_sending_us_the_same_we_just_sent_them_is_ok() { #[test] fn import_approval_happy_path() { - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - let peer_c = PeerId::random(); + let peers = make_peers_and_authority_ids(15); + + let peer_a = peers.get(0).unwrap().0; + let peer_b = peers.get(1).unwrap().0; + let peer_c = peers.get(2).unwrap().0; let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); @@ -791,6 +807,9 @@ fn import_approval_happy_path() { let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); overseer_send(overseer, msg).await; + // Set up a gossip topology, where a, b, and c are topology neighboors to the node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0, 1], &[2, 4], 3)).await; + // import an assignment related to `hash` locally let validator_index = ValidatorIndex(0); let candidate_index = 0u32; @@ -833,14 +852,14 @@ fn import_approval_happy_path() { ); // send the an approval from peer_b - let approval = IndirectSignedApprovalVote { + let approval = IndirectSignedApprovalVoteV2 { block_hash: hash, - candidate_index, + candidate_indices: candidate_index.into(), validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_b, msg).await; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -901,14 +920,14 @@ fn import_approval_bad() { let cert = fake_assignment_cert(hash, validator_index); // send the an approval from peer_b, we don't have an assignment yet - let approval = IndirectSignedApprovalVote { + let approval = IndirectSignedApprovalVoteV2 { block_hash: hash, - candidate_index, + candidate_indices: candidate_index.into(), validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_b, msg).await; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, &peer_b, msg).await; expect_reputation_change(overseer, &peer_b, COST_UNEXPECTED_MESSAGE).await; @@ -933,8 +952,8 @@ fn import_approval_bad() { expect_reputation_change(overseer, &peer_b, BENEFIT_VALID_MESSAGE_FIRST).await; // and try again - let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_b, msg).await; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1033,7 +1052,8 @@ fn update_peer_view() { let hash_b = Hash::repeat_byte(0xBB); let hash_c = Hash::repeat_byte(0xCC); let hash_d = Hash::repeat_byte(0xDD); - let peer_a = PeerId::random(); + let peers = make_peers_and_authority_ids(8); + let peer_a = peers.first().unwrap().0; let peer = &peer_a; let state = test_harness(State::default(), |mut virtual_overseer| async move { @@ -1067,6 +1087,9 @@ fn update_peer_view() { let msg = ApprovalDistributionMessage::NewBlocks(vec![meta_a, meta_b, meta_c]); overseer_send(overseer, msg).await; + // Setup a topology where peer_a is neigboor to current node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0], &[2], 1)).await; + let cert_a = fake_assignment_cert(hash_a, ValidatorIndex(0)); let cert_b = fake_assignment_cert(hash_b, ValidatorIndex(0)); @@ -1249,14 +1272,14 @@ fn import_remotely_then_locally() { assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); // send the approval remotely - let approval = IndirectSignedApprovalVote { + let approval = IndirectSignedApprovalVoteV2 { block_hash: hash, - candidate_index, + candidate_indices: candidate_index.into(), validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, peer, msg).await; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, peer, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1280,7 +1303,8 @@ fn import_remotely_then_locally() { #[test] fn sends_assignments_even_when_state_is_approved() { - let peer_a = PeerId::random(); + let peers = make_peers_and_authority_ids(8); + let peer_a = peers.first().unwrap().0; let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); let peer = &peer_a; @@ -1300,6 +1324,9 @@ fn sends_assignments_even_when_state_is_approved() { let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); overseer_send(overseer, msg).await; + // Setup a topology where peer_a is neigboor to current node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0], &[2], 1)).await; + let validator_index = ValidatorIndex(0); let candidate_index = 0u32; @@ -1321,8 +1348,11 @@ fn sends_assignments_even_when_state_is_approved() { ) .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeApproval(approval.clone())) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; // connect the peer. setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; @@ -1365,7 +1395,8 @@ fn sends_assignments_even_when_state_is_approved() { /// assignemnts. #[test] fn sends_assignments_even_when_state_is_approved_v2() { - let peer_a = PeerId::random(); + let peers = make_peers_and_authority_ids(8); + let peer_a = peers.first().unwrap().0; let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); let peer = &peer_a; @@ -1385,6 +1416,9 @@ fn sends_assignments_even_when_state_is_approved_v2() { let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); overseer_send(overseer, msg).await; + // Setup a topology where peer_a is neigboor to current node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0], &[2], 1)).await; + let validator_index = ValidatorIndex(0); let cores = vec![0, 1, 2, 3]; let candidate_bitfield: CandidateBitfield = cores.clone().try_into().unwrap(); @@ -1401,9 +1435,9 @@ fn sends_assignments_even_when_state_is_approved_v2() { // Assumes candidate index == core index. let approvals = cores .iter() - .map(|core| IndirectSignedApprovalVote { + .map(|core| IndirectSignedApprovalVoteV2 { block_hash: hash, - candidate_index: *core, + candidate_indices: (*core).into(), validator: validator_index, signature: dummy_signature(), }) @@ -1454,8 +1488,8 @@ fn sends_assignments_even_when_state_is_approved_v2() { )) => { // Construct a hashmaps of approvals for comparison. Approval distribution reorders messages because they are kept in a // hashmap as well. - let sent_approvals = sent_approvals.into_iter().map(|approval| (approval.candidate_index, approval)).collect::>(); - let approvals = approvals.into_iter().map(|approval| (approval.candidate_index, approval)).collect::>(); + let sent_approvals = sent_approvals.into_iter().map(|approval| (approval.candidate_indices.clone(), approval)).collect::>(); + let approvals = approvals.into_iter().map(|approval| (approval.candidate_indices.clone(), approval)).collect::>(); assert_eq!(peers, vec![*peer]); assert_eq!(sent_approvals, approvals); @@ -1565,13 +1599,19 @@ fn propagates_locally_generated_assignment_to_both_dimensions() { // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology( + 1, + &peers, + &[0, 10, 20, 30, 40, 60, 70, 80], + &[50, 51, 52, 53, 54, 55, 56, 57], + 1, + ), ) .await; let expected_indices = [ // Both dimensions in the gossip topology - 0, 10, 20, 30, 50, 51, 52, 53, + 0, 10, 20, 30, 40, 60, 70, 80, 50, 51, 52, 53, 54, 55, 56, 57, ]; // new block `hash_a` with 1 candidates @@ -1608,8 +1648,11 @@ fn propagates_locally_generated_assignment_to_both_dimensions() { ) .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeApproval(approval.clone())) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; let assignments = vec![(cert.clone(), candidate_index)]; let approvals = vec![approval.clone()]; @@ -1673,7 +1716,7 @@ fn propagates_assignments_along_unshared_dimension() { // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53], 1), ) .await; @@ -1816,13 +1859,19 @@ fn propagates_to_required_after_connect() { // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology( + 1, + &peers, + &[0, 10, 20, 30, 40, 60, 70, 80], + &[50, 51, 52, 53, 54, 55, 56, 57], + 1, + ), ) .await; let expected_indices = [ // Both dimensions in the gossip topology, minus omitted. - 20, 30, 52, 53, + 20, 30, 40, 60, 70, 80, 52, 53, 54, 55, 56, 57, ]; // new block `hash_a` with 1 candidates @@ -1859,8 +1908,11 @@ fn propagates_to_required_after_connect() { ) .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeApproval(approval.clone())) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; let assignments = vec![(cert.clone(), candidate_index)]; let approvals = vec![approval.clone()]; @@ -1987,53 +2039,21 @@ fn sends_to_more_peers_after_getting_topology() { ) .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeApproval(approval.clone())) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; let assignments = vec![(cert.clone(), candidate_index)]; let approvals = vec![approval.clone()]; - let mut expected_indices = vec![0, 10, 20, 30, 50, 51, 52, 53]; - let assignment_sent_peers = assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - sent_peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) - )) - )) => { - // Only sends to random peers. - assert_eq!(sent_peers.len(), 4); - for peer in &sent_peers { - let i = peers.iter().position(|p| peer == &p.0).unwrap(); - // Random gossip before topology can send to topology-targeted peers. - // Remove them from the expected indices so we don't expect - // them to get the messages again after the assignment. - expected_indices.retain(|&i2| i2 != i); - } - assert_eq!(sent_assignments, assignments); - sent_peers - } - ); - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - sent_peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(sent_approvals) - )) - )) => { - // Random sampling is reused from the assignment. - assert_eq!(sent_peers, assignment_sent_peers); - assert_eq!(sent_approvals, approvals); - } - ); + let expected_indices = vec![0, 10, 20, 30, 50, 51, 52, 53]; // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53], 1), ) .await; @@ -2136,7 +2156,7 @@ fn originator_aggression_l1() { // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53], 1), ) .await; @@ -2149,8 +2169,11 @@ fn originator_aggression_l1() { ) .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeApproval(approval.clone())) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; let assignments = vec![(cert.clone(), candidate_index)]; let approvals = vec![approval.clone()]; @@ -2292,7 +2315,7 @@ fn non_originator_aggression_l1() { // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53], 1), ) .await; @@ -2397,7 +2420,7 @@ fn non_originator_aggression_l2() { // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53], 1), ) .await; @@ -2543,7 +2566,7 @@ fn resends_messages_periodically() { // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53], 1), ) .await; @@ -2686,9 +2709,9 @@ fn batch_test_round(message_count: usize) { .collect(); let approvals: Vec<_> = validators - .map(|index| IndirectSignedApprovalVote { + .map(|index| IndirectSignedApprovalVoteV2 { block_hash: Hash::zero(), - candidate_index: 0, + candidate_indices: 0u32.into(), validator: ValidatorIndex(index as u32), signature: dummy_signature(), }) @@ -2755,7 +2778,7 @@ fn batch_test_round(message_count: usize) { assert_eq!(peers.len(), 1); for (message_index, approval) in sent_approvals.iter().enumerate() { - assert_eq!(approval, &approvals[approval_index + message_index]); + assert_eq!(approval, &approvals[approval_index + message_index].clone().try_into().unwrap()); } } ); diff --git a/polkadot/node/network/availability-recovery/src/lib.rs b/polkadot/node/network/availability-recovery/src/lib.rs index fb0cdb720571..c8a628645f34 100644 --- a/polkadot/node/network/availability-recovery/src/lib.rs +++ b/polkadot/node/network/availability-recovery/src/lib.rs @@ -104,7 +104,8 @@ const TIMEOUT_START_NEW_REQUESTS: Duration = CHUNK_REQUEST_TIMEOUT; const TIMEOUT_START_NEW_REQUESTS: Duration = Duration::from_millis(100); /// PoV size limit in bytes for which prefer fetching from backers. -const SMALL_POV_LIMIT: usize = 128 * 1024; +//TODO: Cleanup increased for versi testing +const SMALL_POV_LIMIT: usize = 3 * 1024 * 1024; #[derive(Clone, PartialEq)] /// The strategy we use to recover the PoV. diff --git a/polkadot/node/network/protocol/src/grid_topology.rs b/polkadot/node/network/protocol/src/grid_topology.rs index 99dd513c4d79..8bd9adbc17c1 100644 --- a/polkadot/node/network/protocol/src/grid_topology.rs +++ b/polkadot/node/network/protocol/src/grid_topology.rs @@ -73,12 +73,20 @@ pub struct SessionGridTopology { shuffled_indices: Vec, /// The canonical shuffling of validators for the session. canonical_shuffling: Vec, + /// The list of peer-ids in an efficient way to search. + peer_ids: HashSet, } impl SessionGridTopology { /// Create a new session grid topology. pub fn new(shuffled_indices: Vec, canonical_shuffling: Vec) -> Self { - SessionGridTopology { shuffled_indices, canonical_shuffling } + let mut peer_ids = HashSet::new(); + for peer_info in canonical_shuffling.iter() { + for peer_id in peer_info.peer_ids.iter() { + peer_ids.insert(*peer_id); + } + } + SessionGridTopology { shuffled_indices, canonical_shuffling, peer_ids } } /// Produces the outgoing routing logic for a particular peer. @@ -111,6 +119,11 @@ impl SessionGridTopology { Some(grid_subset) } + + /// Tells if a given peer id is validator in a session + pub fn is_validator(&self, peer: &PeerId) -> bool { + self.peer_ids.contains(peer) + } } struct MatrixNeighbors { @@ -273,6 +286,11 @@ impl SessionGridTopologyEntry { pub fn get(&self) -> &SessionGridTopology { &self.topology } + + /// Tells if a given peer id is validator in a session + pub fn is_validator(&self, peer: &PeerId) -> bool { + self.topology.is_validator(peer) + } } /// A set of topologies indexed by session @@ -347,6 +365,7 @@ impl Default for SessionBoundGridTopologyStorage { topology: SessionGridTopology { shuffled_indices: Vec::new(), canonical_shuffling: Vec::new(), + peer_ids: Default::default(), }, local_neighbors: GridNeighbors::empty(), }, diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index ba9b9a7f4900..8cd11bafa5f6 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -599,10 +599,7 @@ pub mod vstaging { }; use polkadot_node_primitives::{ - approval::{ - v1::IndirectSignedApprovalVote, - v2::{CandidateBitfield, IndirectAssignmentCertV2}, - }, + approval::v2::{CandidateBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2}, UncheckedSignedFullStatement, }; @@ -780,7 +777,7 @@ pub mod vstaging { Assignments(Vec<(IndirectAssignmentCertV2, CandidateBitfield)>), /// Approvals for candidates in some recent, unfinalized block. #[codec(index = 1)] - Approvals(Vec), + Approvals(Vec), } /// Dummy network message type, so we will receive connect/disconnect events. diff --git a/polkadot/node/primitives/src/approval.rs b/polkadot/node/primitives/src/approval.rs index bfa1f1aa47d2..49635023cd56 100644 --- a/polkadot/node/primitives/src/approval.rs +++ b/polkadot/node/primitives/src/approval.rs @@ -219,7 +219,9 @@ pub mod v2 { use std::ops::BitOr; use bitvec::{prelude::Lsb0, vec::BitVec}; - use polkadot_primitives::{CandidateIndex, CoreIndex, Hash, ValidatorIndex}; + use polkadot_primitives::{ + CandidateIndex, CoreIndex, Hash, ValidatorIndex, ValidatorSignature, + }; /// A static context associated with producing randomness for a core. pub const CORE_RANDOMNESS_CONTEXT: &[u8] = b"A&V CORE v2"; @@ -473,6 +475,59 @@ pub mod v2 { }) } } + + impl From for IndirectSignedApprovalVoteV2 { + fn from(value: super::v1::IndirectSignedApprovalVote) -> Self { + Self { + block_hash: value.block_hash, + validator: value.validator, + candidate_indices: value.candidate_index.into(), + signature: value.signature, + } + } + } + + /// Errors that can occur when trying to convert to/from approvals v1/v2 + #[derive(Debug)] + pub enum ApprovalConversionError { + /// More than one candidate was signed. + MoreThanOneCandidate(usize), + } + + impl TryFrom for super::v1::IndirectSignedApprovalVote { + type Error = ApprovalConversionError; + + fn try_from(value: IndirectSignedApprovalVoteV2) -> Result { + if value.candidate_indices.count_ones() != 1 { + return Err(ApprovalConversionError::MoreThanOneCandidate( + value.candidate_indices.count_ones(), + )) + } + Ok(Self { + block_hash: value.block_hash, + validator: value.validator, + candidate_index: value.candidate_indices.first_one().expect("Qed we checked above") + as u32, + signature: value.signature, + }) + } + } + + /// A signed approval vote which references the candidate indirectly via the block. + /// + /// In practice, we have a look-up from block hash and candidate index to candidate hash, + /// so this can be transformed into a `SignedApprovalVote`. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct IndirectSignedApprovalVoteV2 { + /// A block hash where the candidate appears. + pub block_hash: Hash, + /// The index of the candidate in the list of candidates fully included as-of the block. + pub candidate_indices: CandidateBitfield, + /// The validator index. + pub validator: ValidatorIndex, + /// The signature by the validator. + pub signature: ValidatorSignature, + } } #[cfg(test)] diff --git a/polkadot/node/primitives/src/disputes/message.rs b/polkadot/node/primitives/src/disputes/message.rs index 89d3ea6c0af9..31fe73a7ba1c 100644 --- a/polkadot/node/primitives/src/disputes/message.rs +++ b/polkadot/node/primitives/src/disputes/message.rs @@ -170,7 +170,7 @@ impl DisputeMessage { let valid_vote = ValidDisputeVote { validator_index: valid_index, signature: valid_statement.validator_signature().clone(), - kind: *valid_kind, + kind: valid_kind.clone(), }; let invalid_vote = InvalidDisputeVote { diff --git a/polkadot/node/primitives/src/disputes/mod.rs b/polkadot/node/primitives/src/disputes/mod.rs index ae8602dd5fc4..c267445ad0be 100644 --- a/polkadot/node/primitives/src/disputes/mod.rs +++ b/polkadot/node/primitives/src/disputes/mod.rs @@ -46,6 +46,15 @@ pub struct SignedDisputeStatement { session_index: SessionIndex, } +/// Errors encountered while signing a dispute statement +#[derive(Debug)] +pub enum SignedDisputeStatementError { + /// Encountered a keystore error while signing + KeyStoreError(KeystoreError), + /// Could not generate signing payload + PayloadError, +} + /// Tracked votes on candidates, for the purposes of dispute resolution. #[derive(Debug, Clone)] pub struct CandidateVotes { @@ -107,8 +116,9 @@ impl ValidCandidateVotes { ValidDisputeStatementKind::BackingValid(_) | ValidDisputeStatementKind::BackingSeconded(_) => false, ValidDisputeStatementKind::Explicit | - ValidDisputeStatementKind::ApprovalChecking => { - occupied.insert((kind, sig)); + ValidDisputeStatementKind::ApprovalChecking | + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(_) => { + occupied.insert((kind.clone(), sig)); kind != occupied.get().0 }, }, @@ -213,16 +223,19 @@ impl SignedDisputeStatement { candidate_hash: CandidateHash, session_index: SessionIndex, validator_public: ValidatorId, - ) -> Result, KeystoreError> { + ) -> Result, SignedDisputeStatementError> { let dispute_statement = if valid { DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) } else { DisputeStatement::Invalid(InvalidDisputeStatementKind::Explicit) }; - let data = dispute_statement.payload_data(candidate_hash, session_index); + let data = dispute_statement + .payload_data(candidate_hash, session_index) + .map_err(|_| SignedDisputeStatementError::PayloadError)?; let signature = keystore - .sr25519_sign(ValidatorId::ID, validator_public.as_ref(), &data)? + .sr25519_sign(ValidatorId::ID, validator_public.as_ref(), &data) + .map_err(SignedDisputeStatementError::KeyStoreError)? .map(|sig| Self { dispute_statement, candidate_hash, diff --git a/polkadot/node/service/src/fake_runtime_api.rs b/polkadot/node/service/src/fake_runtime_api.rs index d9553afa024b..f2f2bdefae7a 100644 --- a/polkadot/node/service/src/fake_runtime_api.rs +++ b/polkadot/node/service/src/fake_runtime_api.rs @@ -23,12 +23,12 @@ use beefy_primitives::ecdsa_crypto::{AuthorityId as BeefyId, Signature as BeefyS use grandpa_primitives::AuthorityId as GrandpaId; use pallet_transaction_payment::{FeeDetails, RuntimeDispatchInfo}; use polkadot_primitives::{ - runtime_api, slashing, AccountId, AuthorityDiscoveryId, Balance, Block, BlockNumber, - CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, - DisputeState, ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, - InboundHrmpMessage, Nonce, OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, - ScrapedOnChainVotes, SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, - ValidatorId, ValidatorIndex, ValidatorSignature, + runtime_api, slashing, vstaging::ApprovalVotingParams, AccountId, AuthorityDiscoveryId, + Balance, Block, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Nonce, OccupiedCoreAssumption, + PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, + ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; use sp_core::OpaqueMetadata; use sp_runtime::{ @@ -116,6 +116,7 @@ sp_api::impl_runtime_apis! { } } + #[api_version(5)] impl runtime_api::ParachainHost for Runtime { fn validators() -> Vec { unimplemented!() diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index f561a0e28512..d8fe54f98312 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -33,8 +33,8 @@ use polkadot_node_network_protocol::{ }; use polkadot_node_primitives::{ approval::{ - v1::{BlockApprovalMeta, IndirectSignedApprovalVote}, - v2::{CandidateBitfield, IndirectAssignmentCertV2}, + v1::BlockApprovalMeta, + v2::{CandidateBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2}, }, AvailableData, BabeEpoch, BlockWeight, CandidateVotes, CollationGenerationConfig, CollationSecondedSignal, DisputeMessage, DisputeStatus, ErasureChunk, PoV, @@ -42,14 +42,14 @@ use polkadot_node_primitives::{ ValidationResult, }; use polkadot_primitives::{ - slashing, vstaging as vstaging_primitives, AuthorityDiscoveryId, BackedCandidate, BlockNumber, - CandidateEvent, CandidateHash, CandidateIndex, CandidateReceipt, CollatorId, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupIndex, - GroupRotationInfo, Hash, Header as BlockHeader, Id as ParaId, InboundDownwardMessage, - InboundHrmpMessage, MultiDisputeStatementSet, OccupiedCoreAssumption, PersistedValidationData, - PvfCheckStatement, PvfExecTimeoutKind, SessionIndex, SessionInfo, SignedAvailabilityBitfield, - SignedAvailabilityBitfields, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, - ValidatorSignature, + slashing, vstaging as vstaging_primitives, vstaging::ApprovalVotingParams, + AuthorityDiscoveryId, BackedCandidate, BlockNumber, CandidateEvent, CandidateHash, + CandidateIndex, CandidateReceipt, CollatorId, CommittedCandidateReceipt, CoreState, + DisputeState, ExecutorParams, GroupIndex, GroupRotationInfo, Hash, Header as BlockHeader, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, MultiDisputeStatementSet, + OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, PvfExecTimeoutKind, + SessionIndex, SessionInfo, SignedAvailabilityBitfield, SignedAvailabilityBitfields, + ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; use polkadot_statement_table::v2::Misbehavior; use std::{ @@ -694,6 +694,8 @@ pub enum RuntimeApiRequest { slashing::OpaqueKeyOwnershipProof, RuntimeApiSender>, ), + /// Approval voting params + ApprovalVotingParams(RuntimeApiSender), /// Get the backing state of the given para. /// This is a staging API that will not be available on production runtimes. @@ -911,7 +913,7 @@ pub enum ApprovalVotingMessage { /// protocol. /// /// Should not be sent unless the block hash within the indirect vote is known. - CheckAndImportApproval(IndirectSignedApprovalVote, oneshot::Sender), + CheckAndImportApproval(IndirectSignedApprovalVoteV2, oneshot::Sender), /// Returns the highest possible ancestor hash of the provided block hash which is /// acceptable to vote on finality for. /// The `BlockNumber` provided is the number of the block's ancestor which is the @@ -927,7 +929,7 @@ pub enum ApprovalVotingMessage { /// requires calling into `approval-distribution`: Calls should be infrequent and bounded. GetApprovalSignaturesForCandidate( CandidateHash, - oneshot::Sender>, + oneshot::Sender, ValidatorSignature)>>, ), } @@ -943,7 +945,7 @@ pub enum ApprovalDistributionMessage { /// Distribute an approval vote for the local validator. The approval vote is assumed to be /// valid, relevant, and the corresponding approval already issued. /// If not, the subsystem is free to drop the message. - DistributeApproval(IndirectSignedApprovalVote), + DistributeApproval(IndirectSignedApprovalVoteV2), /// An update from the network bridge. #[from] NetworkBridgeUpdate(NetworkBridgeEvent), @@ -951,7 +953,7 @@ pub enum ApprovalDistributionMessage { /// Get all approval signatures for all chains a candidate appeared in. GetApprovalSignatures( HashSet<(Hash, CandidateIndex)>, - oneshot::Sender>, + oneshot::Sender, ValidatorSignature)>>, ), /// Approval checking lag update measured in blocks. ApprovalCheckingLagUpdate(BlockNumber), diff --git a/polkadot/node/subsystem-types/src/runtime_client.rs b/polkadot/node/subsystem-types/src/runtime_client.rs index 312cc4eec6ce..3e439dd4732d 100644 --- a/polkadot/node/subsystem-types/src/runtime_client.rs +++ b/polkadot/node/subsystem-types/src/runtime_client.rs @@ -16,12 +16,13 @@ use async_trait::async_trait; use polkadot_primitives::{ - runtime_api::ParachainHost, vstaging, Block, BlockNumber, CandidateCommitments, CandidateEvent, - CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, - GroupRotationInfo, Hash, Id, InboundDownwardMessage, InboundHrmpMessage, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, - SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, - ValidatorSignature, + runtime_api::ParachainHost, + vstaging::{self, ApprovalVotingParams}, + Block, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, + Id, InboundDownwardMessage, InboundHrmpMessage, OccupiedCoreAssumption, + PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, + ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; use sc_transaction_pool_api::OffchainTransactionPoolFactory; use sp_api::{ApiError, ApiExt, ProvideRuntimeApi}; @@ -247,6 +248,9 @@ pub trait RuntimeApiSubsystemClient { at: Hash, para_id: Id, ) -> Result, ApiError>; + + /// Approval voting configuration parameters + async fn approval_voting_params(&self, at: Hash) -> Result; } /// Default implementation of [`RuntimeApiSubsystemClient`] using the client. @@ -473,6 +477,11 @@ where runtime_api.submit_report_dispute_lost(at, dispute_proof, key_ownership_proof) } + /// Approval voting configuration parameters + async fn approval_voting_params(&self, at: Hash) -> Result { + self.client.runtime_api().approval_voting_params(at) + } + async fn staging_para_backing_state( &self, at: Hash, diff --git a/polkadot/primitives/src/runtime_api.rs b/polkadot/primitives/src/runtime_api.rs index 483256fe20f3..fb42b2c976ff 100644 --- a/polkadot/primitives/src/runtime_api.rs +++ b/polkadot/primitives/src/runtime_api.rs @@ -114,10 +114,11 @@ //! separated from the stable primitives. use crate::{ - vstaging, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, - SessionIndex, SessionInfo, ValidatorId, ValidatorIndex, ValidatorSignature, + vstaging::{self, ApprovalVotingParams}, + BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, + CoreState, DisputeState, ExecutorParams, GroupRotationInfo, OccupiedCoreAssumption, + PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, + ValidatorId, ValidatorIndex, ValidatorSignature, }; use parity_scale_codec::{Decode, Encode}; use polkadot_core_primitives as pcp; @@ -240,6 +241,9 @@ sp_api::decl_runtime_apis! { key_ownership_proof: vstaging::slashing::OpaqueKeyOwnershipProof, ) -> Option<()>; + /// Approval voting configuration parameters + #[api_version(99)] + fn approval_voting_params() -> ApprovalVotingParams; /***** Asynchronous backing *****/ /// Returns the state of parachain backing for a given para. diff --git a/polkadot/primitives/src/v5/mod.rs b/polkadot/primitives/src/v5/mod.rs index 59fb6c927b2d..1f5aa272fb2c 100644 --- a/polkadot/primitives/src/v5/mod.rs +++ b/polkadot/primitives/src/v5/mod.rs @@ -354,6 +354,13 @@ pub mod well_known_keys { /// Unique identifier for the Parachains Inherent pub const PARACHAINS_INHERENT_IDENTIFIER: InherentIdentifier = *b"parachn0"; +// /// TODO: Make this two a parachain host configuration. +// /// Maximum allowed candidates to be signed withing a signle approval votes. +// pub const MAX_APPROVAL_COALESCE_COUNT: u64 = 6; +// /// The maximum time we await for an approval to be coalesced with other approvals +// /// before we sign it and distribute to our peers +// pub const MAX_APPROVAL_COALESCE_WAIT_MILLIS: u64 = 500; + /// The key type ID for parachain assignment key. pub const ASSIGNMENT_KEY_TYPE_ID: KeyTypeId = KeyTypeId(*b"asgn"); @@ -1115,6 +1122,26 @@ impl ApprovalVote { } } +/// A vote of approvalf for multiple candidates. +#[derive(Clone, RuntimeDebug)] +pub struct ApprovalVoteMultipleCandidates<'a>(pub &'a Vec); + +impl<'a> ApprovalVoteMultipleCandidates<'a> { + /// Yields the signing payload for this approval vote. + pub fn signing_payload(&self, session_index: SessionIndex) -> Vec { + const MAGIC: [u8; 4] = *b"APPR"; + // Make this backwards compatible with `ApprovalVote` so if we have just on candidate the signature + // will look the same. + // This gives us the nice benefit that old nodes can still check signatures when len is 1 and the + // new node can check the signature coming from old nodes. + if self.0.len() == 1 { + (MAGIC, self.0.first().expect("QED: we just checked"), session_index).encode() + } else { + (MAGIC, &self.0, session_index).encode() + } + } +} + /// Custom validity errors used in Polkadot while validating transactions. #[repr(u8)] pub enum ValidityError { @@ -1291,25 +1318,39 @@ pub enum DisputeStatement { impl DisputeStatement { /// Get the payload data for this type of dispute statement. - pub fn payload_data(&self, candidate_hash: CandidateHash, session: SessionIndex) -> Vec { - match *self { + pub fn payload_data( + &self, + candidate_hash: CandidateHash, + session: SessionIndex, + ) -> Result, ()> { + match self { DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) => - ExplicitDisputeStatement { valid: true, candidate_hash, session }.signing_payload(), + Ok(ExplicitDisputeStatement { valid: true, candidate_hash, session } + .signing_payload()), DisputeStatement::Valid(ValidDisputeStatementKind::BackingSeconded( inclusion_parent, - )) => CompactStatement::Seconded(candidate_hash).signing_payload(&SigningContext { + )) => Ok(CompactStatement::Seconded(candidate_hash).signing_payload(&SigningContext { session_index: session, - parent_hash: inclusion_parent, - }), + parent_hash: *inclusion_parent, + })), DisputeStatement::Valid(ValidDisputeStatementKind::BackingValid(inclusion_parent)) => - CompactStatement::Valid(candidate_hash).signing_payload(&SigningContext { + Ok(CompactStatement::Valid(candidate_hash).signing_payload(&SigningContext { session_index: session, - parent_hash: inclusion_parent, - }), + parent_hash: *inclusion_parent, + })), DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) => - ApprovalVote(candidate_hash).signing_payload(session), + Ok(ApprovalVote(candidate_hash).signing_payload(session)), + DisputeStatement::Valid( + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(candidate_hashes), + ) => + if candidate_hashes.contains(&candidate_hash) { + Ok(ApprovalVoteMultipleCandidates(candidate_hashes).signing_payload(session)) + } else { + Err(()) + }, DisputeStatement::Invalid(InvalidDisputeStatementKind::Explicit) => - ExplicitDisputeStatement { valid: false, candidate_hash, session }.signing_payload(), + Ok(ExplicitDisputeStatement { valid: false, candidate_hash, session } + .signing_payload()), } } @@ -1321,7 +1362,7 @@ impl DisputeStatement { session: SessionIndex, validator_signature: &ValidatorSignature, ) -> Result<(), ()> { - let payload = self.payload_data(candidate_hash, session); + let payload = self.payload_data(candidate_hash, session)?; if validator_signature.verify(&payload[..], &validator_public) { Ok(()) @@ -1353,13 +1394,14 @@ impl DisputeStatement { Self::Valid(ValidDisputeStatementKind::BackingValid(_)) => true, Self::Valid(ValidDisputeStatementKind::Explicit) | Self::Valid(ValidDisputeStatementKind::ApprovalChecking) | + Self::Valid(ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(_)) | Self::Invalid(_) => false, } } } /// Different kinds of statements of validity on a candidate. -#[derive(Encode, Decode, Copy, Clone, PartialEq, RuntimeDebug, TypeInfo)] +#[derive(Encode, Decode, Clone, PartialEq, RuntimeDebug, TypeInfo)] pub enum ValidDisputeStatementKind { /// An explicit statement issued as part of a dispute. #[codec(index = 0)] @@ -1373,6 +1415,11 @@ pub enum ValidDisputeStatementKind { /// An approval vote from the approval checking phase. #[codec(index = 3)] ApprovalChecking, + /// An approval vote from the new version. + /// TODO: Fixme this probably means we can't create this version + /// untill all nodes have been updated to support it. + #[codec(index = 4)] + ApprovalCheckingMultipleCandidates(Vec), } /// Different kinds of statements of invalidity on a candidate. diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index ea341ee5b4fc..2b9d14fb199a 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -54,6 +54,26 @@ pub struct AsyncBackingParams { pub allowed_ancestry_len: u32, } +/// Approval voting configuration parameters +#[derive( + RuntimeDebug, + Copy, + Clone, + PartialEq, + Encode, + Decode, + TypeInfo, + serde::Serialize, + serde::Deserialize, +)] +pub struct ApprovalVotingParams { + /// The maximum number of candidates `approval-voting` can vote for with + /// a single signatures. + /// + /// Setting it to 1, means we send the approval as soon as we have it available. + pub max_approval_coalesce_count: u32, +} + /// Constraints on inbound HRMP channels. #[derive(RuntimeDebug, Clone, PartialEq, Encode, Decode, TypeInfo)] pub struct InboundHrmpLimitations { diff --git a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md index 375b8f1f12b3..3f652b55ef9f 100644 --- a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md +++ b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md @@ -2,7 +2,7 @@ Reading the [section on the approval protocol](../../protocol-approval.md) will likely be necessary to understand the aims of this subsystem. -Approval votes are split into two parts: Assignments and Approvals. Validators first broadcast their assignment to indicate intent to check a candidate. Upon successfully checking, they broadcast an approval vote. If a validator doesn't broadcast their approval vote shortly after issuing an assignment, this is an indication that they are being prevented from recovering or validating the block data and that more validators should self-select to check the candidate. This is known as a "no-show". +Approval votes are split into two parts: Assignments and Approvals. Validators first broadcast their assignment to indicate intent to check a candidate. Upon successfully checking, they don't immediately send the vote instead they queue the check for a short period of time `MAX_APPROVALS_COALESCE_TICKS` to give the opportunity of the validator to vote for more than one candidate. Once MAX_APPROVALS_COALESCE_TICKS have passed or at least `MAX_APPROVAL_COALESCE_COUNT` are ready they broadcast an approval vote for all candidates. If a validator doesn't broadcast their approval vote shortly after issuing an assignment, this is an indication that they are being prevented from recovering or validating the block data and that more validators should self-select to check the candidate. This is known as a "no-show". The core of this subsystem is a Tick-based timer loop, where Ticks are 500ms. We also reason about time in terms of `DelayTranche`s, which measure the number of ticks elapsed since a block was produced. We track metadata for all un-finalized but included candidates. We compute our local assignments to check each candidate, as well as which `DelayTranche` those assignments may be minimally triggered at. As the same candidate may appear in more than one block, we must produce our potential assignments for each (Block, Candidate) pair. The timing loop is based on waiting for assignments to become no-shows or waiting to broadcast and begin our own assignment to check. @@ -94,6 +94,13 @@ struct BlockEntry { // this block. The block can be considered approved has all bits set to 1 approved_bitfield: Bitfield, children: Vec, + // A list of candidates that has been approved, but we didn't not sign and + // advertise the vote yet. + candidates_pending_signature: BTreeMap, + // Assignments we already distributed. A 1 bit means the candidate index for which + // we already have sent out an assignment. We need this to avoid distributing + // multiple core assignments more than once. + distributed_assignments: Bitfield, } // slot_duration * 2 + DelayTranche gives the number of delay tranches since the @@ -224,10 +231,10 @@ On receiving a `ApprovalVotingMessage::CheckAndImportAssignment` message, we che On receiving a `CheckAndImportApproval(indirect_approval_vote, response_channel)` message: * Fetch the `BlockEntry` from the indirect approval vote's `block_hash`. If none, return `ApprovalCheckResult::Bad`. - * Fetch the `CandidateEntry` from the indirect approval vote's `candidate_index`. If the block did not trigger inclusion of enough candidates, return `ApprovalCheckResult::Bad`. - * Construct a `SignedApprovalVote` using the candidate hash and check against the validator's approval key, based on the session info of the block. If invalid or no such validator, return `ApprovalCheckResult::Bad`. + * Fetch all `CandidateEntry` from the indirect approval vote's `candidate_indices`. If the block did not trigger inclusion of enough candidates, return `ApprovalCheckResult::Bad`. + * Construct a `SignedApprovalVote` using the candidates hashes and check against the validator's approval key, based on the session info of the block. If invalid or no such validator, return `ApprovalCheckResult::Bad`. * Send `ApprovalCheckResult::Accepted` - * [Import the checked approval vote](#import-checked-approval) + * [Import the checked approval vote](#import-checked-approval) for all candidates #### `ApprovalVotingMessage::ApprovedAncestor` @@ -295,10 +302,24 @@ On receiving an `ApprovedAncestor(Hash, BlockNumber, response_channel)`: #### Issue Approval Vote * Fetch the block entry and candidate entry. Ignore if `None` - we've probably just lost a race with finality. - * Construct a `SignedApprovalVote` with the validator index for the session. * [Import the checked approval vote](#import-checked-approval). It is "checked" as we've just issued the signature. - * Construct a `IndirectSignedApprovalVote` using the information about the vote. - * Dispatch `ApprovalDistributionMessage::DistributeApproval`. + * IF `MAX_APPROVAL_COALESCE_COUNT` candidates are in the waiting queue + * Construct a `SignedApprovalVote` with the validator index for the session and all candidate hashes in the waiting queue. + * Construct a `IndirectSignedApprovalVote` using the information about the vote. + * Dispatch `ApprovalDistributionMessage::DistributeApproval`. + * ELSE + * Queue the candidate in the `BlockEntry::candidates_pending_signature` + * Arm a per BlockEntry timer with latest tick we can send the vote. + +### Delayed vote distribution + * [Issue Approval Vote](#issue-approval-vote) arms once a per block timer if there are no requirements to send the vote immediately. + * When the timer wakes up it will either: + * IF there is a candidate in the queue past its sending tick: + * Construct a `SignedApprovalVote` with the validator index for the session and all candidate hashes in the waiting queue. + * Construct a `IndirectSignedApprovalVote` using the information about the vote. + * Dispatch `ApprovalDistributionMessage::DistributeApproval`. + * ELSE + * Re-arm the timer with latest tick we have the send a the vote. ### Determining Approval of Candidate diff --git a/polkadot/roadmap/implementers-guide/src/protocol-approval.md b/polkadot/roadmap/implementers-guide/src/protocol-approval.md index aa513c16292d..63345032cbb0 100644 --- a/polkadot/roadmap/implementers-guide/src/protocol-approval.md +++ b/polkadot/roadmap/implementers-guide/src/protocol-approval.md @@ -152,6 +152,12 @@ We strongly prefer if postponements come from tranches higher aka less important TODO: When? Is this optimal for the network? etc. +## Approval coalescing +To reduce the necessary network bandwidth and cpu time when a validator has more than one candidate to approve we are doing our best effort to send a single message that approves all available candidates with a single signature. The implemented heuristic, is that each time we are ready to create a signature and send a vote for a candidate we delay the sending of it untill one of three things happen: +- We gathered a maximum of `MAX_APPROVAL_COALESCE_COUNT` candidates that we are ready to vote for. +- `MAX_APPROVALS_COALESCE_TICKS` have passed since the we were ready to approve the candidate. +- We are already in the last third of the now-show period in order to avoid creating accidental no shows, which in turn my trigger other assignments. + ## On-chain verification We should verify approval on-chain to reward approval checkers. We therefore require the "no show" timeout to be longer than a relay chain slot so that we can witness "no shows" on-chain, which helps with this goal. The major challenge with an on-chain record of the off-chain process is adversarial block producers who may either censor votes or publish votes to the chain which cause other votes to be ignored and unrewarded (reward stealing). diff --git a/polkadot/runtime/kusama/src/lib.rs b/polkadot/runtime/kusama/src/lib.rs index e9e3fb2d2026..07bee8cfa853 100644 --- a/polkadot/runtime/kusama/src/lib.rs +++ b/polkadot/runtime/kusama/src/lib.rs @@ -23,12 +23,12 @@ use pallet_nis::WithMaximumOf; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, - OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, - ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, LOWEST_PUBLIC_ID, - PARACHAIN_KEY_TYPE_ID, + slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, + CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, + ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, + ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, + ValidatorIndex, LOWEST_PUBLIC_ID, PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ auctions, claims, crowdloan, impl_runtime_weights, impls::DealWithFees, paras_registrar, @@ -1887,6 +1887,7 @@ sp_api::impl_runtime_apis! { } } + #[api_version(5)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index 4921af5bedda..c2e5c917ff1a 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -634,7 +634,7 @@ impl BenchBuilder { } else { DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) }; - let data = dispute_statement.payload_data(candidate_hash, session); + let data = dispute_statement.payload_data(candidate_hash, session).unwrap(); let statement_sig = validator_public.sign(&data).unwrap(); (dispute_statement, ValidatorIndex(validator_index), statement_sig) diff --git a/polkadot/runtime/parachains/src/configuration.rs b/polkadot/runtime/parachains/src/configuration.rs index accc01a2b180..47f4d1547368 100644 --- a/polkadot/runtime/parachains/src/configuration.rs +++ b/polkadot/runtime/parachains/src/configuration.rs @@ -24,8 +24,9 @@ use frame_system::pallet_prelude::*; use parity_scale_codec::{Decode, Encode}; use polkadot_parachain::primitives::{MAX_HORIZONTAL_MESSAGE_NUM, MAX_UPWARD_MESSAGE_NUM}; use primitives::{ - vstaging::AsyncBackingParams, Balance, ExecutorParams, SessionIndex, MAX_CODE_SIZE, - MAX_HEAD_DATA_SIZE, MAX_POV_SIZE, ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, + vstaging::{ApprovalVotingParams, AsyncBackingParams}, + Balance, ExecutorParams, SessionIndex, MAX_CODE_SIZE, MAX_HEAD_DATA_SIZE, MAX_POV_SIZE, + ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, }; use sp_runtime::{traits::Zero, Perbill}; use sp_std::prelude::*; @@ -242,6 +243,10 @@ pub struct HostConfiguration { /// /// This value should be greater than [`paras_availability_period`]. pub minimum_validation_upgrade_delay: BlockNumber, + + /// Params used by approval-voting + /// TODO: fixme this is not correctly migrated + pub approval_voting_params: ApprovalVotingParams, } impl> Default for HostConfiguration { @@ -287,6 +292,7 @@ impl> Default for HostConfiguration crate::hrmp::HRMP_MAX_INBOUND_CHANNELS_BOUND { return Err(MaxHrmpInboundChannelsExceeded) } - + // TODO: add consistency check for approval-voting-params Ok(()) } @@ -1150,6 +1156,22 @@ pub mod pallet { config.on_demand_ttl = new; }) } + + /// Set approval-voting-params. + #[pallet::call_index(52)] + #[pallet::weight(( + T::WeightInfo::set_config_with_executor_params(), + DispatchClass::Operational, + ))] + pub fn set_approval_voting_params( + origin: OriginFor, + new: ApprovalVotingParams, + ) -> DispatchResult { + ensure_root(origin)?; + Self::schedule_config_update(|config| { + config.approval_voting_params = new; + }) + } } #[pallet::hooks] diff --git a/polkadot/runtime/parachains/src/configuration/migration/v7.rs b/polkadot/runtime/parachains/src/configuration/migration/v7.rs index 113651381207..3624eb982323 100644 --- a/polkadot/runtime/parachains/src/configuration/migration/v7.rs +++ b/polkadot/runtime/parachains/src/configuration/migration/v7.rs @@ -240,6 +240,7 @@ pvf_voting_ttl : pre.pvf_voting_ttl, minimum_validation_upgrade_delay : pre.minimum_validation_upgrade_delay, async_backing_params : pre.async_backing_params, executor_params : pre.executor_params, + } }; diff --git a/polkadot/runtime/parachains/src/configuration/migration/v8.rs b/polkadot/runtime/parachains/src/configuration/migration/v8.rs index 7f7cc1cdefcd..d39f76100a18 100644 --- a/polkadot/runtime/parachains/src/configuration/migration/v8.rs +++ b/polkadot/runtime/parachains/src/configuration/migration/v8.rs @@ -23,7 +23,7 @@ use frame_support::{ weights::Weight, }; use frame_system::pallet_prelude::BlockNumberFor; -use primitives::SessionIndex; +use primitives::{vstaging::ApprovalVotingParams, SessionIndex}; use sp_runtime::Perbill; use sp_std::vec::Vec; @@ -150,6 +150,9 @@ on_demand_base_fee : 10_000_000u128, on_demand_fee_variability : Perbill::from_percent(3), on_demand_target_queue_utilization : Perbill::from_percent(25), on_demand_ttl : 5u32.into(), +approval_voting_params : ApprovalVotingParams { + max_approval_coalesce_count: 1, + } } }; diff --git a/polkadot/runtime/parachains/src/configuration/tests.rs b/polkadot/runtime/parachains/src/configuration/tests.rs index 43c03067a9a7..72d1202db61f 100644 --- a/polkadot/runtime/parachains/src/configuration/tests.rs +++ b/polkadot/runtime/parachains/src/configuration/tests.rs @@ -312,6 +312,7 @@ fn setting_pending_config_members() { pvf_voting_ttl: 3, minimum_validation_upgrade_delay: 20, executor_params: Default::default(), + approval_voting_params: ApprovalVotingParams { max_approval_coalesce_count: 1 }, on_demand_queue_max_size: 10_000u32, on_demand_base_fee: 10_000_000u128, on_demand_fee_variability: Perbill::from_percent(3), diff --git a/polkadot/runtime/parachains/src/disputes.rs b/polkadot/runtime/parachains/src/disputes.rs index cf2e99e7359a..e0ff68f79063 100644 --- a/polkadot/runtime/parachains/src/disputes.rs +++ b/polkadot/runtime/parachains/src/disputes.rs @@ -25,11 +25,11 @@ use frame_system::pallet_prelude::*; use parity_scale_codec::{Decode, Encode}; use polkadot_runtime_metrics::get_current_time; use primitives::{ - byzantine_threshold, supermajority_threshold, ApprovalVote, CandidateHash, - CheckedDisputeStatementSet, CheckedMultiDisputeStatementSet, CompactStatement, ConsensusLog, - DisputeState, DisputeStatement, DisputeStatementSet, ExplicitDisputeStatement, - InvalidDisputeStatementKind, MultiDisputeStatementSet, SessionIndex, SigningContext, - ValidDisputeStatementKind, ValidatorId, ValidatorIndex, ValidatorSignature, + byzantine_threshold, supermajority_threshold, vstaging::ApprovalVoteMultipleCandidates, + ApprovalVote, CandidateHash, CheckedDisputeStatementSet, CheckedMultiDisputeStatementSet, + CompactStatement, ConsensusLog, DisputeState, DisputeStatement, DisputeStatementSet, + ExplicitDisputeStatement, InvalidDisputeStatementKind, MultiDisputeStatementSet, SessionIndex, + SigningContext, ValidDisputeStatementKind, ValidatorId, ValidatorIndex, ValidatorSignature, }; use scale_info::TypeInfo; use sp_runtime::{ @@ -1016,6 +1016,8 @@ impl Pallet { statement, signature, ) { + log::warn!("Failed to check dispute signature"); + importer.undo(undo); filter.remove_index(i); continue @@ -1261,21 +1263,29 @@ fn check_signature( statement: &DisputeStatement, validator_signature: &ValidatorSignature, ) -> Result<(), ()> { - let payload = match *statement { + let payload = match statement { DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) => ExplicitDisputeStatement { valid: true, candidate_hash, session }.signing_payload(), DisputeStatement::Valid(ValidDisputeStatementKind::BackingSeconded(inclusion_parent)) => CompactStatement::Seconded(candidate_hash).signing_payload(&SigningContext { session_index: session, - parent_hash: inclusion_parent, + parent_hash: *inclusion_parent, }), DisputeStatement::Valid(ValidDisputeStatementKind::BackingValid(inclusion_parent)) => CompactStatement::Valid(candidate_hash).signing_payload(&SigningContext { session_index: session, - parent_hash: inclusion_parent, + parent_hash: *inclusion_parent, }), DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) => ApprovalVote(candidate_hash).signing_payload(session), + DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates( + candidates, + )) => + if candidates.contains(&candidate_hash) { + ApprovalVoteMultipleCandidates(candidates).signing_payload(session) + } else { + return Err(()) + }, DisputeStatement::Invalid(InvalidDisputeStatementKind::Explicit) => ExplicitDisputeStatement { valid: false, candidate_hash, session }.signing_payload(), }; diff --git a/polkadot/runtime/parachains/src/runtime_api_impl/vstaging.rs b/polkadot/runtime/parachains/src/runtime_api_impl/vstaging.rs index 5406428377d0..a9c433451808 100644 --- a/polkadot/runtime/parachains/src/runtime_api_impl/vstaging.rs +++ b/polkadot/runtime/parachains/src/runtime_api_impl/vstaging.rs @@ -16,6 +16,8 @@ //! Put implementations of functions from staging APIs here. +use primitives::vstaging::ApprovalVotingParams; + use crate::{configuration, dmp, hrmp, inclusion, initializer, paras, shared}; use frame_system::pallet_prelude::BlockNumberFor; use primitives::{ @@ -27,6 +29,11 @@ use primitives::{ }; use sp_std::prelude::*; +pub fn approval_voting_params() -> ApprovalVotingParams { + let config = >::config(); + config.approval_voting_params +} + /// Implementation for `StagingParaBackingState` function from the runtime API pub fn backing_state( para_id: ParaId, diff --git a/polkadot/runtime/polkadot/src/lib.rs b/polkadot/runtime/polkadot/src/lib.rs index c4458076cb3d..e7adbdc9be16 100644 --- a/polkadot/runtime/polkadot/src/lib.rs +++ b/polkadot/runtime/polkadot/src/lib.rs @@ -60,12 +60,12 @@ use pallet_session::historical as session_historical; use pallet_transaction_payment::{FeeDetails, RuntimeDispatchInfo}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, - OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, - ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, LOWEST_PUBLIC_ID, - PARACHAIN_KEY_TYPE_ID, + slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, + CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, + ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, + ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, + ValidatorIndex, LOWEST_PUBLIC_ID, PARACHAIN_KEY_TYPE_ID, }; use sp_core::OpaqueMetadata; use sp_mmr_primitives as mmr; @@ -1690,6 +1690,7 @@ sp_api::impl_runtime_apis! { } } + #[api_version(5)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 6894bd7bbf44..25cf8f21de69 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -23,11 +23,12 @@ use pallet_nis::WithMaximumOf; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, - OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, - ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, PARACHAIN_KEY_TYPE_ID, + slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, + CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, + ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, + ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, + ValidatorIndex, PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ assigned_slots, auctions, claims, crowdloan, impl_runtime_weights, impls::ToAuthor, @@ -1714,6 +1715,7 @@ sp_api::impl_runtime_apis! { } } + #[api_version(5)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index 9ae30c376010..717423e0b124 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -42,12 +42,12 @@ use pallet_session::historical as session_historical; use pallet_transaction_payment::{CurrencyAdapter, FeeDetails, RuntimeDispatchInfo}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, - SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, - ValidatorSignature, PARACHAIN_KEY_TYPE_ID, + slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, + CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, + ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, + PvfCheckStatement, ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, + ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ assigned_slots, auctions, crowdloan, elections::OnChainAccuracy, impl_runtime_weights, @@ -1561,6 +1561,7 @@ sp_api::impl_runtime_apis! { } } + #[api_version(5)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() diff --git a/polkadot/scripts/ci/gitlab/pipeline/build.yml b/polkadot/scripts/ci/gitlab/pipeline/build.yml index 845ac7970108..70d1b1fc5456 100644 --- a/polkadot/scripts/ci/gitlab/pipeline/build.yml +++ b/polkadot/scripts/ci/gitlab/pipeline/build.yml @@ -21,7 +21,7 @@ build-linux-stable: # Ensure we run the UI tests. RUN_UI_TESTS: 1 script: - - time cargo build --locked --profile testnet --features pyroscope,fast-runtime --verbose --bins + - time cargo build --locked --profile testnet --features pyroscope,fast-runtime,network-protocol-staging --verbose --bins # pack artifacts - mkdir -p ./artifacts - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name diff --git a/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml b/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml index 1c8df44cbaa7..2384a5935661 100644 --- a/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml +++ b/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml @@ -64,6 +64,36 @@ zombienet-tests-parachains-pvf: tags: - zombienet-polkadot-integration-test +zombienet-tests-parachains-approval-coalescing: + stage: zombienet + image: "${ZOMBIENET_IMAGE}" + extends: + - .kubernetes-env + - .zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0006-approval-voting-coalescing.zndsl" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + zombienet-tests-parachains-disputes: stage: zombienet image: "${ZOMBIENET_IMAGE}" diff --git a/polkadot/zombienet_tests/functional/0001-parachains-pvf.zndsl b/polkadot/zombienet_tests/functional/0001-parachains-pvf.zndsl index 46bb8bcdf72b..7859dd6d1156 100644 --- a/polkadot/zombienet_tests/functional/0001-parachains-pvf.zndsl +++ b/polkadot/zombienet_tests/functional/0001-parachains-pvf.zndsl @@ -32,6 +32,8 @@ alice: parachain 2005 block height is at least 10 within 300 seconds alice: parachain 2006 block height is at least 10 within 300 seconds alice: parachain 2007 block height is at least 10 within 300 seconds +alice: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds + # Check preparation time is under 10s. # Check all buckets <= 10. alice: reports histogram polkadot_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds diff --git a/polkadot/zombienet_tests/functional/0002-parachains-disputes.toml b/polkadot/zombienet_tests/functional/0002-parachains-disputes.toml index a0a87d60d4e3..5c649ceaf75f 100644 --- a/polkadot/zombienet_tests/functional/0002-parachains-disputes.toml +++ b/polkadot/zombienet_tests/functional/0002-parachains-disputes.toml @@ -5,6 +5,10 @@ timeout = 1000 max_validators_per_core = 5 needed_approvals = 8 +[relaychain.genesis.runtime.runtime_genesis_config.configuration.config.approval_voting_params] + max_approval_coalesce_count = 5 + + [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" diff --git a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml b/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml new file mode 100644 index 000000000000..ec6d17dff4e7 --- /dev/null +++ b/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml @@ -0,0 +1,195 @@ +[settings] +timeout = 1000 + +[relaychain] +default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" +chain = "rococo-local" +chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default-bootnode" + +[relaychain.genesis.runtime.runtime_genesis_config.configuration.config] + needed_approvals = 5 + relay_vrf_modulo_samples = 3 + +[relaychain.genesis.runtime.runtime_genesis_config.configuration.config.approval_voting_params] + max_approval_coalesce_count = 6 + +[relaychain.default_resources] +limits = { memory = "4G", cpu = "2" } +requests = { memory = "2G", cpu = "1" } + + [[relaychain.nodes]] + name = "alice" + args = [ "--alice", "-lparachain=debug,runtime=debug,peerset=trace" ] + + [[relaychain.nodes]] + name = "bob" + args = [ "--bob", "-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "charlie" + args = [ "--charlie", "-lparachain=debug,runtime=debug" ] + + [[relaychain.nodes]] + name = "dave" + args = [ "--dave", "-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "ferdie" + args = [ "--ferdie", "-lparachain=debug,runtime=debug" ] + + [[relaychain.nodes]] + name = "eve" + args = [ "--eve", "-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "one" + args = [ "--one", "-lparachain=debug,runtime=debug" ] + + [[relaychain.nodes]] + name = "two" + args = [ "--two", "-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "nine" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "eleven" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "twelve" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "thirteen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "fourteen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "fifteen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "sixteen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "seventeen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "eithteen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "nineteen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "twenty" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "twentyone" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "twentytwo" + args = ["-lparachain=debug,runtime=debug"] + +[[parachains]] +id = 2000 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=1" + + [parachains.collator] + name = "collator01" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--pvf-complexity=1", "--parachain-id=2000"] + +[[parachains]] +id = 2001 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=10" + + [parachains.collator] + name = "collator02" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--parachain-id=2001", "--pvf-complexity=10"] + +[[parachains]] +id = 2002 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=100" + + [parachains.collator] + name = "collator03" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--parachain-id=2002", "--pvf-complexity=100"] + +[[parachains]] +id = 2003 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=20000 --pvf-complexity=300" + + [parachains.collator] + name = "collator04" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=20000", "--parachain-id=2003", "--pvf-complexity=300"] + +[[parachains]] +id = 2004 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=300" + + [parachains.collator] + name = "collator05" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--parachain-id=2004", "--pvf-complexity=300"] + +[[parachains]] +id = 2005 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=20000 --pvf-complexity=400" + + [parachains.collator] + name = "collator06" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=20000", "--pvf-complexity=400", "--parachain-id=2005"] + +[[parachains]] +id = 2006 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=300" + + [parachains.collator] + name = "collator07" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--pvf-complexity=300", "--parachain-id=2006"] + +[[parachains]] +id = 2007 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=300" + + [parachains.collator] + name = "collator08" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--pvf-complexity=300", "--parachain-id=2007"] + +[types.Header] +number = "u64" +parent_hash = "Hash" +post_state = "Hash" \ No newline at end of file diff --git a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.zndsl b/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.zndsl new file mode 100644 index 000000000000..272ac4fa2640 --- /dev/null +++ b/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.zndsl @@ -0,0 +1,51 @@ +Description: Approval voting coalescing does not lag finality +Network: ./0006-approval-voting-coalescing.toml +Creds: config + +# Check authority status. +alice: reports node_roles is 4 +bob: reports node_roles is 4 +charlie: reports node_roles is 4 +dave: reports node_roles is 4 +eve: reports node_roles is 4 +ferdie: reports node_roles is 4 +one: reports node_roles is 4 +two: reports node_roles is 4 + +# Ensure parachains are registered. +alice: parachain 2000 is registered within 60 seconds +bob: parachain 2001 is registered within 60 seconds +charlie: parachain 2002 is registered within 60 seconds +dave: parachain 2003 is registered within 60 seconds +ferdie: parachain 2004 is registered within 60 seconds +eve: parachain 2005 is registered within 60 seconds +one: parachain 2006 is registered within 60 seconds +two: parachain 2007 is registered within 60 seconds + +# Ensure parachains made progress. +alice: parachain 2000 block height is at least 10 within 300 seconds +alice: parachain 2001 block height is at least 10 within 300 seconds +alice: parachain 2002 block height is at least 10 within 300 seconds +alice: parachain 2003 block height is at least 10 within 300 seconds +alice: parachain 2004 block height is at least 10 within 300 seconds +alice: parachain 2005 block height is at least 10 within 300 seconds +alice: parachain 2006 block height is at least 10 within 300 seconds +alice: parachain 2007 block height is at least 10 within 300 seconds + +alice: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +bob: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +charlie: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +dave: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +eve: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +ferdie: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +one: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +two: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds + +alice: reports polkadot_parachain_approval_checking_finality_lag is 0 +bob: reports polkadot_parachain_approval_checking_finality_lag is 0 +charlie: reports polkadot_parachain_approval_checking_finality_lag is 0 +dave: reports polkadot_parachain_approval_checking_finality_lag is 0 +ferdie: reports polkadot_parachain_approval_checking_finality_lag is 0 +eve: reports polkadot_parachain_approval_checking_finality_lag is 0 +one: reports polkadot_parachain_approval_checking_finality_lag is 0 +two: reports polkadot_parachain_approval_checking_finality_lag is 0 \ No newline at end of file From 619fff2e6221a704669b45bbef17354626ddd88f Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Sun, 27 Aug 2023 14:05:00 +0300 Subject: [PATCH 004/192] Fix build warnings Signed-off-by: Alexandru Gheorghe --- polkadot/node/service/src/fake_runtime_api.rs | 12 ++++++------ polkadot/runtime/kusama/src/lib.rs | 12 ++++++------ polkadot/runtime/polkadot/src/lib.rs | 12 ++++++------ polkadot/runtime/rococo/src/lib.rs | 11 +++++------ polkadot/runtime/westend/src/lib.rs | 12 ++++++------ 5 files changed, 29 insertions(+), 30 deletions(-) diff --git a/polkadot/node/service/src/fake_runtime_api.rs b/polkadot/node/service/src/fake_runtime_api.rs index f2f2bdefae7a..a8ec22baf791 100644 --- a/polkadot/node/service/src/fake_runtime_api.rs +++ b/polkadot/node/service/src/fake_runtime_api.rs @@ -23,12 +23,12 @@ use beefy_primitives::ecdsa_crypto::{AuthorityId as BeefyId, Signature as BeefyS use grandpa_primitives::AuthorityId as GrandpaId; use pallet_transaction_payment::{FeeDetails, RuntimeDispatchInfo}; use polkadot_primitives::{ - runtime_api, slashing, vstaging::ApprovalVotingParams, AccountId, AuthorityDiscoveryId, - Balance, Block, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Nonce, OccupiedCoreAssumption, - PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, - ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, + runtime_api, slashing, AccountId, AuthorityDiscoveryId, Balance, Block, BlockNumber, + CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, + DisputeState, ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Nonce, OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, + ScrapedOnChainVotes, SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, + ValidatorId, ValidatorIndex, ValidatorSignature, }; use sp_core::OpaqueMetadata; use sp_runtime::{ diff --git a/polkadot/runtime/kusama/src/lib.rs b/polkadot/runtime/kusama/src/lib.rs index 07bee8cfa853..56fdcec7165a 100644 --- a/polkadot/runtime/kusama/src/lib.rs +++ b/polkadot/runtime/kusama/src/lib.rs @@ -23,12 +23,12 @@ use pallet_nis::WithMaximumOf; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, - CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, - ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, - InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, - ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, - ValidatorIndex, LOWEST_PUBLIC_ID, PARACHAIN_KEY_TYPE_ID, + slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, + OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, + ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, LOWEST_PUBLIC_ID, + PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ auctions, claims, crowdloan, impl_runtime_weights, impls::DealWithFees, paras_registrar, diff --git a/polkadot/runtime/polkadot/src/lib.rs b/polkadot/runtime/polkadot/src/lib.rs index e7adbdc9be16..b37ebaeff1a4 100644 --- a/polkadot/runtime/polkadot/src/lib.rs +++ b/polkadot/runtime/polkadot/src/lib.rs @@ -60,12 +60,12 @@ use pallet_session::historical as session_historical; use pallet_transaction_payment::{FeeDetails, RuntimeDispatchInfo}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, - CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, - ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, - InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, - ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, - ValidatorIndex, LOWEST_PUBLIC_ID, PARACHAIN_KEY_TYPE_ID, + slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, + OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, + ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, LOWEST_PUBLIC_ID, + PARACHAIN_KEY_TYPE_ID, }; use sp_core::OpaqueMetadata; use sp_mmr_primitives as mmr; diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 25cf8f21de69..042caeee936a 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -23,12 +23,11 @@ use pallet_nis::WithMaximumOf; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, - CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, - ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, - InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, - ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, - ValidatorIndex, PARACHAIN_KEY_TYPE_ID, + slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, + OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, + ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ assigned_slots, auctions, claims, crowdloan, impl_runtime_weights, impls::ToAuthor, diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index 717423e0b124..a7b2782591b5 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -42,12 +42,12 @@ use pallet_session::historical as session_historical; use pallet_transaction_payment::{CurrencyAdapter, FeeDetails, RuntimeDispatchInfo}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, - CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, - ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, - InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, - PvfCheckStatement, ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, - ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, PARACHAIN_KEY_TYPE_ID, + slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, + OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, + SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, + ValidatorSignature, PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ assigned_slots, auctions, crowdloan, elections::OnChainAccuracy, impl_runtime_weights, From ed1d9d0c3091524694b41abf3eaee38bac248e84 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 28 Aug 2023 09:43:52 +0300 Subject: [PATCH 005/192] ci: fix worker binaries could not be found Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/build.yml | 4 ++++ docker/malus_injected.Dockerfile | 2 +- docker/polkadot_injected_debug.Dockerfile | 6 ++++-- docker/polkadot_injected_release.Dockerfile | 2 ++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 290ef8c8f72d..c46aa895a4eb 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -23,6 +23,8 @@ build-linux-stable: - mkdir -p ./artifacts - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name - mv ./target/testnet/polkadot ./artifacts/. + - mv ./target/testnet/polkadot-prepare-worker ./artifacts/. + - mv ./target/testnet/polkadot-execute-worker ./artifacts/. - pushd artifacts - sha256sum polkadot | tee polkadot.sha256 - shasum -c polkadot.sha256 @@ -68,6 +70,8 @@ build-malus: # pack artifacts - mkdir -p ./artifacts - mv ./target/testnet/malus ./artifacts/. + - mv ./target/testnet/polkadot-execute-worker ./artifacts/. + - mv ./target/testnet/polkadot-prepare-worker ./artifacts/. - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" diff --git a/docker/malus_injected.Dockerfile b/docker/malus_injected.Dockerfile index ecffd2c4f9b4..762a2e442c82 100644 --- a/docker/malus_injected.Dockerfile +++ b/docker/malus_injected.Dockerfile @@ -39,7 +39,7 @@ RUN apt-get update && \ # add adder-collator binary to docker image -COPY ./artifacts/malus /usr/local/bin +COPY ./artifacts/malus ./artifacts/polkadot-execute-worker ./artifacts/polkadot-prepare-worker /usr/local/bin USER nonroot diff --git a/docker/polkadot_injected_debug.Dockerfile b/docker/polkadot_injected_debug.Dockerfile index 3dd62f7ba56f..f7f764d335a2 100644 --- a/docker/polkadot_injected_debug.Dockerfile +++ b/docker/polkadot_injected_debug.Dockerfile @@ -32,13 +32,15 @@ RUN apt-get update && \ chown -R polkadot:polkadot /data && \ ln -s /data /polkadot/.local/share/polkadot -# add polkadot binary to docker image -COPY ./artifacts/polkadot /usr/local/bin +# add polkadot binaries to docker image +COPY ./artifacts/polkadot ./artifacts/polkadot-execute-worker ./artifacts/polkadot-prepare-worker /usr/local/bin USER polkadot # check if executable works in this container RUN /usr/local/bin/polkadot --version +RUN /usr/local/bin/polkadot-execute-worker --version +RUN /usr/local/bin/polkadot-prepare-worker --version EXPOSE 30333 9933 9944 VOLUME ["/polkadot"] diff --git a/docker/polkadot_injected_release.Dockerfile b/docker/polkadot_injected_release.Dockerfile index ba0a79e78187..87ae7ac27dc0 100644 --- a/docker/polkadot_injected_release.Dockerfile +++ b/docker/polkadot_injected_release.Dockerfile @@ -44,6 +44,8 @@ USER polkadot # check if executable works in this container RUN /usr/bin/polkadot --version +RUN /usr/local/bin/polkadot-execute-worker --version +RUN /usr/local/bin/polkadot-prepare-worker --version EXPOSE 30333 9933 9944 VOLUME ["/polkadot"] From 7d7b82c68918183bb549d1d7b759836b762a6304 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 28 Aug 2023 10:20:33 +0300 Subject: [PATCH 006/192] Add missing bits Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/build.yml | 4 ++-- polkadot/Cargo.toml | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index c46aa895a4eb..422bea02efee 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -18,7 +18,7 @@ build-linux-stable: # Ensure we run the UI tests. RUN_UI_TESTS: 1 script: - - time cargo build --locked --profile testnet --features pyroscope,fast-runtime --bin polkadot + - time cargo build --locked --profile testnet --features pyroscope,fast-runtime --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker # pack artifacts - mkdir -p ./artifacts - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name @@ -66,7 +66,7 @@ build-malus: - .run-immediately - .collect-artifacts script: - - time cargo build --locked --profile testnet --verbose -p polkadot-test-malus + - time cargo build --locked --profile testnet --verbose -p polkadot-test-malus --bin polkadot-prepare-worker --bin polkadot-execute-worker # pack artifacts - mkdir -p ./artifacts - mv ./target/testnet/malus ./artifacts/. diff --git a/polkadot/Cargo.toml b/polkadot/Cargo.toml index 561b49ab4204..925d6589dc0b 100644 --- a/polkadot/Cargo.toml +++ b/polkadot/Cargo.toml @@ -159,6 +159,16 @@ assets = [ "/usr/bin/", "755", ], + [ + "target/release/polkadot-prepare-worker", + "/usr/lib/polkadot/", + "755" + ], + [ + "target/release/polkadot-execute-worker", + "/usr/lib/polkadot/", + "755" + ], [ "scripts/packaging/polkadot.service", "/lib/systemd/system/", From 7bc13d311b273238cd6f4606f55e61ae5fbe9970 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 28 Aug 2023 12:21:25 +0300 Subject: [PATCH 007/192] Build with network-protocol-staging Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 422bea02efee..7f5c86ff5f68 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -18,7 +18,7 @@ build-linux-stable: # Ensure we run the UI tests. RUN_UI_TESTS: 1 script: - - time cargo build --locked --profile testnet --features pyroscope,fast-runtime --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker + - time cargo build --locked --profile testnet --features pyroscope,fast-runtime,network-protocol-staging --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker # pack artifacts - mkdir -p ./artifacts - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name From 53f8556c501baaaebf6059366bcbece06294e4cd Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 28 Aug 2023 11:46:22 +0300 Subject: [PATCH 008/192] Validate disconnect theory Signed-off-by: Alexandru Gheorghe --- polkadot/node/network/approval-distribution/src/lib.rs | 6 +++--- .../network/statement-distribution/src/legacy_v1/mod.rs | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 3b8624e9166d..bdc2f859b909 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -583,9 +583,9 @@ impl State { NetworkBridgeEvent::PeerDisconnected(peer_id) => { gum::trace!(target: LOG_TARGET, ?peer_id, "Peer disconnected"); self.peer_views.remove(&peer_id); - self.blocks.iter_mut().for_each(|(_hash, entry)| { - entry.known_by.remove(&peer_id); - }) + // self.blocks.iter_mut().for_each(|(_hash, entry)| { + // entry.known_by.remove(&peer_id); + // }) }, NetworkBridgeEvent::NewGossipTopology(topology) => { self.handle_new_session_topology( diff --git a/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs b/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs index b5895cb9f65b..938eb7fe64c0 100644 --- a/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs +++ b/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs @@ -1195,6 +1195,10 @@ async fn modify_reputation( peer: PeerId, rep: Rep, ) { + //TODO: Test teory in versi + if rep == COST_DUPLICATE_STATEMENT { + return + } reputation.modify(sender, peer, rep).await; } From 5e004e16e23525fbfe423b4ea15cd36d750e5646 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 28 Aug 2023 20:27:18 +0300 Subject: [PATCH 009/192] Log errors when banning peers Signed-off-by: Alexandru Gheorghe --- substrate/client/network/src/peer_store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrate/client/network/src/peer_store.rs b/substrate/client/network/src/peer_store.rs index 2f3d4a1fd1a0..96bcd47cc12e 100644 --- a/substrate/client/network/src/peer_store.rs +++ b/substrate/client/network/src/peer_store.rs @@ -222,7 +222,7 @@ impl PeerStoreInner { if peer_info.reputation < BANNED_THRESHOLD { self.protocols.iter().for_each(|handle| handle.disconnect_peer(peer_id)); - log::trace!( + log::error!( target: LOG_TARGET, "Report {}: {:+} to {}. Reason: {}. Banned, disconnecting.", peer_id, From 9850b2ff909b506f81178776bd277125306cb109 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 28 Aug 2023 23:47:09 +0300 Subject: [PATCH 010/192] fix zombienet test Signed-off-by: Andrei Sandu --- .gitlab/pipeline/zombienet/polkadot.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab/pipeline/zombienet/polkadot.yml b/.gitlab/pipeline/zombienet/polkadot.yml index 82dd13cd290c..7bc23db3259a 100644 --- a/.gitlab/pipeline/zombienet/polkadot.yml +++ b/.gitlab/pipeline/zombienet/polkadot.yml @@ -79,6 +79,14 @@ zombienet-polkadot-functional-0004-beefy-and-mmr: --local-dir="${LOCAL_DIR}/functional" --test="0003-beefy-and-mmr.zndsl" +zombienet-polkadot-functional-0005-max-tranche0: + extends: + - .zombienet-polkadot-common + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh + --local-dir="${LOCAL_DIR}/functional" + --test="0005-parachains-max-tranche0.zndsl" + zombienet-polkadot-smoke-0001-parachains-smoke-test: extends: - .zombienet-polkadot-common From 46cfaf1e55e9e27452e1c9e805d29684eae9bf72 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 29 Aug 2023 19:06:20 +0300 Subject: [PATCH 011/192] cargo lock Signed-off-by: Andrei Sandu --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 0ccf5aa0ba5e..a586275ba332 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11952,6 +11952,7 @@ dependencies = [ "itertools 0.10.5", "kvdb", "kvdb-memorydb", + "log", "merlin 2.0.1", "parity-scale-codec", "parking_lot 0.12.1", From 47beabdf0675289a1a3e4cbb3843fed0937106b8 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 30 Aug 2023 18:07:12 +0300 Subject: [PATCH 012/192] superfluous Signed-off-by: Andrei Sandu --- .../approval-voting/src/approval_db/v2/migration_helpers.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs index 3d1f28ee1937..a70d766fc8c9 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs @@ -15,7 +15,7 @@ // along with Polkadot. If not, see . //! Approval DB migration helpers. -use super::{StoredBlockRange, *}; +use super::*; use crate::backend::Backend; use polkadot_node_primitives::approval::v1::{ AssignmentCert, AssignmentCertKind, VrfOutput, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT, From 3d3e37cf4810090c4a9f43b752ec4a9f4b000694 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 30 Aug 2023 13:27:04 +0300 Subject: [PATCH 013/192] Separate approval Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 69 +- .../network/approval-distribution/src/lib.rs | 593 ++++++++---------- .../approval-distribution/src/metrics.rs | 12 - .../approval-distribution/src/tests.rs | 474 +++++++++++++- 4 files changed, 786 insertions(+), 362 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 69c5603624d3..e8a19cd2bc2f 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -837,7 +837,7 @@ impl State { #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn get_approval_voting_params_or_default( &mut self, - _ctx: &mut Context, + ctx: &mut Context, block_hash: Hash, ) -> ApprovalVotingParams { if let Some(params) = self @@ -847,30 +847,29 @@ impl State { { *params } else { - // let (s_tx, s_rx) = oneshot::channel(); - - // ctx.send_message(RuntimeApiMessage::Request( - // block_hash, - // RuntimeApiRequest::ApprovalVotingParams(s_tx), - // )) - // .await; - - // match s_rx.await { - // Ok(Ok(params)) => { - // self.approval_voting_params_cache - // .as_mut() - // .map(|cache| cache.put(block_hash, params)); - // params - // }, - // _ => { - // gum::error!( - // target: LOG_TARGET, - // "Could not request approval voting params from runtime using defaults" - // ); - // TODO: Uncomment this after versi test - ApprovalVotingParams { max_approval_coalesce_count: 6 } - // }, - // } + let (s_tx, s_rx) = oneshot::channel(); + + ctx.send_message(RuntimeApiMessage::Request( + block_hash, + RuntimeApiRequest::ApprovalVotingParams(s_tx), + )) + .await; + + match s_rx.await { + Ok(Ok(params)) => { + self.approval_voting_params_cache + .as_mut() + .map(|cache| cache.put(block_hash, params)); + params + }, + _ => { + gum::error!( + target: LOG_TARGET, + "Could not request approval voting params from runtime using defaults" + ); + ApprovalVotingParams { max_approval_coalesce_count: 6 } + }, + } } } } @@ -3276,7 +3275,12 @@ async fn issue_approval( .defer_candidate_signature( candidate_index as _, candidate_hash, - compute_delayed_approval_sending_tick(state, &block_entry, session_info), + compute_delayed_approval_sending_tick( + state, + &block_entry, + &candidate_entry, + session_info, + ), ) .is_some() { @@ -3532,9 +3536,17 @@ fn issue_local_invalid_statement( fn compute_delayed_approval_sending_tick( state: &State, block_entry: &BlockEntry, + candidate_entry: &CandidateEntry, session_info: &SessionInfo, ) -> Tick { let current_block_tick = slot_number_to_tick(state.slot_duration_millis, block_entry.slot()); + let assignment_tranche = candidate_entry + .approval_entry(&block_entry.block_hash()) + .and_then(|approval_entry| approval_entry.our_assignment()) + .map(|our_assignment| our_assignment.tranche()) + .unwrap(); + + let assignment_triggered_tick = current_block_tick + assignment_tranche as Tick; let no_show_duration_ticks = slot_number_to_tick( state.slot_duration_millis, @@ -3545,9 +3557,8 @@ fn compute_delayed_approval_sending_tick( min( tick_now + MAX_APPROVAL_COALESCE_WAIT_TICKS as Tick, // We don't want to accidentally cause, no-shows so if we are past - // the 2 thirds of the no show time, force the sending of the + // the seconnd half of the no show time, force the sending of the // approval immediately. - // TODO: TBD the right value here, so that we don't accidentally create no-shows. - current_block_tick + (no_show_duration_ticks * 2) / 3, + assignment_triggered_tick + no_show_duration_ticks / 2, ) } diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index bdc2f859b909..8d28a54dcd89 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -33,9 +33,7 @@ use polkadot_node_network_protocol::{ Versioned, View, }; use polkadot_node_primitives::approval::{ - v1::{ - AssignmentCertKind, BlockApprovalMeta, IndirectAssignmentCert, IndirectSignedApprovalVote, - }, + v1::{AssignmentCertKind, BlockApprovalMeta, IndirectAssignmentCert}, v2::{ AsBitIndex, AssignmentCertKindV2, CandidateBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2, @@ -121,9 +119,9 @@ struct ApprovalEntry { // The assignment certificate. assignment: IndirectAssignmentCertV2, // The candidates claimed by the certificate. A mapping between bit index and candidate index. - candidates: CandidateBitfield, + assignment_claimed_candidates: CandidateBitfield, // The approval signatures for each `CandidateIndex` claimed by the assignment certificate. - approvals: HashMap, + approvals: HashMap, // The validator index of the assignment signer. validator_index: ValidatorIndex, // Information required for gossiping to other peers using the grid topology. @@ -136,6 +134,8 @@ enum ApprovalEntryError { CandidateIndexOutOfBounds, InvalidCandidateIndex, DuplicateApproval, + CouldNotFindAssignment, + AssignmentsFollowedDifferentPaths(RequiredRouting, RequiredRouting), } impl ApprovalEntry { @@ -148,7 +148,7 @@ impl ApprovalEntry { validator_index: assignment.validator, assignment, approvals: HashMap::with_capacity(candidates.len()), - candidates, + assignment_claimed_candidates: candidates, routing_info, } } @@ -156,23 +156,15 @@ impl ApprovalEntry { // Create a `MessageSubject` to reference the assignment. pub fn create_assignment_knowledge(&self, block_hash: Hash) -> (MessageSubject, MessageKind) { ( - MessageSubject(block_hash, self.candidates.clone(), self.validator_index), + MessageSubject( + block_hash, + self.assignment_claimed_candidates.clone(), + self.validator_index, + ), MessageKind::Assignment, ) } - // Create a `MessageSubject` to reference the approval. - pub fn create_approval_knowledge( - &self, - block_hash: Hash, - candidate_index: CandidateIndex, - ) -> (MessageSubject, MessageKind) { - ( - MessageSubject(block_hash, candidate_index.into(), self.validator_index), - MessageKind::Approval, - ) - } - // Updates routing information and returns the previous information if any. pub fn routing_info_mut(&mut self) -> &mut ApprovalRouting { &mut self.routing_info @@ -193,7 +185,6 @@ impl ApprovalEntry { pub fn note_approval( &mut self, approval: IndirectSignedApprovalVoteV2, - candidate_index: CandidateIndex, ) -> Result<(), ApprovalEntryError> { // First do some sanity checks: // - check validator index matches @@ -203,24 +194,28 @@ impl ApprovalEntry { return Err(ApprovalEntryError::InvalidValidatorIndex) } - if self.candidates.len() <= candidate_index as usize { - return Err(ApprovalEntryError::CandidateIndexOutOfBounds) - } - if !self.candidates.bit_at((candidate_index).as_bit_index()) { + // We need at least one of the candidates in the approval to be in this assignment + if !approval + .candidate_indices + .iter_ones() + .fold(false, |accumulator, candidate_index| { + self.assignment_claimed_candidates.bit_at((candidate_index).as_bit_index()) || + accumulator + }) { return Err(ApprovalEntryError::InvalidCandidateIndex) } - if self.approvals.contains_key(&candidate_index) { + if self.approvals.contains_key(&approval.candidate_indices) { return Err(ApprovalEntryError::DuplicateApproval) } - self.approvals.insert(candidate_index, approval.clone()); + self.approvals.insert(approval.candidate_indices.clone(), approval.clone()); Ok(()) } // Get the assignment certiticate and claimed candidates. pub fn assignment(&self) -> (IndirectAssignmentCertV2, CandidateBitfield) { - (self.assignment.clone(), self.candidates.clone()) + (self.assignment.clone(), self.assignment_claimed_candidates.clone()) } // Get all approvals for all candidates claimed by the assignment. @@ -228,14 +223,6 @@ impl ApprovalEntry { self.approvals.values().cloned().collect::>() } - // Get the approval for a specific candidate index. - pub fn approval( - &self, - candidate_index: CandidateIndex, - ) -> Option { - self.approvals.get(&candidate_index).cloned() - } - // Get validator index. pub fn validator_index(&self) -> ValidatorIndex { self.validator_index @@ -433,6 +420,51 @@ impl PeerKnowledge { fn contains(&self, message: &MessageSubject, kind: MessageKind) -> bool { self.sent.contains(message, kind) || self.received.contains(message, kind) } + + // Tells if all assignments for a given approval are included in the knowledge of the peer + fn contains_assignments_for_approval(&self, approval: &IndirectSignedApprovalVoteV2) -> bool { + Self::generate_assignments_keys(&approval).iter().fold( + true, + |accumulator, assignment_key| { + accumulator && self.contains(&assignment_key.0, assignment_key.1) + }, + ) + } + + // Generate the knowledge keys for querying if all assignments of an approval are known + // by this peer. + fn generate_assignments_keys( + approval: &IndirectSignedApprovalVoteV2, + ) -> Vec<(MessageSubject, MessageKind)> { + approval + .candidate_indices + .iter_ones() + .map(|candidate_index| { + ( + MessageSubject( + approval.block_hash, + (candidate_index as CandidateIndex).into(), + approval.validator, + ), + MessageKind::Assignment, + ) + }) + .collect_vec() + } + + // Generate the knowledge keys for querying if an approval is known by peer. + fn generate_approval_key( + approval: &IndirectSignedApprovalVoteV2, + ) -> (MessageSubject, MessageKind) { + ( + MessageSubject( + approval.block_hash, + approval.candidate_indices.clone(), + approval.validator, + ), + MessageKind::Approval, + ) + } } /// Information about blocks in our current view as well as whether peers know of them. @@ -465,13 +497,13 @@ impl BlockEntry { // First map one entry per candidate to the same key we will use in `approval_entries`. // Key is (Validator_index, CandidateBitfield) that links the `ApprovalEntry` to the (K,V) // entry in `candidate_entry.messages`. - for claimed_candidate_index in entry.candidates.iter_ones() { + for claimed_candidate_index in entry.assignment_claimed_candidates.iter_ones() { match self.candidates.get_mut(claimed_candidate_index) { Some(candidate_entry) => { candidate_entry - .messages + .assignments .entry(entry.validator_index()) - .or_insert(entry.candidates.clone()); + .or_insert(entry.assignment_claimed_candidates.clone()); }, None => { // This should never happen, but if it happens, it means the subsystem is @@ -487,50 +519,90 @@ impl BlockEntry { } self.approval_entries - .entry((entry.validator_index, entry.candidates.clone())) + .entry((entry.validator_index, entry.assignment_claimed_candidates.clone())) .or_insert(entry) } - // Returns a mutable reference of `ApprovalEntry` for `candidate_index` from validator - // `validator_index`. - pub fn approval_entry( + // Tels if all candidate_indices are valid candidates + pub fn contains_candidates(&self, candidate_indices: &CandidateBitfield) -> bool { + candidate_indices.iter_ones().fold(true, |accumulator, candidate_index| { + self.candidates.get(candidate_index as usize).is_some() && accumulator + }) + } + // Saves the given approval in all ApprovalEntries that contain an assignment for any of the + // candidates in the approval. + // + // Returns the required routing needed for this approval. + pub fn note_approval( &mut self, - candidate_index: CandidateIndex, - validator_index: ValidatorIndex, - ) -> Option<&mut ApprovalEntry> { - self.candidates - .get(candidate_index as usize) - .map_or(None, |candidate_entry| candidate_entry.messages.get(&validator_index)) - .map_or(None, |candidate_indices| { - self.approval_entries.get_mut(&(validator_index, candidate_indices.clone())) + approval: IndirectSignedApprovalVoteV2, + ) -> Result { + let mut required_routing = None; + + if self.candidates.len() < approval.candidate_indices.len() as usize { + return Err(ApprovalEntryError::CandidateIndexOutOfBounds) + } + + // First determine all assignments bitfields that might be covered by this approval + let covered_assignments_bitfields: HashSet = approval + .candidate_indices + .iter_ones() + .filter_map(|candidate_index| { + self.candidates.get_mut(candidate_index).map_or(None, |candidate_entry| { + candidate_entry.assignments.get(&approval.validator).cloned() + }) }) - } + .collect(); - // Get all approval entries for a given candidate. - pub fn approval_entries(&self, candidate_index: CandidateIndex) -> Vec<&ApprovalEntry> { - // Get the keys for fetching `ApprovalEntry` from `self.approval_entries`, - let approval_entry_keys = self - .candidates - .get(candidate_index as usize) - .map(|candidate_entry| &candidate_entry.messages); - - if let Some(approval_entry_keys) = approval_entry_keys { - // Ensure no duplicates. - let approval_entry_keys = approval_entry_keys.iter().unique().collect::>(); - - let mut entries = Vec::new(); - for (validator_index, candidate_indices) in approval_entry_keys { - if let Some(entry) = - self.approval_entries.get(&(*validator_index, candidate_indices.clone())) - { - entries.push(entry); + // Mark the vote in all approval entries + for assignment_bitfield in covered_assignments_bitfields { + if let Some(approval_entry) = + self.approval_entries.get_mut(&(approval.validator, assignment_bitfield)) + { + approval_entry.note_approval(approval.clone())?; + + if let Some(required_routing) = required_routing { + if required_routing != approval_entry.routing_info().required_routing { + // This shouldn't happen since the required routing is computed based on the + // validator_index, so two assignments from the same validators will have + // the same required routing. + return Err(ApprovalEntryError::AssignmentsFollowedDifferentPaths( + required_routing, + approval_entry.routing_info().required_routing, + )) + } + } else { + required_routing = Some(approval_entry.routing_info().required_routing) } } - entries + } + + if let Some(required_routing) = required_routing { + Ok(required_routing) } else { - vec![] + Err(ApprovalEntryError::CouldNotFindAssignment) } } + + pub fn approval_votes( + &self, + candidate_index: CandidateIndex, + ) -> Vec { + let result: Option> = + self.candidates.get(candidate_index as usize).map(|candidate_entry| { + candidate_entry + .assignments + .iter() + .filter_map(|(validator, assignment_bitfield)| { + self.approval_entries.get(&(*validator, assignment_bitfield.clone())) + }) + .map(|approval_entry| approval_entry.approvals.clone().into_iter()) + .flatten() + .collect() + }); + + result.map(|result| result.into_values().collect_vec()).unwrap_or_default() + } } // Information about candidates in the context of a particular block they are included in. @@ -540,7 +612,7 @@ impl BlockEntry { struct CandidateEntry { // The value represents part of the lookup key in `approval_entries` to fetch the assignment // and existing votes. - messages: HashMap, + assignments: HashMap, } #[derive(Debug, Clone, PartialEq)] @@ -583,9 +655,9 @@ impl State { NetworkBridgeEvent::PeerDisconnected(peer_id) => { gum::trace!(target: LOG_TARGET, ?peer_id, "Peer disconnected"); self.peer_views.remove(&peer_id); - // self.blocks.iter_mut().for_each(|(_hash, entry)| { - // entry.known_by.remove(&peer_id); - // }) + self.blocks.iter_mut().for_each(|(_hash, entry)| { + entry.known_by.remove(&peer_id); + }) }, NetworkBridgeEvent::NewGossipTopology(topology) => { self.handle_new_session_topology( @@ -833,6 +905,7 @@ impl State { } } + // Entry point for processing an approval coming from a peer. async fn process_incoming_approvals( &mut self, ctx: &mut Context, @@ -1377,17 +1450,19 @@ impl State { } } + // Checks if an approval can be processed. + // Returns true if we can continue with processing the approval and false otherwise. async fn check_approval_can_be_processed( ctx: &mut Context, - message_subjects: &Vec, - message_kind: MessageKind, + assignments_knowledge_key: &Vec<(MessageSubject, MessageKind)>, + approval_knowledge_key: &(MessageSubject, MessageKind), entry: &mut BlockEntry, reputation: &mut ReputationAggregator, peer_id: PeerId, metrics: &Metrics, ) -> bool { - for message_subject in message_subjects { - if !entry.knowledge.contains(&message_subject, MessageKind::Assignment) { + for message_subject in assignments_knowledge_key { + if !entry.knowledge.contains(&message_subject.0, message_subject.1) { gum::trace!( target: LOG_TARGET, ?peer_id, @@ -1398,66 +1473,63 @@ impl State { metrics.on_approval_unknown_assignment(); return false } + } - // check if our knowledge of the peer already contains this approval - match entry.known_by.entry(peer_id) { - hash_map::Entry::Occupied(mut knowledge) => { - let peer_knowledge = knowledge.get_mut(); - if peer_knowledge.contains(&message_subject, message_kind) { - if !peer_knowledge.received.insert(message_subject.clone(), message_kind) { - gum::trace!( - target: LOG_TARGET, - ?peer_id, - ?message_subject, - "Duplicate approval", - ); + // check if our knowledge of the peer already contains this approval + match entry.known_by.entry(peer_id) { + hash_map::Entry::Occupied(mut knowledge) => { + let peer_knowledge = knowledge.get_mut(); + if peer_knowledge.contains(&approval_knowledge_key.0, approval_knowledge_key.1) { + if !peer_knowledge + .received + .insert(approval_knowledge_key.0.clone(), approval_knowledge_key.1) + { + gum::trace!( + target: LOG_TARGET, + ?peer_id, + ?approval_knowledge_key, + "Duplicate approval", + ); - modify_reputation( - reputation, - ctx.sender(), - peer_id, - COST_DUPLICATE_MESSAGE, - ) - .await; - metrics.on_approval_duplicate(); - } - return false - } - }, - hash_map::Entry::Vacant(_) => { - gum::debug!( - target: LOG_TARGET, - ?peer_id, - ?message_subject, - "Approval from a peer is out of view", - ); - modify_reputation(reputation, ctx.sender(), peer_id, COST_UNEXPECTED_MESSAGE) + modify_reputation( + reputation, + ctx.sender(), + peer_id, + COST_DUPLICATE_MESSAGE, + ) .await; - metrics.on_approval_out_of_view(); - }, - } - } - - let good_known_approval = - message_subjects.iter().fold(false, |accumulator, message_subject| { - // if the approval is known to be valid, reward the peer - if entry.knowledge.contains(&message_subject, message_kind) { - if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { - peer_knowledge.received.insert(message_subject.clone(), message_kind); + metrics.on_approval_duplicate(); } - // We already processed this approval no need to continue. - true - } else { - accumulator + return false } - }); - if good_known_approval { - gum::trace!(target: LOG_TARGET, ?peer_id, ?message_subjects, "Known approval"); + }, + hash_map::Entry::Vacant(_) => { + gum::debug!( + target: LOG_TARGET, + ?peer_id, + ?approval_knowledge_key, + "Approval from a peer is out of view", + ); + modify_reputation(reputation, ctx.sender(), peer_id, COST_UNEXPECTED_MESSAGE).await; + metrics.on_approval_out_of_view(); + }, + } + + if entry.knowledge.contains(&approval_knowledge_key.0, approval_knowledge_key.1) { + if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { + peer_knowledge + .received + .insert(approval_knowledge_key.0.clone(), approval_knowledge_key.1); + } + + // We already processed this approval no need to continue. + gum::trace!(target: LOG_TARGET, ?peer_id, ?approval_knowledge_key, "Known approval"); metrics.on_approval_good_known(); modify_reputation(reputation, ctx.sender(), peer_id, BENEFIT_VALID_MESSAGE).await; + false + } else { + true } - - !good_known_approval } async fn import_and_circulate_approval( @@ -1486,19 +1558,7 @@ impl State { let validator_index = vote.validator; let candidate_indices = &vote.candidate_indices; let entry = match self.blocks.get_mut(&block_hash) { - Some(entry) - if vote.candidate_indices.iter_ones().fold(true, |result, candidate_index| { - let approval_entry_exists = entry.candidates.get(candidate_index as usize).is_some(); - if !approval_entry_exists { - gum::debug!( - target: LOG_TARGET, ?block_hash, ?candidate_index, validator_index = ?vote.validator, candidate_indices = ?vote.candidate_indices, - peer_id = ?source.peer_id(), "Received approval before assignment" - ); - metrics.on_approval_entry_not_found(); - } - approval_entry_exists && result - }) => - entry, + Some(entry) if entry.contains_candidates(&vote.candidate_indices) => entry, _ => { if let Some(peer_id) = source.peer_id() { if !self.recent_outdated_blocks.is_recent_outdated(&block_hash) { @@ -1518,7 +1578,6 @@ impl State { ) .await; metrics.on_approval_invalid_block(); - } else { metrics.on_approval_recent_outdated(); } @@ -1528,23 +1587,14 @@ impl State { }; // compute metadata on the assignment. - let message_subjects = candidate_indices - .iter_ones() - .map(|candidate_index| { - MessageSubject( - block_hash, - (candidate_index as CandidateIndex).into(), - validator_index, - ) - }) - .collect_vec(); - let message_kind = MessageKind::Approval; + let assignments_knowledge_keys = PeerKnowledge::generate_assignments_keys(&vote); + let approval_knwowledge_key = PeerKnowledge::generate_approval_key(&vote); if let Some(peer_id) = source.peer_id() { if !Self::check_approval_can_be_processed( ctx, - &message_subjects, - message_kind, + &assignments_knowledge_keys, + &approval_knwowledge_key, entry, &mut self.reputation, peer_id, @@ -1574,6 +1624,7 @@ impl State { target: LOG_TARGET, ?peer_id, ?result, + ?vote, "Checked approval", ); match result { @@ -1586,11 +1637,13 @@ impl State { ) .await; - for message_subject in &message_subjects { - entry.knowledge.insert(message_subject.clone(), message_kind); - if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { - peer_knowledge.received.insert(message_subject.clone(), message_kind); - } + entry + .knowledge + .insert(approval_knwowledge_key.0.clone(), approval_knwowledge_key.1); + if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { + peer_knowledge + .received + .insert(approval_knwowledge_key.0.clone(), approval_knwowledge_key.1); } }, ApprovalCheckResult::Bad(error) => { @@ -1612,11 +1665,10 @@ impl State { }, } } else { - let all_approvals_imported_already = - message_subjects.iter().fold(true, |result, message_subject| { - !entry.knowledge.insert(message_subject.clone(), message_kind) && result - }); - if all_approvals_imported_already { + if !entry + .knowledge + .insert(approval_knwowledge_key.0.clone(), approval_knwowledge_key.1) + { // if we already imported all approvals, there is no need to distribute it again gum::warn!( target: LOG_TARGET, @@ -1631,45 +1683,21 @@ impl State { } } - let mut required_routing = RequiredRouting::None; - - for candidate_index in candidate_indices.iter_ones() { - // The entry is created when assignment is imported, so we assume this exists. - let approval_entry = entry.approval_entry(candidate_index as _, validator_index); - if approval_entry.is_none() { - let peer_id = source.peer_id(); - // This indicates a bug in approval-distribution, since we check the knowledge at - // the begining of the function. - gum::warn!( - target: LOG_TARGET, - ?peer_id, - "Unknown approval assignment", - ); - // No rep change as this is caused by an issue - metrics.on_approval_unexpected(); - return - } - - let approval_entry = approval_entry.expect("Just checked above; qed"); - - if let Err(err) = approval_entry.note_approval(vote.clone(), candidate_index as _) { - // this would indicate a bug in approval-voting: - // - validator index mismatch - // - candidate index mismatch - // - duplicate approval + let required_routing = match entry.note_approval(vote.clone()) { + Ok(required_routing) => required_routing, + Err(err) => { gum::warn!( target: LOG_TARGET, hash = ?block_hash, - ?candidate_index, - ?validator_index, + validator_index = ?vote.validator, + candidate_bitfield = ?vote.candidate_indices, ?err, "Possible bug: Vote import failed", ); metrics.on_approval_bug(); return - } - required_routing = approval_entry.routing_info().required_routing; - } + }, + }; // Invariant: to our knowledge, none of the peers except for the `source` know about the // approval. @@ -1680,7 +1708,6 @@ impl State { let topology = self.topologies.get_topology(entry.session); let source_peer = source.peer_id(); - let message_subjects_clone = message_subjects.clone(); let peer_filter = move |peer, knowledge: &PeerKnowledge| { if Some(peer) == source_peer.as_ref() { return false @@ -1698,8 +1725,8 @@ impl State { let in_topology = topology .map_or(false, |t| t.local_grid_neighbors().route_to_peer(required_routing, peer)); in_topology || - message_subjects_clone.iter().fold(true, |result, message_subject| { - result && knowledge.sent.contains(message_subject, MessageKind::Assignment) + assignments_knowledge_keys.iter().fold(true, |result, message_subject| { + result && knowledge.sent.contains(&message_subject.0, message_subject.1) }) }; @@ -1714,9 +1741,7 @@ impl State { for peer in peers.iter() { // we already filtered peers above, so this should always be Some if let Some(entry) = entry.known_by.get_mut(&peer.0) { - for message_subject in &message_subjects { - entry.sent.insert(message_subject.clone(), message_kind); - } + entry.sent.insert(approval_knwowledge_key.0.clone(), approval_knwowledge_key.1); } } @@ -1730,43 +1755,7 @@ impl State { "Sending an approval to peers", ); - let v1_peers = filter_by_peer_version(&peers, ValidationVersion::V1.into()); - let v2_peers = filter_by_peer_version(&peers, ValidationVersion::VStaging.into()); - - if !v1_peers.is_empty() { - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - v1_peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals( - approvals - .clone() - .into_iter() - .filter(|approval| approval.candidate_indices.count_ones() == 1) - .map(|approval| { - approval - .try_into() - .expect("We checked len() == 1 so it should not fail; qed") - }) - .collect::>(), - ), - )), - )) - .await; - metrics.on_approval_sent_v1(); - } - - if !v2_peers.is_empty() { - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - v2_peers, - Versioned::VStaging( - protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals), - ), - ), - )) - .await; - metrics.on_approval_sent_v2(); - } + send_approvals_batched(ctx.sender(), approvals, &peers).await; } } @@ -1798,9 +1787,8 @@ impl State { }; let sigs = block_entry - .approval_entries(index) + .approval_votes(index) .into_iter() - .filter_map(|approval_entry| approval_entry.approval(index)) .map(|approval| { ( approval.validator, @@ -1837,7 +1825,7 @@ impl State { let _timer = metrics.time_unify_with_peer(); let mut assignments_to_send = Vec::new(); - let mut approvals_to_send = HashMap::new(); + let mut approvals_to_send = Vec::new(); let view_finalized_number = view.finalized_number; for head in view.into_iter() { @@ -1860,6 +1848,7 @@ impl State { let peer_knowledge = entry.known_by.entry(peer_id).or_default(); let topology = topologies.get_topology(entry.session); + let mut approvals_to_send_this_block = HashMap::new(); // We want to iterate the `approval_entries` of the block entry as these contain all // assignments that also link all approval votes. for approval_entry in entry.approval_entries.values_mut() { @@ -1908,55 +1897,27 @@ impl State { // Filter approval votes. for approval_message in approval_messages { - let (should_forward_approval, candidates_covered_by_approvals) = - approval_message.candidate_indices.iter_ones().fold( - (true, Vec::new()), - |(should_forward_approval, mut new_covered_approvals), - approval_candidate_index| { - let (message_subject, message_kind) = approval_entry - .create_approval_knowledge( - block, - approval_candidate_index as _, - ); - // The assignments for all candidates signed in the approval - // should already have been sent to the peer, otherwise we can't - // send our approval and risk breaking our reputation. - let should_forward_approval = should_forward_approval && - peer_knowledge - .contains(&message_subject, MessageKind::Assignment); - if !peer_knowledge.contains(&message_subject, message_kind) { - new_covered_approvals.push((message_subject, message_kind)) - } - - (should_forward_approval, new_covered_approvals) - }, - ); + let approval_knowledge = + PeerKnowledge::generate_approval_key(&approval_message); - if should_forward_approval { - if !approvals_to_send.contains_key(&( - approval_message.block_hash, - approval_message.validator, - approval_message.candidate_indices.clone(), - )) { - approvals_to_send.insert( - ( - approval_message.block_hash, - approval_message.validator, - approval_message.candidate_indices.clone(), - ), - approval_message, - ); + if !peer_knowledge.contains(&approval_knowledge.0, approval_knowledge.1) { + if !approvals_to_send_this_block.contains_key(&approval_knowledge.0) { + approvals_to_send_this_block + .insert(approval_knowledge.0.clone(), approval_message); } - - candidates_covered_by_approvals.into_iter().for_each( - |(approval_knowledge, message_kind)| { - peer_knowledge.sent.insert(approval_knowledge, message_kind); - }, - ); } } } + // An approval can span multiple assignments/ApprovalEntries, so after we processed + // all of the assignments decide which of the approvals we can safely send, because + // all of assignments are already sent or about to be sent. + for (approval_key, approval) in approvals_to_send_this_block { + if peer_knowledge.contains_assignments_for_approval(&approval) { + approvals_to_send.push(approval); + peer_knowledge.sent.insert(approval_key, MessageKind::Approval); + } + } block = entry.parent_hash; } } @@ -1987,12 +1948,8 @@ impl State { "Sending approvals to unified peer", ); - send_approvals_batched( - sender, - approvals_to_send.into_values().collect_vec(), - &vec![(peer_id, protocol_version)], - ) - .await; + send_approvals_batched(sender, approvals_to_send, &vec![(peer_id, protocol_version)]) + .await; } } @@ -2223,6 +2180,8 @@ async fn adjust_required_routing_and_propagate continue, }; + let mut approvals_to_send_for_this_block = HashMap::new(); + // We just need to iterate the `approval_entries` of the block entry as these contain all // assignments that also link all approval votes. for approval_entry in block_entry.approval_entries.values_mut() { @@ -2259,32 +2218,30 @@ async fn adjust_required_routing_and_propagate { + assert_eq!(peers.len(), 2); + assert_eq!(assignments.len(), 1); + } + ); + + // send the an approval from peer_b + let approval = IndirectSignedApprovalVoteV2 { + block_hash: hash, + candidate_indices, + validator: validator_index, + signature: dummy_signature(), + }; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, &peer_b, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + vote, + tx, + )) => { + assert_eq!(vote, approval); + tx.send(ApprovalCheckResult::Accepted).unwrap(); + } + ); + + expect_reputation_change(overseer, &peer_b, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) + )) + )) => { + assert_eq!(peers.len(), 1); + assert_eq!(approvals.len(), 1); + } + ); + virtual_overseer + }); +} + +// Tests that votes that cover multiple assignments candidates are correctly processed on importing +#[test] +fn multiple_assignments_covered_with_one_approval_vote() { + let peers = make_peers_and_authority_ids(15); + + let peer_a = peers.get(0).unwrap().0; + let peer_b = peers.get(1).unwrap().0; + let peer_c = peers.get(2).unwrap().0; + let peer_d = peers.get(4).unwrap().0; + let parent_hash = Hash::repeat_byte(0xFF); + let hash = Hash::repeat_byte(0xAA); + + let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // setup peers with V2 protocol versions + setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_d, view![hash], ValidationVersion::VStaging).await; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 2], + slot: 1.into(), + session: 1, + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // Set up a gossip topology, where a, b, and c, d are topology neighboors to the node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0, 1], &[2, 4], 3)).await; + + // import an assignment related to `hash` locally + let validator_index = ValidatorIndex(2); // peer_c is the originator + let candidate_indices: CandidateBitfield = + vec![0 as CandidateIndex, 1 as CandidateIndex].try_into().unwrap(); + + let core_bitfields = vec![CoreIndex(0)].try_into().unwrap(); + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfields); + + // send the candidate 0 assignment from peer_b + let assignment = IndirectAssignmentCertV2 { + block_hash: hash, + validator: validator_index, + cert: cert.cert, + }; + let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(vec![( + assignment, + (0 as CandidateIndex).into(), + )]); + send_message_from_peer_v2(overseer, &peer_d, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + _, _, + tx, + )) => { + tx.send(AssignmentCheckResult::Accepted).unwrap(); + } + ); + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert!(peers.len() >= 2); + assert!(peers.contains(&peer_a)); + assert!(peers.contains(&peer_b)); + assert_eq!(assignments.len(), 1); + } + ); + + let candidate_bitfields = vec![CoreIndex(1)].try_into().unwrap(); + let cert = fake_assignment_cert_v2(hash, validator_index, candidate_bitfields); + + // send the candidate 1 assignment from peer_c + let assignment = IndirectAssignmentCertV2 { + block_hash: hash, + validator: validator_index, + cert: cert.cert, + }; + let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(vec![( + assignment, + (1 as CandidateIndex).into(), + )]); + + send_message_from_peer_v2(overseer, &peer_c, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + _, _, + tx, + )) => { + tx.send(AssignmentCheckResult::Accepted).unwrap(); + } + ); + expect_reputation_change(overseer, &peer_c, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert!(peers.len() >= 2); + assert!(peers.contains(&peer_b)); + assert!(peers.contains(&peer_a)); + assert_eq!(assignments.len(), 1); + } + ); + + // send an approval from peer_b + let approval = IndirectSignedApprovalVoteV2 { + block_hash: hash, + candidate_indices, + validator: validator_index, + signature: dummy_signature(), + }; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, &peer_d, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + vote, + tx, + )) => { + assert_eq!(vote, approval); + tx.send(ApprovalCheckResult::Accepted).unwrap(); + } + ); + + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) + )) + )) => { + assert!(peers.len() >= 2); + assert!(peers.contains(&peer_b)); + assert!(peers.contains(&peer_a)); + assert_eq!(approvals.len(), 1); + } + ); + for candidate_index in 0..1 { + let (tx_distribution, rx_distribution) = oneshot::channel(); + let mut candidates_requesting_signatures = HashSet::new(); + candidates_requesting_signatures.insert((hash, candidate_index)); + overseer_send( + overseer, + ApprovalDistributionMessage::GetApprovalSignatures( + candidates_requesting_signatures, + tx_distribution, + ), + ) + .await; + let signatures = rx_distribution.await.unwrap(); + + assert_eq!(signatures.len(), 1); + for (signing_validator, signature) in signatures { + assert_eq!(validator_index, signing_validator); + assert_eq!(signature.0, hash); + assert_eq!(signature.2, approval.signature); + assert_eq!(signature.1, vec![0, 1]); + } + } + virtual_overseer + }); +} + +// Tests that votes that cover multiple assignments candidates are correctly processed when unify +// with peer view +#[test] +fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { + let peers = make_peers_and_authority_ids(15); + + let peer_a = peers.get(0).unwrap().0; + let peer_b = peers.get(1).unwrap().0; + let peer_d = peers.get(4).unwrap().0; + let parent_hash = Hash::repeat_byte(0xFF); + let hash = Hash::repeat_byte(0xAA); + + let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + setup_peer_with_view(overseer, &peer_d, view![hash], ValidationVersion::VStaging).await; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 2], + slot: 1.into(), + session: 1, + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // Set up a gossip topology, where a, b, and c, d are topology neighboors to the node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0, 1], &[2, 4], 3)).await; + + // import an assignment related to `hash` locally + let validator_index = ValidatorIndex(2); // peer_c is the originator + let candidate_indices: CandidateBitfield = + vec![0 as CandidateIndex, 1 as CandidateIndex].try_into().unwrap(); + + let core_bitfields = vec![CoreIndex(0)].try_into().unwrap(); + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfields); + + // send the candidate 0 assignment from peer_b + let assignment = IndirectAssignmentCertV2 { + block_hash: hash, + validator: validator_index, + cert: cert.cert, + }; + let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(vec![( + assignment, + (0 as CandidateIndex).into(), + )]); + send_message_from_peer_v2(overseer, &peer_d, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + _, _, + tx, + )) => { + tx.send(AssignmentCheckResult::Accepted).unwrap(); + } + ); + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; + + let candidate_bitfields = vec![CoreIndex(1)].try_into().unwrap(); + let cert = fake_assignment_cert_v2(hash, validator_index, candidate_bitfields); + + // send the candidate 1 assignment from peer_c + let assignment = IndirectAssignmentCertV2 { + block_hash: hash, + validator: validator_index, + cert: cert.cert, + }; + let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(vec![( + assignment, + (1 as CandidateIndex).into(), + )]); + + send_message_from_peer_v2(overseer, &peer_d, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + _, _, + tx, + )) => { + tx.send(AssignmentCheckResult::Accepted).unwrap(); + } + ); + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; + + // send an approval from peer_b + let approval = IndirectSignedApprovalVoteV2 { + block_hash: hash, + candidate_indices, + validator: validator_index, + signature: dummy_signature(), + }; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, &peer_d, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + vote, + tx, + )) => { + assert_eq!(vote, approval); + tx.send(ApprovalCheckResult::Accepted).unwrap(); + } + ); + + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; + + // setup peers with V2 protocol versions + setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; + let mut expected_peers_assignments = vec![peer_a, peer_b]; + let mut expected_peers_approvals = vec![peer_a, peer_b]; + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert!(peers.len() == 1); + assert!(expected_peers_assignments.contains(peers.first().unwrap())); + expected_peers_assignments.retain(|peer| peer != peers.first().unwrap()); + assert_eq!(assignments.len(), 2); + } + ); + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) + )) + )) => { + assert!(peers.len() == 1); + assert!(expected_peers_approvals.contains(peers.first().unwrap())); + expected_peers_approvals.retain(|peer| peer != peers.first().unwrap()); + assert_eq!(approvals.len(), 1); + } + ); + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert!(peers.len() == 1); + assert!(expected_peers_assignments.contains(peers.first().unwrap())); + expected_peers_assignments.retain(|peer| peer != peers.first().unwrap()); + assert_eq!(assignments.len(), 2); + } + ); + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) + )) + )) => { + assert!(peers.len() == 1); + assert!(expected_peers_approvals.contains(peers.first().unwrap())); + expected_peers_approvals.retain(|peer| peer != peers.first().unwrap()); + assert_eq!(approvals.len(), 1); + } + ); + + virtual_overseer + }); +} + #[test] fn import_approval_bad() { let peer_a = PeerId::random(); From da61d985a863c8ccfefaf000d2e871a3433e71b0 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 1 Sep 2023 09:17:09 +0300 Subject: [PATCH 014/192] Revert "Log errors when banning peers" This reverts commit 5e004e16e23525fbfe423b4ea15cd36d750e5646. --- substrate/client/network/src/peer_store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrate/client/network/src/peer_store.rs b/substrate/client/network/src/peer_store.rs index 96bcd47cc12e..2f3d4a1fd1a0 100644 --- a/substrate/client/network/src/peer_store.rs +++ b/substrate/client/network/src/peer_store.rs @@ -222,7 +222,7 @@ impl PeerStoreInner { if peer_info.reputation < BANNED_THRESHOLD { self.protocols.iter().for_each(|handle| handle.disconnect_peer(peer_id)); - log::error!( + log::trace!( target: LOG_TARGET, "Report {}: {:+} to {}. Reason: {}. Banned, disconnecting.", peer_id, From f3fee248ace628198101f1d29946af644a215651 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 1 Sep 2023 10:10:44 +0300 Subject: [PATCH 015/192] Cleanup post migrating hacks when migrating from polkadot repo Signed-off-by: Alexandru Gheorghe --- .../src/blockchain_rpc_client.rs | 5 +- .../src/rpc_client.rs | 15 +- .../network/availability-recovery/src/lib.rs | 2 +- .../src/legacy_v1/mod.rs | 4 - polkadot/node/service/src/fake_runtime_api.rs | 1 - polkadot/primitives/src/runtime_api.rs | 1 + polkadot/primitives/src/v5/mod.rs | 15 +- polkadot/runtime/kusama/src/lib.rs | 1 - .../src/configuration/migration/v7.rs | 1 - polkadot/runtime/polkadot/src/lib.rs | 1 - polkadot/runtime/rococo/src/lib.rs | 1 - polkadot/scripts/ci/gitlab/pipeline/build.yml | 174 ------ .../scripts/ci/gitlab/pipeline/zombienet.yml | 502 ------------------ 13 files changed, 23 insertions(+), 700 deletions(-) delete mode 100644 polkadot/scripts/ci/gitlab/pipeline/build.yml delete mode 100644 polkadot/scripts/ci/gitlab/pipeline/zombienet.yml diff --git a/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs b/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs index 855d0bf65c77..a40edc0ed3b1 100644 --- a/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs +++ b/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs @@ -357,9 +357,10 @@ impl RuntimeApiSubsystemClient for BlockChainRpcClient { ) -> Result, ApiError> { Ok(self.rpc_client.parachain_host_staging_para_backing_state(at, para_id).await?) } + /// Approval voting configuration parameters - async fn approval_voting_params(&self, _at: Hash) -> Result { - Ok(ApprovalVotingParams { max_approval_coalesce_count: 1 }) + async fn approval_voting_params(&self, at: Hash) -> Result { + Ok(self.rpc_client.parachain_host_staging_approval_voting_params(at).await?) } } diff --git a/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs b/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs index c1e92b249d77..a8792b7f8127 100644 --- a/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs +++ b/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs @@ -31,7 +31,7 @@ use parity_scale_codec::{Decode, Encode}; use cumulus_primitives_core::{ relay_chain::{ slashing, - vstaging::{AsyncBackingParams, BackingState}, + vstaging::{ApprovalVotingParams, AsyncBackingParams, BackingState}, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash as RelayHash, Header as RelayHeader, InboundHrmpMessage, OccupiedCoreAssumption, @@ -611,6 +611,19 @@ impl RelayChainRpcClient { .await } + #[allow(missing_docs)] + pub async fn parachain_host_staging_approval_voting_params( + &self, + at: RelayHash, + ) -> Result { + self.call_remote_runtime_function( + "ParachainHost_staging_approval_voting_params", + at, + None::<()>, + ) + .await + } + #[allow(missing_docs)] pub async fn parachain_host_staging_para_backing_state( &self, diff --git a/polkadot/node/network/availability-recovery/src/lib.rs b/polkadot/node/network/availability-recovery/src/lib.rs index f765e5680568..e40f8ee1ea44 100644 --- a/polkadot/node/network/availability-recovery/src/lib.rs +++ b/polkadot/node/network/availability-recovery/src/lib.rs @@ -102,7 +102,7 @@ const TIMEOUT_START_NEW_REQUESTS: Duration = Duration::from_millis(100); /// PoV size limit in bytes for which prefer fetching from backers. //TODO: Cleanup increased for versi testing -const SMALL_POV_LIMIT: usize = 3 * 1024 * 1024; +const SMALL_POV_LIMIT: usize = 128 * 1024; #[derive(Clone, PartialEq)] /// The strategy we use to recover the PoV. diff --git a/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs b/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs index 082b777e0270..9ae76047383c 100644 --- a/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs +++ b/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs @@ -1195,10 +1195,6 @@ async fn modify_reputation( peer: PeerId, rep: Rep, ) { - //TODO: Test teory in versi - if rep == COST_DUPLICATE_STATEMENT { - return - } reputation.modify(sender, peer, rep).await; } diff --git a/polkadot/node/service/src/fake_runtime_api.rs b/polkadot/node/service/src/fake_runtime_api.rs index a8ec22baf791..d9553afa024b 100644 --- a/polkadot/node/service/src/fake_runtime_api.rs +++ b/polkadot/node/service/src/fake_runtime_api.rs @@ -116,7 +116,6 @@ sp_api::impl_runtime_apis! { } } - #[api_version(5)] impl runtime_api::ParachainHost for Runtime { fn validators() -> Vec { unimplemented!() diff --git a/polkadot/primitives/src/runtime_api.rs b/polkadot/primitives/src/runtime_api.rs index 30e0ec0b4bda..f275682b50bd 100644 --- a/polkadot/primitives/src/runtime_api.rs +++ b/polkadot/primitives/src/runtime_api.rs @@ -241,6 +241,7 @@ sp_api::decl_runtime_apis! { key_ownership_proof: vstaging::slashing::OpaqueKeyOwnershipProof, ) -> Option<()>; + /***** Staging *****/ /// Get the minimum number of backing votes for a parachain candidate. /// This is a staging method! Do not use on production runtimes! diff --git a/polkadot/primitives/src/v5/mod.rs b/polkadot/primitives/src/v5/mod.rs index c190d406efa3..55e4b0481775 100644 --- a/polkadot/primitives/src/v5/mod.rs +++ b/polkadot/primitives/src/v5/mod.rs @@ -354,13 +354,6 @@ pub mod well_known_keys { /// Unique identifier for the Parachains Inherent pub const PARACHAINS_INHERENT_IDENTIFIER: InherentIdentifier = *b"parachn0"; -// /// TODO: Make this two a parachain host configuration. -// /// Maximum allowed candidates to be signed withing a signle approval votes. -// pub const MAX_APPROVAL_COALESCE_COUNT: u64 = 6; -// /// The maximum time we await for an approval to be coalesced with other approvals -// /// before we sign it and distribute to our peers -// pub const MAX_APPROVAL_COALESCE_WAIT_MILLIS: u64 = 500; - /// The key type ID for parachain assignment key. pub const ASSIGNMENT_KEY_TYPE_ID: KeyTypeId = KeyTypeId(*b"asgn"); @@ -1134,10 +1127,10 @@ impl<'a> ApprovalVoteMultipleCandidates<'a> { /// Yields the signing payload for this approval vote. pub fn signing_payload(&self, session_index: SessionIndex) -> Vec { const MAGIC: [u8; 4] = *b"APPR"; - // Make this backwards compatible with `ApprovalVote` so if we have just on candidate the signature - // will look the same. - // This gives us the nice benefit that old nodes can still check signatures when len is 1 and the - // new node can check the signature coming from old nodes. + // Make this backwards compatible with `ApprovalVote` so if we have just on candidate the + // signature will look the same. + // This gives us the nice benefit that old nodes can still check signatures when len is 1 + // and the new node can check the signature coming from old nodes. if self.0.len() == 1 { (MAGIC, self.0.first().expect("QED: we just checked"), session_index).encode() } else { diff --git a/polkadot/runtime/kusama/src/lib.rs b/polkadot/runtime/kusama/src/lib.rs index e20f898de75c..4b5f03b38c6c 100644 --- a/polkadot/runtime/kusama/src/lib.rs +++ b/polkadot/runtime/kusama/src/lib.rs @@ -1885,7 +1885,6 @@ sp_api::impl_runtime_apis! { } } - #[api_version(5)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() diff --git a/polkadot/runtime/parachains/src/configuration/migration/v7.rs b/polkadot/runtime/parachains/src/configuration/migration/v7.rs index 3624eb982323..113651381207 100644 --- a/polkadot/runtime/parachains/src/configuration/migration/v7.rs +++ b/polkadot/runtime/parachains/src/configuration/migration/v7.rs @@ -240,7 +240,6 @@ pvf_voting_ttl : pre.pvf_voting_ttl, minimum_validation_upgrade_delay : pre.minimum_validation_upgrade_delay, async_backing_params : pre.async_backing_params, executor_params : pre.executor_params, - } }; diff --git a/polkadot/runtime/polkadot/src/lib.rs b/polkadot/runtime/polkadot/src/lib.rs index bd1890a58f75..2efd329eea0c 100644 --- a/polkadot/runtime/polkadot/src/lib.rs +++ b/polkadot/runtime/polkadot/src/lib.rs @@ -1688,7 +1688,6 @@ sp_api::impl_runtime_apis! { } } - #[api_version(5)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index b3897f06a062..a80f45c340d9 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -1711,7 +1711,6 @@ sp_api::impl_runtime_apis! { } } - #[api_version(5)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() diff --git a/polkadot/scripts/ci/gitlab/pipeline/build.yml b/polkadot/scripts/ci/gitlab/pipeline/build.yml deleted file mode 100644 index 70d1b1fc5456..000000000000 --- a/polkadot/scripts/ci/gitlab/pipeline/build.yml +++ /dev/null @@ -1,174 +0,0 @@ -# This file is part of .gitlab-ci.yml -# Here are all jobs that are executed during "build" stage - -build-linux-stable: - stage: build - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - # the job can be found in check.yml - needs: - - job: job-starter - artifacts: false - extends: - - .docker-env - - .common-refs - - .compiler-info - - .collect-artifacts - variables: - RUST_TOOLCHAIN: stable - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - # Ensure we run the UI tests. - RUN_UI_TESTS: 1 - script: - - time cargo build --locked --profile testnet --features pyroscope,fast-runtime,network-protocol-staging --verbose --bins - # pack artifacts - - mkdir -p ./artifacts - - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name - - mv ./target/testnet/polkadot ./artifacts/. - - mv ./target/testnet/polkadot-prepare-worker ./artifacts/. - - mv ./target/testnet/polkadot-execute-worker ./artifacts/. - - pushd artifacts - - sha256sum polkadot | tee polkadot.sha256 - - shasum -c polkadot.sha256 - - popd - - EXTRATAG="${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" - - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})" - - echo -n ${VERSION} > ./artifacts/VERSION - - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG - - echo -n ${CI_JOB_ID} > ./artifacts/BUILD_LINUX_JOB_ID - - RELEASE_VERSION=$(./artifacts/polkadot -V | awk '{print $2}'| awk -F "-" '{print $1}') - - echo -n "v${RELEASE_VERSION}" > ./artifacts/BUILD_RELEASE_VERSION - -build-test-collators: - stage: build - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - # the job can be found in check.yml - needs: - - job: job-starter - artifacts: false - extends: - - .docker-env - - .common-refs - - .compiler-info - - .collect-artifacts - script: - - time cargo build --locked --profile testnet --verbose -p test-parachain-adder-collator - - time cargo build --locked --profile testnet --verbose -p test-parachain-undying-collator - # pack artifacts - - mkdir -p ./artifacts - - mv ./target/testnet/adder-collator ./artifacts/. - - mv ./target/testnet/undying-collator ./artifacts/. - - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - echo "undying-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - -build-malus: - stage: build - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - # the job can be found in check.yml - needs: - - job: job-starter - artifacts: false - extends: - - .docker-env - - .common-refs - - .compiler-info - - .collect-artifacts - script: - - time cargo build --locked --profile testnet --verbose -p polkadot-test-malus - # pack artifacts - - mkdir -p ./artifacts - - mv ./target/testnet/malus ./artifacts/. - - mv ./target/testnet/polkadot-execute-worker ./artifacts/. - - mv ./target/testnet/polkadot-prepare-worker ./artifacts/. - - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - -build-staking-miner: - stage: build - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - # the job can be found in check.yml - needs: - - job: job-starter - artifacts: false - extends: - - .docker-env - - .common-refs - - .compiler-info - - .collect-artifacts - script: - - time cargo build --locked --release --package staking-miner - # pack artifacts - - mkdir -p ./artifacts - - mv ./target/release/staking-miner ./artifacts/. - - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - -build-rustdoc: - stage: build - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - # the job can be found in test.yml - needs: - - job: test-deterministic-wasm - artifacts: false - extends: - - .docker-env - - .test-refs - variables: - SKIP_WASM_BUILD: 1 - artifacts: - name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc" - when: on_success - expire_in: 1 days - paths: - - ./crate-docs/ - script: - # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features` - # FIXME: return to stable when https://github.com/rust-lang/rust/issues/96937 gets into stable - - time cargo doc --workspace --verbose --no-deps - - rm -f ./target/doc/.lock - - mv ./target/doc ./crate-docs - # FIXME: remove me after CI image gets nonroot - - chown -R nonroot:nonroot ./crate-docs - - echo "" > ./crate-docs/index.html - -build-implementers-guide: - stage: build - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - # the job can be found in test.yml - needs: - - job: test-deterministic-wasm - artifacts: false - extends: - - .kubernetes-env - - .test-refs - - .collect-artifacts-short - # git depth is set on purpose: https://github.com/paritytech/polkadot/issues/6284 - variables: - GIT_STRATEGY: clone - GIT_DEPTH: 0 - CI_IMAGE: paritytech/mdbook-utils:e14aae4a-20221123 - script: - - mdbook build ./roadmap/implementers-guide - - mkdir -p artifacts - - mv roadmap/implementers-guide/book artifacts/ - -build-short-benchmark: - stage: build - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - # the job can be found in check.yml - needs: - - job: job-starter - artifacts: false - extends: - - .docker-env - - .test-refs - - .collect-artifacts - script: - - cargo build --profile release --locked --features=runtime-benchmarks - - mkdir artifacts - - cp ./target/release/polkadot ./artifacts/ diff --git a/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml b/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml deleted file mode 100644 index 2384a5935661..000000000000 --- a/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml +++ /dev/null @@ -1,502 +0,0 @@ -# This file is part of .gitlab-ci.yml -# Here are all jobs that are executed during "zombienet" stage - -zombienet-tests-parachains-smoke-test: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-malus-image - - job: publish-test-collators-image - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE="docker.io/paritypr/colander:7292" # The collator image is fixed - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-parachains-smoke-test.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-pvf: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-parachains-pvf.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-approval-coalescing: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0006-approval-voting-coalescing.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-disputes: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: publish-malus-image - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0002-parachains-disputes.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-disputes-garbage-candidate: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: publish-malus-image - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0003-parachains-garbage-candidate.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-disputes-past-session: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: publish-malus-image - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0004-parachains-disputes-past-session.zndsl" - allow_failure: true - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-max-tranche0-approvals: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0005-parachains-max-tranche0.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-test-parachains-upgrade-smoke-test: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-malus-image - - job: publish-test-collators-image - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" - before_script: - - echo "ZombieNet Tests Config" - - echo "${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}" - - echo "docker.io/parity/polkadot-collator:latest" - - echo "${ZOMBIENET_IMAGE}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export COL_IMAGE="docker.io/parity/polkadot-parachain:latest" # Use cumulus lastest image - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0002-parachains-upgrade-smoke-test.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-misc-paritydb: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - artifacts: true - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/misc" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-paritydb.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-misc-upgrade-node: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: build-linux-stable - artifacts: true - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/misc" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE="docker.io/parity/polkadot:latest" - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - - BUILD_LINUX_JOB_ID="$(cat ./artifacts/BUILD_LINUX_JOB_ID)" - - export POLKADOT_PR_ARTIFACTS_URL="https://gitlab.parity.io/parity/mirrors/polkadot/-/jobs/${BUILD_LINUX_JOB_ID}/artifacts/raw/artifacts" - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0002-upgrade-node.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-malus-dispute-valid: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-malus-image - - job: publish-test-collators-image - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/node/malus/integrationtests" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie* - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-dispute-valid-block.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-deregister-register-validator: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - artifacts: true - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie* - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0003-deregister-register-validator-smoke.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-beefy-and-mmr: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie* - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0003-beefy-and-mmr.zndsl" - allow_failure: true - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-async-backing-compatibility: - stage: zombienet - extends: - - .kubernetes-env - - .zombienet-refs - image: "${ZOMBIENET_IMAGE}" - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: build-linux-stable - artifacts: true - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/async_backing" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - BUILD_RELEASE_VERSION="$(cat ./artifacts/BUILD_RELEASE_VERSION)" - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export ZOMBIENET_INTEGRATION_TEST_SECONDARY_IMAGE="docker.io/parity/polkadot:${BUILD_RELEASE_VERSION}" - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="001-async-backing-compatibility.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-async-backing-runtime-upgrade: - stage: zombienet - extends: - - .kubernetes-env - - .zombienet-refs - image: "${ZOMBIENET_IMAGE}" - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: build-linux-stable - artifacts: true - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/async_backing" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - BUILD_RELEASE_VERSION="$(cat ./artifacts/BUILD_RELEASE_VERSION)" - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export ZOMBIENET_INTEGRATION_TEST_SECONDARY_IMAGE="docker.io/parity/polkadot:${BUILD_RELEASE_VERSION}" - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - - export POLKADOT_PR_BIN_URL="https://gitlab.parity.io/parity/mirrors/polkadot/-/jobs/${BUILD_LINUX_JOB_ID}/artifacts/raw/artifacts/polkadot" - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="002-async-backing-runtime-upgrade.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-async-backing-collator-mix: - stage: zombienet - extends: - - .kubernetes-env - - .zombienet-refs - image: "${ZOMBIENET_IMAGE}" - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: build-linux-stable - artifacts: true - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/async_backing" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - BUILD_RELEASE_VERSION="$(cat ./artifacts/BUILD_RELEASE_VERSION)" - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export ZOMBIENET_INTEGRATION_TEST_SECONDARY_IMAGE="docker.io/parity/polkadot:${BUILD_RELEASE_VERSION}" - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="003-async-backing-collator-mix.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test From 6338d3301c7b9bda2ef03ea4405e51a3e666863e Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 1 Sep 2023 12:04:33 +0300 Subject: [PATCH 016/192] Fixup clippy Signed-off-by: Alexandru Gheorghe --- polkadot/node/network/approval-distribution/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 8d28a54dcd89..9b0d42a5ff64 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -596,8 +596,7 @@ impl BlockEntry { .filter_map(|(validator, assignment_bitfield)| { self.approval_entries.get(&(*validator, assignment_bitfield.clone())) }) - .map(|approval_entry| approval_entry.approvals.clone().into_iter()) - .flatten() + .flat_map(|approval_entry| approval_entry.approvals.clone().into_iter()) .collect() }); From 7fdae09a5c1e46fa4d9b520d356c18bb3049725e Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 26 Sep 2023 15:34:57 +0300 Subject: [PATCH 017/192] Add host configuration v10 with approval_voting params ... the param was incorrectly appended to v9 instead of creating a new version as v10. Signed-off-by: Alexandru Gheorghe --- .../runtime/parachains/src/configuration.rs | 1 - .../parachains/src/configuration/migration.rs | 1 + .../src/configuration/migration/v10.rs | 327 ++++++++++++++++++ .../src/configuration/migration/v9.rs | 110 +++++- polkadot/runtime/polkadot/src/lib.rs | 1 + polkadot/runtime/rococo/src/lib.rs | 1 + polkadot/runtime/westend/src/lib.rs | 1 + 7 files changed, 436 insertions(+), 6 deletions(-) create mode 100644 polkadot/runtime/parachains/src/configuration/migration/v10.rs diff --git a/polkadot/runtime/parachains/src/configuration.rs b/polkadot/runtime/parachains/src/configuration.rs index b0e5810ac686..6690e4156c7f 100644 --- a/polkadot/runtime/parachains/src/configuration.rs +++ b/polkadot/runtime/parachains/src/configuration.rs @@ -261,7 +261,6 @@ pub struct HostConfiguration { /// backable. pub minimum_backing_votes: u32, /// Params used by approval-voting - /// TODO: fixme this is not correctly migrated pub approval_voting_params: ApprovalVotingParams, } diff --git a/polkadot/runtime/parachains/src/configuration/migration.rs b/polkadot/runtime/parachains/src/configuration/migration.rs index 26f8a85b496d..db323d3aad93 100644 --- a/polkadot/runtime/parachains/src/configuration/migration.rs +++ b/polkadot/runtime/parachains/src/configuration/migration.rs @@ -16,6 +16,7 @@ //! A module that is responsible for migration of storage. +pub mod v10; pub mod v6; pub mod v7; pub mod v8; diff --git a/polkadot/runtime/parachains/src/configuration/migration/v10.rs b/polkadot/runtime/parachains/src/configuration/migration/v10.rs new file mode 100644 index 000000000000..53c5f4d2c52c --- /dev/null +++ b/polkadot/runtime/parachains/src/configuration/migration/v10.rs @@ -0,0 +1,327 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! A module that is responsible for migration of storage. + +use crate::configuration::{self, Config, Pallet}; +use frame_support::{ + pallet_prelude::*, + traits::{Defensive, StorageVersion}, + weights::Weight, +}; +use frame_system::pallet_prelude::BlockNumberFor; +use primitives::{vstaging::ApprovalVotingParams, SessionIndex, LEGACY_MIN_BACKING_VOTES}; +use sp_runtime::Perbill; +use sp_std::vec::Vec; + +use frame_support::traits::OnRuntimeUpgrade; + +use super::v9::V9HostConfiguration; +type V10HostConfiguration = configuration::HostConfiguration; + +mod v9 { + use super::*; + + #[frame_support::storage_alias] + pub(crate) type ActiveConfig = + StorageValue, V9HostConfiguration>, OptionQuery>; + + #[frame_support::storage_alias] + pub(crate) type PendingConfigs = StorageValue< + Pallet, + Vec<(SessionIndex, V9HostConfiguration>)>, + OptionQuery, + >; +} + +mod v10 { + use super::*; + + #[frame_support::storage_alias] + pub(crate) type ActiveConfig = + StorageValue, V10HostConfiguration>, OptionQuery>; + + #[frame_support::storage_alias] + pub(crate) type PendingConfigs = StorageValue< + Pallet, + Vec<(SessionIndex, V10HostConfiguration>)>, + OptionQuery, + >; +} + +pub struct MigrateToV10(sp_std::marker::PhantomData); +impl OnRuntimeUpgrade for MigrateToV10 { + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result, sp_runtime::TryRuntimeError> { + log::trace!(target: crate::configuration::LOG_TARGET, "Running pre_upgrade() for HostConfiguration MigrateToV10"); + Ok(Vec::new()) + } + + fn on_runtime_upgrade() -> Weight { + log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV10 started"); + if StorageVersion::get::>() == 9 { + let weight_consumed = migrate_to_v10::(); + + log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV10 executed successfully"); + StorageVersion::new(10).put::>(); + + weight_consumed + } else { + log::warn!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV10 should be removed."); + T::DbWeight::get().reads(1) + } + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade(_state: Vec) -> Result<(), sp_runtime::TryRuntimeError> { + log::trace!(target: crate::configuration::LOG_TARGET, "Running post_upgrade() for HostConfiguration MigrateToV10"); + ensure!( + StorageVersion::get::>() >= 10, + "Storage version should be >= 10 after the migration" + ); + + Ok(()) + } +} + +fn migrate_to_v10() -> Weight { + // Unusual formatting is justified: + // - make it easier to verify that fields assign what they supposed to assign. + // - this code is transient and will be removed after all migrations are done. + // - this code is important enough to optimize for legibility sacrificing consistency. + #[rustfmt::skip] + let translate = + |pre: V9HostConfiguration>| -> + V10HostConfiguration> + { + V10HostConfiguration { +max_code_size : pre.max_code_size, +max_head_data_size : pre.max_head_data_size, +max_upward_queue_count : pre.max_upward_queue_count, +max_upward_queue_size : pre.max_upward_queue_size, +max_upward_message_size : pre.max_upward_message_size, +max_upward_message_num_per_candidate : pre.max_upward_message_num_per_candidate, +hrmp_max_message_num_per_candidate : pre.hrmp_max_message_num_per_candidate, +validation_upgrade_cooldown : pre.validation_upgrade_cooldown, +validation_upgrade_delay : pre.validation_upgrade_delay, +max_pov_size : pre.max_pov_size, +max_downward_message_size : pre.max_downward_message_size, +hrmp_sender_deposit : pre.hrmp_sender_deposit, +hrmp_recipient_deposit : pre.hrmp_recipient_deposit, +hrmp_channel_max_capacity : pre.hrmp_channel_max_capacity, +hrmp_channel_max_total_size : pre.hrmp_channel_max_total_size, +hrmp_max_parachain_inbound_channels : pre.hrmp_max_parachain_inbound_channels, +hrmp_max_parachain_outbound_channels : pre.hrmp_max_parachain_outbound_channels, +hrmp_channel_max_message_size : pre.hrmp_channel_max_message_size, +code_retention_period : pre.code_retention_period, +on_demand_cores : pre.on_demand_cores, +on_demand_retries : pre.on_demand_retries, +group_rotation_frequency : pre.group_rotation_frequency, +paras_availability_period : pre.paras_availability_period, +scheduling_lookahead : pre.scheduling_lookahead, +max_validators_per_core : pre.max_validators_per_core, +max_validators : pre.max_validators, +dispute_period : pre.dispute_period, +dispute_post_conclusion_acceptance_period: pre.dispute_post_conclusion_acceptance_period, +no_show_slots : pre.no_show_slots, +n_delay_tranches : pre.n_delay_tranches, +zeroth_delay_tranche_width : pre.zeroth_delay_tranche_width, +needed_approvals : pre.needed_approvals, +relay_vrf_modulo_samples : pre.relay_vrf_modulo_samples, +pvf_voting_ttl : pre.pvf_voting_ttl, +minimum_validation_upgrade_delay : pre.minimum_validation_upgrade_delay, +async_backing_params : pre.async_backing_params, +executor_params : pre.executor_params, +on_demand_queue_max_size : 10_000u32, +on_demand_base_fee : 10_000_000u128, +on_demand_fee_variability : Perbill::from_percent(3), +on_demand_target_queue_utilization : Perbill::from_percent(25), +on_demand_ttl : 5u32.into(), +minimum_backing_votes : LEGACY_MIN_BACKING_VOTES, +approval_voting_params : ApprovalVotingParams { + max_approval_coalesce_count: 1, + } + } + }; + + let v9 = v9::ActiveConfig::::get() + .defensive_proof("Could not decode old config") + .unwrap_or_default(); + let v10 = translate(v9); + v10::ActiveConfig::::set(Some(v10)); + + // Allowed to be empty. + let pending_v9 = v9::PendingConfigs::::get().unwrap_or_default(); + let mut pending_v10 = Vec::new(); + + for (session, v9) in pending_v9.into_iter() { + let v10 = translate(v9); + pending_v10.push((session, v10)); + } + v10::PendingConfigs::::set(Some(pending_v10.clone())); + + let num_configs = (pending_v10.len() + 1) as u64; + T::DbWeight::get().reads_writes(num_configs, num_configs) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mock::{new_test_ext, Test}; + + #[test] + fn v10_deserialized_from_actual_data() { + // Example how to get new `raw_config`: + // We'll obtain the raw_config at a specified a block + // Steps: + // 1. Go to Polkadot.js -> Developer -> Chain state -> Storage: https://polkadot.js.org/apps/#/chainstate + // 2. Set these parameters: + // 2.1. selected state query: configuration; activeConfig(): + // PolkadotRuntimeParachainsConfigurationHostConfiguration + // 2.2. blockhash to query at: + // 0xf89d3ab5312c5f70d396dc59612f0aa65806c798346f9db4b35278baed2e0e53 (the hash of + // the block) + // 2.3. Note the value of encoded storage key -> + // 0x06de3d8a54d27e44a9d5ce189618f22db4b49d95320d9021994c850f25b8e385 for the + // referenced block. + // 2.4. You'll also need the decoded values to update the test. + // 3. Go to Polkadot.js -> Developer -> Chain state -> Raw storage + // 3.1 Enter the encoded storage key and you get the raw config. + + // This exceeds the maximal line width length, but that's fine, since this is not code and + // doesn't need to be read and also leaving it as one line allows to easily copy it. + let raw_config = + hex_literal::hex![" + 0000300000800000080000000000100000c8000005000000050000000200000002000000000000000000000000005000000010000400000000000000000000000000000000000000000000000000000000000000000000000800000000200000040000000000100000b004000000000000000000001027000080b2e60e80c3c90180969800000000000000000000000000050000001400000004000000010000000101000000000600000064000000020000001900000000000000030000000200000002000000050000000200000001000000" + ]; + + let v10 = + V10HostConfiguration::::decode(&mut &raw_config[..]).unwrap(); + + // We check only a sample of the values here. If we missed any fields or messed up data + // types that would skew all the fields coming after. + assert_eq!(v10.max_code_size, 3_145_728); + assert_eq!(v10.validation_upgrade_cooldown, 2); + assert_eq!(v10.max_pov_size, 5_242_880); + assert_eq!(v10.hrmp_channel_max_message_size, 1_048_576); + assert_eq!(v10.n_delay_tranches, 25); + assert_eq!(v10.minimum_validation_upgrade_delay, 5); + assert_eq!(v10.group_rotation_frequency, 20); + assert_eq!(v10.on_demand_cores, 0); + assert_eq!(v10.on_demand_base_fee, 10_000_000); + assert_eq!(v10.minimum_backing_votes, LEGACY_MIN_BACKING_VOTES); + assert_eq!(v10.approval_voting_params.max_approval_coalesce_count, 1); + } + + #[test] + fn test_migrate_to_v10() { + // Host configuration has lots of fields. However, in this migration we only add one + // field. The most important part to check are a couple of the last fields. We also pick + // extra fields to check arbitrarily, e.g. depending on their position (i.e. the middle) and + // also their type. + // + // We specify only the picked fields and the rest should be provided by the `Default` + // implementation. That implementation is copied over between the two types and should work + // fine. + let v9 = V9HostConfiguration:: { + needed_approvals: 69, + paras_availability_period: 55, + hrmp_recipient_deposit: 1337, + max_pov_size: 1111, + minimum_validation_upgrade_delay: 20, + ..Default::default() + }; + + let mut pending_configs = Vec::new(); + pending_configs.push((100, v9.clone())); + pending_configs.push((300, v9.clone())); + + new_test_ext(Default::default()).execute_with(|| { + // Implant the v9 version in the state. + v9::ActiveConfig::::set(Some(v9)); + v9::PendingConfigs::::set(Some(pending_configs)); + + migrate_to_v10::(); + + let v10 = v10::ActiveConfig::::get().unwrap(); + assert_eq!(v10.approval_voting_params.max_approval_coalesce_count, 1); + + let mut configs_to_check = v10::PendingConfigs::::get().unwrap(); + configs_to_check.push((0, v10.clone())); + + for (_, v9) in configs_to_check { + #[rustfmt::skip] + { + assert_eq!(v9.max_code_size , v10.max_code_size); + assert_eq!(v9.max_head_data_size , v10.max_head_data_size); + assert_eq!(v9.max_upward_queue_count , v10.max_upward_queue_count); + assert_eq!(v9.max_upward_queue_size , v10.max_upward_queue_size); + assert_eq!(v9.max_upward_message_size , v10.max_upward_message_size); + assert_eq!(v9.max_upward_message_num_per_candidate , v10.max_upward_message_num_per_candidate); + assert_eq!(v9.hrmp_max_message_num_per_candidate , v10.hrmp_max_message_num_per_candidate); + assert_eq!(v9.validation_upgrade_cooldown , v10.validation_upgrade_cooldown); + assert_eq!(v9.validation_upgrade_delay , v10.validation_upgrade_delay); + assert_eq!(v9.max_pov_size , v10.max_pov_size); + assert_eq!(v9.max_downward_message_size , v10.max_downward_message_size); + assert_eq!(v9.hrmp_max_parachain_outbound_channels , v10.hrmp_max_parachain_outbound_channels); + assert_eq!(v9.hrmp_sender_deposit , v10.hrmp_sender_deposit); + assert_eq!(v9.hrmp_recipient_deposit , v10.hrmp_recipient_deposit); + assert_eq!(v9.hrmp_channel_max_capacity , v10.hrmp_channel_max_capacity); + assert_eq!(v9.hrmp_channel_max_total_size , v10.hrmp_channel_max_total_size); + assert_eq!(v9.hrmp_max_parachain_inbound_channels , v10.hrmp_max_parachain_inbound_channels); + assert_eq!(v9.hrmp_channel_max_message_size , v10.hrmp_channel_max_message_size); + assert_eq!(v9.code_retention_period , v10.code_retention_period); + assert_eq!(v9.on_demand_cores , v10.on_demand_cores); + assert_eq!(v9.on_demand_retries , v10.on_demand_retries); + assert_eq!(v9.group_rotation_frequency , v10.group_rotation_frequency); + assert_eq!(v9.paras_availability_period , v10.paras_availability_period); + assert_eq!(v9.scheduling_lookahead , v10.scheduling_lookahead); + assert_eq!(v9.max_validators_per_core , v10.max_validators_per_core); + assert_eq!(v9.max_validators , v10.max_validators); + assert_eq!(v9.dispute_period , v10.dispute_period); + assert_eq!(v9.no_show_slots , v10.no_show_slots); + assert_eq!(v9.n_delay_tranches , v10.n_delay_tranches); + assert_eq!(v9.zeroth_delay_tranche_width , v10.zeroth_delay_tranche_width); + assert_eq!(v9.needed_approvals , v10.needed_approvals); + assert_eq!(v9.relay_vrf_modulo_samples , v10.relay_vrf_modulo_samples); + assert_eq!(v9.pvf_voting_ttl , v10.pvf_voting_ttl); + assert_eq!(v9.minimum_validation_upgrade_delay , v10.minimum_validation_upgrade_delay); + assert_eq!(v9.async_backing_params.allowed_ancestry_len, v10.async_backing_params.allowed_ancestry_len); + assert_eq!(v9.async_backing_params.max_candidate_depth , v10.async_backing_params.max_candidate_depth); + assert_eq!(v9.executor_params , v10.executor_params); + assert_eq!(v9.minimum_backing_votes , v10.minimum_backing_votes); + }; // ; makes this a statement. `rustfmt::skip` cannot be put on an expression. + } + }); + } + + // Test that migration doesn't panic in case there're no pending configurations upgrades in + // pallet's storage. + #[test] + fn test_migrate_to_v10_no_pending() { + let v9 = V9HostConfiguration::::default(); + + new_test_ext(Default::default()).execute_with(|| { + // Implant the v9 version in the state. + v9::ActiveConfig::::set(Some(v9)); + // Ensure there're no pending configs. + v10::PendingConfigs::::set(None); + + // Shouldn't fail. + migrate_to_v10::(); + }); + } +} diff --git a/polkadot/runtime/parachains/src/configuration/migration/v9.rs b/polkadot/runtime/parachains/src/configuration/migration/v9.rs index 37cb451ab36b..909a63ae6291 100644 --- a/polkadot/runtime/parachains/src/configuration/migration/v9.rs +++ b/polkadot/runtime/parachains/src/configuration/migration/v9.rs @@ -23,14 +23,117 @@ use frame_support::{ weights::Weight, }; use frame_system::pallet_prelude::BlockNumberFor; -use primitives::{vstaging::ApprovalVotingParams, SessionIndex, LEGACY_MIN_BACKING_VOTES}; +use primitives::{ + vstaging::AsyncBackingParams, Balance, ExecutorParams, SessionIndex, LEGACY_MIN_BACKING_VOTES, + ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, +}; + use sp_runtime::Perbill; use sp_std::vec::Vec; use frame_support::traits::OnRuntimeUpgrade; use super::v8::V8HostConfiguration; -type V9HostConfiguration = configuration::HostConfiguration; +/// All configuration of the runtime with respect to paras. +#[derive(Clone, Encode, Decode, Debug)] +pub struct V9HostConfiguration { + pub max_code_size: u32, + pub max_head_data_size: u32, + pub max_upward_queue_count: u32, + pub max_upward_queue_size: u32, + pub max_upward_message_size: u32, + pub max_upward_message_num_per_candidate: u32, + pub hrmp_max_message_num_per_candidate: u32, + pub validation_upgrade_cooldown: BlockNumber, + pub validation_upgrade_delay: BlockNumber, + pub async_backing_params: AsyncBackingParams, + pub max_pov_size: u32, + pub max_downward_message_size: u32, + pub hrmp_max_parachain_outbound_channels: u32, + pub hrmp_sender_deposit: Balance, + pub hrmp_recipient_deposit: Balance, + pub hrmp_channel_max_capacity: u32, + pub hrmp_channel_max_total_size: u32, + pub hrmp_max_parachain_inbound_channels: u32, + pub hrmp_channel_max_message_size: u32, + pub executor_params: ExecutorParams, + pub code_retention_period: BlockNumber, + pub on_demand_cores: u32, + pub on_demand_retries: u32, + pub on_demand_queue_max_size: u32, + pub on_demand_target_queue_utilization: Perbill, + pub on_demand_fee_variability: Perbill, + pub on_demand_base_fee: Balance, + pub on_demand_ttl: BlockNumber, + pub group_rotation_frequency: BlockNumber, + pub paras_availability_period: BlockNumber, + pub scheduling_lookahead: u32, + pub max_validators_per_core: Option, + pub max_validators: Option, + pub dispute_period: SessionIndex, + pub dispute_post_conclusion_acceptance_period: BlockNumber, + pub no_show_slots: u32, + pub n_delay_tranches: u32, + pub zeroth_delay_tranche_width: u32, + pub needed_approvals: u32, + pub relay_vrf_modulo_samples: u32, + pub pvf_voting_ttl: SessionIndex, + pub minimum_validation_upgrade_delay: BlockNumber, + pub minimum_backing_votes: u32, +} + +impl> Default for V9HostConfiguration { + fn default() -> Self { + Self { + async_backing_params: AsyncBackingParams { + max_candidate_depth: 0, + allowed_ancestry_len: 0, + }, + group_rotation_frequency: 1u32.into(), + paras_availability_period: 1u32.into(), + no_show_slots: 1u32.into(), + validation_upgrade_cooldown: Default::default(), + validation_upgrade_delay: 2u32.into(), + code_retention_period: Default::default(), + max_code_size: Default::default(), + max_pov_size: Default::default(), + max_head_data_size: Default::default(), + on_demand_cores: Default::default(), + on_demand_retries: Default::default(), + scheduling_lookahead: 1, + max_validators_per_core: Default::default(), + max_validators: None, + dispute_period: 6, + dispute_post_conclusion_acceptance_period: 100.into(), + n_delay_tranches: Default::default(), + zeroth_delay_tranche_width: Default::default(), + needed_approvals: Default::default(), + relay_vrf_modulo_samples: Default::default(), + max_upward_queue_count: Default::default(), + max_upward_queue_size: Default::default(), + max_downward_message_size: Default::default(), + max_upward_message_size: Default::default(), + max_upward_message_num_per_candidate: Default::default(), + hrmp_sender_deposit: Default::default(), + hrmp_recipient_deposit: Default::default(), + hrmp_channel_max_capacity: Default::default(), + hrmp_channel_max_total_size: Default::default(), + hrmp_max_parachain_inbound_channels: Default::default(), + hrmp_channel_max_message_size: Default::default(), + hrmp_max_parachain_outbound_channels: Default::default(), + hrmp_max_message_num_per_candidate: Default::default(), + pvf_voting_ttl: 2u32.into(), + minimum_validation_upgrade_delay: 2.into(), + executor_params: Default::default(), + on_demand_queue_max_size: ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, + on_demand_base_fee: 10_000_000u128, + on_demand_fee_variability: Perbill::from_percent(3), + on_demand_target_queue_utilization: Perbill::from_percent(25), + on_demand_ttl: 5u32.into(), + minimum_backing_votes: LEGACY_MIN_BACKING_VOTES, + } + } +} mod v8 { use super::*; @@ -151,9 +254,6 @@ on_demand_fee_variability : Perbill::from_percent(3), on_demand_target_queue_utilization : Perbill::from_percent(25), on_demand_ttl : 5u32.into(), minimum_backing_votes : LEGACY_MIN_BACKING_VOTES, -approval_voting_params : ApprovalVotingParams { - max_approval_coalesce_count: 1, - } } }; diff --git a/polkadot/runtime/polkadot/src/lib.rs b/polkadot/runtime/polkadot/src/lib.rs index 5956b0e155bb..9c84c8840cd5 100644 --- a/polkadot/runtime/polkadot/src/lib.rs +++ b/polkadot/runtime/polkadot/src/lib.rs @@ -1560,6 +1560,7 @@ pub mod migrations { parachains_configuration::migration::v9::MigrateToV9, // Migrate parachain info format paras_registrar::migration::VersionCheckedMigrateToV1, + parachains_configuration::migration::v10::MigrateToV10, ); } diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 7046c4640c04..83eb3314b49c 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -1561,6 +1561,7 @@ pub mod migrations { parachains_configuration::migration::v8::MigrateToV8, parachains_configuration::migration::v9::MigrateToV9, paras_registrar::migration::VersionCheckedMigrateToV1, + parachains_configuration::migration::v10::MigrateToV10, ); } diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index 9af18b5be2bb..c70d88f766ba 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -1445,6 +1445,7 @@ pub mod migrations { UpgradeSessionKeys, parachains_configuration::migration::v9::MigrateToV9, paras_registrar::migration::VersionCheckedMigrateToV1, + parachains_configuration::migration::v10::MigrateToV10, ); } From e70b11326f73ca7fbc0d52ee2d444352a48902c9 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 26 Sep 2023 18:01:03 +0300 Subject: [PATCH 018/192] Fixup failure of lint-markdown E.g: https://github.com/paritytech/polkadot-sdk/actions/runs/6313437255/job/17141490461?pr=1178 Signed-off-by: Alexandru Gheorghe --- .../src/node/approval/approval-voting.md | 26 +++++++++---------- .../src/protocol-approval.md | 26 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md index e5b3cfd79751..1a17f90d9ba3 100644 --- a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md +++ b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md @@ -267,25 +267,25 @@ entry. The cert itself contains information necessary to determine the candidate * Determine the claimed core index by looking up the candidate with given index in `block_entry.candidates`. Return `AssignmentCheckResult::Bad` if missing. * Check the assignment cert - * If the cert kind is `RelayVRFModulo`, then the certificate is valid as long as `sample < - session_info.relay_vrf_samples` and the VRF is valid for the validator's key with the input - `block_entry.relay_vrf_story ++ sample.encode()` as described with - [the approvals protocol section](../../protocol-approval.md#assignment-criteria). We set - `core_index = vrf.make_bytes().to_u32() % session_info.n_cores`. If the `BlockEntry` causes + * If the cert kind is `RelayVRFModulo`, then the certificate is valid as long as `sample < + session_info.relay_vrf_samples` and the VRF is valid for the validator's key with the input + `block_entry.relay_vrf_story ++ sample.encode()` as described with + [the approvals protocol section](../../protocol-approval.md#assignment-criteria). We set + `core_index = vrf.make_bytes().to_u32() % session_info.n_cores`. If the `BlockEntry` causes inclusion of a candidate at `core_index`, then this is a valid assignment for the candidate at `core_index` and has delay tranche 0. Otherwise, it can be ignored. * If the cert kind is `RelayVRFModuloCompact`, then the certificate is valid as long as the VRF - is valid for the validator's key with the input `block_entry.relay_vrf_story ++ relay_vrf_samples.encode()` - as described with [the approvals protocol section](../../protocol-approval.md#assignment-criteria). - We enforce that all `core_bitfield` indices are included in the set of the core indices sampled from the + is valid for the validator's key with the input `block_entry.relay_vrf_story ++ relay_vrf_samples.encode()` + as described with [the approvals protocol section](../../protocol-approval.md#assignment-criteria). + We enforce that all `core_bitfield` indices are included in the set of the core indices sampled from the VRF Output. The assignment is considered a valid tranche0 assignment for all claimed candidates if all `core_bitfield` indices match the core indices where the claimed candidates were included at. - * If the cert kind is `RelayVRFDelay`, then we check if the VRF is valid for the validator's key with the - input `block_entry.relay_vrf_story ++ cert.core_index.encode()` as described in [the approvals protocol - section](../../protocol-approval.md#assignment-criteria). The cert can be ignored if the block did not - cause inclusion of a candidate on that core index. Otherwise, this is a valid assignment for the included - candidate. The delay tranche for the assignment is determined by reducing + * If the cert kind is `RelayVRFDelay`, then we check if the VRF is valid for the validator's key with the + input `block_entry.relay_vrf_story ++ cert.core_index.encode()` as described in [the approvals protocol + section](../../protocol-approval.md#assignment-criteria). The cert can be ignored if the block did not + cause inclusion of a candidate on that core index. Otherwise, this is a valid assignment for the included + candidate. The delay tranche for the assignment is determined by reducing `(vrf.make_bytes().to_u64() % (session_info.n_delay_tranches + session_info.zeroth_delay_tranche_width)).saturating_sub(session_info.zeroth_delay_tranche_width)`. * We also check that the core index derived by the output is covered by the `VRFProof` by means of an auxiliary signature. * If the delay tranche is too far in the future, return `AssignmentCheckResult::TooFarInFuture`. diff --git a/polkadot/roadmap/implementers-guide/src/protocol-approval.md b/polkadot/roadmap/implementers-guide/src/protocol-approval.md index c614b64410ec..4fac84591456 100644 --- a/polkadot/roadmap/implementers-guide/src/protocol-approval.md +++ b/polkadot/roadmap/implementers-guide/src/protocol-approval.md @@ -189,9 +189,9 @@ Assignment criteria compute actual assignments using stories and the validators' Assignment criteria output a `Position` consisting of both a `ParaId` to be checked, as well as a precedence `DelayTranche` for when the assignment becomes valid. -Assignment criteria come in four flavors, `RelayVRFModuloCompact`, `RelayVRFDelay`, `RelayEquivocation` and the -deprecated `RelayVRFModulo`. Among these, `RelayVRFModulo`, `RelayVRFModuloCompact` and `RelayVRFDelay` run a -VRF whose input is the output of a `RelayVRFStory`, while `RelayEquivocation` runs a VRF whose input is the +Assignment criteria come in four flavors, `RelayVRFModuloCompact`, `RelayVRFDelay`, `RelayEquivocation` and the +deprecated `RelayVRFModulo`. Among these, `RelayVRFModulo`, `RelayVRFModuloCompact` and `RelayVRFDelay` run a +VRF whose input is the output of a `RelayVRFStory`, while `RelayEquivocation` runs a VRF whose input is the output of a `RelayEquivocationStory`. Among these, we have two distinct VRF output computations: @@ -204,14 +204,14 @@ sampled availability core in this relay chain block. We choose three samples in secure and efficient by increasing this to four or five, and reducing the backing checks accordingly. All successful `RelayVRFModulo` samples are assigned delay tranche zero. -`RelayVRFModuloCompact` runs a single samples whose VRF input is the `RelayVRFStory` and the sample count. Similar -to `RelayVRFModulo` introduces multiple core assignments for tranche zero. It computes the VRF output with -`schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Core v2" and samples up to 160 bytes of the output -as an array of `u32`. Then reduces each `u32` modulo the number of availability cores, and outputs up +`RelayVRFModuloCompact` runs a single samples whose VRF input is the `RelayVRFStory` and the sample count. Similar +to `RelayVRFModulo` introduces multiple core assignments for tranche zero. It computes the VRF output with +`schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Core v2" and samples up to 160 bytes of the output +as an array of `u32`. Then reduces each `u32` modulo the number of availability cores, and outputs up to `relay_vrf_modulo_samples` availability core indices. There is no sampling process for `RelayVRFDelay` and `RelayEquivocation`. We instead run them on specific candidates -and they compute a delay from their VRF output. `RelayVRFDelay` runs for all candidates included under, aka declared +and they compute a delay from their VRF output. `RelayVRFDelay` runs for all candidates included under, aka declared available by, a relay chain block, and inputs the associated VRF output via `RelayVRFStory`. `RelayEquivocation` runs only on candidate block equivocations, and inputs their block hashes via the `RelayEquivocation` story. @@ -229,14 +229,14 @@ cannot merge those with the same delay and different stories because `RelayEquiv We track all validators' announced approval assignments for each candidate associated to each relay chain block, which tells us which validators were assigned to which candidates. -We permit at most one assignment per candidate per story per validator, so one validator could be assigned under both -the `RelayVRFDelay` and `RelayEquivocation` criteria, but not under both `RelayVRFModulo/RelayVRFModuloCompact` +We permit at most one assignment per candidate per story per validator, so one validator could be assigned under both +the `RelayVRFDelay` and `RelayEquivocation` criteria, but not under both `RelayVRFModulo/RelayVRFModuloCompact` and `RelayVRFDelay` criteria, since those both use the same story. We permit only one approval vote per candidate per validator, which counts for any applicable criteria. -We announce, and start checking for, our own assignments when the delay of their tranche is reached, but only if the +We announce, and start checking for, our own assignments when the delay of their tranche is reached, but only if the tracker says the assignee candidate requires more approval checkers. We never announce an assignment we believe unnecessary -because early announcements gives an adversary information. All delay tranche zero assignments always get announced, +because early announcements gives an adversary information. All delay tranche zero assignments always get announced, which includes all `RelayVRFModulo` and `RelayVRFModuloCompact` assignments. In other words, if some candidate `C` needs more approval checkers by the time we reach round `t` then any validators @@ -325,7 +325,7 @@ finality. We might explore limits on postponement too, but this sounds much har ## Parameters -We prefer doing approval checkers assignments under `RelayVRFModulo` or `RelayVRFModuloCompact` as opposed to +We prefer doing approval checkers assignments under `RelayVRFModulo` or `RelayVRFModuloCompact` as opposed to `RelayVRFDelay` because `RelayVRFModulo` avoids giving individual checkers too many assignments and tranche zero assignments benefit security the most. We suggest assigning at least 16 checkers under `RelayVRFModulo` or `RelayVRFModuloCompact` although assignment levels have never been properly analyzed. From 85939bb864ebd68638fbaa9e120a99e7a417733d Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 26 Sep 2023 18:26:04 +0300 Subject: [PATCH 019/192] Fixup test-rustdoc job ... failure example here: https://gitlab.parity.io/parity/mirrors/polkadot-sdk/-/jobs/3799036 Signed-off-by: Alexandru Gheorghe --- polkadot/node/primitives/src/approval.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/polkadot/node/primitives/src/approval.rs b/polkadot/node/primitives/src/approval.rs index bfa1f1aa47d2..e5ae24f7a51e 100644 --- a/polkadot/node/primitives/src/approval.rs +++ b/polkadot/node/primitives/src/approval.rs @@ -365,7 +365,7 @@ pub mod v2 { /// Multiple assignment stories based on the VRF that authorized the relay-chain block /// where the candidates were included. /// - /// The context is [`v2::RELAY_VRF_MODULO_CONTEXT`] + /// The context is [`super::v2::RELAY_VRF_MODULO_CONTEXT`] #[codec(index = 0)] RelayVRFModuloCompact { /// A bitfield representing the core indices claimed by this assignment. @@ -374,7 +374,7 @@ pub mod v2 { /// An assignment story based on the VRF that authorized the relay-chain block where the /// candidate was included combined with the index of a particular core. /// - /// The context is [`v2::RELAY_VRF_DELAY_CONTEXT`] + /// The context is [`super::v1::RELAY_VRF_DELAY_CONTEXT`] #[codec(index = 1)] RelayVRFDelay { /// The core index chosen in this cert. @@ -384,7 +384,7 @@ pub mod v2 { /// An assignment story based on the VRF that authorized the relay-chain block where the /// candidate was included combined with a sample number. /// - /// The context used to produce bytes is [`v1::RELAY_VRF_MODULO_CONTEXT`] + /// The context used to produce bytes is [`super::v1::RELAY_VRF_MODULO_CONTEXT`] #[codec(index = 2)] RelayVRFModulo { /// The sample number used in this cert. From d42f3734e2344cf117f2ec5126f9729fbb04a947 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 27 Sep 2023 09:55:19 +0300 Subject: [PATCH 020/192] Cosmetic fixes Signed-off-by: Alexandru Gheorghe --- polkadot/node/network/availability-recovery/src/lib.rs | 1 - polkadot/primitives/src/runtime_api.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/polkadot/node/network/availability-recovery/src/lib.rs b/polkadot/node/network/availability-recovery/src/lib.rs index a040b0fe0aaa..e2146981da92 100644 --- a/polkadot/node/network/availability-recovery/src/lib.rs +++ b/polkadot/node/network/availability-recovery/src/lib.rs @@ -78,7 +78,6 @@ const LRU_SIZE: u32 = 16; const COST_INVALID_REQUEST: Rep = Rep::CostMajor("Peer sent unparsable request"); /// PoV size limit in bytes for which prefer fetching from backers. -//TODO: Cleanup increased for versi testing const SMALL_POV_LIMIT: usize = 128 * 1024; #[derive(Clone, PartialEq)] diff --git a/polkadot/primitives/src/runtime_api.rs b/polkadot/primitives/src/runtime_api.rs index efa6246250bb..17191d2f7722 100644 --- a/polkadot/primitives/src/runtime_api.rs +++ b/polkadot/primitives/src/runtime_api.rs @@ -241,7 +241,7 @@ sp_api::decl_runtime_apis! { key_ownership_proof: vstaging::slashing::OpaqueKeyOwnershipProof, ) -> Option<()>; - /***** Staging *****/ + /***** Staging *****/ /// Get the minimum number of backing votes for a parachain candidate. /// This is a staging method! Do not use on production runtimes! From 6dd51735462615a328580e7f9244b1b76a062e32 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 27 Sep 2023 10:26:50 +0300 Subject: [PATCH 021/192] pipeline/zombinet: fix 0006 test name Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/zombienet/polkadot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/zombienet/polkadot.yml b/.gitlab/pipeline/zombienet/polkadot.yml index 8d3b21eeff7a..5914f50c3ede 100644 --- a/.gitlab/pipeline/zombienet/polkadot.yml +++ b/.gitlab/pipeline/zombienet/polkadot.yml @@ -111,7 +111,7 @@ zombienet-polkadot-functional-0006-parachains-max-tranche0: script: - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh --local-dir="${LOCAL_DIR}/functional" - --test="0006-parachains-0006-parachains-max-tranche0.zndsl" + --test="0006-parachains-max-tranche0.zndsl" zombienet-polkadot-smoke-0001-parachains-smoke-test: extends: From 0bdc06e9d0cbf03f35e637d01d3691acabb001f5 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 27 Sep 2023 10:30:58 +0300 Subject: [PATCH 022/192] fixup cargo fmt nightly Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/tests.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index eaebac187f18..d3c3fa67a186 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -3847,7 +3847,8 @@ async fn handle_approval_on_max_wait_time( .lock() .set_tick(MAX_APPROVAL_COALESCE_WAIT_TICKS as Tick + TICK_NOW_BEGIN); - // Third time we fetch the configuration when timer expires and we are ready to sent the approval + // Third time we fetch the configuration when timer expires and we are ready to sent the + // approval assert_matches!( overseer_recv(virtual_overseer).await, AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { From 218748db342ee6242d0295a62482a05489cdec42 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 27 Sep 2023 10:45:12 +0300 Subject: [PATCH 023/192] fixup clippy Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index d3c3fa67a186..ec7ec7b9bba8 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -3740,7 +3740,7 @@ async fn handle_approval_on_max_coalesce_count( for _ in &candidate_indicies { assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, timeout, tx)) if timeout == PvfExecTimeoutKind::Approval => { + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, _, timeout, tx)) if timeout == PvfExecTimeoutKind::Approval => { tx.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } @@ -3804,7 +3804,7 @@ async fn handle_approval_on_max_wait_time( for _ in &candidate_indicies { assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, timeout, tx)) if timeout == PvfExecTimeoutKind::Approval => { + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, _, timeout, tx)) if timeout == PvfExecTimeoutKind::Approval => { tx.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } From 0335cf1f5adc519829bf6f18a7b9156583cc0cd9 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 28 Sep 2023 11:15:11 +0300 Subject: [PATCH 024/192] Fixup markdownlint Signed-off-by: Alexandru Gheorghe --- .../src/node/approval/approval-voting.md | 19 ++++++++++--------- .../src/protocol-approval.md | 8 ++++++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md index 40d959b0beb9..8d7daa173abc 100644 --- a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md +++ b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md @@ -3,13 +3,13 @@ Reading the [section on the approval protocol](../../protocol-approval.md) will likely be necessary to understand the aims of this subsystem. -Approval votes are split into two parts: Assignments and Approvals. Validators first broadcast their assignment to -indicate intent to check a candidate. Upon successfully checking, they don't immediately send the vote instead -they queue the check for a short period of time `MAX_APPROVALS_COALESCE_TICKS` to give the opportunity of the +Approval votes are split into two parts: Assignments and Approvals. Validators first broadcast their assignment to +indicate intent to check a candidate. Upon successfully checking, they don't immediately send the vote instead +they queue the check for a short period of time `MAX_APPROVALS_COALESCE_TICKS` to give the opportunity of the validator to vote for more than one candidate. Once MAX_APPROVALS_COALESCE_TICKS have passed or at least -`MAX_APPROVAL_COALESCE_COUNT` are ready they broadcast an approval vote for all candidates. If a validator -doesn't broadcast their approval vote shortly after issuing an assignment, this is an indication that they are -being prevented from recovering or validating the block data and that more validators should self-select to +`MAX_APPROVAL_COALESCE_COUNT` are ready they broadcast an approval vote for all candidates. If a validator +doesn't broadcast their approval vote shortly after issuing an assignment, this is an indication that they are +being prevented from recovering or validating the block data and that more validators should self-select to check the candidate. This is known as a "no-show". The core of this subsystem is a Tick-based timer loop, where Ticks are 500ms. We also reason about time in terms of @@ -313,9 +313,9 @@ entry. The cert itself contains information necessary to determine the candidate On receiving a `CheckAndImportApproval(indirect_approval_vote, response_channel)` message: * Fetch the `BlockEntry` from the indirect approval vote's `block_hash`. If none, return `ApprovalCheckResult::Bad`. - * Fetch all `CandidateEntry` from the indirect approval vote's `candidate_indices`. If the block did not trigger + * Fetch all `CandidateEntry` from the indirect approval vote's `candidate_indices`. If the block did not trigger inclusion of enough candidates, return `ApprovalCheckResult::Bad`. - * Construct a `SignedApprovalVote` using the candidates hashes and check against the validator's approval key, + * Construct a `SignedApprovalVote` using the candidates hashes and check against the validator's approval key, based on the session info of the block. If invalid or no such validator, return `ApprovalCheckResult::Bad`. * Send `ApprovalCheckResult::Accepted` * [Import the checked approval vote](#import-checked-approval) for all candidates @@ -422,7 +422,8 @@ On receiving an `ApprovedAncestor(Hash, BlockNumber, response_channel)`: * Arm a per BlockEntry timer with latest tick we can send the vote. ### Delayed vote distribution - * [Issue Approval Vote](#issue-approval-vote) arms once a per block timer if there are no requirements to send the vote immediately. + * [Issue Approval Vote](#issue-approval-vote) arms once a per block timer if there are no requirements to send the + vote immediately. * When the timer wakes up it will either: * IF there is a candidate in the queue past its sending tick: * Construct a `SignedApprovalVote` with the validator index for the session and all candidate hashes in the waiting queue. diff --git a/polkadot/roadmap/implementers-guide/src/protocol-approval.md b/polkadot/roadmap/implementers-guide/src/protocol-approval.md index 86e496fb9eba..1a217afb9b71 100644 --- a/polkadot/roadmap/implementers-guide/src/protocol-approval.md +++ b/polkadot/roadmap/implementers-guide/src/protocol-approval.md @@ -297,10 +297,14 @@ provide somewhat more security. TODO: When? Is this optimal for the network? etc. ## Approval coalescing -To reduce the necessary network bandwidth and cpu time when a validator has more than one candidate to approve we are doing our best effort to send a single message that approves all available candidates with a single signature. The implemented heuristic, is that each time we are ready to create a signature and send a vote for a candidate we delay the sending of it untill one of three things happen: +To reduce the necessary network bandwidth and cpu time when a validator has more than one candidate to approve we are +doing our best effort to send a single message that approves all available candidates with a single signature. +The implemented heuristic, is that each time we are ready to create a signature and send a vote for a candidate we +delay the sending of it untill one of three things happen: - We gathered a maximum of `MAX_APPROVAL_COALESCE_COUNT` candidates that we are ready to vote for. - `MAX_APPROVALS_COALESCE_TICKS` have passed since the we were ready to approve the candidate. -- We are already in the last third of the now-show period in order to avoid creating accidental no shows, which in turn my trigger other assignments. +- We are already in the last third of the now-show period in order to avoid creating accidental no shows, which in + turn my trigger other assignments. ## On-chain verification From 0f0faf0e39680e503ab333eabe64dcd38bf8e997 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 28 Sep 2023 11:58:57 +0300 Subject: [PATCH 025/192] Fixup runtime-migration-westend Signed-off-by: Alexandru Gheorghe --- polkadot/runtime/parachains/src/configuration.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/polkadot/runtime/parachains/src/configuration.rs b/polkadot/runtime/parachains/src/configuration.rs index 6690e4156c7f..32b3bb665669 100644 --- a/polkadot/runtime/parachains/src/configuration.rs +++ b/polkadot/runtime/parachains/src/configuration.rs @@ -505,7 +505,8 @@ pub mod pallet { /// v6-v7: /// v7-v8: /// v8-v9: - const STORAGE_VERSION: StorageVersion = StorageVersion::new(9); + /// v9-10: + const STORAGE_VERSION: StorageVersion = StorageVersion::new(10); #[pallet::pallet] #[pallet::storage_version(STORAGE_VERSION)] From 6eabb634a52da1a7003aa63de14d5e661f714bb3 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 28 Sep 2023 16:23:46 +0300 Subject: [PATCH 026/192] approval-voting: address round 1 of feedback Signed-off-by: Alexandru Gheorghe --- .../approval-voting/src/approval_checking.rs | 64 ++++---- .../src/approval_db/v2/migration_helpers.rs | 6 +- .../approval-voting/src/approval_db/v2/mod.rs | 10 +- .../node/core/approval-voting/src/backend.rs | 3 +- .../node/core/approval-voting/src/criteria.rs | 2 +- polkadot/node/core/approval-voting/src/lib.rs | 56 +++---- .../approval-voting/src/persisted_entries.rs | 146 ++++++++++++------ .../node/core/approval-voting/src/tests.rs | 2 + .../network/approval-distribution/src/lib.rs | 4 +- 9 files changed, 166 insertions(+), 127 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/approval_checking.rs b/polkadot/node/core/approval-voting/src/approval_checking.rs index 2f0ea4160ae1..374b55aa0b4f 100644 --- a/polkadot/node/core/approval-voting/src/approval_checking.rs +++ b/polkadot/node/core/approval-voting/src/approval_checking.rs @@ -467,13 +467,15 @@ mod tests { #[test] fn pending_is_not_approved() { - let candidate = approval_db::v1::CandidateEntry { - candidate: dummy_candidate_receipt(dummy_hash()), - session: 0, - block_assignments: BTreeMap::default(), - approvals: BitVec::default(), - } - .into(); + let candidate = CandidateEntry::from_v1( + approval_db::v1::CandidateEntry { + candidate: dummy_candidate_receipt(dummy_hash()), + session: 0, + block_assignments: BTreeMap::default(), + approvals: BitVec::default(), + }, + 0, + ); let approval_entry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), @@ -500,13 +502,15 @@ mod tests { #[test] fn exact_takes_only_assignments_up_to() { - let mut candidate: CandidateEntry = approval_db::v1::CandidateEntry { - candidate: dummy_candidate_receipt(dummy_hash()), - session: 0, - block_assignments: BTreeMap::default(), - approvals: bitvec![u8, BitOrderLsb0; 0; 10], - } - .into(); + let mut candidate: CandidateEntry = CandidateEntry::from_v1( + approval_db::v1::CandidateEntry { + candidate: dummy_candidate_receipt(dummy_hash()), + session: 0, + block_assignments: BTreeMap::default(), + approvals: bitvec![u8, BitOrderLsb0; 0; 10], + }, + 0, + ); for i in 0..3 { candidate.mark_approval(ValidatorIndex(i)); @@ -572,13 +576,15 @@ mod tests { #[test] fn one_honest_node_always_approves() { - let mut candidate: CandidateEntry = approval_db::v1::CandidateEntry { - candidate: dummy_candidate_receipt(dummy_hash()), - session: 0, - block_assignments: BTreeMap::default(), - approvals: bitvec![u8, BitOrderLsb0; 0; 10], - } - .into(); + let mut candidate: CandidateEntry = CandidateEntry::from_v1( + approval_db::v1::CandidateEntry { + candidate: dummy_candidate_receipt(dummy_hash()), + session: 0, + block_assignments: BTreeMap::default(), + approvals: bitvec![u8, BitOrderLsb0; 0; 10], + }, + 0, + ); for i in 0..3 { candidate.mark_approval(ValidatorIndex(i)); @@ -1043,13 +1049,15 @@ mod tests { let no_show_duration = 10; let needed_approvals = 3; - let mut candidate: CandidateEntry = approval_db::v1::CandidateEntry { - candidate: dummy_candidate_receipt(dummy_hash()), - session: 0, - block_assignments: BTreeMap::default(), - approvals: bitvec![u8, BitOrderLsb0; 0; 3], - } - .into(); + let mut candidate: CandidateEntry = CandidateEntry::from_v1( + approval_db::v1::CandidateEntry { + candidate: dummy_candidate_receipt(dummy_hash()), + session: 0, + block_assignments: BTreeMap::default(), + approvals: bitvec![u8, BitOrderLsb0; 0; 3], + }, + 0, + ); for i in 0..3 { candidate.mark_approval(ValidatorIndex(i)); diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs index a70d766fc8c9..54efa3dc8efc 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs @@ -86,11 +86,13 @@ pub fn v1_to_v2(db: Arc, config: Config) -> Result<()> { let mut counter = 0; // Get all candidate entries, approval entries and convert each of them. for block in all_blocks { - for (_core_index, candidate_hash) in block.candidates() { + for (candidate_index, (_core_index, candidate_hash)) in + block.candidates().iter().enumerate() + { // Loading the candidate will also perform the conversion to the updated format and // return that represantation. if let Some(candidate_entry) = backend - .load_candidate_entry_v1(&candidate_hash) + .load_candidate_entry_v1(&candidate_hash, candidate_index as CandidateIndex) .map_err(|e| Error::InternalError(e))? { // Write the updated representation. diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs index 88650a539130..2846b0ca499b 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs @@ -62,9 +62,10 @@ impl V1ReadBackend for DbBackend { fn load_candidate_entry_v1( &self, candidate_hash: &CandidateHash, + candidate_index: CandidateIndex, ) -> SubsystemResult> { load_candidate_entry_v1(&*self.inner, &self.config, candidate_hash) - .map(|e| e.map(Into::into)) + .map(|e| e.map(|e| persisted_entries::CandidateEntry::from_v1(e, candidate_index))) } fn load_block_entry_v1( @@ -205,9 +206,8 @@ pub struct TrancheEntry { pub struct OurApproval { /// The signature for the candidates hashes pointed by indices. pub signature: ValidatorSignature, - /// The indices of the candidates signed in this approval, an empty value means only - /// the candidate referred by this approval entry was signed. - pub signed_candidates_indices: Option, + /// The indices of the candidates signed in this approval. + pub signed_candidates_indices: CandidateBitfield, } /// Metadata regarding approval of a particular candidate within the context of some @@ -270,7 +270,7 @@ pub struct CandidateSigningContext { /// The candidate hash, to be included in the signature. pub candidate_hash: CandidateHash, /// The latest tick we have to create and send the approval. - pub send_no_later_than_tick: Tick, + pub sign_no_later_than_tick: Tick, } impl From for Tick { diff --git a/polkadot/node/core/approval-voting/src/backend.rs b/polkadot/node/core/approval-voting/src/backend.rs index 374e7a826d19..37d9c76d252f 100644 --- a/polkadot/node/core/approval-voting/src/backend.rs +++ b/polkadot/node/core/approval-voting/src/backend.rs @@ -22,7 +22,7 @@ //! before any commit to the underlying storage is made. use polkadot_node_subsystem::SubsystemResult; -use polkadot_primitives::{BlockNumber, CandidateHash, Hash}; +use polkadot_primitives::{BlockNumber, CandidateHash, CandidateIndex, Hash}; use std::collections::HashMap; @@ -72,6 +72,7 @@ pub trait V1ReadBackend: Backend { fn load_candidate_entry_v1( &self, candidate_hash: &CandidateHash, + candidate_index: CandidateIndex, ) -> SubsystemResult>; /// Load a block entry from the DB with scheme version 1. diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index db74302f0225..1f751e2bf140 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -278,7 +278,7 @@ impl AssignmentCriteria for RealAssignmentCriteria { config: &Config, leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>, ) -> HashMap { - compute_assignments(keystore, relay_vrf_story, config, leaving_cores, true) + compute_assignments(keystore, relay_vrf_story, config, leaving_cores, false) } fn check_assignment_cert( diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 95d04cc0d147..cb46782d4878 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -1346,7 +1346,7 @@ fn distribution_messages_for_activation( session: block_entry.session(), }); let mut signatures_queued = HashSet::new(); - for (i, (_, candidate_hash)) in block_entry.candidates().iter().enumerate() { + for (_, candidate_hash) in block_entry.candidates() { let _candidate_span = distribution_message_span.child("candidate").with_candidate(*candidate_hash); let candidate_entry = match db.load_candidate_entry(&candidate_hash)? { @@ -1449,14 +1449,14 @@ fn distribution_messages_for_activation( continue }, } - let candidate_indices = approval_sig - .signed_candidates_indices - .unwrap_or((i as CandidateIndex).into()); - if signatures_queued.insert(candidate_indices.clone()) { + if signatures_queued + .insert(approval_sig.signed_candidates_indices.clone()) + { messages.push(ApprovalDistributionMessage::DistributeApproval( IndirectSignedApprovalVoteV2 { block_hash, - candidate_indices, + candidate_indices: approval_sig + .signed_candidates_indices, validator: assignment.validator_index(), signature: approval_sig.signature, }, @@ -3384,23 +3384,15 @@ async fn maybe_create_signature( target: LOG_TARGET, "Candidates pending signatures {:}", block_entry.num_candidates_pending_signature() ); - - let oldest_candidate_to_sign = match block_entry.longest_waiting_candidate_signature() { - Some(candidate) => candidate, - // No cached candidates, nothing to do here, this just means the timer fired, - // but the signatures were already sent because we gathered more than - // max_approval_coalesce_count. - None => return Ok(None), - }; - let tick_now = state.clock.tick_now(); - if oldest_candidate_to_sign.send_no_later_than_tick > tick_now && - (block_entry.num_candidates_pending_signature() as u32) < - approval_params.max_approval_coalesce_count - { - return Ok(Some(oldest_candidate_to_sign.send_no_later_than_tick)) - } + let (candidates_to_sign, sign_no_later_then) = block_entry + .get_candidates_that_need_signature(tick_now, approval_params.max_approval_coalesce_count); + + let (candidates_hashes, candidates_indices) = match candidates_to_sign { + Some(candidates_to_sign) => candidates_to_sign, + None => return Ok(sign_no_later_then), + }; let session_info = match get_session_info( session_info_provider, @@ -3436,12 +3428,10 @@ async fn maybe_create_signature( }, }; - let candidate_hashes = block_entry.candidate_hashes_pending_signature(); - let signature = match sign_approval( &state.keystore, &validator_pubkey, - candidate_hashes.clone(), + candidates_hashes.clone(), block_entry.session(), ) { Some(sig) => sig, @@ -3457,14 +3447,13 @@ async fn maybe_create_signature( return Ok(None) }, }; + metrics.on_approval_coalesce(candidates_hashes.len() as u32); - let candidate_entries = candidate_hashes + let candidate_entries = candidates_hashes .iter() .map(|candidate_hash| db.load_candidate_entry(candidate_hash)) .collect::>>>()?; - let candidate_indices = block_entry.candidate_indices_pending_signature(); - for candidate_entry in candidate_entries { let mut candidate_entry = candidate_entry .expect("Candidate was scheduled to be signed entry in db should exist; qed"); @@ -3473,26 +3462,17 @@ async fn maybe_create_signature( .expect("Candidate was scheduled to be signed entry in db should exist; qed"); approval_entry.import_approval_sig(OurApproval { signature: signature.clone(), - signed_candidates_indices: Some( - candidate_indices - .clone() - .try_into() - .expect("Fails only of array empty, it can't be, qed"), - ), + signed_candidates_indices: candidates_indices.clone(), }); db.write_candidate_entry(candidate_entry); } - metrics.on_approval_coalesce(candidate_indices.len() as u32); - metrics.on_approval_produced(); ctx.send_unbounded_message(ApprovalDistributionMessage::DistributeApproval( IndirectSignedApprovalVoteV2 { block_hash: block_entry.block_hash(), - candidate_indices: candidate_indices - .try_into() - .expect("Fails only of array empty, it can't be, qed"), + candidate_indices: candidates_indices, validator: validator_index, signature, }, diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index e441c23be122..d01d3c8fe9d8 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -20,6 +20,7 @@ //! Within that context, things are plain-old-data. Within this module, //! data and logic are intertwined. +use itertools::Itertools; use polkadot_node_primitives::approval::{ v1::{DelayTranche, RelayVRFStory}, v2::{AssignmentCertV2, CandidateBitfield}, @@ -93,22 +94,22 @@ impl From for crate::approval_db::v2::OurApproval { } } -impl From for OurApproval { - fn from(value: ValidatorSignature) -> Self { - Self { signature: value, signed_candidates_indices: Default::default() } - } -} - /// Metadata about our approval signature #[derive(Debug, Clone, PartialEq)] pub struct OurApproval { /// The signature for the candidates hashes pointed by indices. pub signature: ValidatorSignature, - /// The indices of the candidates signed in this approval, an empty value means only - /// the candidate referred by this approval entry was signed. - pub signed_candidates_indices: Option, + /// The indices of the candidates signed in this approval. + pub signed_candidates_indices: CandidateBitfield, } +impl OurApproval { + /// Converts a ValidatorSignature to an OurApproval. + /// It used in converting the database from v1 to v2. + pub fn from_v1(value: ValidatorSignature, candidate_index: CandidateIndex) -> Self { + Self { signature: value, signed_candidates_indices: candidate_index.into() } + } +} /// Metadata regarding approval of a particular candidate within the context of some /// particular block. #[derive(Debug, Clone, PartialEq)] @@ -265,6 +266,23 @@ impl ApprovalEntry { (None, None) } } + + // Convert an ApprovalEntry from v1 version to a v2 version + pub fn from_v1( + value: crate::approval_db::v1::ApprovalEntry, + candidate_index: CandidateIndex, + ) -> Self { + ApprovalEntry { + tranches: value.tranches.into_iter().map(|tranche| tranche.into()).collect(), + backing_group: value.backing_group, + our_assignment: value.our_assignment.map(|assignment| assignment.into()), + our_approval_sig: value + .our_approval_sig + .map(|sig| OurApproval::from_v1(sig, candidate_index)), + assigned_validators: value.assignments, + approved: value.approved, + } + } } impl From for ApprovalEntry { @@ -336,6 +354,23 @@ impl CandidateEntry { pub fn approval_entry(&self, block_hash: &Hash) -> Option<&ApprovalEntry> { self.block_assignments.get(block_hash) } + + /// Convert a CandidateEntry from a v1 to its v2 equivalent. + pub fn from_v1( + value: crate::approval_db::v1::CandidateEntry, + candidate_index: CandidateIndex, + ) -> Self { + Self { + approvals: value.approvals, + block_assignments: value + .block_assignments + .into_iter() + .map(|(h, ae)| (h, ApprovalEntry::from_v1(ae, candidate_index))) + .collect(), + candidate: value.candidate, + session: value.session, + } + } } impl From for CandidateEntry { @@ -398,7 +433,7 @@ pub struct BlockEntry { #[derive(Debug, Clone, PartialEq)] pub struct CandidateSigningContext { pub candidate_hash: CandidateHash, - pub send_no_later_than_tick: Tick, + pub sign_no_later_than_tick: Tick, } impl BlockEntry { @@ -495,11 +530,11 @@ impl BlockEntry { &mut self, candidate_index: CandidateIndex, candidate_hash: CandidateHash, - send_no_later_than_tick: Tick, + sign_no_later_than_tick: Tick, ) -> Option { self.candidates_pending_signature.insert( candidate_index, - CandidateSigningContext { candidate_hash, send_no_later_than_tick }, + CandidateSigningContext { candidate_hash, sign_no_later_than_tick }, ) } @@ -514,7 +549,7 @@ impl BlockEntry { } /// Candidate hashes for candidates pending signatures - pub fn candidate_hashes_pending_signature(&self) -> Vec { + fn candidate_hashes_pending_signature(&self) -> Vec { self.candidates_pending_signature .values() .map(|unsigned_approval| unsigned_approval.candidate_hash) @@ -522,15 +557,55 @@ impl BlockEntry { } /// Candidate indices for candidates pending signature - pub fn candidate_indices_pending_signature(&self) -> Vec { - self.candidates_pending_signature.keys().map(|val| *val).collect() + fn candidate_indices_pending_signature(&self) -> CandidateBitfield { + self.candidates_pending_signature + .keys() + .map(|val| *val) + .collect_vec() + .try_into() + .expect("Fails only of array empty, it can't be, qed") } - /// Returns the candidate that has been longest in the queue. - pub fn longest_waiting_candidate_signature(&self) -> Option<&CandidateSigningContext> { - self.candidates_pending_signature + /// Returns a list of candidates hashes that need need signature created at the current tick: + /// This might happen in other of the two reasons: + /// 1. We queued more than max_approval_coalesce_count candidates. + /// 2. We have candidates that waiting in the queue past their `sign_no_later_than_tick` + /// + /// Additionally, we also return the first tick when we will have to create a signature, + /// so that the caller can arm the timer if it is not already armed. + pub fn get_candidates_that_need_signature( + &self, + tick_now: Tick, + max_approval_coalesce_count: u32, + ) -> (Option<(Vec, CandidateBitfield)>, Option) { + let sign_no_later_than_tick = self + .candidates_pending_signature .values() - .min_by(|a, b| a.send_no_later_than_tick.cmp(&b.send_no_later_than_tick)) + .min_by(|a, b| a.sign_no_later_than_tick.cmp(&b.sign_no_later_than_tick)) + .map(|val| val.sign_no_later_than_tick); + + if let Some(sign_no_later_than_tick) = sign_no_later_than_tick { + if sign_no_later_than_tick <= tick_now || + self.num_candidates_pending_signature() >= max_approval_coalesce_count as usize + { + ( + Some(( + self.candidate_hashes_pending_signature(), + self.candidate_indices_pending_signature(), + )), + Some(sign_no_later_than_tick), + ) + } else { + // We can still wait for other candidates to queue in, so just make sure + // we wake up at the tick we have to sign the longest waiting candidate. + (Default::default(), Some(sign_no_later_than_tick)) + } + } else { + // No cached candidates, nothing to do here, this just means the timer fired, + // but the signatures were already sent because we gathered more than + // max_approval_coalesce_count. + (Default::default(), sign_no_later_than_tick) + } } /// Signals the approval was issued for the candidates pending signature @@ -605,7 +680,7 @@ impl From for CandidateSigningC fn from(signing_context: crate::approval_db::v2::CandidateSigningContext) -> Self { Self { candidate_hash: signing_context.candidate_hash, - send_no_later_than_tick: signing_context.send_no_later_than_tick.into(), + sign_no_later_than_tick: signing_context.sign_no_later_than_tick.into(), } } } @@ -614,36 +689,7 @@ impl From for crate::approval_db::v2::CandidateSigningC fn from(signing_context: CandidateSigningContext) -> Self { Self { candidate_hash: signing_context.candidate_hash, - send_no_later_than_tick: signing_context.send_no_later_than_tick.into(), - } - } -} - -/// Migration helpers. -impl From for CandidateEntry { - fn from(value: crate::approval_db::v1::CandidateEntry) -> Self { - Self { - approvals: value.approvals, - block_assignments: value - .block_assignments - .into_iter() - .map(|(h, ae)| (h, ae.into())) - .collect(), - candidate: value.candidate, - session: value.session, - } - } -} - -impl From for ApprovalEntry { - fn from(value: crate::approval_db::v1::ApprovalEntry) -> Self { - ApprovalEntry { - tranches: value.tranches.into_iter().map(|tranche| tranche.into()).collect(), - backing_group: value.backing_group, - our_assignment: value.our_assignment.map(|assignment| assignment.into()), - our_approval_sig: value.our_approval_sig.map(Into::into), - assigned_validators: value.assignments, - approved: value.approved, + sign_no_later_than_tick: signing_context.sign_no_later_than_tick.into(), } } } diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index ec7ec7b9bba8..04aef90aae97 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -280,6 +280,7 @@ impl V1ReadBackend for TestStoreInner { fn load_candidate_entry_v1( &self, candidate_hash: &CandidateHash, + _candidate_index: CandidateIndex, ) -> SubsystemResult> { self.load_candidate_entry(candidate_hash) } @@ -363,6 +364,7 @@ impl V1ReadBackend for TestStore { fn load_candidate_entry_v1( &self, candidate_hash: &CandidateHash, + _candidate_index: CandidateIndex, ) -> SubsystemResult> { self.load_candidate_entry(candidate_hash) } diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index d63732bbe04b..8162128755a6 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -134,7 +134,7 @@ enum ApprovalEntryError { CandidateIndexOutOfBounds, InvalidCandidateIndex, DuplicateApproval, - CouldNotFindAssignment, + UnknownAssignment, AssignmentsFollowedDifferentPaths(RequiredRouting, RequiredRouting), } @@ -580,7 +580,7 @@ impl BlockEntry { if let Some(required_routing) = required_routing { Ok(required_routing) } else { - Err(ApprovalEntryError::CouldNotFindAssignment) + Err(ApprovalEntryError::UnknownAssignment) } } From 9dc4a93441898d7d7f3031fc788406da322beecd Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 28 Sep 2023 18:29:50 +0300 Subject: [PATCH 027/192] approval-distribution: cosmetic improvements Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 43 +++++++++++-------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 8162128755a6..7411b9c7b545 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -180,6 +180,16 @@ impl ApprovalEntry { self.routing_info.required_routing = required_routing; } + // Tells if this entry assignment covers at least one candidate in the approval + pub fn includes_approval_candidates(&self, approval: &IndirectSignedApprovalVoteV2) -> bool { + for candidate_index in approval.candidate_indices.iter_ones() { + if self.assignment_claimed_candidates.bit_at((candidate_index).as_bit_index()) { + return true + } + } + return false + } + // Records a new approval. Returns false if the claimed candidate is not found or we already // have received the approval. pub fn note_approval( @@ -195,13 +205,7 @@ impl ApprovalEntry { } // We need at least one of the candidates in the approval to be in this assignment - if !approval - .candidate_indices - .iter_ones() - .fold(false, |accumulator, candidate_index| { - self.assignment_claimed_candidates.bit_at((candidate_index).as_bit_index()) || - accumulator - }) { + if !self.includes_approval_candidates(&approval) { return Err(ApprovalEntryError::InvalidCandidateIndex) } @@ -405,6 +409,13 @@ impl Knowledge { } success } + + // Tells if all keys are contained by this peer_knowledge + pub fn contains_all_keys(&self, keys: &Vec<(MessageSubject, MessageKind)>) -> bool { + keys.iter().fold(true, |accumulator, assignment_key| { + accumulator && self.contains(&assignment_key.0, assignment_key.1) + }) + } } /// Information that has been circulated to and from a peer. @@ -423,12 +434,12 @@ impl PeerKnowledge { // Tells if all assignments for a given approval are included in the knowledge of the peer fn contains_assignments_for_approval(&self, approval: &IndirectSignedApprovalVoteV2) -> bool { - Self::generate_assignments_keys(&approval).iter().fold( - true, - |accumulator, assignment_key| { - accumulator && self.contains(&assignment_key.0, assignment_key.1) - }, - ) + self.contains_all_keys(&Self::generate_assignments_keys(&approval)) + } + + // Tells if all keys are contained by this peer_knowledge + pub fn contains_all_keys(&self, keys: &Vec<(MessageSubject, MessageKind)>) -> bool { + self.sent.contains_all_keys(keys) || self.received.contains_all_keys(keys) } // Generate the knowledge keys for querying if all assignments of an approval are known @@ -584,6 +595,7 @@ impl BlockEntry { } } + /// Returns the list of approval votes covering this candidate pub fn approval_votes( &self, candidate_index: CandidateIndex, @@ -1723,10 +1735,7 @@ impl State { // 3. Any randomly selected peers have been sent the assignment already. let in_topology = topology .map_or(false, |t| t.local_grid_neighbors().route_to_peer(required_routing, peer)); - in_topology || - assignments_knowledge_keys.iter().fold(true, |result, message_subject| { - result && knowledge.sent.contains(&message_subject.0, message_subject.1) - }) + in_topology || knowledge.sent.contains_all_keys(&assignments_knowledge_keys) }; let peers = entry From 566d7f5c439a356fedae415691d08315ade22735 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 29 Sep 2023 12:39:10 +0300 Subject: [PATCH 028/192] Comment vstaging Signed-off-by: Alexandru Gheorghe --- polkadot/node/network/protocol/src/lib.rs | 516 +++++++++++----------- 1 file changed, 258 insertions(+), 258 deletions(-) diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index ba9b9a7f4900..5d8b3cd550ca 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -588,264 +588,264 @@ pub mod v1 { } } -/// vstaging network protocol types. -pub mod vstaging { - use bitvec::{order::Lsb0, slice::BitSlice, vec::BitVec}; - use parity_scale_codec::{Decode, Encode}; - - use polkadot_primitives::vstaging::{ - CandidateHash, CollatorId, CollatorSignature, GroupIndex, Hash, Id as ParaId, - UncheckedSignedAvailabilityBitfield, UncheckedSignedStatement, - }; - - use polkadot_node_primitives::{ - approval::{ - v1::IndirectSignedApprovalVote, - v2::{CandidateBitfield, IndirectAssignmentCertV2}, - }, - UncheckedSignedFullStatement, - }; - - /// Network messages used by the bitfield distribution subsystem. - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] - pub enum BitfieldDistributionMessage { - /// A signed availability bitfield for a given relay-parent hash. - #[codec(index = 0)] - Bitfield(Hash, UncheckedSignedAvailabilityBitfield), - } - - /// Bitfields indicating the statements that are known or undesired - /// about a candidate. - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] - pub struct StatementFilter { - /// Seconded statements. '1' is known or undesired. - pub seconded_in_group: BitVec, - /// Valid statements. '1' is known or undesired. - pub validated_in_group: BitVec, - } - - impl StatementFilter { - /// Create a new blank filter with the given group size. - pub fn blank(group_size: usize) -> Self { - StatementFilter { - seconded_in_group: BitVec::repeat(false, group_size), - validated_in_group: BitVec::repeat(false, group_size), - } - } - - /// Create a new full filter with the given group size. - pub fn full(group_size: usize) -> Self { - StatementFilter { - seconded_in_group: BitVec::repeat(true, group_size), - validated_in_group: BitVec::repeat(true, group_size), - } - } - - /// Whether the filter has a specific expected length, consistent across both - /// bitfields. - pub fn has_len(&self, len: usize) -> bool { - self.seconded_in_group.len() == len && self.validated_in_group.len() == len - } - - /// Determine the number of backing validators in the statement filter. - pub fn backing_validators(&self) -> usize { - self.seconded_in_group - .iter() - .by_vals() - .zip(self.validated_in_group.iter().by_vals()) - .filter(|&(s, v)| s || v) // no double-counting - .count() - } - - /// Whether the statement filter has at least one seconded statement. - pub fn has_seconded(&self) -> bool { - self.seconded_in_group.iter().by_vals().any(|x| x) - } - - /// Mask out `Seconded` statements in `self` according to the provided - /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. - pub fn mask_seconded(&mut self, mask: &BitSlice) { - for (mut x, mask) in self - .seconded_in_group - .iter_mut() - .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) - { - // (x, mask) => x - // (true, true) => false - // (true, false) => true - // (false, true) => false - // (false, false) => false - *x = *x && !mask; - } - } - - /// Mask out `Valid` statements in `self` according to the provided - /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. - pub fn mask_valid(&mut self, mask: &BitSlice) { - for (mut x, mask) in self - .validated_in_group - .iter_mut() - .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) - { - // (x, mask) => x - // (true, true) => false - // (true, false) => true - // (false, true) => false - // (false, false) => false - *x = *x && !mask; - } - } - } - - /// A manifest of a known backed candidate, along with a description - /// of the statements backing it. - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] - pub struct BackedCandidateManifest { - /// The relay-parent of the candidate. - pub relay_parent: Hash, - /// The hash of the candidate. - pub candidate_hash: CandidateHash, - /// The group index backing the candidate at the relay-parent. - pub group_index: GroupIndex, - /// The para ID of the candidate. It is illegal for this to - /// be a para ID which is not assigned to the group indicated - /// in this manifest. - pub para_id: ParaId, - /// The head-data corresponding to the candidate. - pub parent_head_data_hash: Hash, - /// A statement filter which indicates which validators in the - /// para's group at the relay-parent have validated this candidate - /// and issued statements about it, to the advertiser's knowledge. - /// - /// This MUST have exactly the minimum amount of bytes - /// necessary to represent the number of validators in the assigned - /// backing group as-of the relay-parent. - pub statement_knowledge: StatementFilter, - } - - /// An acknowledgement of a backed candidate being known. - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] - pub struct BackedCandidateAcknowledgement { - /// The hash of the candidate. - pub candidate_hash: CandidateHash, - /// A statement filter which indicates which validators in the - /// para's group at the relay-parent have validated this candidate - /// and issued statements about it, to the advertiser's knowledge. - /// - /// This MUST have exactly the minimum amount of bytes - /// necessary to represent the number of validators in the assigned - /// backing group as-of the relay-parent. - pub statement_knowledge: StatementFilter, - } - - /// Network messages used by the statement distribution subsystem. - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] - pub enum StatementDistributionMessage { - /// A notification of a signed statement in compact form, for a given relay parent. - #[codec(index = 0)] - Statement(Hash, UncheckedSignedStatement), - - /// A notification of a backed candidate being known by the - /// sending node, for the purpose of being requested by the receiving node - /// if needed. - #[codec(index = 1)] - BackedCandidateManifest(BackedCandidateManifest), - - /// A notification of a backed candidate being known by the sending node, - /// for the purpose of informing a receiving node which already has the candidate. - #[codec(index = 2)] - BackedCandidateKnown(BackedCandidateAcknowledgement), - - /// All messages for V1 for compatibility with the statement distribution - /// protocol, for relay-parents that don't support asynchronous backing. - /// - /// These are illegal to send to V1 peers, and illegal to send concerning relay-parents - /// which support asynchronous backing. This backwards compatibility should be - /// considered immediately deprecated and can be removed once the node software - /// is not required to support logic from before asynchronous backing anymore. - #[codec(index = 255)] - V1Compatibility(crate::v1::StatementDistributionMessage), - } - - /// Network messages used by the approval distribution subsystem. - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] - pub enum ApprovalDistributionMessage { - /// Assignments for candidates in recent, unfinalized blocks. - /// We use a bitfield to reference claimed candidates, where the bit index is equal to - /// candidate index. - /// - /// Actually checking the assignment may yield a different result. - /// TODO: Look at getting rid of bitfield in the future. - #[codec(index = 0)] - Assignments(Vec<(IndirectAssignmentCertV2, CandidateBitfield)>), - /// Approvals for candidates in some recent, unfinalized block. - #[codec(index = 1)] - Approvals(Vec), - } - - /// Dummy network message type, so we will receive connect/disconnect events. - #[derive(Debug, Clone, PartialEq, Eq)] - pub enum GossipSupportNetworkMessage {} - - /// Network messages used by the collator protocol subsystem - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] - pub enum CollatorProtocolMessage { - /// Declare the intent to advertise collations under a collator ID, attaching a - /// signature of the `PeerId` of the node using the given collator ID key. - #[codec(index = 0)] - Declare(CollatorId, ParaId, CollatorSignature), - /// Advertise a collation to a validator. Can only be sent once the peer has - /// declared that they are a collator with given ID. - #[codec(index = 1)] - AdvertiseCollation { - /// Hash of the relay parent advertised collation is based on. - relay_parent: Hash, - /// Candidate hash. - candidate_hash: CandidateHash, - /// Parachain head data hash before candidate execution. - parent_head_data_hash: Hash, - }, - /// A collation sent to a validator was seconded. - #[codec(index = 4)] - CollationSeconded(Hash, UncheckedSignedFullStatement), - } - - /// All network messages on the validation peer-set. - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] - pub enum ValidationProtocol { - /// Bitfield distribution messages - #[codec(index = 1)] - #[from] - BitfieldDistribution(BitfieldDistributionMessage), - /// Statement distribution messages - #[codec(index = 3)] - #[from] - StatementDistribution(StatementDistributionMessage), - /// Approval distribution messages - #[codec(index = 4)] - #[from] - ApprovalDistribution(ApprovalDistributionMessage), - } - - /// All network messages on the collation peer-set. - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] - pub enum CollationProtocol { - /// Collator protocol messages - #[codec(index = 0)] - #[from] - CollatorProtocol(CollatorProtocolMessage), - } - - /// Get the payload that should be signed and included in a `Declare` message. - /// - /// The payload is the local peer id of the node, which serves to prove that it - /// controls the collator key it is declaring an intention to collate under. - pub fn declare_signature_payload(peer_id: &sc_network::PeerId) -> Vec { - let mut payload = peer_id.to_bytes(); - payload.extend_from_slice(b"COLL"); - payload - } -} +// /// vstaging network protocol types. +// pub mod vstaging { +// use bitvec::{order::Lsb0, slice::BitSlice, vec::BitVec}; +// use parity_scale_codec::{Decode, Encode}; + +// use polkadot_primitives::vstaging::{ +// CandidateHash, CollatorId, CollatorSignature, GroupIndex, Hash, Id as ParaId, +// UncheckedSignedAvailabilityBitfield, UncheckedSignedStatement, +// }; + +// use polkadot_node_primitives::{ +// approval::{ +// v1::IndirectSignedApprovalVote, +// v2::{CandidateBitfield, IndirectAssignmentCertV2}, +// }, +// UncheckedSignedFullStatement, +// }; + +// /// Network messages used by the bitfield distribution subsystem. +// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +// pub enum BitfieldDistributionMessage { +// /// A signed availability bitfield for a given relay-parent hash. +// #[codec(index = 0)] +// Bitfield(Hash, UncheckedSignedAvailabilityBitfield), +// } + +// /// Bitfields indicating the statements that are known or undesired +// /// about a candidate. +// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +// pub struct StatementFilter { +// /// Seconded statements. '1' is known or undesired. +// pub seconded_in_group: BitVec, +// /// Valid statements. '1' is known or undesired. +// pub validated_in_group: BitVec, +// } + +// impl StatementFilter { +// /// Create a new blank filter with the given group size. +// pub fn blank(group_size: usize) -> Self { +// StatementFilter { +// seconded_in_group: BitVec::repeat(false, group_size), +// validated_in_group: BitVec::repeat(false, group_size), +// } +// } + +// /// Create a new full filter with the given group size. +// pub fn full(group_size: usize) -> Self { +// StatementFilter { +// seconded_in_group: BitVec::repeat(true, group_size), +// validated_in_group: BitVec::repeat(true, group_size), +// } +// } + +// /// Whether the filter has a specific expected length, consistent across both +// /// bitfields. +// pub fn has_len(&self, len: usize) -> bool { +// self.seconded_in_group.len() == len && self.validated_in_group.len() == len +// } + +// /// Determine the number of backing validators in the statement filter. +// pub fn backing_validators(&self) -> usize { +// self.seconded_in_group +// .iter() +// .by_vals() +// .zip(self.validated_in_group.iter().by_vals()) +// .filter(|&(s, v)| s || v) // no double-counting +// .count() +// } + +// /// Whether the statement filter has at least one seconded statement. +// pub fn has_seconded(&self) -> bool { +// self.seconded_in_group.iter().by_vals().any(|x| x) +// } + +// /// Mask out `Seconded` statements in `self` according to the provided +// /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. +// pub fn mask_seconded(&mut self, mask: &BitSlice) { +// for (mut x, mask) in self +// .seconded_in_group +// .iter_mut() +// .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) +// { +// // (x, mask) => x +// // (true, true) => false +// // (true, false) => true +// // (false, true) => false +// // (false, false) => false +// *x = *x && !mask; +// } +// } + +// /// Mask out `Valid` statements in `self` according to the provided +// /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. +// pub fn mask_valid(&mut self, mask: &BitSlice) { +// for (mut x, mask) in self +// .validated_in_group +// .iter_mut() +// .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) +// { +// // (x, mask) => x +// // (true, true) => false +// // (true, false) => true +// // (false, true) => false +// // (false, false) => false +// *x = *x && !mask; +// } +// } +// } + +// /// A manifest of a known backed candidate, along with a description +// /// of the statements backing it. +// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +// pub struct BackedCandidateManifest { +// /// The relay-parent of the candidate. +// pub relay_parent: Hash, +// /// The hash of the candidate. +// pub candidate_hash: CandidateHash, +// /// The group index backing the candidate at the relay-parent. +// pub group_index: GroupIndex, +// /// The para ID of the candidate. It is illegal for this to +// /// be a para ID which is not assigned to the group indicated +// /// in this manifest. +// pub para_id: ParaId, +// /// The head-data corresponding to the candidate. +// pub parent_head_data_hash: Hash, +// /// A statement filter which indicates which validators in the +// /// para's group at the relay-parent have validated this candidate +// /// and issued statements about it, to the advertiser's knowledge. +// /// +// /// This MUST have exactly the minimum amount of bytes +// /// necessary to represent the number of validators in the assigned +// /// backing group as-of the relay-parent. +// pub statement_knowledge: StatementFilter, +// } + +// /// An acknowledgement of a backed candidate being known. +// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +// pub struct BackedCandidateAcknowledgement { +// /// The hash of the candidate. +// pub candidate_hash: CandidateHash, +// /// A statement filter which indicates which validators in the +// /// para's group at the relay-parent have validated this candidate +// /// and issued statements about it, to the advertiser's knowledge. +// /// +// /// This MUST have exactly the minimum amount of bytes +// /// necessary to represent the number of validators in the assigned +// /// backing group as-of the relay-parent. +// pub statement_knowledge: StatementFilter, +// } + +// /// Network messages used by the statement distribution subsystem. +// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +// pub enum StatementDistributionMessage { +// /// A notification of a signed statement in compact form, for a given relay parent. +// #[codec(index = 0)] +// Statement(Hash, UncheckedSignedStatement), + +// /// A notification of a backed candidate being known by the +// /// sending node, for the purpose of being requested by the receiving node +// /// if needed. +// #[codec(index = 1)] +// BackedCandidateManifest(BackedCandidateManifest), + +// /// A notification of a backed candidate being known by the sending node, +// /// for the purpose of informing a receiving node which already has the candidate. +// #[codec(index = 2)] +// BackedCandidateKnown(BackedCandidateAcknowledgement), + +// /// All messages for V1 for compatibility with the statement distribution +// /// protocol, for relay-parents that don't support asynchronous backing. +// /// +// /// These are illegal to send to V1 peers, and illegal to send concerning relay-parents +// /// which support asynchronous backing. This backwards compatibility should be +// /// considered immediately deprecated and can be removed once the node software +// /// is not required to support logic from before asynchronous backing anymore. +// #[codec(index = 255)] +// V1Compatibility(crate::v1::StatementDistributionMessage), +// } + +// /// Network messages used by the approval distribution subsystem. +// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +// pub enum ApprovalDistributionMessage { +// /// Assignments for candidates in recent, unfinalized blocks. +// /// We use a bitfield to reference claimed candidates, where the bit index is equal to +// /// candidate index. +// /// +// /// Actually checking the assignment may yield a different result. +// /// TODO: Look at getting rid of bitfield in the future. +// #[codec(index = 0)] +// Assignments(Vec<(IndirectAssignmentCertV2, CandidateBitfield)>), +// /// Approvals for candidates in some recent, unfinalized block. +// #[codec(index = 1)] +// Approvals(Vec), +// } + +// /// Dummy network message type, so we will receive connect/disconnect events. +// #[derive(Debug, Clone, PartialEq, Eq)] +// pub enum GossipSupportNetworkMessage {} + +// /// Network messages used by the collator protocol subsystem +// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +// pub enum CollatorProtocolMessage { +// /// Declare the intent to advertise collations under a collator ID, attaching a +// /// signature of the `PeerId` of the node using the given collator ID key. +// #[codec(index = 0)] +// Declare(CollatorId, ParaId, CollatorSignature), +// /// Advertise a collation to a validator. Can only be sent once the peer has +// /// declared that they are a collator with given ID. +// #[codec(index = 1)] +// AdvertiseCollation { +// /// Hash of the relay parent advertised collation is based on. +// relay_parent: Hash, +// /// Candidate hash. +// candidate_hash: CandidateHash, +// /// Parachain head data hash before candidate execution. +// parent_head_data_hash: Hash, +// }, +// /// A collation sent to a validator was seconded. +// #[codec(index = 4)] +// CollationSeconded(Hash, UncheckedSignedFullStatement), +// } + +// /// All network messages on the validation peer-set. +// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] +// pub enum ValidationProtocol { +// /// Bitfield distribution messages +// #[codec(index = 1)] +// #[from] +// BitfieldDistribution(BitfieldDistributionMessage), +// /// Statement distribution messages +// #[codec(index = 3)] +// #[from] +// StatementDistribution(StatementDistributionMessage), +// /// Approval distribution messages +// #[codec(index = 4)] +// #[from] +// ApprovalDistribution(ApprovalDistributionMessage), +// } + +// /// All network messages on the collation peer-set. +// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] +// pub enum CollationProtocol { +// /// Collator protocol messages +// #[codec(index = 0)] +// #[from] +// CollatorProtocol(CollatorProtocolMessage), +// } + +// /// Get the payload that should be signed and included in a `Declare` message. +// /// +// /// The payload is the local peer id of the node, which serves to prove that it +// /// controls the collator key it is declaring an intention to collate under. +// pub fn declare_signature_payload(peer_id: &sc_network::PeerId) -> Vec { +// let mut payload = peer_id.to_bytes(); +// payload.extend_from_slice(b"COLL"); +// payload +// } +// } /// Returns the subset of `peers` with the specified `version`. pub fn filter_by_peer_version( From 460bab2508c9b8bc2e1c06d1091581857e8d4982 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 29 Sep 2023 12:43:58 +0300 Subject: [PATCH 029/192] Revert "Comment vstaging" This reverts commit 566d7f5c439a356fedae415691d08315ade22735. --- polkadot/node/network/protocol/src/lib.rs | 516 +++++++++++----------- 1 file changed, 258 insertions(+), 258 deletions(-) diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index 5d8b3cd550ca..ba9b9a7f4900 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -588,264 +588,264 @@ pub mod v1 { } } -// /// vstaging network protocol types. -// pub mod vstaging { -// use bitvec::{order::Lsb0, slice::BitSlice, vec::BitVec}; -// use parity_scale_codec::{Decode, Encode}; - -// use polkadot_primitives::vstaging::{ -// CandidateHash, CollatorId, CollatorSignature, GroupIndex, Hash, Id as ParaId, -// UncheckedSignedAvailabilityBitfield, UncheckedSignedStatement, -// }; - -// use polkadot_node_primitives::{ -// approval::{ -// v1::IndirectSignedApprovalVote, -// v2::{CandidateBitfield, IndirectAssignmentCertV2}, -// }, -// UncheckedSignedFullStatement, -// }; - -// /// Network messages used by the bitfield distribution subsystem. -// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -// pub enum BitfieldDistributionMessage { -// /// A signed availability bitfield for a given relay-parent hash. -// #[codec(index = 0)] -// Bitfield(Hash, UncheckedSignedAvailabilityBitfield), -// } - -// /// Bitfields indicating the statements that are known or undesired -// /// about a candidate. -// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -// pub struct StatementFilter { -// /// Seconded statements. '1' is known or undesired. -// pub seconded_in_group: BitVec, -// /// Valid statements. '1' is known or undesired. -// pub validated_in_group: BitVec, -// } - -// impl StatementFilter { -// /// Create a new blank filter with the given group size. -// pub fn blank(group_size: usize) -> Self { -// StatementFilter { -// seconded_in_group: BitVec::repeat(false, group_size), -// validated_in_group: BitVec::repeat(false, group_size), -// } -// } - -// /// Create a new full filter with the given group size. -// pub fn full(group_size: usize) -> Self { -// StatementFilter { -// seconded_in_group: BitVec::repeat(true, group_size), -// validated_in_group: BitVec::repeat(true, group_size), -// } -// } - -// /// Whether the filter has a specific expected length, consistent across both -// /// bitfields. -// pub fn has_len(&self, len: usize) -> bool { -// self.seconded_in_group.len() == len && self.validated_in_group.len() == len -// } - -// /// Determine the number of backing validators in the statement filter. -// pub fn backing_validators(&self) -> usize { -// self.seconded_in_group -// .iter() -// .by_vals() -// .zip(self.validated_in_group.iter().by_vals()) -// .filter(|&(s, v)| s || v) // no double-counting -// .count() -// } - -// /// Whether the statement filter has at least one seconded statement. -// pub fn has_seconded(&self) -> bool { -// self.seconded_in_group.iter().by_vals().any(|x| x) -// } - -// /// Mask out `Seconded` statements in `self` according to the provided -// /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. -// pub fn mask_seconded(&mut self, mask: &BitSlice) { -// for (mut x, mask) in self -// .seconded_in_group -// .iter_mut() -// .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) -// { -// // (x, mask) => x -// // (true, true) => false -// // (true, false) => true -// // (false, true) => false -// // (false, false) => false -// *x = *x && !mask; -// } -// } - -// /// Mask out `Valid` statements in `self` according to the provided -// /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. -// pub fn mask_valid(&mut self, mask: &BitSlice) { -// for (mut x, mask) in self -// .validated_in_group -// .iter_mut() -// .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) -// { -// // (x, mask) => x -// // (true, true) => false -// // (true, false) => true -// // (false, true) => false -// // (false, false) => false -// *x = *x && !mask; -// } -// } -// } - -// /// A manifest of a known backed candidate, along with a description -// /// of the statements backing it. -// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -// pub struct BackedCandidateManifest { -// /// The relay-parent of the candidate. -// pub relay_parent: Hash, -// /// The hash of the candidate. -// pub candidate_hash: CandidateHash, -// /// The group index backing the candidate at the relay-parent. -// pub group_index: GroupIndex, -// /// The para ID of the candidate. It is illegal for this to -// /// be a para ID which is not assigned to the group indicated -// /// in this manifest. -// pub para_id: ParaId, -// /// The head-data corresponding to the candidate. -// pub parent_head_data_hash: Hash, -// /// A statement filter which indicates which validators in the -// /// para's group at the relay-parent have validated this candidate -// /// and issued statements about it, to the advertiser's knowledge. -// /// -// /// This MUST have exactly the minimum amount of bytes -// /// necessary to represent the number of validators in the assigned -// /// backing group as-of the relay-parent. -// pub statement_knowledge: StatementFilter, -// } - -// /// An acknowledgement of a backed candidate being known. -// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -// pub struct BackedCandidateAcknowledgement { -// /// The hash of the candidate. -// pub candidate_hash: CandidateHash, -// /// A statement filter which indicates which validators in the -// /// para's group at the relay-parent have validated this candidate -// /// and issued statements about it, to the advertiser's knowledge. -// /// -// /// This MUST have exactly the minimum amount of bytes -// /// necessary to represent the number of validators in the assigned -// /// backing group as-of the relay-parent. -// pub statement_knowledge: StatementFilter, -// } - -// /// Network messages used by the statement distribution subsystem. -// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -// pub enum StatementDistributionMessage { -// /// A notification of a signed statement in compact form, for a given relay parent. -// #[codec(index = 0)] -// Statement(Hash, UncheckedSignedStatement), - -// /// A notification of a backed candidate being known by the -// /// sending node, for the purpose of being requested by the receiving node -// /// if needed. -// #[codec(index = 1)] -// BackedCandidateManifest(BackedCandidateManifest), - -// /// A notification of a backed candidate being known by the sending node, -// /// for the purpose of informing a receiving node which already has the candidate. -// #[codec(index = 2)] -// BackedCandidateKnown(BackedCandidateAcknowledgement), - -// /// All messages for V1 for compatibility with the statement distribution -// /// protocol, for relay-parents that don't support asynchronous backing. -// /// -// /// These are illegal to send to V1 peers, and illegal to send concerning relay-parents -// /// which support asynchronous backing. This backwards compatibility should be -// /// considered immediately deprecated and can be removed once the node software -// /// is not required to support logic from before asynchronous backing anymore. -// #[codec(index = 255)] -// V1Compatibility(crate::v1::StatementDistributionMessage), -// } - -// /// Network messages used by the approval distribution subsystem. -// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -// pub enum ApprovalDistributionMessage { -// /// Assignments for candidates in recent, unfinalized blocks. -// /// We use a bitfield to reference claimed candidates, where the bit index is equal to -// /// candidate index. -// /// -// /// Actually checking the assignment may yield a different result. -// /// TODO: Look at getting rid of bitfield in the future. -// #[codec(index = 0)] -// Assignments(Vec<(IndirectAssignmentCertV2, CandidateBitfield)>), -// /// Approvals for candidates in some recent, unfinalized block. -// #[codec(index = 1)] -// Approvals(Vec), -// } - -// /// Dummy network message type, so we will receive connect/disconnect events. -// #[derive(Debug, Clone, PartialEq, Eq)] -// pub enum GossipSupportNetworkMessage {} - -// /// Network messages used by the collator protocol subsystem -// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -// pub enum CollatorProtocolMessage { -// /// Declare the intent to advertise collations under a collator ID, attaching a -// /// signature of the `PeerId` of the node using the given collator ID key. -// #[codec(index = 0)] -// Declare(CollatorId, ParaId, CollatorSignature), -// /// Advertise a collation to a validator. Can only be sent once the peer has -// /// declared that they are a collator with given ID. -// #[codec(index = 1)] -// AdvertiseCollation { -// /// Hash of the relay parent advertised collation is based on. -// relay_parent: Hash, -// /// Candidate hash. -// candidate_hash: CandidateHash, -// /// Parachain head data hash before candidate execution. -// parent_head_data_hash: Hash, -// }, -// /// A collation sent to a validator was seconded. -// #[codec(index = 4)] -// CollationSeconded(Hash, UncheckedSignedFullStatement), -// } - -// /// All network messages on the validation peer-set. -// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] -// pub enum ValidationProtocol { -// /// Bitfield distribution messages -// #[codec(index = 1)] -// #[from] -// BitfieldDistribution(BitfieldDistributionMessage), -// /// Statement distribution messages -// #[codec(index = 3)] -// #[from] -// StatementDistribution(StatementDistributionMessage), -// /// Approval distribution messages -// #[codec(index = 4)] -// #[from] -// ApprovalDistribution(ApprovalDistributionMessage), -// } - -// /// All network messages on the collation peer-set. -// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] -// pub enum CollationProtocol { -// /// Collator protocol messages -// #[codec(index = 0)] -// #[from] -// CollatorProtocol(CollatorProtocolMessage), -// } - -// /// Get the payload that should be signed and included in a `Declare` message. -// /// -// /// The payload is the local peer id of the node, which serves to prove that it -// /// controls the collator key it is declaring an intention to collate under. -// pub fn declare_signature_payload(peer_id: &sc_network::PeerId) -> Vec { -// let mut payload = peer_id.to_bytes(); -// payload.extend_from_slice(b"COLL"); -// payload -// } -// } +/// vstaging network protocol types. +pub mod vstaging { + use bitvec::{order::Lsb0, slice::BitSlice, vec::BitVec}; + use parity_scale_codec::{Decode, Encode}; + + use polkadot_primitives::vstaging::{ + CandidateHash, CollatorId, CollatorSignature, GroupIndex, Hash, Id as ParaId, + UncheckedSignedAvailabilityBitfield, UncheckedSignedStatement, + }; + + use polkadot_node_primitives::{ + approval::{ + v1::IndirectSignedApprovalVote, + v2::{CandidateBitfield, IndirectAssignmentCertV2}, + }, + UncheckedSignedFullStatement, + }; + + /// Network messages used by the bitfield distribution subsystem. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum BitfieldDistributionMessage { + /// A signed availability bitfield for a given relay-parent hash. + #[codec(index = 0)] + Bitfield(Hash, UncheckedSignedAvailabilityBitfield), + } + + /// Bitfields indicating the statements that are known or undesired + /// about a candidate. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct StatementFilter { + /// Seconded statements. '1' is known or undesired. + pub seconded_in_group: BitVec, + /// Valid statements. '1' is known or undesired. + pub validated_in_group: BitVec, + } + + impl StatementFilter { + /// Create a new blank filter with the given group size. + pub fn blank(group_size: usize) -> Self { + StatementFilter { + seconded_in_group: BitVec::repeat(false, group_size), + validated_in_group: BitVec::repeat(false, group_size), + } + } + + /// Create a new full filter with the given group size. + pub fn full(group_size: usize) -> Self { + StatementFilter { + seconded_in_group: BitVec::repeat(true, group_size), + validated_in_group: BitVec::repeat(true, group_size), + } + } + + /// Whether the filter has a specific expected length, consistent across both + /// bitfields. + pub fn has_len(&self, len: usize) -> bool { + self.seconded_in_group.len() == len && self.validated_in_group.len() == len + } + + /// Determine the number of backing validators in the statement filter. + pub fn backing_validators(&self) -> usize { + self.seconded_in_group + .iter() + .by_vals() + .zip(self.validated_in_group.iter().by_vals()) + .filter(|&(s, v)| s || v) // no double-counting + .count() + } + + /// Whether the statement filter has at least one seconded statement. + pub fn has_seconded(&self) -> bool { + self.seconded_in_group.iter().by_vals().any(|x| x) + } + + /// Mask out `Seconded` statements in `self` according to the provided + /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. + pub fn mask_seconded(&mut self, mask: &BitSlice) { + for (mut x, mask) in self + .seconded_in_group + .iter_mut() + .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) + { + // (x, mask) => x + // (true, true) => false + // (true, false) => true + // (false, true) => false + // (false, false) => false + *x = *x && !mask; + } + } + + /// Mask out `Valid` statements in `self` according to the provided + /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. + pub fn mask_valid(&mut self, mask: &BitSlice) { + for (mut x, mask) in self + .validated_in_group + .iter_mut() + .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) + { + // (x, mask) => x + // (true, true) => false + // (true, false) => true + // (false, true) => false + // (false, false) => false + *x = *x && !mask; + } + } + } + + /// A manifest of a known backed candidate, along with a description + /// of the statements backing it. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct BackedCandidateManifest { + /// The relay-parent of the candidate. + pub relay_parent: Hash, + /// The hash of the candidate. + pub candidate_hash: CandidateHash, + /// The group index backing the candidate at the relay-parent. + pub group_index: GroupIndex, + /// The para ID of the candidate. It is illegal for this to + /// be a para ID which is not assigned to the group indicated + /// in this manifest. + pub para_id: ParaId, + /// The head-data corresponding to the candidate. + pub parent_head_data_hash: Hash, + /// A statement filter which indicates which validators in the + /// para's group at the relay-parent have validated this candidate + /// and issued statements about it, to the advertiser's knowledge. + /// + /// This MUST have exactly the minimum amount of bytes + /// necessary to represent the number of validators in the assigned + /// backing group as-of the relay-parent. + pub statement_knowledge: StatementFilter, + } + + /// An acknowledgement of a backed candidate being known. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct BackedCandidateAcknowledgement { + /// The hash of the candidate. + pub candidate_hash: CandidateHash, + /// A statement filter which indicates which validators in the + /// para's group at the relay-parent have validated this candidate + /// and issued statements about it, to the advertiser's knowledge. + /// + /// This MUST have exactly the minimum amount of bytes + /// necessary to represent the number of validators in the assigned + /// backing group as-of the relay-parent. + pub statement_knowledge: StatementFilter, + } + + /// Network messages used by the statement distribution subsystem. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum StatementDistributionMessage { + /// A notification of a signed statement in compact form, for a given relay parent. + #[codec(index = 0)] + Statement(Hash, UncheckedSignedStatement), + + /// A notification of a backed candidate being known by the + /// sending node, for the purpose of being requested by the receiving node + /// if needed. + #[codec(index = 1)] + BackedCandidateManifest(BackedCandidateManifest), + + /// A notification of a backed candidate being known by the sending node, + /// for the purpose of informing a receiving node which already has the candidate. + #[codec(index = 2)] + BackedCandidateKnown(BackedCandidateAcknowledgement), + + /// All messages for V1 for compatibility with the statement distribution + /// protocol, for relay-parents that don't support asynchronous backing. + /// + /// These are illegal to send to V1 peers, and illegal to send concerning relay-parents + /// which support asynchronous backing. This backwards compatibility should be + /// considered immediately deprecated and can be removed once the node software + /// is not required to support logic from before asynchronous backing anymore. + #[codec(index = 255)] + V1Compatibility(crate::v1::StatementDistributionMessage), + } + + /// Network messages used by the approval distribution subsystem. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum ApprovalDistributionMessage { + /// Assignments for candidates in recent, unfinalized blocks. + /// We use a bitfield to reference claimed candidates, where the bit index is equal to + /// candidate index. + /// + /// Actually checking the assignment may yield a different result. + /// TODO: Look at getting rid of bitfield in the future. + #[codec(index = 0)] + Assignments(Vec<(IndirectAssignmentCertV2, CandidateBitfield)>), + /// Approvals for candidates in some recent, unfinalized block. + #[codec(index = 1)] + Approvals(Vec), + } + + /// Dummy network message type, so we will receive connect/disconnect events. + #[derive(Debug, Clone, PartialEq, Eq)] + pub enum GossipSupportNetworkMessage {} + + /// Network messages used by the collator protocol subsystem + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum CollatorProtocolMessage { + /// Declare the intent to advertise collations under a collator ID, attaching a + /// signature of the `PeerId` of the node using the given collator ID key. + #[codec(index = 0)] + Declare(CollatorId, ParaId, CollatorSignature), + /// Advertise a collation to a validator. Can only be sent once the peer has + /// declared that they are a collator with given ID. + #[codec(index = 1)] + AdvertiseCollation { + /// Hash of the relay parent advertised collation is based on. + relay_parent: Hash, + /// Candidate hash. + candidate_hash: CandidateHash, + /// Parachain head data hash before candidate execution. + parent_head_data_hash: Hash, + }, + /// A collation sent to a validator was seconded. + #[codec(index = 4)] + CollationSeconded(Hash, UncheckedSignedFullStatement), + } + + /// All network messages on the validation peer-set. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] + pub enum ValidationProtocol { + /// Bitfield distribution messages + #[codec(index = 1)] + #[from] + BitfieldDistribution(BitfieldDistributionMessage), + /// Statement distribution messages + #[codec(index = 3)] + #[from] + StatementDistribution(StatementDistributionMessage), + /// Approval distribution messages + #[codec(index = 4)] + #[from] + ApprovalDistribution(ApprovalDistributionMessage), + } + + /// All network messages on the collation peer-set. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] + pub enum CollationProtocol { + /// Collator protocol messages + #[codec(index = 0)] + #[from] + CollatorProtocol(CollatorProtocolMessage), + } + + /// Get the payload that should be signed and included in a `Declare` message. + /// + /// The payload is the local peer id of the node, which serves to prove that it + /// controls the collator key it is declaring an intention to collate under. + pub fn declare_signature_payload(peer_id: &sc_network::PeerId) -> Vec { + let mut payload = peer_id.to_bytes(); + payload.extend_from_slice(b"COLL"); + payload + } +} /// Returns the subset of `peers` with the specified `version`. pub fn filter_by_peer_version( From 85acb511da241c80c27ea47c823ed69ad3d0b2f2 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 29 Sep 2023 12:58:59 +0300 Subject: [PATCH 030/192] Add v3 protocol Signed-off-by: Alexandru Gheorghe --- polkadot/node/network/protocol/src/lib.rs | 309 ++++++++++++++++++++-- 1 file changed, 294 insertions(+), 15 deletions(-) diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index ba9b9a7f4900..a17a483b97de 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -253,18 +253,21 @@ impl View { /// A protocol-versioned type. #[derive(Debug, Clone, PartialEq, Eq)] -pub enum Versioned { +pub enum Versioned { /// V1 type. V1(V1), + /// V2 type. + V2(V2), /// VStaging type. VStaging(VStaging), } -impl Versioned<&'_ V1, &'_ VStaging> { +impl Versioned<&'_ V1, &'_ V2, &'_ VStaging> { /// Convert to a fully-owned version of the message. - pub fn clone_inner(&self) -> Versioned { + pub fn clone_inner(&self) -> Versioned { match *self { Versioned::V1(inner) => Versioned::V1(inner.clone()), + Versioned::V2(inner) => Versioned::V2(inner.clone()), Versioned::VStaging(inner) => Versioned::VStaging(inner.clone()), } } @@ -272,7 +275,7 @@ impl Versioned<&'_ V1, &'_ VStaging> { /// All supported versions of the validation protocol message. pub type VersionedValidationProtocol = - Versioned; + Versioned; impl From for VersionedValidationProtocol { fn from(v1: v1::ValidationProtocol) -> Self { @@ -287,7 +290,8 @@ impl From for VersionedValidationProtocol { } /// All supported versions of the collation protocol message. -pub type VersionedCollationProtocol = Versioned; +pub type VersionedCollationProtocol = + Versioned; impl From for VersionedCollationProtocol { fn from(v1: v1::CollationProtocol) -> Self { @@ -307,6 +311,7 @@ macro_rules! impl_versioned_full_protocol_from { fn from(versioned_from: $from) -> $out { match versioned_from { Versioned::V1(x) => Versioned::V1(x.into()), + Versioned::V2(x) => Versioned::V2(x.into()), Versioned::VStaging(x) => Versioned::VStaging(x.into()), } } @@ -320,6 +325,7 @@ macro_rules! impl_versioned_try_from { $from:ty, $out:ty, $v1_pat:pat => $v1_out:expr, + $v2_pat:pat => $v2_out:expr, $vstaging_pat:pat => $vstaging_out:expr ) => { impl TryFrom<$from> for $out { @@ -329,6 +335,7 @@ macro_rules! impl_versioned_try_from { #[allow(unreachable_patterns)] // when there is only one variant match x { Versioned::V1($v1_pat) => Ok(Versioned::V1($v1_out)), + Versioned::V2($v2_pat) => Ok(Versioned::V2($v2_out)), Versioned::VStaging($vstaging_pat) => Ok(Versioned::VStaging($vstaging_out)), _ => Err(crate::WrongVariant), } @@ -352,8 +359,11 @@ macro_rules! impl_versioned_try_from { } /// Version-annotated messages used by the bitfield distribution subsystem. -pub type BitfieldDistributionMessage = - Versioned; +pub type BitfieldDistributionMessage = Versioned< + v1::BitfieldDistributionMessage, + v2::BitfieldDistributionMessage, + vstaging::BitfieldDistributionMessage, +>; impl_versioned_full_protocol_from!( BitfieldDistributionMessage, VersionedValidationProtocol, @@ -363,12 +373,16 @@ impl_versioned_try_from!( VersionedValidationProtocol, BitfieldDistributionMessage, v1::ValidationProtocol::BitfieldDistribution(x) => x, + v2::ValidationProtocol::BitfieldDistribution(x) => x, vstaging::ValidationProtocol::BitfieldDistribution(x) => x ); /// Version-annotated messages used by the statement distribution subsystem. -pub type StatementDistributionMessage = - Versioned; +pub type StatementDistributionMessage = Versioned< + v1::StatementDistributionMessage, + v2::StatementDistributionMessage, + vstaging::StatementDistributionMessage, +>; impl_versioned_full_protocol_from!( StatementDistributionMessage, VersionedValidationProtocol, @@ -378,12 +392,16 @@ impl_versioned_try_from!( VersionedValidationProtocol, StatementDistributionMessage, v1::ValidationProtocol::StatementDistribution(x) => x, + v2::ValidationProtocol::StatementDistribution(x) => x, vstaging::ValidationProtocol::StatementDistribution(x) => x ); /// Version-annotated messages used by the approval distribution subsystem. -pub type ApprovalDistributionMessage = - Versioned; +pub type ApprovalDistributionMessage = Versioned< + v1::ApprovalDistributionMessage, + v2::ApprovalDistributionMessage, + vstaging::ApprovalDistributionMessage, +>; impl_versioned_full_protocol_from!( ApprovalDistributionMessage, VersionedValidationProtocol, @@ -393,13 +411,17 @@ impl_versioned_try_from!( VersionedValidationProtocol, ApprovalDistributionMessage, v1::ValidationProtocol::ApprovalDistribution(x) => x, + v2::ValidationProtocol::ApprovalDistribution(x) => x, vstaging::ValidationProtocol::ApprovalDistribution(x) => x ); /// Version-annotated messages used by the gossip-support subsystem (this is void). -pub type GossipSupportNetworkMessage = - Versioned; +pub type GossipSupportNetworkMessage = Versioned< + v1::GossipSupportNetworkMessage, + v2::GossipSupportNetworkMessage, + vstaging::GossipSupportNetworkMessage, +>; // This is a void enum placeholder, so never gets sent over the wire. impl TryFrom for GossipSupportNetworkMessage { type Error = WrongVariant; @@ -416,8 +438,11 @@ impl<'a> TryFrom<&'a VersionedValidationProtocol> for GossipSupportNetworkMessag } /// Version-annotated messages used by the bitfield distribution subsystem. -pub type CollatorProtocolMessage = - Versioned; +pub type CollatorProtocolMessage = Versioned< + v1::CollatorProtocolMessage, + v2::CollatorProtocolMessage, + vstaging::CollatorProtocolMessage, +>; impl_versioned_full_protocol_from!( CollatorProtocolMessage, VersionedCollationProtocol, @@ -427,6 +452,7 @@ impl_versioned_try_from!( VersionedCollationProtocol, CollatorProtocolMessage, v1::CollationProtocol::CollatorProtocol(x) => x, + v2::CollationProtocol::CollatorProtocol(x) => x, vstaging::CollationProtocol::CollatorProtocol(x) => x ); @@ -588,6 +614,259 @@ pub mod v1 { } } +/// v2 network protocol types. +pub mod v2 { + use bitvec::{order::Lsb0, slice::BitSlice, vec::BitVec}; + use parity_scale_codec::{Decode, Encode}; + + use polkadot_primitives::{ + CandidateHash, CandidateIndex, CollatorId, CollatorSignature, GroupIndex, Hash, + Id as ParaId, UncheckedSignedAvailabilityBitfield, UncheckedSignedStatement, + }; + + use polkadot_node_primitives::{ + approval::{IndirectAssignmentCert, IndirectSignedApprovalVote}, + UncheckedSignedFullStatement, + }; + + /// Network messages used by the bitfield distribution subsystem. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum BitfieldDistributionMessage { + /// A signed availability bitfield for a given relay-parent hash. + #[codec(index = 0)] + Bitfield(Hash, UncheckedSignedAvailabilityBitfield), + } + + /// Bitfields indicating the statements that are known or undesired + /// about a candidate. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct StatementFilter { + /// Seconded statements. '1' is known or undesired. + pub seconded_in_group: BitVec, + /// Valid statements. '1' is known or undesired. + pub validated_in_group: BitVec, + } + + impl StatementFilter { + /// Create a new blank filter with the given group size. + pub fn blank(group_size: usize) -> Self { + StatementFilter { + seconded_in_group: BitVec::repeat(false, group_size), + validated_in_group: BitVec::repeat(false, group_size), + } + } + + /// Create a new full filter with the given group size. + pub fn full(group_size: usize) -> Self { + StatementFilter { + seconded_in_group: BitVec::repeat(true, group_size), + validated_in_group: BitVec::repeat(true, group_size), + } + } + + /// Whether the filter has a specific expected length, consistent across both + /// bitfields. + pub fn has_len(&self, len: usize) -> bool { + self.seconded_in_group.len() == len && self.validated_in_group.len() == len + } + + /// Determine the number of backing validators in the statement filter. + pub fn backing_validators(&self) -> usize { + self.seconded_in_group + .iter() + .by_vals() + .zip(self.validated_in_group.iter().by_vals()) + .filter(|&(s, v)| s || v) // no double-counting + .count() + } + + /// Whether the statement filter has at least one seconded statement. + pub fn has_seconded(&self) -> bool { + self.seconded_in_group.iter().by_vals().any(|x| x) + } + + /// Mask out `Seconded` statements in `self` according to the provided + /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. + pub fn mask_seconded(&mut self, mask: &BitSlice) { + for (mut x, mask) in self + .seconded_in_group + .iter_mut() + .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) + { + // (x, mask) => x + // (true, true) => false + // (true, false) => true + // (false, true) => false + // (false, false) => false + *x = *x && !mask; + } + } + + /// Mask out `Valid` statements in `self` according to the provided + /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. + pub fn mask_valid(&mut self, mask: &BitSlice) { + for (mut x, mask) in self + .validated_in_group + .iter_mut() + .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) + { + // (x, mask) => x + // (true, true) => false + // (true, false) => true + // (false, true) => false + // (false, false) => false + *x = *x && !mask; + } + } + } + + /// A manifest of a known backed candidate, along with a description + /// of the statements backing it. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct BackedCandidateManifest { + /// The relay-parent of the candidate. + pub relay_parent: Hash, + /// The hash of the candidate. + pub candidate_hash: CandidateHash, + /// The group index backing the candidate at the relay-parent. + pub group_index: GroupIndex, + /// The para ID of the candidate. It is illegal for this to + /// be a para ID which is not assigned to the group indicated + /// in this manifest. + pub para_id: ParaId, + /// The head-data corresponding to the candidate. + pub parent_head_data_hash: Hash, + /// A statement filter which indicates which validators in the + /// para's group at the relay-parent have validated this candidate + /// and issued statements about it, to the advertiser's knowledge. + /// + /// This MUST have exactly the minimum amount of bytes + /// necessary to represent the number of validators in the assigned + /// backing group as-of the relay-parent. + pub statement_knowledge: StatementFilter, + } + + /// An acknowledgement of a backed candidate being known. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct BackedCandidateAcknowledgement { + /// The hash of the candidate. + pub candidate_hash: CandidateHash, + /// A statement filter which indicates which validators in the + /// para's group at the relay-parent have validated this candidate + /// and issued statements about it, to the advertiser's knowledge. + /// + /// This MUST have exactly the minimum amount of bytes + /// necessary to represent the number of validators in the assigned + /// backing group as-of the relay-parent. + pub statement_knowledge: StatementFilter, + } + + /// Network messages used by the statement distribution subsystem. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum StatementDistributionMessage { + /// A notification of a signed statement in compact form, for a given relay parent. + #[codec(index = 0)] + Statement(Hash, UncheckedSignedStatement), + + /// A notification of a backed candidate being known by the + /// sending node, for the purpose of being requested by the receiving node + /// if needed. + #[codec(index = 1)] + BackedCandidateManifest(BackedCandidateManifest), + + /// A notification of a backed candidate being known by the sending node, + /// for the purpose of informing a receiving node which already has the candidate. + #[codec(index = 2)] + BackedCandidateKnown(BackedCandidateAcknowledgement), + + /// All messages for V1 for compatibility with the statement distribution + /// protocol, for relay-parents that don't support asynchronous backing. + /// + /// These are illegal to send to V1 peers, and illegal to send concerning relay-parents + /// which support asynchronous backing. This backwards compatibility should be + /// considered immediately deprecated and can be removed once the node software + /// is not required to support logic from before asynchronous backing anymore. + #[codec(index = 255)] + V1Compatibility(crate::v1::StatementDistributionMessage), + } + + /// Network messages used by the approval distribution subsystem. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum ApprovalDistributionMessage { + /// Assignments for candidates in recent, unfinalized blocks. + /// + /// Actually checking the assignment may yield a different result. + #[codec(index = 0)] + Assignments(Vec<(IndirectAssignmentCert, CandidateIndex)>), + /// Approvals for candidates in some recent, unfinalized block. + #[codec(index = 1)] + Approvals(Vec), + } + + /// Dummy network message type, so we will receive connect/disconnect events. + #[derive(Debug, Clone, PartialEq, Eq)] + pub enum GossipSupportNetworkMessage {} + + /// Network messages used by the collator protocol subsystem + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum CollatorProtocolMessage { + /// Declare the intent to advertise collations under a collator ID, attaching a + /// signature of the `PeerId` of the node using the given collator ID key. + #[codec(index = 0)] + Declare(CollatorId, ParaId, CollatorSignature), + /// Advertise a collation to a validator. Can only be sent once the peer has + /// declared that they are a collator with given ID. + #[codec(index = 1)] + AdvertiseCollation { + /// Hash of the relay parent advertised collation is based on. + relay_parent: Hash, + /// Candidate hash. + candidate_hash: CandidateHash, + /// Parachain head data hash before candidate execution. + parent_head_data_hash: Hash, + }, + /// A collation sent to a validator was seconded. + #[codec(index = 4)] + CollationSeconded(Hash, UncheckedSignedFullStatement), + } + + /// All network messages on the validation peer-set. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] + pub enum ValidationProtocol { + /// Bitfield distribution messages + #[codec(index = 1)] + #[from] + BitfieldDistribution(BitfieldDistributionMessage), + /// Statement distribution messages + #[codec(index = 3)] + #[from] + StatementDistribution(StatementDistributionMessage), + /// Approval distribution messages + #[codec(index = 4)] + #[from] + ApprovalDistribution(ApprovalDistributionMessage), + } + + /// All network messages on the collation peer-set. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] + pub enum CollationProtocol { + /// Collator protocol messages + #[codec(index = 0)] + #[from] + CollatorProtocol(CollatorProtocolMessage), + } + + /// Get the payload that should be signed and included in a `Declare` message. + /// + /// The payload is the local peer id of the node, which serves to prove that it + /// controls the collator key it is declaring an intention to collate under. + pub fn declare_signature_payload(peer_id: &sc_network::PeerId) -> Vec { + let mut payload = peer_id.to_bytes(); + payload.extend_from_slice(b"COLL"); + payload + } +} + /// vstaging network protocol types. pub mod vstaging { use bitvec::{order::Lsb0, slice::BitSlice, vec::BitVec}; From 56055c1c37dab805a2f6c6dd68b01d9570dfa1ba Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Sat, 30 Sep 2023 17:09:40 +0300 Subject: [PATCH 031/192] Fixup Signed-off-by: Alexandru Gheorghe --- polkadot/Cargo.toml | 2 + .../network/bitfield-distribution/src/lib.rs | 40 ++- polkadot/node/network/bridge/src/network.rs | 2 +- polkadot/node/network/bridge/src/rx/mod.rs | 59 +++- .../src/collator_side/mod.rs | 24 +- .../src/validator_side/mod.rs | 3 +- polkadot/node/network/protocol/src/lib.rs | 7 + .../node/network/protocol/src/peer_set.rs | 6 +- .../src/legacy_v1/mod.rs | 47 ++- .../statement-distribution/src/v2/mod.rs | 271 +++++++++++++----- .../undying/collator/Cargo.toml | 3 + .../0006-parachains-max-tranche0.toml | 2 +- 12 files changed, 361 insertions(+), 105 deletions(-) diff --git a/polkadot/Cargo.toml b/polkadot/Cargo.toml index 6e82cb69f6ec..06ac13f1bf87 100644 --- a/polkadot/Cargo.toml +++ b/polkadot/Cargo.toml @@ -64,6 +64,8 @@ jemalloc-allocator = [ "polkadot-node-core-pvf/jemalloc-allocator", "polkadot-overseer/jemalloc-allocator", ] +network-protocol-staging = [ "polkadot-cli/network-protocol-staging" ] + # Enables timeout-based tests supposed to be run only in CI environment as they may be flaky # when run locally depending on system load diff --git a/polkadot/node/network/bitfield-distribution/src/lib.rs b/polkadot/node/network/bitfield-distribution/src/lib.rs index 7a1a32770a2a..9cc79aee8490 100644 --- a/polkadot/node/network/bitfield-distribution/src/lib.rs +++ b/polkadot/node/network/bitfield-distribution/src/lib.rs @@ -25,14 +25,15 @@ use always_assert::never; use futures::{channel::oneshot, FutureExt}; +use net_protocol::filter_by_peer_version; use polkadot_node_network_protocol::{ self as net_protocol, grid_topology::{ GridNeighbors, RandomRouting, RequiredRouting, SessionBoundGridTopologyStorage, }, peer_set::{ProtocolVersion, ValidationVersion}, - v1 as protocol_v1, v2 as protocol_v2, OurView, PeerId, UnifiedReputationChange as Rep, - Versioned, View, + v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, OurView, PeerId, + UnifiedReputationChange as Rep, Versioned, View, }; use polkadot_node_subsystem::{ jaeger, messages::*, overseer, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, PerLeafSpan, @@ -96,11 +97,16 @@ impl BitfieldGossipMessage { self.relay_parent, self.signed_availability.into(), )), - Some(ValidationVersion::V2) | Some(ValidationVersion::VStaging) => + Some(ValidationVersion::V2) => Versioned::V2(protocol_v2::BitfieldDistributionMessage::Bitfield( self.relay_parent, self.signed_availability.into(), )), + Some(ValidationVersion::VStaging) => + Versioned::VStaging(protocol_vstaging::BitfieldDistributionMessage::Bitfield( + self.relay_parent, + self.signed_availability.into(), + )), None => { never!("Peers should only have supported protocol versions."); @@ -492,17 +498,13 @@ async fn relay_message( } else { let _span = span.child("gossip"); - let filter_by_version = |peers: &[(PeerId, ProtocolVersion)], - version: ValidationVersion| { - peers - .iter() - .filter(|(_, v)| v == &version.into()) - .map(|(peer_id, _)| *peer_id) - .collect::>() - }; + let v1_interested_peers = + filter_by_peer_version(&interested_peers, ValidationVersion::V1.into()); + let v2_interested_peers = + filter_by_peer_version(&interested_peers, ValidationVersion::V2.into()); - let v1_interested_peers = filter_by_version(&interested_peers, ValidationVersion::V1); - let v2_interested_peers = filter_by_version(&interested_peers, ValidationVersion::V2); + let vstaging_interested_peers = + filter_by_peer_version(&interested_peers, ValidationVersion::VStaging.into()); if !v1_interested_peers.is_empty() { ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( @@ -515,7 +517,15 @@ async fn relay_message( if !v2_interested_peers.is_empty() { ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( v2_interested_peers, - message.into_validation_protocol(ValidationVersion::V2.into()), + message.clone().into_validation_protocol(ValidationVersion::V2.into()), + )) + .await + } + + if !vstaging_interested_peers.is_empty() { + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + vstaging_interested_peers, + message.into_validation_protocol(ValidationVersion::VStaging.into()), )) .await } @@ -541,7 +551,7 @@ async fn process_incoming_peer_message( relay_parent, bitfield, )) | - Versioned::VStaging(protocol_v2::BitfieldDistributionMessage::Bitfield( + Versioned::VStaging(protocol_vstaging::BitfieldDistributionMessage::Bitfield( relay_parent, bitfield, )) => (relay_parent, bitfield), diff --git a/polkadot/node/network/bridge/src/network.rs b/polkadot/node/network/bridge/src/network.rs index 3cf6741d4293..a13045285522 100644 --- a/polkadot/node/network/bridge/src/network.rs +++ b/polkadot/node/network/bridge/src/network.rs @@ -72,7 +72,7 @@ pub(crate) fn send_validation_message_vstaging( message: WireMessage, metrics: &Metrics, ) { - gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation v2 message to peers",); + gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation vstaging message to peers",); send_message( net, diff --git a/polkadot/node/network/bridge/src/rx/mod.rs b/polkadot/node/network/bridge/src/rx/mod.rs index 0d32ece88e28..83bf10bbb670 100644 --- a/polkadot/node/network/bridge/src/rx/mod.rs +++ b/polkadot/node/network/bridge/src/rx/mod.rs @@ -21,6 +21,7 @@ use super::*; use always_assert::never; use bytes::Bytes; use futures::stream::{BoxStream, StreamExt}; +use net_protocol::filter_by_peer_version; use parity_scale_codec::{Decode, DecodeAll}; use sc_network::Event as NetworkEvent; @@ -323,7 +324,7 @@ where &mut network_service, vec![peer], &peerset_protocol_names, - WireMessage::::ViewUpdate( + WireMessage::::ViewUpdate( local_view, ), &metrics, @@ -495,6 +496,16 @@ where v_messages, &metrics, ) + } else if expected_versions[PeerSet::Validation] == + Some(ValidationVersion::VStaging.into()) + { + handle_peer_messages::( + remote, + PeerSet::Validation, + &mut shared.0.lock().validation_peers, + v_messages, + &metrics, + ) } else { gum::warn!( target: LOG_TARGET, @@ -537,6 +548,16 @@ where c_messages, &metrics, ) + } else if expected_versions[PeerSet::Collation] == + Some(CollationVersion::VStaging.into()) + { + handle_peer_messages::( + remote, + PeerSet::Collation, + &mut shared.0.lock().collation_peers, + c_messages, + &metrics, + ) } else { gum::warn!( target: LOG_TARGET, @@ -827,15 +848,18 @@ fn update_our_view( ) }; - let filter_by_version = |peers: &[(PeerId, ProtocolVersion)], version| { - peers.iter().filter(|(_, v)| v == &version).map(|(p, _)| *p).collect::>() - }; + let v1_validation_peers = + filter_by_peer_version(&validation_peers, ValidationVersion::V1.into()); + let v1_collation_peers = filter_by_peer_version(&collation_peers, CollationVersion::V1.into()); - let v1_validation_peers = filter_by_version(&validation_peers, ValidationVersion::V1.into()); - let v1_collation_peers = filter_by_version(&collation_peers, CollationVersion::V1.into()); + let v2_validation_peers = + filter_by_peer_version(&validation_peers, ValidationVersion::V2.into()); + let v2_collation_peers = filter_by_peer_version(&collation_peers, CollationVersion::V2.into()); - let v2_validation_peers = filter_by_version(&validation_peers, ValidationVersion::V2.into()); - let v2_collation_peers = filter_by_version(&collation_peers, ValidationVersion::V2.into()); + let vstaging_validation_peers = + filter_by_peer_version(&validation_peers, ValidationVersion::VStaging.into()); + let vstaging_collation_peers = + filter_by_peer_version(&collation_peers, CollationVersion::VStaging.into()); send_validation_message_v1( net, @@ -853,7 +877,7 @@ fn update_our_view( metrics, ); - send_validation_message_vstaging( + send_validation_message_v2( net, v2_validation_peers, peerset_protocol_names, @@ -865,10 +889,25 @@ fn update_our_view( net, v2_collation_peers, peerset_protocol_names, - WireMessage::ViewUpdate(new_view), + WireMessage::ViewUpdate(new_view.clone()), + metrics, + ); + + send_validation_message_vstaging( + net, + vstaging_validation_peers, + peerset_protocol_names, + WireMessage::ViewUpdate(new_view.clone()), metrics, ); + send_collation_message_vstaging( + net, + vstaging_collation_peers, + peerset_protocol_names, + WireMessage::ViewUpdate(new_view.clone()), + metrics, + ); let our_view = OurView::new( live_heads.iter().take(MAX_VIEW_HEADS).cloned().map(|a| (a.hash, a.span)), finalized_number, diff --git a/polkadot/node/network/collator-protocol/src/collator_side/mod.rs b/polkadot/node/network/collator-protocol/src/collator_side/mod.rs index ba9f5c0120e7..1c40759dde7f 100644 --- a/polkadot/node/network/collator-protocol/src/collator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/collator_side/mod.rs @@ -596,7 +596,7 @@ fn declare_message( ); Versioned::V1(protocol_v1::CollationProtocol::CollatorProtocol(wire_message)) }, - CollationVersion::V2 | CollationVersion::VStaging => { + CollationVersion::V2 => { let declare_signature_payload = protocol_v2::declare_signature_payload(&state.local_peer_id); let wire_message = protocol_v2::CollatorProtocolMessage::Declare( @@ -606,6 +606,16 @@ fn declare_message( ); Versioned::V2(protocol_v2::CollationProtocol::CollatorProtocol(wire_message)) }, + CollationVersion::VStaging => { + let declare_signature_payload = + protocol_vstaging::declare_signature_payload(&state.local_peer_id); + let wire_message = protocol_vstaging::CollatorProtocolMessage::Declare( + state.collator_pair.public(), + para_id, + state.collator_pair.sign(&declare_signature_payload), + ); + Versioned::VStaging(protocol_v2::CollationProtocol::CollatorProtocol(wire_message)) + }, }) } @@ -710,7 +720,7 @@ async fn advertise_collation( collation.status.advance_to_advertised(); let collation_message = match protocol_version { - CollationVersion::V2 | CollationVersion::VStaging => { + CollationVersion::V2 => { let wire_message = protocol_v2::CollatorProtocolMessage::AdvertiseCollation { relay_parent, candidate_hash: *candidate_hash, @@ -718,6 +728,16 @@ async fn advertise_collation( }; Versioned::V2(protocol_v2::CollationProtocol::CollatorProtocol(wire_message)) }, + CollationVersion::VStaging => { + let wire_message = protocol_vstaging::CollatorProtocolMessage::AdvertiseCollation { + relay_parent, + candidate_hash: *candidate_hash, + parent_head_data_hash: collation.parent_head_data_hash, + }; + Versioned::VStaging(protocol_vstaging::CollationProtocol::CollatorProtocol( + wire_message, + )) + }, CollationVersion::V1 => { let wire_message = protocol_v1::CollatorProtocolMessage::AdvertiseCollation(relay_parent); diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index bbc37d3ee2ae..0567b33dfe85 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -697,7 +697,8 @@ async fn request_collation( let requests = Requests::CollationFetchingV1(req); (requests, response_recv.boxed()) }, - (CollationVersion::V2, Some(ProspectiveCandidate { candidate_hash, .. })) => { + (CollationVersion::V2, Some(ProspectiveCandidate { candidate_hash, .. })) | + (CollationVersion::VStaging, Some(ProspectiveCandidate { candidate_hash, .. })) => { let (req, response_recv) = OutgoingRequest::new( Recipient::Peer(peer_id), request_v2::CollationFetchingRequest { relay_parent, para_id, candidate_hash }, diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index 329c380ed6a4..c628b1e6ce01 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -289,6 +289,12 @@ impl From for VersionedValidationProtocol { } } +impl From for VersionedValidationProtocol { + fn from(vstaging: vstaging::ValidationProtocol) -> Self { + VersionedValidationProtocol::VStaging(vstaging) + } +} + /// All supported versions of the collation protocol message. pub type VersionedCollationProtocol = Versioned; @@ -350,6 +356,7 @@ macro_rules! impl_versioned_try_from { match x { Versioned::V1($v1_pat) => Ok(Versioned::V1($v1_out.clone())), Versioned::V2($v2_pat) => Ok(Versioned::V2($v2_out.clone())), + Versioned::VStaging($vstaging_pat) => Ok(Versioned::VStaging($vstaging_out.clone())), _ => Err(crate::WrongVariant), } } diff --git a/polkadot/node/network/protocol/src/peer_set.rs b/polkadot/node/network/protocol/src/peer_set.rs index a86f8ce9a4a7..0f1d9de2a565 100644 --- a/polkadot/node/network/protocol/src/peer_set.rs +++ b/polkadot/node/network/protocol/src/peer_set.rs @@ -154,6 +154,8 @@ impl PeerSet { Some("validation/1") } else if version == ValidationVersion::V2.into() { Some("validation/2") + } else if version == ValidationVersion::VStaging.into() { + Some("validation/3") } else { None }, @@ -162,6 +164,8 @@ impl PeerSet { Some("collation/1") } else if version == CollationVersion::V2.into() { Some("collation/2") + } else if version == CollationVersion::VStaging.into() { + Some("collation/3") } else { None }, @@ -238,7 +242,7 @@ pub enum CollationVersion { /// The second version. V2 = 2, /// Same format as V2, - VStaging, + VStaging = 3, } /// Marker indicating the version is unknown. diff --git a/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs b/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs index f7f0834aba09..d9866af1ee23 100644 --- a/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs +++ b/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use net_protocol::{filter_by_peer_version, peer_set::ProtocolVersion}; use parity_scale_codec::Encode; use polkadot_node_network_protocol::{ @@ -1062,7 +1063,7 @@ async fn circulate_statement<'a, Context>( "We filter out duplicates above. qed.", ); - let (v1_peers_to_send, v2_peers_to_send) = peers_to_send + let (v1_peers_to_send, non_v1_peers_to_send) = peers_to_send .into_iter() .map(|peer_id| { let peer_data = @@ -1094,6 +1095,22 @@ async fn circulate_statement<'a, Context>( )) .await; } + + let peers_to_send: Vec<(PeerId, ProtocolVersion)> = non_v1_peers_to_send + .iter() + .map(|(p, _, version)| (*p, (*version).into())) + .collect(); + + let peer_needs_dependent_statement = v1_peers_to_send + .into_iter() + .chain(non_v1_peers_to_send) + .filter_map(|(peer, needs_dependent, _)| if needs_dependent { Some(peer) } else { None }) + .collect(); + + let v2_peers_to_send = filter_by_peer_version(&peers_to_send, ValidationVersion::V2.into()); + let vstaging_to_send = + filter_by_peer_version(&peers_to_send, ValidationVersion::VStaging.into()); + if !v2_peers_to_send.is_empty() { gum::trace!( target: LOG_TARGET, @@ -1103,17 +1120,28 @@ async fn circulate_statement<'a, Context>( "Sending statement to v2 peers", ); ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - v2_peers_to_send.iter().map(|(p, _, _)| *p).collect(), + v2_peers_to_send, compatible_v1_message(ValidationVersion::V2, payload.clone()).into(), )) .await; } - v1_peers_to_send - .into_iter() - .chain(v2_peers_to_send) - .filter_map(|(peer, needs_dependent, _)| if needs_dependent { Some(peer) } else { None }) - .collect() + if !vstaging_to_send.is_empty() { + gum::trace!( + target: LOG_TARGET, + ?vstaging_to_send, + ?relay_parent, + statement = ?stored.statement, + "Sending statement to vstaging peers", + ); + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + vstaging_to_send, + compatible_v1_message(ValidationVersion::VStaging, payload.clone()).into(), + )) + .await; + } + + peer_needs_dependent_statement } /// Send all statements about a given candidate hash to a peer. @@ -2171,7 +2199,10 @@ fn compatible_v1_message( ) -> net_protocol::StatementDistributionMessage { match version { ValidationVersion::V1 => Versioned::V1(message), - ValidationVersion::V2 | ValidationVersion::VStaging => + ValidationVersion::V2 => Versioned::V2(protocol_v2::StatementDistributionMessage::V1Compatibility(message)), + ValidationVersion::VStaging => Versioned::VStaging( + protocol_vstaging::StatementDistributionMessage::V1Compatibility(message), + ), } } diff --git a/polkadot/node/network/statement-distribution/src/v2/mod.rs b/polkadot/node/network/statement-distribution/src/v2/mod.rs index 51ae89eed781..0f29e33676b5 100644 --- a/polkadot/node/network/statement-distribution/src/v2/mod.rs +++ b/polkadot/node/network/statement-distribution/src/v2/mod.rs @@ -17,6 +17,7 @@ //! Implementation of the v2 statement distribution protocol, //! designed for asynchronous backing. +use net_protocol::{filter_by_peer_version, peer_set::ProtocolVersion}; use polkadot_node_network_protocol::{ self as net_protocol, grid_topology::SessionGridTopology, @@ -252,6 +253,7 @@ fn connected_validator_peer( struct PeerState { view: View, + protocol_version: ValidationVersion, implicit_view: HashSet, discovery_ids: Option>, } @@ -324,9 +326,13 @@ pub(crate) async fn handle_network_update( NetworkBridgeEvent::PeerConnected(peer_id, role, protocol_version, mut authority_ids) => { gum::trace!(target: LOG_TARGET, ?peer_id, ?role, ?protocol_version, "Peer connected"); - if protocol_version != ValidationVersion::V2.into() { + let versioned_protocol = if protocol_version != ValidationVersion::V2.into() && + protocol_version != ValidationVersion::VStaging.into() + { return - } + } else { + protocol_version.try_into().expect("Qed, we checked above") + }; if let Some(ref mut authority_ids) = authority_ids { authority_ids.retain(|a| match state.authorities.entry(a.clone()) { @@ -353,6 +359,7 @@ pub(crate) async fn handle_network_update( PeerState { view: View::default(), implicit_view: HashSet::new(), + protocol_version: versioned_protocol, discovery_ids: authority_ids, }, ); @@ -708,7 +715,7 @@ async fn send_peer_messages_for_relay_parent( send_pending_cluster_statements( ctx, relay_parent, - &peer, + &(peer, peer_data.protocol_version), validator_id, &mut local_validator_state.cluster_tracker, &state.candidates, @@ -720,7 +727,7 @@ async fn send_peer_messages_for_relay_parent( send_pending_grid_messages( ctx, relay_parent, - &peer, + &(peer, peer_data.protocol_version), validator_id, &per_session_state.groups, relay_parent_state, @@ -733,15 +740,26 @@ async fn send_peer_messages_for_relay_parent( fn pending_statement_network_message( statement_store: &StatementStore, relay_parent: Hash, - peer: &PeerId, + peer: &(PeerId, ValidationVersion), originator: ValidatorIndex, compact: CompactStatement, ) -> Option<(Vec, net_protocol::VersionedValidationProtocol)> { - statement_store - .validator_statement(originator, compact) - .map(|s| s.as_unchecked().clone()) - .map(|signed| protocol_v2::StatementDistributionMessage::Statement(relay_parent, signed)) - .map(|msg| (vec![*peer], Versioned::V2(msg).into())) + match peer.1 { + ValidationVersion::V2 => statement_store + .validator_statement(originator, compact) + .map(|s| s.as_unchecked().clone()) + .map(|signed| { + protocol_v2::StatementDistributionMessage::Statement(relay_parent, signed) + }) + .map(|msg| (vec![peer.0], Versioned::V2(msg).into())), + _ => statement_store + .validator_statement(originator, compact) + .map(|s| s.as_unchecked().clone()) + .map(|signed| { + protocol_vstaging::StatementDistributionMessage::Statement(relay_parent, signed) + }) + .map(|msg| (vec![peer.0], Versioned::VStaging(msg).into())), + } } /// Send a peer all pending cluster statements for a relay parent. @@ -749,7 +767,7 @@ fn pending_statement_network_message( async fn send_pending_cluster_statements( ctx: &mut Context, relay_parent: Hash, - peer_id: &PeerId, + peer_id: &(PeerId, ValidationVersion), peer_validator_id: ValidatorIndex, cluster_tracker: &mut ClusterTracker, candidates: &Candidates, @@ -793,7 +811,7 @@ async fn send_pending_cluster_statements( async fn send_pending_grid_messages( ctx: &mut Context, relay_parent: Hash, - peer_id: &PeerId, + peer_id: &(PeerId, ValidationVersion), peer_validator_id: ValidatorIndex, groups: &Groups, relay_parent_state: &mut PerRelayParentState, @@ -855,20 +873,30 @@ async fn send_pending_grid_messages( candidate_hash, local_knowledge.clone(), ); - - messages.push(( - vec![*peer_id], - Versioned::V2( - protocol_v2::StatementDistributionMessage::BackedCandidateManifest( - manifest, - ), - ) - .into(), - )); + match peer_id.1 { + ValidationVersion::V2 => messages.push(( + vec![peer_id.0], + Versioned::V2( + protocol_v2::StatementDistributionMessage::BackedCandidateManifest( + manifest, + ), + ) + .into(), + )), + _ => messages.push(( + vec![peer_id.0], + Versioned::VStaging( + protocol_vstaging::StatementDistributionMessage::BackedCandidateManifest( + manifest, + ), + ) + .into(), + )), + }; }, grid::ManifestKind::Acknowledgement => { messages.extend(acknowledgement_and_statement_messages( - *peer_id, + peer_id, peer_validator_id, groups, relay_parent_state, @@ -1155,11 +1183,18 @@ async fn circulate_statement( (local_validator, targets) }; - let mut statement_to = Vec::new(); + let mut statement_to_peers: Vec<(PeerId, ProtocolVersion)> = Vec::new(); for (target, authority_id, kind) in targets { // Find peer ID based on authority ID, and also filter to connected. - let peer_id: PeerId = match authorities.get(&authority_id) { - Some(p) if peers.get(p).map_or(false, |p| p.knows_relay_parent(&relay_parent)) => *p, + let peer_id: (PeerId, ProtocolVersion) = match authorities.get(&authority_id) { + Some(p) if peers.get(p).map_or(false, |p| p.knows_relay_parent(&relay_parent)) => ( + *p, + peers + .get(p) + .expect("Qed, can't fail because it was checked above") + .protocol_version + .into(), + ), None | Some(_) => continue, }; @@ -1177,11 +1212,11 @@ async fn circulate_statement( originator, compact_statement.clone(), ); - statement_to.push(peer_id); + statement_to_peers.push(peer_id); } }, DirectTargetKind::Grid => { - statement_to.push(peer_id); + statement_to_peers.push(peer_id); local_validator.grid_tracker.sent_or_received_direct_statement( &per_session.groups, originator, @@ -1192,17 +1227,23 @@ async fn circulate_statement( } } + let statement_to_v2_peers = + filter_by_peer_version(&statement_to_peers, ValidationVersion::V2.into()); + + let statement_to_vstaging_peers = + filter_by_peer_version(&statement_to_peers, ValidationVersion::VStaging.into()); + // ship off the network messages to the network bridge. - if !statement_to.is_empty() { + if !statement_to_v2_peers.is_empty() { gum::debug!( target: LOG_TARGET, ?compact_statement, - n_peers = ?statement_to.len(), - "Sending statement to peers", + n_peers = ?statement_to_v2_peers.len(), + "Sending statement to v2 peers", ); ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - statement_to, + statement_to_v2_peers, Versioned::V2(protocol_v2::StatementDistributionMessage::Statement( relay_parent, statement.as_unchecked().clone(), @@ -1211,6 +1252,25 @@ async fn circulate_statement( )) .await; } + + if !statement_to_vstaging_peers.is_empty() { + gum::debug!( + target: LOG_TARGET, + ?compact_statement, + n_peers = ?statement_to_peers.len(), + "Sending statement to vstaging peers", + ); + + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + statement_to_vstaging_peers, + Versioned::VStaging(protocol_vstaging::StatementDistributionMessage::Statement( + relay_parent, + statement.as_unchecked().clone(), + )) + .into(), + )) + .await; + } } /// Check a statement signature under this parent hash. fn check_statement_signature( @@ -1696,14 +1756,8 @@ async fn provide_candidate_to_grid( statement_knowledge: filter.clone(), }; - let manifest_message = - Versioned::V2(protocol_v2::StatementDistributionMessage::BackedCandidateManifest(manifest)); - let ack_message = Versioned::V2( - protocol_v2::StatementDistributionMessage::BackedCandidateKnown(acknowledgement), - ); - - let mut manifest_peers = Vec::new(); - let mut ack_peers = Vec::new(); + let mut manifest_peers: Vec<(PeerId, ProtocolVersion)> = Vec::new(); + let mut ack_peers: Vec<(PeerId, ProtocolVersion)> = Vec::new(); let mut post_statements = Vec::new(); for (v, action) in actions { @@ -1711,7 +1765,7 @@ async fn provide_candidate_to_grid( None => continue, Some(p) => if peers.get(&p).map_or(false, |d| d.knows_relay_parent(&relay_parent)) { - p + (p, peers.get(&p).expect("Qed, was checked above").protocol_version.into()) } else { continue }, @@ -1737,42 +1791,91 @@ async fn provide_candidate_to_grid( &per_session.groups, group_index, candidate_hash, + &(p.0, p.1.try_into().expect("Qed, can not fail was checked above")), ) .into_iter() - .map(|m| (vec![p], m)), + .map(|m| (vec![p.0], m)), ); } - if !manifest_peers.is_empty() { + let manifest_peers_v2 = filter_by_peer_version(&manifest_peers, ValidationVersion::V2.into()); + let manifest_peers_vstaging = + filter_by_peer_version(&manifest_peers, ValidationVersion::VStaging.into()); + if !manifest_peers_v2.is_empty() { gum::debug!( target: LOG_TARGET, ?candidate_hash, - n_peers = manifest_peers.len(), - "Sending manifest to peers" + n_peers = manifest_peers_v2.len(), + "Sending manifest to v2 peers" ); ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - manifest_peers, - manifest_message.into(), + manifest_peers_v2, + Versioned::V2(protocol_v2::StatementDistributionMessage::BackedCandidateManifest( + manifest.clone(), + )) + .into(), )) .await; } - if !ack_peers.is_empty() { + if !manifest_peers_vstaging.is_empty() { gum::debug!( target: LOG_TARGET, ?candidate_hash, - n_peers = ack_peers.len(), - "Sending acknowledgement to peers" + n_peers = manifest_peers_vstaging.len(), + "Sending manifest to vstaging peers" ); ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - ack_peers, - ack_message.into(), + manifest_peers_vstaging, + Versioned::VStaging( + protocol_vstaging::StatementDistributionMessage::BackedCandidateManifest(manifest), + ) + .into(), )) .await; } + let ack_peers_v2 = filter_by_peer_version(&ack_peers, ValidationVersion::V2.into()); + let ack_peers_vstaging = filter_by_peer_version(&ack_peers, ValidationVersion::VStaging.into()); + if !ack_peers_v2.is_empty() { + gum::debug!( + target: LOG_TARGET, + ?candidate_hash, + n_peers = ack_peers_v2.len(), + "Sending acknowledgement to v2 peers" + ); + + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + ack_peers_v2, + Versioned::V2(protocol_v2::StatementDistributionMessage::BackedCandidateKnown( + acknowledgement.clone(), + )) + .into(), + )) + .await; + } + + if !ack_peers_vstaging.is_empty() { + gum::debug!( + target: LOG_TARGET, + ?candidate_hash, + n_peers = ack_peers_vstaging.len(), + "Sending acknowledgement to vstaging peers" + ); + + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + ack_peers_vstaging, + Versioned::VStaging( + protocol_vstaging::StatementDistributionMessage::BackedCandidateKnown( + acknowledgement, + ), + ) + .into(), + )) + .await; + } if !post_statements.is_empty() { ctx.send_message(NetworkBridgeTxMessage::SendValidationMessages(post_statements)) .await; @@ -2055,6 +2158,7 @@ fn post_acknowledgement_statement_messages( groups: &Groups, group_index: GroupIndex, candidate_hash: CandidateHash, + peer: &(PeerId, ValidationVersion), ) -> Vec { let sending_filter = match grid_tracker.pending_statements_for(recipient, candidate_hash) { None => return Vec::new(), @@ -2071,14 +2175,22 @@ fn post_acknowledgement_statement_messages( recipient, statement.payload(), ); - - messages.push(Versioned::V2( - protocol_v2::StatementDistributionMessage::Statement( - relay_parent, - statement.as_unchecked().clone(), - ) - .into(), - )); + match peer.1.into() { + ValidationVersion::V2 => messages.push(Versioned::V2( + protocol_v2::StatementDistributionMessage::Statement( + relay_parent, + statement.as_unchecked().clone(), + ) + .into(), + )), + _ => messages.push(Versioned::VStaging( + protocol_vstaging::StatementDistributionMessage::Statement( + relay_parent, + statement.as_unchecked().clone(), + ) + .into(), + )), + }; } messages @@ -2148,7 +2260,15 @@ async fn handle_incoming_manifest( }; let messages = acknowledgement_and_statement_messages( - peer, + &( + peer, + state + .peers + .get(&peer) + .map(|val| val.protocol_version) + // Assume the latest stable version, if we don't have info about peer version. + .unwrap_or(ValidationVersion::V2), + ), sender_index, &per_session.groups, relay_parent_state, @@ -2179,7 +2299,7 @@ async fn handle_incoming_manifest( /// Produces acknowledgement and statement messages to be sent over the network, /// noting that they have been sent within the grid topology tracker as well. fn acknowledgement_and_statement_messages( - peer: PeerId, + peer: &(PeerId, ValidationVersion), validator_index: ValidatorIndex, groups: &Groups, relay_parent_state: &mut PerRelayParentState, @@ -2198,11 +2318,20 @@ fn acknowledgement_and_statement_messages( statement_knowledge: local_knowledge.clone(), }; - let msg = Versioned::V2(protocol_v2::StatementDistributionMessage::BackedCandidateKnown( - acknowledgement, + let msg_v2 = Versioned::V2(protocol_v2::StatementDistributionMessage::BackedCandidateKnown( + acknowledgement.clone(), )); - let mut messages = vec![(vec![peer], msg.into())]; + let mut messages = match peer.1 { + ValidationVersion::V2 => vec![(vec![peer.0], msg_v2.into())], + _ => vec![( + vec![peer.0], + Versioned::VStaging(protocol_v2::StatementDistributionMessage::BackedCandidateKnown( + acknowledgement, + )) + .into(), + )], + }; local_validator.grid_tracker.manifest_sent_to( groups, @@ -2219,9 +2348,10 @@ fn acknowledgement_and_statement_messages( &groups, group_index, candidate_hash, + peer, ); - messages.extend(statement_messages.into_iter().map(|m| (vec![peer], m))); + messages.extend(statement_messages.into_iter().map(|m| (vec![peer.0], m))); messages } @@ -2301,6 +2431,15 @@ async fn handle_incoming_acknowledgement( &per_session.groups, group_index, candidate_hash, + &( + peer, + state + .peers + .get(&peer) + .map(|val| val.protocol_version) + // Assume the latest stable version, if we don't have info about peer version. + .unwrap_or(ValidationVersion::V2), + ), ); if !messages.is_empty() { diff --git a/polkadot/parachain/test-parachains/undying/collator/Cargo.toml b/polkadot/parachain/test-parachains/undying/collator/Cargo.toml index 3fbed4046bde..5ee088dcc082 100644 --- a/polkadot/parachain/test-parachains/undying/collator/Cargo.toml +++ b/polkadot/parachain/test-parachains/undying/collator/Cargo.toml @@ -39,3 +39,6 @@ sc-service = { path = "../../../../../substrate/client/service" } sp-keyring = { path = "../../../../../substrate/primitives/keyring" } tokio = { version = "1.24.2", features = ["macros"] } + +[features] +network-protocol-staging = [ "polkadot-cli/network-protocol-staging" ] \ No newline at end of file diff --git a/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml b/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml index ab7fa0195d13..0b75b1a48743 100644 --- a/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml +++ b/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml @@ -2,7 +2,7 @@ timeout = 1000 bootnode = true -[relaychain.genesis.runtime.runtime_genesis_config.configuration.config] +[relaychain.genesis.runtime.configuration.config] max_validators_per_core = 1 needed_approvals = 7 relay_vrf_modulo_samples = 5 From 55797197862d968f831d8f7a3ec91c53eb3a4811 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 11 Oct 2023 10:58:07 +0200 Subject: [PATCH 032/192] Fix cargo fmt Signed-off-by: Alexandru Gheorghe --- polkadot/cli/Cargo.toml | 2 +- polkadot/node/network/protocol/Cargo.toml | 2 +- polkadot/node/network/protocol/src/lib.rs | 3 ++- polkadot/node/service/Cargo.toml | 2 +- polkadot/parachain/test-parachains/undying/collator/Cargo.toml | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/polkadot/cli/Cargo.toml b/polkadot/cli/Cargo.toml index ca1c954ba47c..8abc09dd4e26 100644 --- a/polkadot/cli/Cargo.toml +++ b/polkadot/cli/Cargo.toml @@ -74,4 +74,4 @@ runtime-metrics = [ "service/runtime-metrics", ] -network-protocol-staging = [ "service/network-protocol-staging" ] \ No newline at end of file +network-protocol-staging = [ "service/network-protocol-staging" ] diff --git a/polkadot/node/network/protocol/Cargo.toml b/polkadot/node/network/protocol/Cargo.toml index f4a64d274060..c33b9eae3252 100644 --- a/polkadot/node/network/protocol/Cargo.toml +++ b/polkadot/node/network/protocol/Cargo.toml @@ -29,4 +29,4 @@ bitvec = "1" rand_chacha = "0.3.1" [features] -network-protocol-staging = [] \ No newline at end of file +network-protocol-staging = [] diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index c628b1e6ce01..f01de9862ce8 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -356,7 +356,8 @@ macro_rules! impl_versioned_try_from { match x { Versioned::V1($v1_pat) => Ok(Versioned::V1($v1_out.clone())), Versioned::V2($v2_pat) => Ok(Versioned::V2($v2_out.clone())), - Versioned::VStaging($vstaging_pat) => Ok(Versioned::VStaging($vstaging_out.clone())), + Versioned::VStaging($vstaging_pat) => + Ok(Versioned::VStaging($vstaging_out.clone())), _ => Err(crate::WrongVariant), } } diff --git a/polkadot/node/service/Cargo.toml b/polkadot/node/service/Cargo.toml index d3a7b2f15725..ee092e277332 100644 --- a/polkadot/node/service/Cargo.toml +++ b/polkadot/node/service/Cargo.toml @@ -226,4 +226,4 @@ runtime-metrics = [ network-protocol-staging = [ "polkadot-node-network-protocol/network-protocol-staging", -] \ No newline at end of file +] diff --git a/polkadot/parachain/test-parachains/undying/collator/Cargo.toml b/polkadot/parachain/test-parachains/undying/collator/Cargo.toml index b694895f3743..578c3d6715dc 100644 --- a/polkadot/parachain/test-parachains/undying/collator/Cargo.toml +++ b/polkadot/parachain/test-parachains/undying/collator/Cargo.toml @@ -41,4 +41,4 @@ sp-keyring = { path = "../../../../../substrate/primitives/keyring" } tokio = { version = "1.24.2", features = ["macros"] } [features] -network-protocol-staging = [ "polkadot-cli/network-protocol-staging" ] \ No newline at end of file +network-protocol-staging = [ "polkadot-cli/network-protocol-staging" ] From ee3b70267d53d25e330f355b8523f98410437b6e Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 16 Oct 2023 15:47:25 +0300 Subject: [PATCH 033/192] approval-distribution: fix unittests Signed-off-by: Alexandru Gheorghe --- .../approval-distribution/src/tests.rs | 41 +++++++++++-------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/tests.rs b/polkadot/node/network/approval-distribution/src/tests.rs index 58601de327cd..27514277986d 100644 --- a/polkadot/node/network/approval-distribution/src/tests.rs +++ b/polkadot/node/network/approval-distribution/src/tests.rs @@ -873,8 +873,9 @@ fn import_approval_happy_path_v1_v2_peers() { validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v2(overseer, &peer_b, msg).await; + let msg: protocol_vstaging::ApprovalDistributionMessage = + protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_vstaging(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -975,7 +976,7 @@ fn import_approval_happy_path_v2() { signature: dummy_signature(), }; let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v2(overseer, &peer_b, msg).await; + send_message_from_peer_vstaging(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1059,7 +1060,7 @@ fn multiple_assignments_covered_with_one_approval_vote() { assignment, (0 as CandidateIndex).into(), )]); - send_message_from_peer_v2(overseer, &peer_d, msg).await; + send_message_from_peer_vstaging(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1101,7 +1102,7 @@ fn multiple_assignments_covered_with_one_approval_vote() { (1 as CandidateIndex).into(), )]); - send_message_from_peer_v2(overseer, &peer_c, msg).await; + send_message_from_peer_vstaging(overseer, &peer_c, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1137,7 +1138,7 @@ fn multiple_assignments_covered_with_one_approval_vote() { signature: dummy_signature(), }; let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v2(overseer, &peer_d, msg).await; + send_message_from_peer_vstaging(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1241,7 +1242,7 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { assignment, (0 as CandidateIndex).into(), )]); - send_message_from_peer_v2(overseer, &peer_d, msg).await; + send_message_from_peer_vstaging(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1268,7 +1269,7 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { (1 as CandidateIndex).into(), )]); - send_message_from_peer_v2(overseer, &peer_d, msg).await; + send_message_from_peer_vstaging(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1289,7 +1290,7 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { signature: dummy_signature(), }; let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v2(overseer, &peer_d, msg).await; + send_message_from_peer_vstaging(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1410,7 +1411,7 @@ fn import_approval_bad() { signature: dummy_signature(), }; let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v2(overseer, &peer_b, msg).await; + send_message_from_peer_vstaging(overseer, &peer_b, msg).await; expect_reputation_change(overseer, &peer_b, COST_UNEXPECTED_MESSAGE).await; @@ -1436,7 +1437,7 @@ fn import_approval_bad() { // and try again let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v2(overseer, &peer_b, msg).await; + send_message_from_peer_vstaging(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1762,7 +1763,7 @@ fn import_remotely_then_locally() { signature: dummy_signature(), }; let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v2(overseer, peer, msg).await; + send_message_from_peer_vstaging(overseer, peer, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -3172,12 +3173,13 @@ fn resends_messages_periodically() { /// Tests that peers correctly receive versioned messages. #[test] fn import_versioned_approval() { - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - let peer_c = PeerId::random(); + let peers = make_peers_and_authority_ids(15); + let peer_a = peers.get(0).unwrap().0; + let peer_b = peers.get(1).unwrap().0; + let peer_c = peers.get(2).unwrap().0; + let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); - let state = state_without_reputation_delay(); let _ = test_harness(state, |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -3186,6 +3188,10 @@ fn import_versioned_approval() { setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V1).await; setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V2).await; + // Set up a gossip topology, where a, b, c and d are topology neighboors to the node under + // testing. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0, 1], &[2, 4], 3)).await; + // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { hash, @@ -3253,7 +3259,7 @@ fn import_versioned_approval() { vote, tx, )) => { - assert_eq!(vote, approval); + assert_eq!(vote, approval.into()); tx.send(ApprovalCheckResult::Accepted).unwrap(); } ); @@ -3273,6 +3279,7 @@ fn import_versioned_approval() { assert_eq!(approvals.len(), 1); } ); + assert_matches!( overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( From 0d2fbba53313a9c1e58e99e224e59624808dd7b4 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 16 Oct 2023 17:44:30 +0300 Subject: [PATCH 034/192] Approval voting fixup Signed-off-by: Alexandru Gheorghe --- .../functional/0006-approval-voting-coalescing.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml b/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml index ec6d17dff4e7..acc9290eb7c8 100644 --- a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml +++ b/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml @@ -6,11 +6,11 @@ default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default-bootnode" -[relaychain.genesis.runtime.runtime_genesis_config.configuration.config] +[relaychain.genesis.runtime.configuration.config] needed_approvals = 5 relay_vrf_modulo_samples = 3 -[relaychain.genesis.runtime.runtime_genesis_config.configuration.config.approval_voting_params] +[relaychain.genesis.runtime.configuration.config.approval_voting_params] max_approval_coalesce_count = 6 [relaychain.default_resources] From e672c1daeb50428a537ee8c6a790407b4bd8fff2 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 16 Oct 2023 19:12:17 +0300 Subject: [PATCH 035/192] Fixup runtime api post merge Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/runtime-api/src/lib.rs | 2 +- .../subsystem-types/src/runtime_client.rs | 1 + polkadot/primitives/src/runtime_api.rs | 2 +- polkadot/runtime/rococo/src/lib.rs | 21 ++++++++++++------ polkadot/runtime/westend/src/lib.rs | 22 ++++++++++++------- .../0006-approval-voting-coalescing.toml | 2 +- 6 files changed, 32 insertions(+), 18 deletions(-) diff --git a/polkadot/node/core/runtime-api/src/lib.rs b/polkadot/node/core/runtime-api/src/lib.rs index 9610d6b57fb0..1cd6d4ced267 100644 --- a/polkadot/node/core/runtime-api/src/lib.rs +++ b/polkadot/node/core/runtime-api/src/lib.rs @@ -558,7 +558,7 @@ where sender ), Request::ApprovalVotingParams(sender) => { - query!(ApprovalVotingParams, approval_voting_params(), ver = 6, sender) + query!(ApprovalVotingParams, approval_voting_params(), ver = 8, sender) }, Request::SubmitReportDisputeLost(dispute_proof, key_ownership_proof, sender) => query!( SubmitReportDisputeLost, diff --git a/polkadot/node/subsystem-types/src/runtime_client.rs b/polkadot/node/subsystem-types/src/runtime_client.rs index 18ef31c02e14..141b2108eb9a 100644 --- a/polkadot/node/subsystem-types/src/runtime_client.rs +++ b/polkadot/node/subsystem-types/src/runtime_client.rs @@ -256,6 +256,7 @@ pub trait RuntimeApiSubsystemClient { para_id: Id, ) -> Result, ApiError>; + // == v8: Approval voting params == /// Approval voting configuration parameters async fn approval_voting_params(&self, at: Hash) -> Result; } diff --git a/polkadot/primitives/src/runtime_api.rs b/polkadot/primitives/src/runtime_api.rs index c178730511c8..b242e04d3425 100644 --- a/polkadot/primitives/src/runtime_api.rs +++ b/polkadot/primitives/src/runtime_api.rs @@ -259,7 +259,7 @@ sp_api::decl_runtime_apis! { fn async_backing_params() -> AsyncBackingParams; /// Approval voting configuration parameters - #[api_version(99)] + #[api_version(8)] fn approval_voting_params() -> ApprovalVotingParams; } } diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 0a36a41517b7..2285a7928521 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -23,11 +23,12 @@ use pallet_nis::WithMaximumOf; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, - OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, - ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, PARACHAIN_KEY_TYPE_ID, + slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, + CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, + ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, + ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, + ValidatorIndex, PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ assigned_slots, auctions, claims, crowdloan, impl_runtime_weights, @@ -49,7 +50,9 @@ use runtime_parachains::{ inclusion::{AggregateMessageOrigin, UmpQueueId}, initializer as parachains_initializer, origin as parachains_origin, paras as parachains_paras, paras_inherent as parachains_paras_inherent, - runtime_api_impl::v7 as parachains_runtime_api_impl, + runtime_api_impl::{ + v7 as parachains_runtime_api_impl, vstaging as parachains_runtime_api_impl_staging, + }, scheduler as parachains_scheduler, session_info as parachains_session_info, shared as parachains_shared, }; @@ -1586,7 +1589,7 @@ sp_api::impl_runtime_apis! { } } - #[api_version(7)] + #[api_version(8)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() @@ -1729,6 +1732,10 @@ sp_api::impl_runtime_apis! { fn async_backing_params() -> primitives::AsyncBackingParams { parachains_runtime_api_impl::async_backing_params::() } + + fn approval_voting_params() -> ApprovalVotingParams { + parachains_runtime_api_impl_staging::approval_voting_params::() + } } #[api_version(3)] diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index 37accc166262..26ee750f0ec3 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -45,12 +45,12 @@ use pallet_session::historical as session_historical; use pallet_transaction_payment::{CurrencyAdapter, FeeDetails, RuntimeDispatchInfo}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, - SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, - ValidatorSignature, PARACHAIN_KEY_TYPE_ID, + slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, + CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, + ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, + PvfCheckStatement, ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, + ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ assigned_slots, auctions, crowdloan, @@ -70,7 +70,9 @@ use runtime_parachains::{ inclusion::{AggregateMessageOrigin, UmpQueueId}, initializer as parachains_initializer, origin as parachains_origin, paras as parachains_paras, paras_inherent as parachains_paras_inherent, reward_points as parachains_reward_points, - runtime_api_impl::v7 as parachains_runtime_api_impl, + runtime_api_impl::{ + v7 as parachains_runtime_api_impl, vstaging as parachains_runtime_api_impl_staging, + }, scheduler as parachains_scheduler, session_info as parachains_session_info, shared as parachains_shared, }; @@ -1696,7 +1698,7 @@ sp_api::impl_runtime_apis! { } } - #[api_version(7)] + #[api_version(8)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() @@ -1839,6 +1841,10 @@ sp_api::impl_runtime_apis! { fn async_backing_params() -> primitives::AsyncBackingParams { parachains_runtime_api_impl::async_backing_params::() } + + fn approval_voting_params() -> ApprovalVotingParams { + parachains_runtime_api_impl_staging::approval_voting_params::() + } } impl beefy_primitives::BeefyApi for Runtime { diff --git a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml b/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml index acc9290eb7c8..99aa723ec191 100644 --- a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml +++ b/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml @@ -11,7 +11,7 @@ chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default relay_vrf_modulo_samples = 3 [relaychain.genesis.runtime.configuration.config.approval_voting_params] - max_approval_coalesce_count = 6 + max_approval_coalesce_count = 5 [relaychain.default_resources] limits = { memory = "4G", cpu = "2" } From deccd9861ee4c1c4ab92c00d899ebfafebeba274 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 17 Oct 2023 12:23:03 +0300 Subject: [PATCH 036/192] Fixup clippy Signed-off-by: Alexandru Gheorghe --- polkadot/node/service/src/parachains_db/upgrade.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/service/src/parachains_db/upgrade.rs b/polkadot/node/service/src/parachains_db/upgrade.rs index b99f885176b0..a22ec675d0e6 100644 --- a/polkadot/node/service/src/parachains_db/upgrade.rs +++ b/polkadot/node/service/src/parachains_db/upgrade.rs @@ -595,7 +595,7 @@ mod tests { // We need to properly set db version for upgrade to work. fs::write(version_file_path(db_dir.path()), "3").expect("Failed to write DB version"); let expected_candidates = { - let db = Database::open(&db_cfg, db_path.clone()).unwrap(); + let db = Database::open(&db_cfg, db_path).unwrap(); assert_eq!(db.num_columns(), super::columns::v3::NUM_COLUMNS as u32); let db = DbAdapter::new(db, columns::v3::ORDERED_COL); // Fill the approval voting column with test data. From cc5a8fee46d89978af9a8cf7b29123ff5f3d6ca5 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 17 Oct 2023 15:03:19 +0300 Subject: [PATCH 037/192] Minor cleanups Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 2 +- polkadot/node/core/pvf/src/host.rs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index cb46782d4878..0b95ebbad7b7 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -3541,7 +3541,7 @@ fn compute_delayed_approval_sending_tick( .approval_entry(&block_entry.block_hash()) .and_then(|approval_entry| approval_entry.our_assignment()) .map(|our_assignment| our_assignment.tranche()) - .unwrap(); + .unwrap_or_default(); let assignment_triggered_tick = current_block_tick + assignment_tranche as Tick; diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs index fc46622dba91..81695829122b 100644 --- a/polkadot/node/core/pvf/src/host.rs +++ b/polkadot/node/core/pvf/src/host.rs @@ -187,8 +187,7 @@ impl Config { prepare_workers_hard_max_num: 1, execute_worker_program_path, execute_worker_spawn_timeout: Duration::from_secs(3), - // TODO: cleanup increased for versi experimenting. - execute_workers_max_num: 4, + execute_workers_max_num: 2, } } } From e24d8894d541b4f0c50ab45afa24f89333707b88 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 17 Oct 2023 15:24:27 +0300 Subject: [PATCH 038/192] Add metric to see average delayed tick Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 0b95ebbad7b7..2a59796b5470 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -181,6 +181,7 @@ struct MetricsInner { approved_by_one_third: prometheus::Counter, wakeups_triggered_total: prometheus::Counter, coalesced_approvals_buckets: prometheus::Histogram, + coalesced_approvals_waiting_times: prometheus::Histogram, candidate_approval_time_ticks: prometheus::Histogram, block_approval_time_ticks: prometheus::Histogram, time_db_transaction: prometheus::Histogram, @@ -216,6 +217,12 @@ impl Metrics { } } + fn on_delayed_approval(&self, delayed_ticks: u64) { + if let Some(metrics) = &self.0 { + metrics.coalesced_approvals_waiting_times.observe(delayed_ticks as f64) + } + } + fn on_approval_stale(&self) { if let Some(metrics) = &self.0 { metrics.approvals_produced_total.with_label_values(&["stale"]).inc() @@ -373,6 +380,15 @@ impl metrics::Metrics for Metrics { )?, registry, )?, + coalesced_approvals_waiting_times: prometheus::register( + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "polkadot_parachain_approvals_coalesced_approvals_waiting_times", + "Number of ticks we delay the sending of a candidate approval", + ).buckets(vec![1.1, 2.1, 3.1, 4.1, 6.1, 8.1, 12.1, 20.1, 32.1]), + )?, + registry, + )?, approved_by_one_third: prometheus::register( prometheus::Counter::new( "polkadot_parachain_approved_by_one_third", @@ -3297,6 +3313,7 @@ async fn issue_approval( &block_entry, &candidate_entry, session_info, + &metrics, ), ) .is_some() @@ -3535,6 +3552,7 @@ fn compute_delayed_approval_sending_tick( block_entry: &BlockEntry, candidate_entry: &CandidateEntry, session_info: &SessionInfo, + metrics: &Metrics, ) -> Tick { let current_block_tick = slot_number_to_tick(state.slot_duration_millis, block_entry.slot()); let assignment_tranche = candidate_entry @@ -3551,11 +3569,14 @@ fn compute_delayed_approval_sending_tick( ); let tick_now = state.clock.tick_now(); - min( + let sign_no_later_than = min( tick_now + MAX_APPROVAL_COALESCE_WAIT_TICKS as Tick, // We don't want to accidentally cause, no-shows so if we are past // the seconnd half of the no show time, force the sending of the // approval immediately. assignment_triggered_tick + no_show_duration_ticks / 2, - ) + ); + + metrics.on_delayed_approval(sign_no_later_than.checked_sub(tick_now).unwrap_or_default()); + sign_no_later_than } From eebde478bc5f8a87d419ab160698d63be0b72051 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 17 Oct 2023 16:06:23 +0300 Subject: [PATCH 039/192] Fixup CI Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/runtime-api/src/tests.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/polkadot/node/core/runtime-api/src/tests.rs b/polkadot/node/core/runtime-api/src/tests.rs index d6875194bccf..5207d03a412d 100644 --- a/polkadot/node/core/runtime-api/src/tests.rs +++ b/polkadot/node/core/runtime-api/src/tests.rs @@ -20,13 +20,12 @@ use polkadot_node_primitives::{BabeAllowedSlots, BabeEpoch, BabeEpochConfigurati use polkadot_node_subsystem::SpawnGlue; use polkadot_node_subsystem_test_helpers::make_subsystem_context; use polkadot_primitives::{ - async_backing, slashing, - vstaging::{self, ApprovalVotingParams}, - AuthorityDiscoveryId, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, OccupiedCoreAssumption, - PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, - Slot, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, + async_backing, slashing, vstaging::ApprovalVotingParams, AuthorityDiscoveryId, BlockNumber, + CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, + DisputeState, ExecutorParams, GroupRotationInfo, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, + ScrapedOnChainVotes, SessionIndex, SessionInfo, Slot, ValidationCode, ValidationCodeHash, + ValidatorId, ValidatorIndex, ValidatorSignature, }; use sp_api::ApiError; use sp_core::testing::TaskExecutor; From f400d5af1143abb920844e3286d8d64c654afeb2 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 17 Oct 2023 17:52:52 +0300 Subject: [PATCH 040/192] ApprovalVotingParams sane default Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 2a59796b5470..869ae9518166 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -892,7 +892,7 @@ impl State { target: LOG_TARGET, "Could not request approval voting params from runtime using defaults" ); - ApprovalVotingParams { max_approval_coalesce_count: 6 } + ApprovalVotingParams { max_approval_coalesce_count: 1 } }, } } From 0a952c0ed309e09d3651c44744ae82bde1ca8514 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 18 Oct 2023 15:41:11 +0300 Subject: [PATCH 041/192] Add zombienet to check approval coalescing Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/zombienet/polkadot.yml | 8 ++++++ ...l => 0007-approval-voting-coalescing.toml} | 4 +-- ... => 0007-approval-voting-coalescing.zndsl} | 27 ++++++++++++------- 3 files changed, 28 insertions(+), 11 deletions(-) rename polkadot/zombienet_tests/functional/{0006-approval-voting-coalescing.toml => 0007-approval-voting-coalescing.toml} (98%) rename polkadot/zombienet_tests/functional/{0006-approval-voting-coalescing.zndsl => 0007-approval-voting-coalescing.zndsl} (70%) diff --git a/.gitlab/pipeline/zombienet/polkadot.yml b/.gitlab/pipeline/zombienet/polkadot.yml index 8fc8b280bba8..1a7b45e52d83 100644 --- a/.gitlab/pipeline/zombienet/polkadot.yml +++ b/.gitlab/pipeline/zombienet/polkadot.yml @@ -113,6 +113,14 @@ zombienet-polkadot-functional-0006-parachains-max-tranche0: --local-dir="${LOCAL_DIR}/functional" --test="0006-parachains-max-tranche0.zndsl" +zombienet-polkadot-functional-0007-approval-voting-coalescing: + extends: + - .zombienet-polkadot-common + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh + --local-dir="${LOCAL_DIR}/functional" + --test="0007-approval-voting-coalescing.zndsl" + zombienet-polkadot-smoke-0001-parachains-smoke-test: extends: - .zombienet-polkadot-common diff --git a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml similarity index 98% rename from polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml rename to polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml index 99aa723ec191..c7eeeb5383e0 100644 --- a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml +++ b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml @@ -8,7 +8,7 @@ chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default [relaychain.genesis.runtime.configuration.config] needed_approvals = 5 - relay_vrf_modulo_samples = 3 + relay_vrf_modulo_samples = 6 [relaychain.genesis.runtime.configuration.config.approval_voting_params] max_approval_coalesce_count = 5 @@ -19,7 +19,7 @@ requests = { memory = "2G", cpu = "1" } [[relaychain.nodes]] name = "alice" - args = [ "--alice", "-lparachain=debug,runtime=debug,peerset=trace" ] + args = [ "--alice", "-lparachain=trace,runtime=debug" ] [[relaychain.nodes]] name = "bob" diff --git a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.zndsl b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl similarity index 70% rename from polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.zndsl rename to polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl index 272ac4fa2640..a4cc454f38cc 100644 --- a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.zndsl +++ b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl @@ -1,5 +1,5 @@ Description: Approval voting coalescing does not lag finality -Network: ./0006-approval-voting-coalescing.toml +Network: ./0007-approval-voting-coalescing.toml Creds: config # Check authority status. @@ -41,11 +41,20 @@ ferdie: reports substrate_block_height{status="finalized"} is at least 30 within one: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds two: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds -alice: reports polkadot_parachain_approval_checking_finality_lag is 0 -bob: reports polkadot_parachain_approval_checking_finality_lag is 0 -charlie: reports polkadot_parachain_approval_checking_finality_lag is 0 -dave: reports polkadot_parachain_approval_checking_finality_lag is 0 -ferdie: reports polkadot_parachain_approval_checking_finality_lag is 0 -eve: reports polkadot_parachain_approval_checking_finality_lag is 0 -one: reports polkadot_parachain_approval_checking_finality_lag is 0 -two: reports polkadot_parachain_approval_checking_finality_lag is 0 \ No newline at end of file +alice: reports polkadot_parachain_approval_checking_finality_lag < 3 +bob: reports polkadot_parachain_approval_checking_finality_lag < 3 +charlie: reports polkadot_parachain_approval_checking_finality_lag < 3 +dave: reports polkadot_parachain_approval_checking_finality_lag < 3 +ferdie: reports polkadot_parachain_approval_checking_finality_lag < 3 +eve: reports polkadot_parachain_approval_checking_finality_lag < 3 +one: reports polkadot_parachain_approval_checking_finality_lag < 3 +two: reports polkadot_parachain_approval_checking_finality_lag < 3 + +alice: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds +bob: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds +charlie: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds +dave: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds +ferdie: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds +eve: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds +one: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds +two: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds \ No newline at end of file From 183ba64aff9a48ef20701e7814c401156be77447 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 19 Oct 2023 12:48:50 +0300 Subject: [PATCH 042/192] Fix review typo Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/backend.rs | 2 +- polkadot/node/core/approval-voting/src/lib.rs | 2 +- polkadot/node/core/approval-voting/src/persisted_entries.rs | 2 +- polkadot/node/network/approval-distribution/src/lib.rs | 2 +- polkadot/node/service/src/parachains_db/upgrade.rs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/backend.rs b/polkadot/node/core/approval-voting/src/backend.rs index 374e7a826d19..d98f3c5fd202 100644 --- a/polkadot/node/core/approval-voting/src/backend.rs +++ b/polkadot/node/core/approval-voting/src/backend.rs @@ -66,7 +66,7 @@ pub trait Backend { I: IntoIterator; } -/// A read only backed to enable db migration from version 1 of DB. +/// A read only backend to enable db migration from version 1 of DB. pub trait V1ReadBackend: Backend { /// Load a candidate entry from the DB with scheme version 1. fn load_candidate_entry_v1( diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 4ec006a1d81c..e7cbde567392 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -2146,7 +2146,7 @@ where // Since we don't account for tranche in distribution message fingerprinting, some // validators can be assigned to the same core (VRF modulo vs VRF delay). These can be - // safely ignored ignored. However, if an assignment is for multiple cores (these are only + // safely ignored. However, if an assignment is for multiple cores (these are only // tranche0), we cannot ignore it, because it would mean ignoring other non duplicate // assignments. if is_duplicate { diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index 155b2f9c4e02..9cfe1c4cf8da 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -353,7 +353,7 @@ pub struct BlockEntry { // block. The block can be considered approved if the bitfield has all bits set to `true`. pub approved_bitfield: Bitfield, pub children: Vec, - // A list of assignments for which wea already distributed the assignment. + // A list of assignments for which we already distributed the assignment. // We use this to ensure we don't distribute multiple core assignments twice as we track // individual wakeups for each core. distributed_assignments: Bitfield, diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 87e8a072bf68..46d687035f51 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -188,7 +188,7 @@ impl ApprovalEntry { self.routing_info.required_routing = required_routing; } - // Records a new approval. Returns false if the claimed candidate is not found or we already + // Records a new approval. Returns error if the claimed candidate is not found or we already // have received the approval. pub fn note_approval( &mut self, diff --git a/polkadot/node/service/src/parachains_db/upgrade.rs b/polkadot/node/service/src/parachains_db/upgrade.rs index a22ec675d0e6..78e53a251eed 100644 --- a/polkadot/node/service/src/parachains_db/upgrade.rs +++ b/polkadot/node/service/src/parachains_db/upgrade.rs @@ -63,7 +63,7 @@ pub(crate) fn try_upgrade_db( db_kind: DatabaseKind, target_version: Version, ) -> Result<(), Error> { - // Ensure we don't loop forever below befcause of a bug. + // Ensure we don't loop forever below because of a bug. const MAX_MIGRATIONS: u32 = 30; #[cfg(test)] From 3fe8c0d6561dbdd71b4e36f8b22b15d8e7db44c8 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 19 Oct 2023 12:56:33 +0300 Subject: [PATCH 043/192] Remove file accidentally added which was removed from master Signed-off-by: Alexandru Gheorghe --- polkadot/runtime/polkadot/src/lib.rs | 2621 -------------------------- 1 file changed, 2621 deletions(-) delete mode 100644 polkadot/runtime/polkadot/src/lib.rs diff --git a/polkadot/runtime/polkadot/src/lib.rs b/polkadot/runtime/polkadot/src/lib.rs deleted file mode 100644 index 9c84c8840cd5..000000000000 --- a/polkadot/runtime/polkadot/src/lib.rs +++ /dev/null @@ -1,2621 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! The Polkadot runtime. This can be compiled with `#[no_std]`, ready for Wasm. - -#![cfg_attr(not(feature = "std"), no_std)] -// `construct_runtime!` does a lot of recursion and requires us to increase the limit to 256. -#![recursion_limit = "512"] - -use pallet_transaction_payment::CurrencyAdapter; -use runtime_common::{ - auctions, claims, crowdloan, impl_runtime_weights, impls::DealWithFees, paras_registrar, - prod_or_fast, slots, BlockHashCount, BlockLength, CurrencyToVote, SlowAdjustingFeeUpdate, -}; - -use runtime_parachains::{ - assigner_parachains as parachains_assigner_parachains, - configuration as parachains_configuration, disputes as parachains_disputes, - disputes::slashing as parachains_slashing, - dmp as parachains_dmp, hrmp as parachains_hrmp, inclusion as parachains_inclusion, - inclusion::{AggregateMessageOrigin, UmpQueueId}, - initializer as parachains_initializer, origin as parachains_origin, paras as parachains_paras, - paras_inherent as parachains_paras_inherent, reward_points as parachains_reward_points, - runtime_api_impl::v5 as parachains_runtime_api_impl, - scheduler as parachains_scheduler, session_info as parachains_session_info, - shared as parachains_shared, -}; - -use authority_discovery_primitives::AuthorityId as AuthorityDiscoveryId; -use beefy_primitives::ecdsa_crypto::{AuthorityId as BeefyId, Signature as BeefySignature}; -use frame_election_provider_support::{ - bounds::ElectionBoundsBuilder, generate_solution_type, onchain, SequentialPhragmen, -}; -use frame_support::{ - construct_runtime, parameter_types, - traits::{ - fungible::HoldConsideration, ConstU32, Contains, EitherOf, EitherOfDiverse, EverythingBut, - InstanceFilter, KeyOwnerProofSystem, LinearStoragePrice, PrivilegeCmp, ProcessMessage, - ProcessMessageError, WithdrawReasons, - }, - weights::{ConstantMultiplier, WeightMeter}, - PalletId, -}; -use frame_system::EnsureRoot; -use pallet_grandpa::{fg_primitives, AuthorityId as GrandpaId}; -use pallet_im_online::sr25519::AuthorityId as ImOnlineId; -use pallet_session::historical as session_historical; -use pallet_transaction_payment::{FeeDetails, RuntimeDispatchInfo}; -use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; -use primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, - OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, - ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, LOWEST_PUBLIC_ID, - PARACHAIN_KEY_TYPE_ID, -}; -use sp_core::OpaqueMetadata; -use sp_mmr_primitives as mmr; -use sp_runtime::{ - create_runtime_str, - curve::PiecewiseLinear, - generic, impl_opaque_keys, - traits::{ - AccountIdLookup, BlakeTwo256, Block as BlockT, ConvertInto, Extrinsic as ExtrinsicT, - OpaqueKeys, SaturatedConversion, Verify, - }, - transaction_validity::{TransactionPriority, TransactionSource, TransactionValidity}, - ApplyExtrinsicResult, FixedU128, KeyTypeId, Perbill, Percent, Permill, RuntimeDebug, -}; -use sp_staking::SessionIndex; -use sp_std::{cmp::Ordering, collections::btree_map::BTreeMap, prelude::*}; -#[cfg(any(feature = "std", test))] -use sp_version::NativeVersion; -use sp_version::RuntimeVersion; -use xcm::latest::Junction; - -pub use frame_system::Call as SystemCall; -pub use pallet_balances::Call as BalancesCall; -pub use pallet_election_provider_multi_phase::{Call as EPMCall, GeometricDepositBase}; -#[cfg(feature = "std")] -pub use pallet_staking::StakerStatus; -use pallet_staking::UseValidatorsMap; -pub use pallet_timestamp::Call as TimestampCall; -use sp_runtime::traits::Get; -#[cfg(any(feature = "std", test))] -pub use sp_runtime::BuildStorage; - -/// Constant values used within the runtime. -use polkadot_runtime_constants::{currency::*, fee::*, time::*}; - -// Weights used in the runtime. -mod weights; - -mod bag_thresholds; - -// Governance configurations. -pub mod governance; -use governance::{ - pallet_custom_origins, AuctionAdmin, FellowshipAdmin, GeneralAdmin, LeaseAdmin, StakingAdmin, - Treasurer, TreasurySpender, -}; - -pub mod xcm_config; - -impl_runtime_weights!(polkadot_runtime_constants); - -// Make the WASM binary available. -#[cfg(feature = "std")] -include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); - -// Polkadot version identifier; -/// Runtime version (Polkadot). -#[sp_version::runtime_version] -pub const VERSION: RuntimeVersion = RuntimeVersion { - spec_name: create_runtime_str!("polkadot"), - impl_name: create_runtime_str!("parity-polkadot"), - authoring_version: 0, - spec_version: 9430, - impl_version: 0, - apis: RUNTIME_API_VERSIONS, - transaction_version: 24, - state_version: 0, -}; - -/// The BABE epoch configuration at genesis. -pub const BABE_GENESIS_EPOCH_CONFIG: babe_primitives::BabeEpochConfiguration = - babe_primitives::BabeEpochConfiguration { - c: PRIMARY_PROBABILITY, - allowed_slots: babe_primitives::AllowedSlots::PrimaryAndSecondaryVRFSlots, - }; - -/// Native version. -#[cfg(any(feature = "std", test))] -pub fn native_version() -> NativeVersion { - NativeVersion { runtime_version: VERSION, can_author_with: Default::default() } -} - -/// A type to identify calls to the Identity pallet. These will be filtered to prevent invocation, -/// locking the state of the pallet and preventing further updates to identities and sub-identities. -/// The locked state will be the genesis state of a new system chain and then removed from the Relay -/// Chain. -pub struct IdentityCalls; -impl Contains for IdentityCalls { - fn contains(c: &RuntimeCall) -> bool { - matches!(c, RuntimeCall::Identity(_)) - } -} - -parameter_types! { - pub const Version: RuntimeVersion = VERSION; - pub const SS58Prefix: u8 = 0; -} - -impl frame_system::Config for Runtime { - type BaseCallFilter = EverythingBut; - type BlockWeights = BlockWeights; - type BlockLength = BlockLength; - type RuntimeOrigin = RuntimeOrigin; - type RuntimeCall = RuntimeCall; - type Nonce = Nonce; - type Hash = Hash; - type Hashing = BlakeTwo256; - type AccountId = AccountId; - type Lookup = AccountIdLookup; - type Block = Block; - type RuntimeEvent = RuntimeEvent; - type BlockHashCount = BlockHashCount; - type DbWeight = RocksDbWeight; - type Version = Version; - type PalletInfo = PalletInfo; - type AccountData = pallet_balances::AccountData; - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = weights::frame_system::WeightInfo; - type SS58Prefix = SS58Prefix; - type OnSetCode = (); - type MaxConsumers = frame_support::traits::ConstU32<16>; -} - -parameter_types! { - pub MaximumSchedulerWeight: Weight = Perbill::from_percent(80) * - BlockWeights::get().max_block; - pub const MaxScheduledPerBlock: u32 = 50; - pub const NoPreimagePostponement: Option = Some(10); -} - -/// Used the compare the privilege of an origin inside the scheduler. -pub struct OriginPrivilegeCmp; - -impl PrivilegeCmp for OriginPrivilegeCmp { - fn cmp_privilege(left: &OriginCaller, right: &OriginCaller) -> Option { - if left == right { - return Some(Ordering::Equal) - } - - match (left, right) { - // Root is greater than anything. - (OriginCaller::system(frame_system::RawOrigin::Root), _) => Some(Ordering::Greater), - // For every other origin we don't care, as they are not used for `ScheduleOrigin`. - _ => None, - } - } -} - -impl pallet_scheduler::Config for Runtime { - type RuntimeOrigin = RuntimeOrigin; - type RuntimeEvent = RuntimeEvent; - type PalletsOrigin = OriginCaller; - type RuntimeCall = RuntimeCall; - type MaximumWeight = MaximumSchedulerWeight; - // The goal of having ScheduleOrigin include AuctionAdmin is to allow the auctions track of - // OpenGov to schedule periodic auctions. - // Also allow Treasurer to schedule recurring payments. - type ScheduleOrigin = EitherOf, AuctionAdmin>, Treasurer>; - type MaxScheduledPerBlock = MaxScheduledPerBlock; - type WeightInfo = weights::pallet_scheduler::WeightInfo; - type OriginPrivilegeCmp = OriginPrivilegeCmp; - type Preimages = Preimage; -} - -parameter_types! { - pub const PreimageBaseDeposit: Balance = deposit(2, 64); - pub const PreimageByteDeposit: Balance = deposit(0, 1); - pub const PreimageHoldReason: RuntimeHoldReason = RuntimeHoldReason::Preimage(pallet_preimage::HoldReason::Preimage); -} - -impl pallet_preimage::Config for Runtime { - type WeightInfo = weights::pallet_preimage::WeightInfo; - type RuntimeEvent = RuntimeEvent; - type Currency = Balances; - type ManagerOrigin = EnsureRoot; - type Consideration = HoldConsideration< - AccountId, - Balances, - PreimageHoldReason, - LinearStoragePrice, - >; -} - -parameter_types! { - pub EpochDuration: u64 = prod_or_fast!( - EPOCH_DURATION_IN_SLOTS as u64, - 2 * MINUTES as u64, - "DOT_EPOCH_DURATION" - ); - pub const ExpectedBlockTime: Moment = MILLISECS_PER_BLOCK; - pub ReportLongevity: u64 = - BondingDuration::get() as u64 * SessionsPerEra::get() as u64 * EpochDuration::get(); -} - -impl pallet_babe::Config for Runtime { - type EpochDuration = EpochDuration; - type ExpectedBlockTime = ExpectedBlockTime; - - // session module is the trigger - type EpochChangeTrigger = pallet_babe::ExternalTrigger; - - type DisabledValidators = Session; - - type WeightInfo = (); - - type MaxAuthorities = MaxAuthorities; - type MaxNominators = MaxNominatorRewardedPerValidator; - - type KeyOwnerProof = - >::Proof; - - type EquivocationReportSystem = - pallet_babe::EquivocationReportSystem; -} - -parameter_types! { - pub const IndexDeposit: Balance = 10 * DOLLARS; -} - -impl pallet_indices::Config for Runtime { - type AccountIndex = AccountIndex; - type Currency = Balances; - type Deposit = IndexDeposit; - type RuntimeEvent = RuntimeEvent; - type WeightInfo = weights::pallet_indices::WeightInfo; -} - -parameter_types! { - pub const ExistentialDeposit: Balance = EXISTENTIAL_DEPOSIT; - pub const MaxLocks: u32 = 50; - pub const MaxReserves: u32 = 50; -} - -impl pallet_balances::Config for Runtime { - type Balance = Balance; - type DustRemoval = (); - type RuntimeEvent = RuntimeEvent; - type ExistentialDeposit = ExistentialDeposit; - type AccountStore = System; - type MaxLocks = MaxLocks; - type MaxReserves = MaxReserves; - type ReserveIdentifier = [u8; 8]; - type WeightInfo = weights::pallet_balances::WeightInfo; - type RuntimeHoldReason = RuntimeHoldReason; - type FreezeIdentifier = (); - type MaxHolds = ConstU32<1>; - type MaxFreezes = ConstU32<0>; -} - -parameter_types! { - pub const TransactionByteFee: Balance = 10 * MILLICENTS; - /// This value increases the priority of `Operational` transactions by adding - /// a "virtual tip" that's equal to the `OperationalFeeMultiplier * final_fee`. - pub const OperationalFeeMultiplier: u8 = 5; -} - -impl pallet_transaction_payment::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type OnChargeTransaction = CurrencyAdapter>; - type OperationalFeeMultiplier = OperationalFeeMultiplier; - type WeightToFee = WeightToFee; - type LengthToFee = ConstantMultiplier; - type FeeMultiplierUpdate = SlowAdjustingFeeUpdate; -} - -parameter_types! { - pub const MinimumPeriod: u64 = SLOT_DURATION / 2; -} -impl pallet_timestamp::Config for Runtime { - type Moment = u64; - type OnTimestampSet = Babe; - type MinimumPeriod = MinimumPeriod; - type WeightInfo = weights::pallet_timestamp::WeightInfo; -} - -impl pallet_authorship::Config for Runtime { - type FindAuthor = pallet_session::FindAccountFromAuthorIndex; - type EventHandler = (Staking, ImOnline); -} - -impl_opaque_keys! { - pub struct SessionKeys { - pub grandpa: Grandpa, - pub babe: Babe, - pub im_online: ImOnline, - pub para_validator: Initializer, - pub para_assignment: ParaSessionInfo, - pub authority_discovery: AuthorityDiscovery, - } -} - -impl pallet_session::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type ValidatorId = AccountId; - type ValidatorIdOf = pallet_staking::StashOf; - type ShouldEndSession = Babe; - type NextSessionRotation = Babe; - type SessionManager = pallet_session::historical::NoteHistoricalRoot; - type SessionHandler = ::KeyTypeIdProviders; - type Keys = SessionKeys; - type WeightInfo = weights::pallet_session::WeightInfo; -} - -impl pallet_session::historical::Config for Runtime { - type FullIdentification = pallet_staking::Exposure; - type FullIdentificationOf = pallet_staking::ExposureOf; -} - -parameter_types! { - // phase durations. 1/4 of the last session for each. - // in testing: 1min or half of the session for each - pub SignedPhase: u32 = prod_or_fast!( - EPOCH_DURATION_IN_SLOTS / 4, - (1 * MINUTES).min(EpochDuration::get().saturated_into::() / 2), - "DOT_SIGNED_PHASE" - ); - pub UnsignedPhase: u32 = prod_or_fast!( - EPOCH_DURATION_IN_SLOTS / 4, - (1 * MINUTES).min(EpochDuration::get().saturated_into::() / 2), - "DOT_UNSIGNED_PHASE" - ); - - // signed config - pub const SignedMaxSubmissions: u32 = 16; - pub const SignedMaxRefunds: u32 = 16 / 4; - pub const SignedFixedDeposit: Balance = deposit(2, 0); - pub const SignedDepositIncreaseFactor: Percent = Percent::from_percent(10); - // 40 DOTs fixed deposit.. - pub const SignedDepositBase: Balance = deposit(2, 0); - // 0.01 DOT per KB of solution data. - pub const SignedDepositByte: Balance = deposit(0, 10) / 1024; - // Each good submission will get 1 DOT as reward - pub SignedRewardBase: Balance = 1 * UNITS; - pub BetterUnsignedThreshold: Perbill = Perbill::from_rational(5u32, 10_000); - - // 4 hour session, 1 hour unsigned phase, 32 offchain executions. - pub OffchainRepeat: BlockNumber = UnsignedPhase::get() / 32; - - pub const MaxElectingVoters: u32 = 22_500; - /// We take the top 22500 nominators as electing voters and all of the validators as electable - /// targets. Whilst this is the case, we cannot and shall not increase the size of the - /// validator intentions. - pub ElectionBounds: frame_election_provider_support::bounds::ElectionBounds = - ElectionBoundsBuilder::default().voters_count(MaxElectingVoters::get().into()).build(); - /// Setup election pallet to support maximum winners upto 1200. This will mean Staking Pallet - /// cannot have active validators higher than this count. - pub const MaxActiveValidators: u32 = 1200; -} - -generate_solution_type!( - #[compact] - pub struct NposCompactSolution16::< - VoterIndex = u32, - TargetIndex = u16, - Accuracy = sp_runtime::PerU16, - MaxVoters = MaxElectingVoters, - >(16) -); - -pub struct OnChainSeqPhragmen; -impl onchain::Config for OnChainSeqPhragmen { - type System = Runtime; - type Solver = SequentialPhragmen; - type DataProvider = Staking; - type WeightInfo = weights::frame_election_provider_support::WeightInfo; - type MaxWinners = MaxActiveValidators; - type Bounds = ElectionBounds; -} - -impl pallet_election_provider_multi_phase::MinerConfig for Runtime { - type AccountId = AccountId; - type MaxLength = OffchainSolutionLengthLimit; - type MaxWeight = OffchainSolutionWeightLimit; - type Solution = NposCompactSolution16; - type MaxVotesPerVoter = < - ::DataProvider - as - frame_election_provider_support::ElectionDataProvider - >::MaxVotesPerVoter; - type MaxWinners = MaxActiveValidators; - - // The unsigned submissions have to respect the weight of the submit_unsigned call, thus their - // weight estimate function is wired to this call's weight. - fn solution_weight(v: u32, t: u32, a: u32, d: u32) -> Weight { - < - ::WeightInfo - as - pallet_election_provider_multi_phase::WeightInfo - >::submit_unsigned(v, t, a, d) - } -} - -impl pallet_election_provider_multi_phase::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type Currency = Balances; - type EstimateCallFee = TransactionPayment; - type SignedPhase = SignedPhase; - type UnsignedPhase = UnsignedPhase; - type SignedMaxSubmissions = SignedMaxSubmissions; - type SignedMaxRefunds = SignedMaxRefunds; - type SignedRewardBase = SignedRewardBase; - type SignedDepositBase = - GeometricDepositBase; - type SignedDepositByte = SignedDepositByte; - type SignedDepositWeight = (); - type SignedMaxWeight = - ::MaxWeight; - type MinerConfig = Self; - type SlashHandler = (); // burn slashes - type RewardHandler = (); // nothing to do upon rewards - type BetterUnsignedThreshold = BetterUnsignedThreshold; - type BetterSignedThreshold = (); - type OffchainRepeat = OffchainRepeat; - type MinerTxPriority = NposSolutionPriority; - type DataProvider = Staking; - #[cfg(any(feature = "fast-runtime", feature = "runtime-benchmarks"))] - type Fallback = onchain::OnChainExecution; - #[cfg(not(any(feature = "fast-runtime", feature = "runtime-benchmarks")))] - type Fallback = frame_election_provider_support::NoElection<( - AccountId, - BlockNumber, - Staking, - MaxActiveValidators, - )>; - type GovernanceFallback = onchain::OnChainExecution; - type Solver = SequentialPhragmen< - AccountId, - pallet_election_provider_multi_phase::SolutionAccuracyOf, - (), - >; - type BenchmarkingConfig = runtime_common::elections::BenchmarkConfig; - type ForceOrigin = EitherOf, StakingAdmin>; - type WeightInfo = weights::pallet_election_provider_multi_phase::WeightInfo; - type MaxWinners = MaxActiveValidators; - type ElectionBounds = ElectionBounds; -} - -parameter_types! { - pub const BagThresholds: &'static [u64] = &bag_thresholds::THRESHOLDS; -} - -type VoterBagsListInstance = pallet_bags_list::Instance1; -impl pallet_bags_list::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type ScoreProvider = Staking; - type WeightInfo = weights::pallet_bags_list::WeightInfo; - type BagThresholds = BagThresholds; - type Score = sp_npos_elections::VoteWeight; -} - -// TODO #6469: This shouldn't be static, but a lazily cached value, not built unless needed, and -// re-built in case input parameters have changed. The `ideal_stake` should be determined by the -// amount of parachain slots being bid on: this should be around `(75 - 25.min(slots / 4))%`. -pallet_staking_reward_curve::build! { - const REWARD_CURVE: PiecewiseLinear<'static> = curve!( - min_inflation: 0_025_000, - max_inflation: 0_100_000, - // 3:2:1 staked : parachains : float. - // while there's no parachains, then this is 75% staked : 25% float. - ideal_stake: 0_750_000, - falloff: 0_050_000, - max_piece_count: 40, - test_precision: 0_005_000, - ); -} - -parameter_types! { - // Six sessions in an era (24 hours). - pub const SessionsPerEra: SessionIndex = prod_or_fast!(6, 1); - - // 28 eras for unbonding (28 days). - pub BondingDuration: sp_staking::EraIndex = prod_or_fast!( - 28, - 28, - "DOT_BONDING_DURATION" - ); - pub SlashDeferDuration: sp_staking::EraIndex = prod_or_fast!( - 27, - 27, - "DOT_SLASH_DEFER_DURATION" - ); - pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE; - pub const MaxNominatorRewardedPerValidator: u32 = 512; - pub const OffendingValidatorsThreshold: Perbill = Perbill::from_percent(17); - // 16 - pub const MaxNominations: u32 = ::LIMIT as u32; -} - -pub struct EraPayout; -impl pallet_staking::EraPayout for EraPayout { - fn era_payout( - total_staked: Balance, - total_issuance: Balance, - era_duration_millis: u64, - ) -> (Balance, Balance) { - // all para-ids that are not active. - let auctioned_slots = Paras::parachains() - .into_iter() - // all active para-ids that do not belong to a system chain is the number - // of parachains that we should take into account for inflation. - .filter(|i| *i >= LOWEST_PUBLIC_ID) - .count() as u64; - - const MAX_ANNUAL_INFLATION: Perquintill = Perquintill::from_percent(10); - const MILLISECONDS_PER_YEAR: u64 = 1000 * 3600 * 24 * 36525 / 100; - - runtime_common::impls::era_payout( - total_staked, - total_issuance, - MAX_ANNUAL_INFLATION, - Perquintill::from_rational(era_duration_millis, MILLISECONDS_PER_YEAR), - auctioned_slots, - ) - } -} - -impl pallet_staking::Config for Runtime { - type Currency = Balances; - type CurrencyBalance = Balance; - type UnixTime = Timestamp; - type CurrencyToVote = CurrencyToVote; - type RewardRemainder = Treasury; - type RuntimeEvent = RuntimeEvent; - type Slash = Treasury; - type Reward = (); - type SessionsPerEra = SessionsPerEra; - type BondingDuration = BondingDuration; - type SlashDeferDuration = SlashDeferDuration; - type AdminOrigin = EitherOf, StakingAdmin>; - type SessionInterface = Self; - type EraPayout = EraPayout; - type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; - type OffendingValidatorsThreshold = OffendingValidatorsThreshold; - type NextNewSession = Session; - type ElectionProvider = ElectionProviderMultiPhase; - type GenesisElectionProvider = onchain::OnChainExecution; - type VoterList = VoterList; - type TargetList = UseValidatorsMap; - type NominationsQuota = pallet_staking::FixedNominationsQuota<{ MaxNominations::get() }>; - type MaxUnlockingChunks = frame_support::traits::ConstU32<32>; - type HistoryDepth = frame_support::traits::ConstU32<84>; - type BenchmarkingConfig = runtime_common::StakingBenchmarkingConfig; - type EventListeners = NominationPools; - type WeightInfo = weights::pallet_staking::WeightInfo; -} - -impl pallet_fast_unstake::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type Currency = Balances; - type BatchSize = frame_support::traits::ConstU32<16>; - type Deposit = frame_support::traits::ConstU128<{ UNITS }>; - type ControlOrigin = EnsureRoot; - type Staking = Staking; - type MaxErasToCheckPerBlock = ConstU32<1>; - #[cfg(feature = "runtime-benchmarks")] - type MaxBackersPerValidator = MaxNominatorRewardedPerValidator; - type WeightInfo = weights::pallet_fast_unstake::WeightInfo; -} - -parameter_types! { - // Minimum 4 CENTS/byte - pub const BasicDeposit: Balance = deposit(1, 258); - pub const FieldDeposit: Balance = deposit(0, 66); - pub const SubAccountDeposit: Balance = deposit(1, 53); - pub const MaxSubAccounts: u32 = 100; - pub const MaxAdditionalFields: u32 = 100; - pub const MaxRegistrars: u32 = 20; -} - -impl pallet_identity::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type Currency = Balances; - type BasicDeposit = BasicDeposit; - type FieldDeposit = FieldDeposit; - type SubAccountDeposit = SubAccountDeposit; - type MaxSubAccounts = MaxSubAccounts; - type MaxAdditionalFields = MaxAdditionalFields; - type MaxRegistrars = MaxRegistrars; - type Slashed = Treasury; - type ForceOrigin = EitherOf, GeneralAdmin>; - type RegistrarOrigin = EitherOf, GeneralAdmin>; - type WeightInfo = weights::pallet_identity::WeightInfo; -} - -parameter_types! { - pub const ProposalBond: Permill = Permill::from_percent(5); - pub const ProposalBondMinimum: Balance = 100 * DOLLARS; - pub const ProposalBondMaximum: Balance = 500 * DOLLARS; - pub const SpendPeriod: BlockNumber = 24 * DAYS; - pub const Burn: Permill = Permill::from_percent(1); - pub const TreasuryPalletId: PalletId = PalletId(*b"py/trsry"); - - pub const TipCountdown: BlockNumber = 1 * DAYS; - pub const TipFindersFee: Percent = Percent::from_percent(20); - pub const TipReportDepositBase: Balance = 1 * DOLLARS; - pub const DataDepositPerByte: Balance = 1 * CENTS; - pub const MaxApprovals: u32 = 100; - pub const MaxAuthorities: u32 = 100_000; - pub const MaxKeys: u32 = 10_000; - pub const MaxPeerInHeartbeats: u32 = 10_000; - pub const RootSpendOriginMaxAmount: Balance = Balance::MAX; - pub const CouncilSpendOriginMaxAmount: Balance = Balance::MAX; -} - -impl pallet_treasury::Config for Runtime { - type PalletId = TreasuryPalletId; - type Currency = Balances; - type ApproveOrigin = EitherOfDiverse, Treasurer>; - type RejectOrigin = EitherOfDiverse, Treasurer>; - type RuntimeEvent = RuntimeEvent; - type OnSlash = Treasury; - type ProposalBond = ProposalBond; - type ProposalBondMinimum = ProposalBondMinimum; - type ProposalBondMaximum = ProposalBondMaximum; - type SpendPeriod = SpendPeriod; - type Burn = Burn; - type BurnDestination = (); - type SpendFunds = Bounties; - type MaxApprovals = MaxApprovals; - type WeightInfo = weights::pallet_treasury::WeightInfo; - type SpendOrigin = TreasurySpender; -} - -parameter_types! { - pub const BountyDepositBase: Balance = 1 * DOLLARS; - pub const BountyDepositPayoutDelay: BlockNumber = 8 * DAYS; - pub const BountyUpdatePeriod: BlockNumber = 90 * DAYS; - pub const MaximumReasonLength: u32 = 16384; - pub const CuratorDepositMultiplier: Permill = Permill::from_percent(50); - pub const CuratorDepositMin: Balance = 10 * DOLLARS; - pub const CuratorDepositMax: Balance = 200 * DOLLARS; - pub const BountyValueMinimum: Balance = 10 * DOLLARS; -} - -impl pallet_bounties::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type BountyDepositBase = BountyDepositBase; - type BountyDepositPayoutDelay = BountyDepositPayoutDelay; - type BountyUpdatePeriod = BountyUpdatePeriod; - type CuratorDepositMultiplier = CuratorDepositMultiplier; - type CuratorDepositMin = CuratorDepositMin; - type CuratorDepositMax = CuratorDepositMax; - type BountyValueMinimum = BountyValueMinimum; - type ChildBountyManager = ChildBounties; - type DataDepositPerByte = DataDepositPerByte; - type MaximumReasonLength = MaximumReasonLength; - type WeightInfo = weights::pallet_bounties::WeightInfo; -} - -parameter_types! { - pub const MaxActiveChildBountyCount: u32 = 100; - pub const ChildBountyValueMinimum: Balance = BountyValueMinimum::get() / 10; -} - -impl pallet_child_bounties::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type MaxActiveChildBountyCount = MaxActiveChildBountyCount; - type ChildBountyValueMinimum = ChildBountyValueMinimum; - type WeightInfo = weights::pallet_child_bounties::WeightInfo; -} - -impl pallet_offences::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type IdentificationTuple = pallet_session::historical::IdentificationTuple; - type OnOffenceHandler = Staking; -} - -impl pallet_authority_discovery::Config for Runtime { - type MaxAuthorities = MaxAuthorities; -} - -parameter_types! { - pub NposSolutionPriority: TransactionPriority = - Perbill::from_percent(90) * TransactionPriority::max_value(); - pub const ImOnlineUnsignedPriority: TransactionPriority = TransactionPriority::max_value(); -} - -impl pallet_im_online::Config for Runtime { - type AuthorityId = ImOnlineId; - type RuntimeEvent = RuntimeEvent; - type ValidatorSet = Historical; - type NextSessionRotation = Babe; - type ReportUnresponsiveness = Offences; - type UnsignedPriority = ImOnlineUnsignedPriority; - type WeightInfo = weights::pallet_im_online::WeightInfo; - type MaxKeys = MaxKeys; - type MaxPeerInHeartbeats = MaxPeerInHeartbeats; -} - -parameter_types! { - pub MaxSetIdSessionEntries: u32 = BondingDuration::get() * SessionsPerEra::get(); -} - -impl pallet_grandpa::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - - type WeightInfo = (); - type MaxAuthorities = MaxAuthorities; - type MaxNominators = MaxNominatorRewardedPerValidator; - type MaxSetIdSessionEntries = MaxSetIdSessionEntries; - - type KeyOwnerProof = >::Proof; - - type EquivocationReportSystem = - pallet_grandpa::EquivocationReportSystem; -} - -/// Submits a transaction with the node's public and signature type. Adheres to the signed extension -/// format of the chain. -impl frame_system::offchain::CreateSignedTransaction for Runtime -where - RuntimeCall: From, -{ - fn create_transaction>( - call: RuntimeCall, - public: ::Signer, - account: AccountId, - nonce: ::Nonce, - ) -> Option<(RuntimeCall, ::SignaturePayload)> { - use sp_runtime::traits::StaticLookup; - // take the biggest period possible. - let period = - BlockHashCount::get().checked_next_power_of_two().map(|c| c / 2).unwrap_or(2) as u64; - - let current_block = System::block_number() - .saturated_into::() - // The `System::block_number` is initialized with `n+1`, - // so the actual block number is `n`. - .saturating_sub(1); - let tip = 0; - let extra: SignedExtra = ( - frame_system::CheckNonZeroSender::::new(), - frame_system::CheckSpecVersion::::new(), - frame_system::CheckTxVersion::::new(), - frame_system::CheckGenesis::::new(), - frame_system::CheckMortality::::from(generic::Era::mortal( - period, - current_block, - )), - frame_system::CheckNonce::::from(nonce), - frame_system::CheckWeight::::new(), - pallet_transaction_payment::ChargeTransactionPayment::::from(tip), - claims::PrevalidateAttests::::new(), - ); - let raw_payload = SignedPayload::new(call, extra) - .map_err(|e| { - log::warn!("Unable to create signed payload: {:?}", e); - }) - .ok()?; - let signature = raw_payload.using_encoded(|payload| C::sign(payload, public))?; - let (call, extra, _) = raw_payload.deconstruct(); - let address = ::Lookup::unlookup(account); - Some((call, (address, signature, extra))) - } -} - -impl frame_system::offchain::SigningTypes for Runtime { - type Public = ::Signer; - type Signature = Signature; -} - -impl frame_system::offchain::SendTransactionTypes for Runtime -where - RuntimeCall: From, -{ - type Extrinsic = UncheckedExtrinsic; - type OverarchingCall = RuntimeCall; -} - -parameter_types! { - // Deposit for a parathread (on-demand parachain) - pub const ParathreadDeposit: Balance = 500 * DOLLARS; - pub const MaxRetries: u32 = 3; -} - -parameter_types! { - pub Prefix: &'static [u8] = b"Pay DOTs to the Polkadot account:"; -} - -impl claims::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type VestingSchedule = Vesting; - type Prefix = Prefix; - /// Only Root can move a claim. - type MoveClaimOrigin = EnsureRoot; - type WeightInfo = weights::runtime_common_claims::WeightInfo; -} - -parameter_types! { - pub const MinVestedTransfer: Balance = 1 * DOLLARS; - pub UnvestedFundsAllowedWithdrawReasons: WithdrawReasons = - WithdrawReasons::except(WithdrawReasons::TRANSFER | WithdrawReasons::RESERVE); -} - -impl pallet_vesting::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type Currency = Balances; - type BlockNumberToBalance = ConvertInto; - type MinVestedTransfer = MinVestedTransfer; - type WeightInfo = weights::pallet_vesting::WeightInfo; - type UnvestedFundsAllowedWithdrawReasons = UnvestedFundsAllowedWithdrawReasons; - const MAX_VESTING_SCHEDULES: u32 = 28; -} - -impl pallet_utility::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type RuntimeCall = RuntimeCall; - type PalletsOrigin = OriginCaller; - type WeightInfo = weights::pallet_utility::WeightInfo; -} - -parameter_types! { - // One storage item; key size is 32; value is size 4+4+16+32 bytes = 56 bytes. - pub const DepositBase: Balance = deposit(1, 88); - // Additional storage item size of 32 bytes. - pub const DepositFactor: Balance = deposit(0, 32); - pub const MaxSignatories: u32 = 100; -} - -impl pallet_multisig::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type RuntimeCall = RuntimeCall; - type Currency = Balances; - type DepositBase = DepositBase; - type DepositFactor = DepositFactor; - type MaxSignatories = MaxSignatories; - type WeightInfo = weights::pallet_multisig::WeightInfo; -} - -parameter_types! { - // One storage item; key size 32, value size 8; . - pub const ProxyDepositBase: Balance = deposit(1, 8); - // Additional storage item size of 33 bytes. - pub const ProxyDepositFactor: Balance = deposit(0, 33); - pub const MaxProxies: u16 = 32; - pub const AnnouncementDepositBase: Balance = deposit(1, 8); - pub const AnnouncementDepositFactor: Balance = deposit(0, 66); - pub const MaxPending: u16 = 32; -} - -/// The type used to represent the kinds of proxying allowed. -#[derive( - Copy, - Clone, - Eq, - PartialEq, - Ord, - PartialOrd, - Encode, - Decode, - RuntimeDebug, - MaxEncodedLen, - scale_info::TypeInfo, -)] -pub enum ProxyType { - Any = 0, - NonTransfer = 1, - Governance = 2, - Staking = 3, - // Skip 4 as it is now removed (was SudoBalances) - IdentityJudgement = 5, - CancelProxy = 6, - Auction = 7, - NominationPools = 8, -} - -#[cfg(test)] -mod proxy_type_tests { - use super::*; - - #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Encode, Decode, RuntimeDebug)] - pub enum OldProxyType { - Any, - NonTransfer, - Governance, - Staking, - SudoBalances, - IdentityJudgement, - } - - #[test] - fn proxy_type_decodes_correctly() { - for (i, j) in vec![ - (OldProxyType::Any, ProxyType::Any), - (OldProxyType::NonTransfer, ProxyType::NonTransfer), - (OldProxyType::Governance, ProxyType::Governance), - (OldProxyType::Staking, ProxyType::Staking), - (OldProxyType::IdentityJudgement, ProxyType::IdentityJudgement), - ] - .into_iter() - { - assert_eq!(i.encode(), j.encode()); - } - assert!(ProxyType::decode(&mut &OldProxyType::SudoBalances.encode()[..]).is_err()); - } -} - -impl Default for ProxyType { - fn default() -> Self { - Self::Any - } -} -impl InstanceFilter for ProxyType { - fn filter(&self, c: &RuntimeCall) -> bool { - match self { - ProxyType::Any => true, - ProxyType::NonTransfer => matches!( - c, - RuntimeCall::System(..) | - RuntimeCall::Scheduler(..) | - RuntimeCall::Babe(..) | - RuntimeCall::Timestamp(..) | - RuntimeCall::Indices(pallet_indices::Call::claim{..}) | - RuntimeCall::Indices(pallet_indices::Call::free{..}) | - RuntimeCall::Indices(pallet_indices::Call::freeze{..}) | - // Specifically omitting Indices `transfer`, `force_transfer` - // Specifically omitting the entire Balances pallet - RuntimeCall::Staking(..) | - RuntimeCall::Session(..) | - RuntimeCall::Grandpa(..) | - RuntimeCall::ImOnline(..) | - RuntimeCall::Treasury(..) | - RuntimeCall::Bounties(..) | - RuntimeCall::ChildBounties(..) | - RuntimeCall::ConvictionVoting(..) | - RuntimeCall::Referenda(..) | - RuntimeCall::Whitelist(..) | - RuntimeCall::Claims(..) | - RuntimeCall::Vesting(pallet_vesting::Call::vest{..}) | - RuntimeCall::Vesting(pallet_vesting::Call::vest_other{..}) | - // Specifically omitting Vesting `vested_transfer`, and `force_vested_transfer` - RuntimeCall::Utility(..) | - RuntimeCall::Identity(..) | - RuntimeCall::Proxy(..) | - RuntimeCall::Multisig(..) | - RuntimeCall::Registrar(paras_registrar::Call::register {..}) | - RuntimeCall::Registrar(paras_registrar::Call::deregister {..}) | - // Specifically omitting Registrar `swap` - RuntimeCall::Registrar(paras_registrar::Call::reserve {..}) | - RuntimeCall::Crowdloan(..) | - RuntimeCall::Slots(..) | - RuntimeCall::Auctions(..) | // Specifically omitting the entire XCM Pallet - RuntimeCall::VoterList(..) | - RuntimeCall::NominationPools(..) | - RuntimeCall::FastUnstake(..) - ), - ProxyType::Governance => matches!( - c, - RuntimeCall::Treasury(..) | - RuntimeCall::Bounties(..) | - RuntimeCall::Utility(..) | - RuntimeCall::ChildBounties(..) | - RuntimeCall::ConvictionVoting(..) | - RuntimeCall::Referenda(..) | - RuntimeCall::Whitelist(..) - ), - ProxyType::Staking => { - matches!( - c, - RuntimeCall::Staking(..) | - RuntimeCall::Session(..) | RuntimeCall::Utility(..) | - RuntimeCall::FastUnstake(..) | - RuntimeCall::VoterList(..) | - RuntimeCall::NominationPools(..) - ) - }, - ProxyType::NominationPools => { - matches!(c, RuntimeCall::NominationPools(..) | RuntimeCall::Utility(..)) - }, - ProxyType::IdentityJudgement => matches!( - c, - RuntimeCall::Identity(pallet_identity::Call::provide_judgement { .. }) | - RuntimeCall::Utility(..) - ), - ProxyType::CancelProxy => { - matches!(c, RuntimeCall::Proxy(pallet_proxy::Call::reject_announcement { .. })) - }, - ProxyType::Auction => matches!( - c, - RuntimeCall::Auctions(..) | - RuntimeCall::Crowdloan(..) | - RuntimeCall::Registrar(..) | - RuntimeCall::Slots(..) - ), - } - } - fn is_superset(&self, o: &Self) -> bool { - match (self, o) { - (x, y) if x == y => true, - (ProxyType::Any, _) => true, - (_, ProxyType::Any) => false, - (ProxyType::NonTransfer, _) => true, - _ => false, - } - } -} - -impl pallet_proxy::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type RuntimeCall = RuntimeCall; - type Currency = Balances; - type ProxyType = ProxyType; - type ProxyDepositBase = ProxyDepositBase; - type ProxyDepositFactor = ProxyDepositFactor; - type MaxProxies = MaxProxies; - type WeightInfo = weights::pallet_proxy::WeightInfo; - type MaxPending = MaxPending; - type CallHasher = BlakeTwo256; - type AnnouncementDepositBase = AnnouncementDepositBase; - type AnnouncementDepositFactor = AnnouncementDepositFactor; -} - -impl parachains_origin::Config for Runtime {} - -impl parachains_configuration::Config for Runtime { - type WeightInfo = weights::runtime_parachains_configuration::WeightInfo; -} - -impl parachains_shared::Config for Runtime {} - -impl parachains_session_info::Config for Runtime { - type ValidatorSet = Historical; -} - -impl parachains_inclusion::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type DisputesHandler = ParasDisputes; - type RewardValidators = parachains_reward_points::RewardValidatorsWithEraPoints; - type MessageQueue = MessageQueue; - type WeightInfo = weights::runtime_parachains_inclusion::WeightInfo; -} - -parameter_types! { - pub const ParasUnsignedPriority: TransactionPriority = TransactionPriority::max_value(); -} - -impl parachains_paras::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type WeightInfo = weights::runtime_parachains_paras::WeightInfo; - type UnsignedPriority = ParasUnsignedPriority; - type QueueFootprinter = ParaInclusion; - type NextSessionRotation = Babe; - type OnNewHead = Registrar; -} - -parameter_types! { - /// Amount of weight that can be spent per block to service messages. - /// - /// # WARNING - /// - /// This is not a good value for para-chains since the `Scheduler` already uses up to 80% block weight. - pub MessageQueueServiceWeight: Weight = Perbill::from_percent(20) * BlockWeights::get().max_block; - pub const MessageQueueHeapSize: u32 = 65_536; - pub const MessageQueueMaxStale: u32 = 8; -} - -/// Message processor to handle any messages that were enqueued into the `MessageQueue` pallet. -pub struct MessageProcessor; -impl ProcessMessage for MessageProcessor { - type Origin = AggregateMessageOrigin; - - fn process_message( - message: &[u8], - origin: Self::Origin, - meter: &mut WeightMeter, - id: &mut [u8; 32], - ) -> Result { - let para = match origin { - AggregateMessageOrigin::Ump(UmpQueueId::Para(para)) => para, - }; - xcm_builder::ProcessXcmMessage::< - Junction, - xcm_executor::XcmExecutor, - RuntimeCall, - >::process_message(message, Junction::Parachain(para.into()), meter, id) - } -} - -impl pallet_message_queue::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type Size = u32; - type HeapSize = MessageQueueHeapSize; - type MaxStale = MessageQueueMaxStale; - type ServiceWeight = MessageQueueServiceWeight; - #[cfg(not(feature = "runtime-benchmarks"))] - type MessageProcessor = MessageProcessor; - #[cfg(feature = "runtime-benchmarks")] - type MessageProcessor = - pallet_message_queue::mock_helpers::NoopMessageProcessor; - type QueueChangeHandler = ParaInclusion; - type QueuePausedQuery = (); - type WeightInfo = weights::pallet_message_queue::WeightInfo; -} - -impl parachains_dmp::Config for Runtime {} - -impl parachains_hrmp::Config for Runtime { - type RuntimeOrigin = RuntimeOrigin; - type RuntimeEvent = RuntimeEvent; - type ChannelManager = EitherOf, GeneralAdmin>; - type Currency = Balances; - type WeightInfo = weights::runtime_parachains_hrmp::WeightInfo; -} - -impl parachains_paras_inherent::Config for Runtime { - type WeightInfo = weights::runtime_parachains_paras_inherent::WeightInfo; -} - -impl parachains_scheduler::Config for Runtime { - type AssignmentProvider = ParaAssignmentProvider; -} - -impl parachains_assigner_parachains::Config for Runtime {} - -impl parachains_initializer::Config for Runtime { - type Randomness = pallet_babe::RandomnessFromOneEpochAgo; - type ForceOrigin = EnsureRoot; - type WeightInfo = weights::runtime_parachains_initializer::WeightInfo; -} - -impl parachains_disputes::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type RewardValidators = parachains_reward_points::RewardValidatorsWithEraPoints; - type SlashingHandler = parachains_slashing::SlashValidatorsForDisputes; - type WeightInfo = weights::runtime_parachains_disputes::WeightInfo; -} - -impl parachains_slashing::Config for Runtime { - type KeyOwnerProofSystem = Historical; - type KeyOwnerProof = - >::Proof; - type KeyOwnerIdentification = >::IdentificationTuple; - type HandleReports = parachains_slashing::SlashingReportHandler< - Self::KeyOwnerIdentification, - Offences, - ReportLongevity, - >; - type WeightInfo = weights::runtime_parachains_disputes_slashing::WeightInfo; - type BenchmarkingConfig = parachains_slashing::BenchConfig<1000>; -} - -parameter_types! { - // Mostly arbitrary deposit price, but should provide an adequate incentive not to spam reserve - // `ParaId`s. - pub const ParaDeposit: Balance = 100 * DOLLARS; - pub const ParaDataByteDeposit: Balance = deposit(0, 1); -} - -impl paras_registrar::Config for Runtime { - type RuntimeOrigin = RuntimeOrigin; - type RuntimeEvent = RuntimeEvent; - type Currency = Balances; - type OnSwap = (Crowdloan, Slots); - type ParaDeposit = ParaDeposit; - type DataDepositPerByte = ParaDataByteDeposit; - type WeightInfo = weights::runtime_common_paras_registrar::WeightInfo; -} - -parameter_types! { - // 12 weeks = 3 months per lease period -> 8 lease periods ~ 2 years - pub LeasePeriod: BlockNumber = prod_or_fast!(12 * WEEKS, 12 * WEEKS, "DOT_LEASE_PERIOD"); - // Polkadot Genesis was on May 26, 2020. - // Target Parachain Onboarding Date: Dec 15, 2021. - // Difference is 568 days. - // We want a lease period to start on the target onboarding date. - // 568 % (12 * 7) = 64 day offset - pub LeaseOffset: BlockNumber = prod_or_fast!(64 * DAYS, 0, "DOT_LEASE_OFFSET"); -} - -impl slots::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type Currency = Balances; - type Registrar = Registrar; - type LeasePeriod = LeasePeriod; - type LeaseOffset = LeaseOffset; - type ForceOrigin = EitherOf, LeaseAdmin>; - type WeightInfo = weights::runtime_common_slots::WeightInfo; -} - -parameter_types! { - pub const CrowdloanId: PalletId = PalletId(*b"py/cfund"); - // Accounts for 10_000 contributions, each using 48 bytes (16 bytes for balance, and 32 bytes - // for a memo). - pub const SubmissionDeposit: Balance = deposit(1, 480_000); - // The minimum crowdloan contribution. - pub const MinContribution: Balance = 5 * DOLLARS; - pub const RemoveKeysLimit: u32 = 1000; - // Allow 32 bytes for an additional memo to a crowdloan. - pub const MaxMemoLength: u8 = 32; -} - -impl crowdloan::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type PalletId = CrowdloanId; - type SubmissionDeposit = SubmissionDeposit; - type MinContribution = MinContribution; - type RemoveKeysLimit = RemoveKeysLimit; - type Registrar = Registrar; - type Auctioneer = Auctions; - type MaxMemoLength = MaxMemoLength; - type WeightInfo = weights::runtime_common_crowdloan::WeightInfo; -} - -parameter_types! { - // The average auction is 7 days long, so this will be 70% for ending period. - // 5 Days = 72000 Blocks @ 6 sec per block - pub const EndingPeriod: BlockNumber = 5 * DAYS; - // ~ 1000 samples per day -> ~ 20 blocks per sample -> 2 minute samples - pub const SampleLength: BlockNumber = 2 * MINUTES; -} - -impl auctions::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type Leaser = Slots; - type Registrar = Registrar; - type EndingPeriod = EndingPeriod; - type SampleLength = SampleLength; - type Randomness = pallet_babe::RandomnessFromOneEpochAgo; - type InitiateOrigin = EitherOf, AuctionAdmin>; - type WeightInfo = weights::runtime_common_auctions::WeightInfo; -} - -parameter_types! { - pub const PoolsPalletId: PalletId = PalletId(*b"py/nopls"); - // Allow pools that got slashed up to 90% to remain operational. - pub const MaxPointsToBalance: u8 = 10; -} - -impl pallet_nomination_pools::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type Currency = Balances; - type RewardCounter = FixedU128; - type BalanceToU256 = runtime_common::BalanceToU256; - type U256ToBalance = runtime_common::U256ToBalance; - type Staking = Staking; - type PostUnbondingPoolsWindow = frame_support::traits::ConstU32<4>; - type MaxMetadataLen = frame_support::traits::ConstU32<256>; - // we use the same number of allowed unlocking chunks as with staking. - type MaxUnbonding = ::MaxUnlockingChunks; - type PalletId = PoolsPalletId; - type MaxPointsToBalance = MaxPointsToBalance; - type WeightInfo = weights::pallet_nomination_pools::WeightInfo; -} - -pub struct InitiateNominationPools; -impl frame_support::traits::OnRuntimeUpgrade for InitiateNominationPools { - fn on_runtime_upgrade() -> frame_support::weights::Weight { - // we use one as an indicator if this has already been set. - if pallet_nomination_pools::MaxPools::::get().is_none() { - // 5 DOT to join a pool. - pallet_nomination_pools::MinJoinBond::::put(5 * UNITS); - // 100 DOT to create a pool. - pallet_nomination_pools::MinCreateBond::::put(100 * UNITS); - - // Initialize with limits for now. - pallet_nomination_pools::MaxPools::::put(0); - pallet_nomination_pools::MaxPoolMembersPerPool::::put(0); - pallet_nomination_pools::MaxPoolMembers::::put(0); - - log::info!(target: "runtime::polkadot", "pools config initiated 🎉"); - ::DbWeight::get().reads_writes(1, 5) - } else { - log::info!(target: "runtime::polkadot", "pools config already initiated 😏"); - ::DbWeight::get().reads(1) - } - } -} - -construct_runtime! { - pub enum Runtime - { - // Basic stuff; balances is uncallable initially. - System: frame_system::{Pallet, Call, Storage, Config, Event} = 0, - Scheduler: pallet_scheduler::{Pallet, Call, Storage, Event} = 1, - Preimage: pallet_preimage::{Pallet, Call, Storage, Event, HoldReason} = 10, - - // Babe must be before session. - Babe: pallet_babe::{Pallet, Call, Storage, Config, ValidateUnsigned} = 2, - - Timestamp: pallet_timestamp::{Pallet, Call, Storage, Inherent} = 3, - Indices: pallet_indices::{Pallet, Call, Storage, Config, Event} = 4, - Balances: pallet_balances::{Pallet, Call, Storage, Config, Event} = 5, - TransactionPayment: pallet_transaction_payment::{Pallet, Storage, Event} = 32, - - // Consensus support. - // Authorship must be before session in order to note author in the correct session and era - // for im-online and staking. - Authorship: pallet_authorship::{Pallet, Storage} = 6, - Staking: pallet_staking::{Pallet, Call, Storage, Config, Event} = 7, - Offences: pallet_offences::{Pallet, Storage, Event} = 8, - Historical: session_historical::{Pallet} = 33, - Session: pallet_session::{Pallet, Call, Storage, Event, Config} = 9, - Grandpa: pallet_grandpa::{Pallet, Call, Storage, Config, Event, ValidateUnsigned} = 11, - ImOnline: pallet_im_online::{Pallet, Call, Storage, Event, ValidateUnsigned, Config} = 12, - AuthorityDiscovery: pallet_authority_discovery::{Pallet, Config} = 13, - - // OpenGov stuff. - Treasury: pallet_treasury::{Pallet, Call, Storage, Config, Event} = 19, - ConvictionVoting: pallet_conviction_voting::{Pallet, Call, Storage, Event} = 20, - Referenda: pallet_referenda::{Pallet, Call, Storage, Event} = 21, - Origins: pallet_custom_origins::{Origin} = 22, - Whitelist: pallet_whitelist::{Pallet, Call, Storage, Event} = 23, - - // Claims. Usable initially. - Claims: claims::{Pallet, Call, Storage, Event, Config, ValidateUnsigned} = 24, - // Vesting. Usable initially, but removed once all vesting is finished. - Vesting: pallet_vesting::{Pallet, Call, Storage, Event, Config} = 25, - // Cunning utilities. Usable initially. - Utility: pallet_utility::{Pallet, Call, Event} = 26, - - // Identity. Late addition. - Identity: pallet_identity::{Pallet, Call, Storage, Event} = 28, - - // Proxy module. Late addition. - Proxy: pallet_proxy::{Pallet, Call, Storage, Event} = 29, - - // Multisig dispatch. Late addition. - Multisig: pallet_multisig::{Pallet, Call, Storage, Event} = 30, - - // Bounties modules. - Bounties: pallet_bounties::{Pallet, Call, Storage, Event} = 34, - ChildBounties: pallet_child_bounties = 38, - - // Election pallet. Only works with staking, but placed here to maintain indices. - ElectionProviderMultiPhase: pallet_election_provider_multi_phase::{Pallet, Call, Storage, Event, ValidateUnsigned} = 36, - - // Provides a semi-sorted list of nominators for staking. - VoterList: pallet_bags_list::::{Pallet, Call, Storage, Event} = 37, - - // Nomination pools: extension to staking. - NominationPools: pallet_nomination_pools::{Pallet, Call, Storage, Event, Config} = 39, - - // Fast unstake pallet: extension to staking. - FastUnstake: pallet_fast_unstake = 40, - - // Parachains pallets. Start indices at 50 to leave room. - ParachainsOrigin: parachains_origin::{Pallet, Origin} = 50, - Configuration: parachains_configuration::{Pallet, Call, Storage, Config} = 51, - ParasShared: parachains_shared::{Pallet, Call, Storage} = 52, - ParaInclusion: parachains_inclusion::{Pallet, Call, Storage, Event} = 53, - ParaInherent: parachains_paras_inherent::{Pallet, Call, Storage, Inherent} = 54, - ParaScheduler: parachains_scheduler::{Pallet, Storage} = 55, - Paras: parachains_paras::{Pallet, Call, Storage, Event, Config, ValidateUnsigned} = 56, - Initializer: parachains_initializer::{Pallet, Call, Storage} = 57, - Dmp: parachains_dmp::{Pallet, Storage} = 58, - // Ump 59 - Hrmp: parachains_hrmp::{Pallet, Call, Storage, Event, Config} = 60, - ParaSessionInfo: parachains_session_info::{Pallet, Storage} = 61, - ParasDisputes: parachains_disputes::{Pallet, Call, Storage, Event} = 62, - ParasSlashing: parachains_slashing::{Pallet, Call, Storage, ValidateUnsigned} = 63, - ParaAssignmentProvider: parachains_assigner_parachains::{Pallet} = 64, - - // Parachain Onboarding Pallets. Start indices at 70 to leave room. - Registrar: paras_registrar::{Pallet, Call, Storage, Event} = 70, - Slots: slots::{Pallet, Call, Storage, Event} = 71, - Auctions: auctions::{Pallet, Call, Storage, Event} = 72, - Crowdloan: crowdloan::{Pallet, Call, Storage, Event} = 73, - - // Pallet for sending XCM. - XcmPallet: pallet_xcm::{Pallet, Call, Storage, Event, Origin, Config} = 99, - - // Generalized message queue - MessageQueue: pallet_message_queue::{Pallet, Call, Storage, Event} = 100, - } -} - -/// The address format for describing accounts. -pub type Address = sp_runtime::MultiAddress; -/// Block header type as expected by this runtime. -pub type Header = generic::Header; -/// Block type as expected by this runtime. -pub type Block = generic::Block; -/// A Block signed with a Justification -pub type SignedBlock = generic::SignedBlock; -/// `BlockId` type as expected by this runtime. -pub type BlockId = generic::BlockId; -/// The `SignedExtension` to the basic transaction logic. -pub type SignedExtra = ( - frame_system::CheckNonZeroSender, - frame_system::CheckSpecVersion, - frame_system::CheckTxVersion, - frame_system::CheckGenesis, - frame_system::CheckMortality, - frame_system::CheckNonce, - frame_system::CheckWeight, - pallet_transaction_payment::ChargeTransactionPayment, - claims::PrevalidateAttests, -); - -pub struct NominationPoolsMigrationV4OldPallet; -impl Get for NominationPoolsMigrationV4OldPallet { - fn get() -> Perbill { - Perbill::zero() - } -} - -/// All migrations that will run on the next runtime upgrade. -/// -/// This contains the combined migrations of the last 10 releases. It allows to skip runtime -/// upgrades in case governance decides to do so. THE ORDER IS IMPORTANT. -pub type Migrations = migrations::Unreleased; - -/// The runtime migrations per release. -#[allow(deprecated, missing_docs)] -pub mod migrations { - use super::*; - use frame_support::traits::LockIdentifier; - use frame_system::pallet_prelude::BlockNumberFor; - - parameter_types! { - pub const DemocracyPalletName: &'static str = "Democracy"; - pub const CouncilPalletName: &'static str = "Council"; - pub const TechnicalCommitteePalletName: &'static str = "TechnicalCommittee"; - pub const PhragmenElectionPalletName: &'static str = "PhragmenElection"; - pub const TechnicalMembershipPalletName: &'static str = "TechnicalMembership"; - pub const TipsPalletName: &'static str = "Tips"; - pub const PhragmenElectionPalletId: LockIdentifier = *b"phrelect"; - } - - // Special Config for Gov V1 pallets, allowing us to run migrations for them without - // implementing their configs on [`Runtime`]. - pub struct UnlockConfig; - impl pallet_democracy::migrations::unlock_and_unreserve_all_funds::UnlockConfig for UnlockConfig { - type Currency = Balances; - type MaxVotes = ConstU32<100>; - type MaxDeposits = ConstU32<100>; - type AccountId = AccountId; - type BlockNumber = BlockNumberFor; - type DbWeight = ::DbWeight; - type PalletName = DemocracyPalletName; - } - impl pallet_elections_phragmen::migrations::unlock_and_unreserve_all_funds::UnlockConfig - for UnlockConfig - { - type Currency = Balances; - type MaxVotesPerVoter = ConstU32<16>; - type PalletId = PhragmenElectionPalletId; - type AccountId = AccountId; - type DbWeight = ::DbWeight; - type PalletName = PhragmenElectionPalletName; - } - impl pallet_tips::migrations::unreserve_deposits::UnlockConfig<()> for UnlockConfig { - type Currency = Balances; - type Hash = Hash; - type DataDepositPerByte = DataDepositPerByte; - type TipReportDepositBase = TipReportDepositBase; - type AccountId = AccountId; - type BlockNumber = BlockNumberFor; - type DbWeight = ::DbWeight; - type PalletName = TipsPalletName; - } - - pub struct ParachainsToUnlock; - impl Contains for ParachainsToUnlock { - fn contains(id: &ParaId) -> bool { - let id: u32 = (*id).into(); - // polkadot parachains/parathreads that are locked and never produced block - match id { - 2003 | 2015 | 2017 | 2018 | 2025 | 2028 | 2036 | 2038 | 2053 | 2055 | 2090 | - 2097 | 2106 | 3336 | 3338 | 3342 => true, - _ => false, - } - } - } - - /// Unreleased migrations. Add new ones here: - pub type Unreleased = ( - pallet_im_online::migration::v1::Migration, - parachains_configuration::migration::v7::MigrateToV7, - parachains_scheduler::migration::v1::MigrateToV1, - parachains_configuration::migration::v8::MigrateToV8, - - // Gov v1 storage migrations - // https://github.com/paritytech/polkadot/issues/6749 - pallet_elections_phragmen::migrations::unlock_and_unreserve_all_funds::UnlockAndUnreserveAllFunds, - pallet_democracy::migrations::unlock_and_unreserve_all_funds::UnlockAndUnreserveAllFunds, - pallet_tips::migrations::unreserve_deposits::UnreserveDeposits, - - // Delete all Gov v1 pallet storage key/values. - frame_support::migrations::RemovePallet::DbWeight>, - frame_support::migrations::RemovePallet::DbWeight>, - frame_support::migrations::RemovePallet::DbWeight>, - frame_support::migrations::RemovePallet::DbWeight>, - frame_support::migrations::RemovePallet::DbWeight>, - frame_support::migrations::RemovePallet::DbWeight>, - - parachains_configuration::migration::v9::MigrateToV9, - // Migrate parachain info format - paras_registrar::migration::VersionCheckedMigrateToV1, - parachains_configuration::migration::v10::MigrateToV10, - ); -} - -/// Unchecked extrinsic type as expected by this runtime. -pub type UncheckedExtrinsic = - generic::UncheckedExtrinsic; -/// Executive: handles dispatch to the various modules. -pub type Executive = frame_executive::Executive< - Runtime, - Block, - frame_system::ChainContext, - Runtime, - AllPalletsWithSystem, - Migrations, ->; - -/// The payload being signed in transactions. -pub type SignedPayload = generic::SignedPayload; - -#[cfg(feature = "runtime-benchmarks")] -mod benches { - frame_benchmarking::define_benchmarks!( - // Polkadot - // NOTE: Make sure to prefix these with `runtime_common::` so - // the that path resolves correctly in the generated file. - [runtime_common::auctions, Auctions] - [runtime_common::claims, Claims] - [runtime_common::crowdloan, Crowdloan] - [runtime_common::slots, Slots] - [runtime_common::paras_registrar, Registrar] - [runtime_parachains::configuration, Configuration] - [runtime_parachains::disputes, ParasDisputes] - [runtime_parachains::disputes::slashing, ParasSlashing] - [runtime_parachains::hrmp, Hrmp] - [runtime_parachains::inclusion, ParaInclusion] - [runtime_parachains::initializer, Initializer] - [runtime_parachains::paras, Paras] - [runtime_parachains::paras_inherent, ParaInherent] - // Substrate - [pallet_bags_list, VoterList] - [pallet_balances, Balances] - [frame_benchmarking::baseline, Baseline::] - [pallet_bounties, Bounties] - [pallet_child_bounties, ChildBounties] - [pallet_election_provider_multi_phase, ElectionProviderMultiPhase] - [frame_election_provider_support, ElectionProviderBench::] - [pallet_fast_unstake, FastUnstake] - [pallet_identity, Identity] - [pallet_im_online, ImOnline] - [pallet_indices, Indices] - [pallet_message_queue, MessageQueue] - [pallet_multisig, Multisig] - [pallet_nomination_pools, NominationPoolsBench::] - [pallet_offences, OffencesBench::] - [pallet_preimage, Preimage] - [pallet_proxy, Proxy] - [pallet_scheduler, Scheduler] - [pallet_session, SessionBench::] - [pallet_staking, Staking] - [frame_system, SystemBench::] - [pallet_timestamp, Timestamp] - [pallet_treasury, Treasury] - [pallet_utility, Utility] - [pallet_vesting, Vesting] - [pallet_conviction_voting, ConvictionVoting] - [pallet_referenda, Referenda] - [pallet_whitelist, Whitelist] - // XCM - [pallet_xcm, XcmPallet] - [pallet_xcm_benchmarks::fungible, pallet_xcm_benchmarks::fungible::Pallet::] - [pallet_xcm_benchmarks::generic, pallet_xcm_benchmarks::generic::Pallet::] - ); -} - -sp_api::impl_runtime_apis! { - impl sp_api::Core for Runtime { - fn version() -> RuntimeVersion { - VERSION - } - - fn execute_block(block: Block) { - Executive::execute_block(block); - } - - fn initialize_block(header: &::Header) { - Executive::initialize_block(header) - } - } - - impl sp_api::Metadata for Runtime { - fn metadata() -> OpaqueMetadata { - OpaqueMetadata::new(Runtime::metadata().into()) - } - - fn metadata_at_version(version: u32) -> Option { - Runtime::metadata_at_version(version) - } - - fn metadata_versions() -> sp_std::vec::Vec { - Runtime::metadata_versions() - } - } - - impl block_builder_api::BlockBuilder for Runtime { - fn apply_extrinsic(extrinsic: ::Extrinsic) -> ApplyExtrinsicResult { - Executive::apply_extrinsic(extrinsic) - } - - fn finalize_block() -> ::Header { - Executive::finalize_block() - } - - fn inherent_extrinsics(data: inherents::InherentData) -> Vec<::Extrinsic> { - data.create_extrinsics() - } - - fn check_inherents( - block: Block, - data: inherents::InherentData, - ) -> inherents::CheckInherentsResult { - data.check_extrinsics(&block) - } - } - - impl pallet_nomination_pools_runtime_api::NominationPoolsApi< - Block, - AccountId, - Balance, - > for Runtime { - fn pending_rewards(member: AccountId) -> Balance { - NominationPools::api_pending_rewards(member).unwrap_or_default() - } - - fn points_to_balance(pool_id: pallet_nomination_pools::PoolId, points: Balance) -> Balance { - NominationPools::api_points_to_balance(pool_id, points) - } - - fn balance_to_points(pool_id: pallet_nomination_pools::PoolId, new_funds: Balance) -> Balance { - NominationPools::api_balance_to_points(pool_id, new_funds) - } - } - - impl pallet_staking_runtime_api::StakingApi for Runtime { - fn nominations_quota(balance: Balance) -> u32 { - Staking::api_nominations_quota(balance) - } - } - - impl tx_pool_api::runtime_api::TaggedTransactionQueue for Runtime { - fn validate_transaction( - source: TransactionSource, - tx: ::Extrinsic, - block_hash: ::Hash, - ) -> TransactionValidity { - Executive::validate_transaction(source, tx, block_hash) - } - } - - impl offchain_primitives::OffchainWorkerApi for Runtime { - fn offchain_worker(header: &::Header) { - Executive::offchain_worker(header) - } - } - - impl primitives::runtime_api::ParachainHost for Runtime { - fn validators() -> Vec { - parachains_runtime_api_impl::validators::() - } - - fn validator_groups() -> (Vec>, GroupRotationInfo) { - parachains_runtime_api_impl::validator_groups::() - } - - fn availability_cores() -> Vec> { - parachains_runtime_api_impl::availability_cores::() - } - - fn persisted_validation_data(para_id: ParaId, assumption: OccupiedCoreAssumption) - -> Option> { - parachains_runtime_api_impl::persisted_validation_data::(para_id, assumption) - } - - fn assumed_validation_data( - para_id: ParaId, - expected_persisted_validation_data_hash: Hash, - ) -> Option<(PersistedValidationData, ValidationCodeHash)> { - parachains_runtime_api_impl::assumed_validation_data::( - para_id, - expected_persisted_validation_data_hash, - ) - } - - fn check_validation_outputs( - para_id: ParaId, - outputs: primitives::CandidateCommitments, - ) -> bool { - parachains_runtime_api_impl::check_validation_outputs::(para_id, outputs) - } - - fn session_index_for_child() -> SessionIndex { - parachains_runtime_api_impl::session_index_for_child::() - } - - fn validation_code(para_id: ParaId, assumption: OccupiedCoreAssumption) - -> Option { - parachains_runtime_api_impl::validation_code::(para_id, assumption) - } - - fn candidate_pending_availability(para_id: ParaId) -> Option> { - parachains_runtime_api_impl::candidate_pending_availability::(para_id) - } - - fn candidate_events() -> Vec> { - parachains_runtime_api_impl::candidate_events::(|ev| { - match ev { - RuntimeEvent::ParaInclusion(ev) => { - Some(ev) - } - _ => None, - } - }) - } - - fn session_info(index: SessionIndex) -> Option { - parachains_runtime_api_impl::session_info::(index) - } - - fn session_executor_params(session_index: SessionIndex) -> Option { - parachains_runtime_api_impl::session_executor_params::(session_index) - } - - fn dmq_contents(recipient: ParaId) -> Vec> { - parachains_runtime_api_impl::dmq_contents::(recipient) - } - - fn inbound_hrmp_channels_contents( - recipient: ParaId - ) -> BTreeMap>> { - parachains_runtime_api_impl::inbound_hrmp_channels_contents::(recipient) - } - - fn validation_code_by_hash(hash: ValidationCodeHash) -> Option { - parachains_runtime_api_impl::validation_code_by_hash::(hash) - } - - fn on_chain_votes() -> Option> { - parachains_runtime_api_impl::on_chain_votes::() - } - - fn submit_pvf_check_statement( - stmt: primitives::PvfCheckStatement, - signature: primitives::ValidatorSignature, - ) { - parachains_runtime_api_impl::submit_pvf_check_statement::(stmt, signature) - } - - fn pvfs_require_precheck() -> Vec { - parachains_runtime_api_impl::pvfs_require_precheck::() - } - - fn validation_code_hash(para_id: ParaId, assumption: OccupiedCoreAssumption) - -> Option - { - parachains_runtime_api_impl::validation_code_hash::(para_id, assumption) - } - - fn disputes() -> Vec<(SessionIndex, CandidateHash, DisputeState)> { - parachains_runtime_api_impl::get_session_disputes::() - } - - fn unapplied_slashes( - ) -> Vec<(SessionIndex, CandidateHash, slashing::PendingSlashes)> { - parachains_runtime_api_impl::unapplied_slashes::() - } - - fn key_ownership_proof( - validator_id: ValidatorId, - ) -> Option { - use parity_scale_codec::Encode; - - Historical::prove((PARACHAIN_KEY_TYPE_ID, validator_id)) - .map(|p| p.encode()) - .map(slashing::OpaqueKeyOwnershipProof::new) - } - - fn submit_report_dispute_lost( - dispute_proof: slashing::DisputeProof, - key_ownership_proof: slashing::OpaqueKeyOwnershipProof, - ) -> Option<()> { - parachains_runtime_api_impl::submit_unsigned_slashing_report::( - dispute_proof, - key_ownership_proof, - ) - } - } - - impl beefy_primitives::BeefyApi for Runtime { - fn beefy_genesis() -> Option { - // dummy implementation due to lack of BEEFY pallet. - None - } - - fn validator_set() -> Option> { - // dummy implementation due to lack of BEEFY pallet. - None - } - - fn submit_report_equivocation_unsigned_extrinsic( - _equivocation_proof: beefy_primitives::EquivocationProof< - BlockNumber, - BeefyId, - BeefySignature, - >, - _key_owner_proof: beefy_primitives::OpaqueKeyOwnershipProof, - ) -> Option<()> { - None - } - - fn generate_key_ownership_proof( - _set_id: beefy_primitives::ValidatorSetId, - _authority_id: BeefyId, - ) -> Option { - None - } - } - - impl mmr::MmrApi for Runtime { - fn mmr_root() -> Result { - Err(mmr::Error::PalletNotIncluded) - } - - fn mmr_leaf_count() -> Result { - Err(mmr::Error::PalletNotIncluded) - } - - fn generate_proof( - _block_numbers: Vec, - _best_known_block_number: Option, - ) -> Result<(Vec, mmr::Proof), mmr::Error> { - Err(mmr::Error::PalletNotIncluded) - } - - fn verify_proof(_leaves: Vec, _proof: mmr::Proof) - -> Result<(), mmr::Error> - { - Err(mmr::Error::PalletNotIncluded) - } - - fn verify_proof_stateless( - _root: Hash, - _leaves: Vec, - _proof: mmr::Proof - ) -> Result<(), mmr::Error> { - Err(mmr::Error::PalletNotIncluded) - } - } - - impl fg_primitives::GrandpaApi for Runtime { - fn grandpa_authorities() -> Vec<(GrandpaId, u64)> { - Grandpa::grandpa_authorities() - } - - fn current_set_id() -> fg_primitives::SetId { - Grandpa::current_set_id() - } - - fn submit_report_equivocation_unsigned_extrinsic( - equivocation_proof: fg_primitives::EquivocationProof< - ::Hash, - sp_runtime::traits::NumberFor, - >, - key_owner_proof: fg_primitives::OpaqueKeyOwnershipProof, - ) -> Option<()> { - let key_owner_proof = key_owner_proof.decode()?; - - Grandpa::submit_unsigned_equivocation_report( - equivocation_proof, - key_owner_proof, - ) - } - - fn generate_key_ownership_proof( - _set_id: fg_primitives::SetId, - authority_id: fg_primitives::AuthorityId, - ) -> Option { - use parity_scale_codec::Encode; - - Historical::prove((fg_primitives::KEY_TYPE, authority_id)) - .map(|p| p.encode()) - .map(fg_primitives::OpaqueKeyOwnershipProof::new) - } - } - - impl babe_primitives::BabeApi for Runtime { - fn configuration() -> babe_primitives::BabeConfiguration { - let epoch_config = Babe::epoch_config().unwrap_or(BABE_GENESIS_EPOCH_CONFIG); - babe_primitives::BabeConfiguration { - slot_duration: Babe::slot_duration(), - epoch_length: EpochDuration::get(), - c: epoch_config.c, - authorities: Babe::authorities().to_vec(), - randomness: Babe::randomness(), - allowed_slots: epoch_config.allowed_slots, - } - } - - fn current_epoch_start() -> babe_primitives::Slot { - Babe::current_epoch_start() - } - - fn current_epoch() -> babe_primitives::Epoch { - Babe::current_epoch() - } - - fn next_epoch() -> babe_primitives::Epoch { - Babe::next_epoch() - } - - fn generate_key_ownership_proof( - _slot: babe_primitives::Slot, - authority_id: babe_primitives::AuthorityId, - ) -> Option { - use parity_scale_codec::Encode; - - Historical::prove((babe_primitives::KEY_TYPE, authority_id)) - .map(|p| p.encode()) - .map(babe_primitives::OpaqueKeyOwnershipProof::new) - } - - fn submit_report_equivocation_unsigned_extrinsic( - equivocation_proof: babe_primitives::EquivocationProof<::Header>, - key_owner_proof: babe_primitives::OpaqueKeyOwnershipProof, - ) -> Option<()> { - let key_owner_proof = key_owner_proof.decode()?; - - Babe::submit_unsigned_equivocation_report( - equivocation_proof, - key_owner_proof, - ) - } - } - - impl authority_discovery_primitives::AuthorityDiscoveryApi for Runtime { - fn authorities() -> Vec { - parachains_runtime_api_impl::relevant_authority_ids::() - } - } - - impl sp_session::SessionKeys for Runtime { - fn generate_session_keys(seed: Option>) -> Vec { - SessionKeys::generate(seed) - } - - fn decode_session_keys( - encoded: Vec, - ) -> Option, sp_core::crypto::KeyTypeId)>> { - SessionKeys::decode_into_raw_public_keys(&encoded) - } - } - - impl frame_system_rpc_runtime_api::AccountNonceApi for Runtime { - fn account_nonce(account: AccountId) -> Nonce { - System::account_nonce(account) - } - } - - impl pallet_transaction_payment_rpc_runtime_api::TransactionPaymentApi< - Block, - Balance, - > for Runtime { - fn query_info(uxt: ::Extrinsic, len: u32) -> RuntimeDispatchInfo { - TransactionPayment::query_info(uxt, len) - } - fn query_fee_details(uxt: ::Extrinsic, len: u32) -> FeeDetails { - TransactionPayment::query_fee_details(uxt, len) - } - fn query_weight_to_fee(weight: Weight) -> Balance { - TransactionPayment::weight_to_fee(weight) - } - fn query_length_to_fee(length: u32) -> Balance { - TransactionPayment::length_to_fee(length) - } - } - - impl pallet_transaction_payment_rpc_runtime_api::TransactionPaymentCallApi - for Runtime - { - fn query_call_info(call: RuntimeCall, len: u32) -> RuntimeDispatchInfo { - TransactionPayment::query_call_info(call, len) - } - fn query_call_fee_details(call: RuntimeCall, len: u32) -> FeeDetails { - TransactionPayment::query_call_fee_details(call, len) - } - fn query_weight_to_fee(weight: Weight) -> Balance { - TransactionPayment::weight_to_fee(weight) - } - fn query_length_to_fee(length: u32) -> Balance { - TransactionPayment::length_to_fee(length) - } - } - - #[cfg(feature = "try-runtime")] - impl frame_try_runtime::TryRuntime for Runtime { - fn on_runtime_upgrade(checks: frame_try_runtime::UpgradeCheckSelect) -> (Weight, Weight) { - log::info!("try-runtime::on_runtime_upgrade polkadot."); - let weight = Executive::try_runtime_upgrade(checks).unwrap(); - (weight, BlockWeights::get().max_block) - } - - fn execute_block( - block: Block, - state_root_check: bool, - signature_check: bool, - select: frame_try_runtime::TryStateSelect, - ) -> Weight { - // NOTE: intentional unwrap: we don't want to propagate the error backwards, and want to - // have a backtrace here. - Executive::try_execute_block(block, state_root_check, signature_check, select).unwrap() - } - } - - #[cfg(feature = "runtime-benchmarks")] - impl frame_benchmarking::Benchmark for Runtime { - fn benchmark_metadata(extra: bool) -> ( - Vec, - Vec, - ) { - use frame_benchmarking::{Benchmarking, BenchmarkList}; - use frame_support::traits::StorageInfoTrait; - - use pallet_session_benchmarking::Pallet as SessionBench; - use pallet_offences_benchmarking::Pallet as OffencesBench; - use pallet_election_provider_support_benchmarking::Pallet as ElectionProviderBench; - use pallet_nomination_pools_benchmarking::Pallet as NominationPoolsBench; - use frame_system_benchmarking::Pallet as SystemBench; - use frame_benchmarking::baseline::Pallet as Baseline; - - let mut list = Vec::::new(); - list_benchmarks!(list, extra); - - let storage_info = AllPalletsWithSystem::storage_info(); - return (list, storage_info) - } - - fn dispatch_benchmark( - config: frame_benchmarking::BenchmarkConfig - ) -> Result< - Vec, - sp_runtime::RuntimeString, - > { - use frame_support::traits::WhitelistedStorageKeys; - use frame_benchmarking::{Benchmarking, BenchmarkBatch, BenchmarkError}; - use sp_storage::TrackedStorageKey; - // Trying to add benchmarks directly to some pallets caused cyclic dependency issues. - // To get around that, we separated the benchmarks into its own crate. - use pallet_session_benchmarking::Pallet as SessionBench; - use pallet_offences_benchmarking::Pallet as OffencesBench; - use pallet_election_provider_support_benchmarking::Pallet as ElectionProviderBench; - use pallet_nomination_pools_benchmarking::Pallet as NominationPoolsBench; - use frame_system_benchmarking::Pallet as SystemBench; - use frame_benchmarking::baseline::Pallet as Baseline; - use xcm::latest::prelude::*; - use xcm_config::{XcmConfig, StatemintLocation, TokenLocation, LocalCheckAccount, SovereignAccountOf}; - - impl pallet_session_benchmarking::Config for Runtime {} - impl pallet_offences_benchmarking::Config for Runtime {} - impl pallet_election_provider_support_benchmarking::Config for Runtime {} - impl frame_system_benchmarking::Config for Runtime {} - impl frame_benchmarking::baseline::Config for Runtime {} - impl pallet_nomination_pools_benchmarking::Config for Runtime {} - impl runtime_parachains::disputes::slashing::benchmarking::Config for Runtime {} - - let mut whitelist: Vec = AllPalletsWithSystem::whitelisted_storage_keys(); - let treasury_key = frame_system::Account::::hashed_key_for(Treasury::account_id()); - whitelist.push(treasury_key.to_vec().into()); - - impl pallet_xcm_benchmarks::Config for Runtime { - type XcmConfig = XcmConfig; - type AccountIdConverter = SovereignAccountOf; - fn valid_destination() -> Result { - Ok(StatemintLocation::get()) - } - fn worst_case_holding(_depositable_count: u32) -> MultiAssets { - // Polkadot only knows about DOT - vec![MultiAsset { id: Concrete(TokenLocation::get()), fun: Fungible(1_000_000 * UNITS) }].into() - } - } - - parameter_types! { - pub const TrustedTeleporter: Option<(MultiLocation, MultiAsset)> = Some(( - StatemintLocation::get(), - MultiAsset { id: Concrete(TokenLocation::get()), fun: Fungible(1 * UNITS) } - )); - pub const TrustedReserve: Option<(MultiLocation, MultiAsset)> = None; - } - - impl pallet_xcm_benchmarks::fungible::Config for Runtime { - type TransactAsset = Balances; - - type CheckedAccount = LocalCheckAccount; - type TrustedTeleporter = TrustedTeleporter; - type TrustedReserve = TrustedReserve; - - fn get_multi_asset() -> MultiAsset { - MultiAsset { - id: Concrete(TokenLocation::get()), - fun: Fungible(1 * UNITS) - } - } - } - - impl pallet_xcm_benchmarks::generic::Config for Runtime { - type RuntimeCall = RuntimeCall; - - fn worst_case_response() -> (u64, Response) { - (0u64, Response::Version(Default::default())) - } - - fn worst_case_asset_exchange() -> Result<(MultiAssets, MultiAssets), BenchmarkError> { - // Polkadot doesn't support asset exchanges - Err(BenchmarkError::Skip) - } - - fn universal_alias() -> Result<(MultiLocation, Junction), BenchmarkError> { - // The XCM executor of Polkadot doesn't have a configured `UniversalAliases` - Err(BenchmarkError::Skip) - } - - fn transact_origin_and_runtime_call() -> Result<(MultiLocation, RuntimeCall), BenchmarkError> { - Ok((StatemintLocation::get(), frame_system::Call::remark_with_event { remark: vec![] }.into())) - } - - fn subscribe_origin() -> Result { - Ok(StatemintLocation::get()) - } - - fn claimable_asset() -> Result<(MultiLocation, MultiLocation, MultiAssets), BenchmarkError> { - let origin = StatemintLocation::get(); - let assets: MultiAssets = (Concrete(TokenLocation::get()), 1_000 * UNITS).into(); - let ticket = MultiLocation { parents: 0, interior: Here }; - Ok((origin, ticket, assets)) - } - - fn unlockable_asset() -> Result<(MultiLocation, MultiLocation, MultiAsset), BenchmarkError> { - // Polkadot doesn't support asset locking - Err(BenchmarkError::Skip) - } - - fn export_message_origin_and_destination( - ) -> Result<(MultiLocation, NetworkId, InteriorMultiLocation), BenchmarkError> { - // Polkadot doesn't support exporting messages - Err(BenchmarkError::Skip) - } - - fn alias_origin() -> Result<(MultiLocation, MultiLocation), BenchmarkError> { - // The XCM executor of Polkadot doesn't have a configured `Aliasers` - Err(BenchmarkError::Skip) - } - } - - let mut batches = Vec::::new(); - let params = (&config, &whitelist); - - add_benchmarks!(params, batches); - - Ok(batches) - } - } -} - -#[cfg(test)] -mod test_fees { - use super::*; - use frame_support::{dispatch::GetDispatchInfo, weights::WeightToFee as WeightToFeeT}; - use keyring::Sr25519Keyring::{Alice, Charlie}; - use pallet_transaction_payment::Multiplier; - use runtime_common::MinimumMultiplier; - use separator::Separatable; - use sp_runtime::{assert_eq_error_rate, FixedPointNumber, MultiAddress, MultiSignature}; - - #[test] - fn payout_weight_portion() { - use pallet_staking::WeightInfo; - let payout_weight = - ::WeightInfo::payout_stakers_alive_staked( - MaxNominatorRewardedPerValidator::get(), - ) - .ref_time() as f64; - let block_weight = BlockWeights::get().max_block.ref_time() as f64; - - println!( - "a full payout takes {:.2} of the block weight [{} / {}]", - payout_weight / block_weight, - payout_weight, - block_weight - ); - assert!(payout_weight * 2f64 < block_weight); - } - - #[test] - fn block_cost() { - let max_block_weight = BlockWeights::get().max_block; - let raw_fee = WeightToFee::weight_to_fee(&max_block_weight); - - let fee_with_multiplier = |m: Multiplier| { - println!( - "Full Block weight == {} // multiplier: {:?} // WeightToFee(full_block) == {} plank", - max_block_weight, - m, - m.saturating_mul_int(raw_fee).separated_string(), - ); - }; - fee_with_multiplier(MinimumMultiplier::get()); - fee_with_multiplier(Multiplier::from_rational(1, 2)); - fee_with_multiplier(Multiplier::from_u32(1)); - fee_with_multiplier(Multiplier::from_u32(2)); - } - - #[test] - fn transfer_cost_min_multiplier() { - let min_multiplier = MinimumMultiplier::get(); - let call = pallet_balances::Call::::transfer_keep_alive { - dest: Charlie.to_account_id().into(), - value: Default::default(), - }; - let info = call.get_dispatch_info(); - println!("call = {:?} / info = {:?}", call, info); - // convert to runtime call. - let call = RuntimeCall::Balances(call); - let extra: SignedExtra = ( - frame_system::CheckNonZeroSender::::new(), - frame_system::CheckSpecVersion::::new(), - frame_system::CheckTxVersion::::new(), - frame_system::CheckGenesis::::new(), - frame_system::CheckMortality::::from(generic::Era::immortal()), - frame_system::CheckNonce::::from(1), - frame_system::CheckWeight::::new(), - pallet_transaction_payment::ChargeTransactionPayment::::from(0), - claims::PrevalidateAttests::::new(), - ); - let uxt = UncheckedExtrinsic { - function: call, - signature: Some(( - MultiAddress::Id(Alice.to_account_id()), - MultiSignature::Sr25519(Alice.sign(b"foo")), - extra, - )), - }; - let len = uxt.encoded_size(); - - let mut ext = sp_io::TestExternalities::new_empty(); - let mut test_with_multiplier = |m: Multiplier| { - ext.execute_with(|| { - pallet_transaction_payment::NextFeeMultiplier::::put(m); - let fee = TransactionPayment::query_fee_details(uxt.clone(), len as u32); - println!( - "multiplier = {:?} // fee details = {:?} // final fee = {:?}", - pallet_transaction_payment::NextFeeMultiplier::::get(), - fee, - fee.final_fee().separated_string(), - ); - }); - }; - - test_with_multiplier(min_multiplier); - test_with_multiplier(Multiplier::saturating_from_rational(1u128, 1u128)); - test_with_multiplier(Multiplier::saturating_from_rational(1u128, 1_0u128)); - test_with_multiplier(Multiplier::saturating_from_rational(1u128, 1_00u128)); - test_with_multiplier(Multiplier::saturating_from_rational(1u128, 1_000u128)); - test_with_multiplier(Multiplier::saturating_from_rational(1u128, 1_000_000u128)); - test_with_multiplier(Multiplier::saturating_from_rational(1u128, 1_000_000_000u128)); - } - - #[test] - fn nominator_limit() { - use pallet_election_provider_multi_phase::WeightInfo; - // starting point of the nominators. - let target_voters: u32 = 50_000; - - // assuming we want around 5k candidates and 1k active validators. (March 31, 2021) - let all_targets: u32 = 5_000; - let desired: u32 = 1_000; - let weight_with = |active| { - ::WeightInfo::submit_unsigned( - active, - all_targets, - active, - desired, - ) - }; - - let mut active = target_voters; - while weight_with(active).all_lte(OffchainSolutionWeightLimit::get()) || - active == target_voters - { - active += 1; - } - - println!("can support {} nominators to yield a weight of {}", active, weight_with(active)); - assert!(active > target_voters, "we need to reevaluate the weight of the election system"); - } - - #[test] - fn signed_deposit_is_sensible() { - // ensure this number does not change, or that it is checked after each change. - // a 1 MB solution should take (40 + 10) DOTs of deposit. - let deposit = SignedFixedDeposit::get() + (SignedDepositByte::get() * 1024 * 1024); - assert_eq_error_rate!(deposit, 50 * DOLLARS, DOLLARS); - } -} - -#[cfg(test)] -mod test { - use std::collections::HashSet; - - use super::*; - use frame_support::traits::WhitelistedStorageKeys; - use sp_core::hexdisplay::HexDisplay; - - #[test] - fn call_size() { - RuntimeCall::assert_size_under(230); - } - - #[test] - fn check_whitelist() { - let whitelist: HashSet = AllPalletsWithSystem::whitelisted_storage_keys() - .iter() - .map(|e| HexDisplay::from(&e.key).to_string()) - .collect(); - - // Block number - assert!( - whitelist.contains("26aa394eea5630e07c48ae0c9558cef702a5c1b19ab7a04f536c519aca4983ac") - ); - // Total issuance - assert!( - whitelist.contains("c2261276cc9d1f8598ea4b6a74b15c2f57c875e4cff74148e4628f264b974c80") - ); - // Execution phase - assert!( - whitelist.contains("26aa394eea5630e07c48ae0c9558cef7ff553b5a9862a516939d82b3d3d8661a") - ); - // Event count - assert!( - whitelist.contains("26aa394eea5630e07c48ae0c9558cef70a98fdbe9ce6c55837576c60c7af3850") - ); - // System events - assert!( - whitelist.contains("26aa394eea5630e07c48ae0c9558cef780d41e5e16056765bc8461851072c9d7") - ); - // XcmPallet VersionDiscoveryQueue - assert!( - whitelist.contains("1405f2411d0af5a7ff397e7c9dc68d194a222ba0333561192e474c59ed8e30e1") - ); - // XcmPallet SafeXcmVersion - assert!( - whitelist.contains("1405f2411d0af5a7ff397e7c9dc68d196323ae84c43568be0d1394d5d0d522c4") - ); - } -} - -#[cfg(test)] -mod multiplier_tests { - use super::*; - use frame_support::{dispatch::DispatchInfo, traits::OnFinalize}; - use runtime_common::{MinimumMultiplier, TargetBlockFullness}; - use separator::Separatable; - use sp_runtime::traits::Convert; - - fn run_with_system_weight(w: Weight, mut assertions: F) - where - F: FnMut() -> (), - { - let mut t: sp_io::TestExternalities = frame_system::GenesisConfig::::default() - .build_storage() - .unwrap() - .into(); - t.execute_with(|| { - System::set_block_consumed_resources(w, 0); - assertions() - }); - } - - #[test] - fn multiplier_can_grow_from_zero() { - let minimum_multiplier = MinimumMultiplier::get(); - let target = TargetBlockFullness::get() * - BlockWeights::get().get(DispatchClass::Normal).max_total.unwrap(); - // if the min is too small, then this will not change, and we are doomed forever. - // the weight is 1/100th bigger than target. - run_with_system_weight(target.saturating_mul(101) / 100, || { - let next = SlowAdjustingFeeUpdate::::convert(minimum_multiplier); - assert!(next > minimum_multiplier, "{:?} !>= {:?}", next, minimum_multiplier); - }) - } - - #[test] - fn fast_unstake_estimate() { - use pallet_fast_unstake::WeightInfo; - let block_time = BlockWeights::get().max_block.ref_time() as f32; - let on_idle = weights::pallet_fast_unstake::WeightInfo::::on_idle_check( - 300, - ::BatchSize::get(), - ) - .ref_time() as f32; - println!("ratio of block weight for full batch fast-unstake {}", on_idle / block_time); - assert!(on_idle / block_time <= 0.5f32) - } - - #[test] - #[ignore] - fn multiplier_growth_simulator() { - // assume the multiplier is initially set to its minimum. We update it with values twice the - //target (target is 25%, thus 50%) and we see at which point it reaches 1. - let mut multiplier = MinimumMultiplier::get(); - let block_weight = BlockWeights::get().get(DispatchClass::Normal).max_total.unwrap(); - let mut blocks = 0; - let mut fees_paid = 0; - - frame_system::Pallet::::set_block_consumed_resources(Weight::MAX, 0); - let info = DispatchInfo { weight: Weight::MAX, ..Default::default() }; - - let mut t: sp_io::TestExternalities = frame_system::GenesisConfig::::default() - .build_storage() - .unwrap() - .into(); - // set the minimum - t.execute_with(|| { - pallet_transaction_payment::NextFeeMultiplier::::set(MinimumMultiplier::get()); - }); - - while multiplier <= Multiplier::from_u32(1) { - t.execute_with(|| { - // imagine this tx was called. - let fee = TransactionPayment::compute_fee(0, &info, 0); - fees_paid += fee; - - // this will update the multiplier. - System::set_block_consumed_resources(block_weight, 0); - TransactionPayment::on_finalize(1); - let next = TransactionPayment::next_fee_multiplier(); - - assert!(next > multiplier, "{:?} !>= {:?}", next, multiplier); - multiplier = next; - - println!( - "block = {} / multiplier {:?} / fee = {:?} / fess so far {:?}", - blocks, - multiplier, - fee.separated_string(), - fees_paid.separated_string() - ); - }); - blocks += 1; - } - } - - #[test] - #[ignore] - fn multiplier_cool_down_simulator() { - // assume the multiplier is initially set to its minimum. We update it with values twice the - //target (target is 25%, thus 50%) and we see at which point it reaches 1. - let mut multiplier = Multiplier::from_u32(2); - let mut blocks = 0; - - let mut t: sp_io::TestExternalities = frame_system::GenesisConfig::::default() - .build_storage() - .unwrap() - .into(); - // set the minimum - t.execute_with(|| { - pallet_transaction_payment::NextFeeMultiplier::::set(multiplier); - }); - - while multiplier > Multiplier::from_u32(0) { - t.execute_with(|| { - // this will update the multiplier. - TransactionPayment::on_finalize(1); - let next = TransactionPayment::next_fee_multiplier(); - - assert!(next < multiplier, "{:?} !>= {:?}", next, multiplier); - multiplier = next; - - println!("block = {} / multiplier {:?}", blocks, multiplier); - }); - blocks += 1; - } - } -} - -#[cfg(all(test, feature = "try-runtime"))] -mod remote_tests { - use super::*; - use frame_try_runtime::{runtime_decl_for_try_runtime::TryRuntime, UpgradeCheckSelect}; - use remote_externalities::{ - Builder, Mode, OfflineConfig, OnlineConfig, SnapshotConfig, Transport, - }; - use std::env::var; - - #[tokio::test] - async fn run_migrations() { - if var("RUN_MIGRATION_TESTS").is_err() { - return - } - - sp_tracing::try_init_simple(); - let transport: Transport = - var("WS").unwrap_or("wss://rpc.polkadot.io:443".to_string()).into(); - let maybe_state_snapshot: Option = var("SNAP").map(|s| s.into()).ok(); - let mut ext = Builder::::default() - .mode(if let Some(state_snapshot) = maybe_state_snapshot { - Mode::OfflineOrElseOnline( - OfflineConfig { state_snapshot: state_snapshot.clone() }, - OnlineConfig { - transport, - state_snapshot: Some(state_snapshot), - ..Default::default() - }, - ) - } else { - Mode::Online(OnlineConfig { transport, ..Default::default() }) - }) - .build() - .await - .unwrap(); - ext.execute_with(|| Runtime::on_runtime_upgrade(UpgradeCheckSelect::PreAndPost)); - } - - #[tokio::test] - #[ignore = "this test is meant to be executed manually"] - async fn try_fast_unstake_all() { - sp_tracing::try_init_simple(); - let transport: Transport = - var("WS").unwrap_or("wss://rpc.polkadot.io:443".to_string()).into(); - let maybe_state_snapshot: Option = var("SNAP").map(|s| s.into()).ok(); - let mut ext = Builder::::default() - .mode(if let Some(state_snapshot) = maybe_state_snapshot { - Mode::OfflineOrElseOnline( - OfflineConfig { state_snapshot: state_snapshot.clone() }, - OnlineConfig { - transport, - state_snapshot: Some(state_snapshot), - ..Default::default() - }, - ) - } else { - Mode::Online(OnlineConfig { transport, ..Default::default() }) - }) - .build() - .await - .unwrap(); - ext.execute_with(|| { - pallet_fast_unstake::ErasToCheckPerBlock::::put(1); - runtime_common::try_runtime::migrate_all_inactive_nominators::() - }); - } -} From fc286188b61dfa351cf41b48318b77eba3512878 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 19 Oct 2023 15:21:38 +0300 Subject: [PATCH 044/192] Remove the need for CollationVersion::VStaging Signed-off-by: Alexandru Gheorghe --- polkadot/node/network/bridge/src/network.rs | 20 --------- polkadot/node/network/bridge/src/rx/mod.rs | 33 +------------- polkadot/node/network/bridge/src/tx/mod.rs | 20 ++------- .../src/collator_side/mod.rs | 45 +++---------------- .../src/validator_side/mod.rs | 27 +++-------- polkadot/node/network/protocol/src/lib.rs | 18 +++----- .../node/network/protocol/src/peer_set.rs | 4 -- 7 files changed, 26 insertions(+), 141 deletions(-) diff --git a/polkadot/node/network/bridge/src/network.rs b/polkadot/node/network/bridge/src/network.rs index a13045285522..c264c94cc19b 100644 --- a/polkadot/node/network/bridge/src/network.rs +++ b/polkadot/node/network/bridge/src/network.rs @@ -145,26 +145,6 @@ pub(crate) fn send_collation_message_v2( ); } -// Helper function to send a collation vstaging message to a list of peers. -// Messages are always sent via the main protocol, even legacy protocol messages. -pub(crate) fn send_collation_message_vstaging( - net: &mut impl Network, - peers: Vec, - peerset_protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Collation, - CollationVersion::VStaging.into(), - peerset_protocol_names, - message, - metrics, - ); -} - /// Lower level function that sends a message to the network using the main protocol version. /// /// This function is only used internally by the network-bridge, which is responsible to only send diff --git a/polkadot/node/network/bridge/src/rx/mod.rs b/polkadot/node/network/bridge/src/rx/mod.rs index 83bf10bbb670..acb664a8ab29 100644 --- a/polkadot/node/network/bridge/src/rx/mod.rs +++ b/polkadot/node/network/bridge/src/rx/mod.rs @@ -66,9 +66,8 @@ use super::validator_discovery; /// /// Defines the `Network` trait with an implementation for an `Arc`. use crate::network::{ - send_collation_message_v1, send_collation_message_v2, send_collation_message_vstaging, - send_validation_message_v1, send_validation_message_v2, send_validation_message_vstaging, - Network, + send_collation_message_v1, send_collation_message_v2, send_validation_message_v1, + send_validation_message_v2, send_validation_message_vstaging, Network, }; use crate::{network::get_peer_id_by_authority_id, WireMessage}; @@ -320,15 +319,6 @@ where ), &metrics, ), - CollationVersion::VStaging => send_collation_message_vstaging( - &mut network_service, - vec![peer], - &peerset_protocol_names, - WireMessage::::ViewUpdate( - local_view, - ), - &metrics, - ), } }, } @@ -548,16 +538,6 @@ where c_messages, &metrics, ) - } else if expected_versions[PeerSet::Collation] == - Some(CollationVersion::VStaging.into()) - { - handle_peer_messages::( - remote, - PeerSet::Collation, - &mut shared.0.lock().collation_peers, - c_messages, - &metrics, - ) } else { gum::warn!( target: LOG_TARGET, @@ -858,8 +838,6 @@ fn update_our_view( let vstaging_validation_peers = filter_by_peer_version(&validation_peers, ValidationVersion::VStaging.into()); - let vstaging_collation_peers = - filter_by_peer_version(&collation_peers, CollationVersion::VStaging.into()); send_validation_message_v1( net, @@ -901,13 +879,6 @@ fn update_our_view( metrics, ); - send_collation_message_vstaging( - net, - vstaging_collation_peers, - peerset_protocol_names, - WireMessage::ViewUpdate(new_view.clone()), - metrics, - ); let our_view = OurView::new( live_heads.iter().take(MAX_VIEW_HEADS).cloned().map(|a| (a.hash, a.span)), finalized_number, diff --git a/polkadot/node/network/bridge/src/tx/mod.rs b/polkadot/node/network/bridge/src/tx/mod.rs index 603869a4e868..5f222ad59c75 100644 --- a/polkadot/node/network/bridge/src/tx/mod.rs +++ b/polkadot/node/network/bridge/src/tx/mod.rs @@ -34,7 +34,7 @@ pub use polkadot_node_network_protocol::peer_set::{peer_sets_info, IsAuthority}; use polkadot_node_network_protocol::request_response::Requests; use sc_network::ReputationChange; -use crate::{network::send_collation_message_vstaging, validator_discovery}; +use crate::validator_discovery; /// Actual interfacing to the network based on the `Network` trait. /// @@ -265,14 +265,7 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), - Versioned::V2(msg) => send_collation_message_v2( - &mut network_service, - peers, - peerset_protocol_names, - WireMessage::ProtocolMessage(msg), - &metrics, - ), - Versioned::VStaging(msg) => send_collation_message_vstaging( + Versioned::V2(msg) | Versioned::VStaging(msg) => send_collation_message_v2( &mut network_service, peers, peerset_protocol_names, @@ -297,14 +290,7 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), - Versioned::V2(msg) => send_collation_message_v2( - &mut network_service, - peers, - peerset_protocol_names, - WireMessage::ProtocolMessage(msg), - &metrics, - ), - Versioned::VStaging(msg) => send_collation_message_vstaging( + Versioned::V2(msg) | Versioned::VStaging(msg) => send_collation_message_v2( &mut network_service, peers, peerset_protocol_names, diff --git a/polkadot/node/network/collator-protocol/src/collator_side/mod.rs b/polkadot/node/network/collator-protocol/src/collator_side/mod.rs index 1c40759dde7f..4fb24ef30bab 100644 --- a/polkadot/node/network/collator-protocol/src/collator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/collator_side/mod.rs @@ -33,8 +33,8 @@ use polkadot_node_network_protocol::{ incoming::{self, OutgoingResponse}, v1 as request_v1, v2 as request_v2, IncomingRequestReceiver, }, - v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, OurView, PeerId, - UnifiedReputationChange as Rep, Versioned, View, + v1 as protocol_v1, v2 as protocol_v2, OurView, PeerId, UnifiedReputationChange as Rep, + Versioned, View, }; use polkadot_node_primitives::{CollationSecondedSignal, PoV, Statement}; use polkadot_node_subsystem::{ @@ -577,13 +577,7 @@ async fn determine_our_validators( fn declare_message( state: &mut State, version: CollationVersion, -) -> Option< - Versioned< - protocol_v1::CollationProtocol, - protocol_v2::CollationProtocol, - protocol_vstaging::CollationProtocol, - >, -> { +) -> Option> { let para_id = state.collating_on?; Some(match version { CollationVersion::V1 => { @@ -606,16 +600,6 @@ fn declare_message( ); Versioned::V2(protocol_v2::CollationProtocol::CollatorProtocol(wire_message)) }, - CollationVersion::VStaging => { - let declare_signature_payload = - protocol_vstaging::declare_signature_payload(&state.local_peer_id); - let wire_message = protocol_vstaging::CollatorProtocolMessage::Declare( - state.collator_pair.public(), - para_id, - state.collator_pair.sign(&declare_signature_payload), - ); - Versioned::VStaging(protocol_v2::CollationProtocol::CollatorProtocol(wire_message)) - }, }) } @@ -728,16 +712,6 @@ async fn advertise_collation( }; Versioned::V2(protocol_v2::CollationProtocol::CollatorProtocol(wire_message)) }, - CollationVersion::VStaging => { - let wire_message = protocol_vstaging::CollatorProtocolMessage::AdvertiseCollation { - relay_parent, - candidate_hash: *candidate_hash, - parent_head_data_hash: collation.parent_head_data_hash, - }; - Versioned::VStaging(protocol_vstaging::CollationProtocol::CollatorProtocol( - wire_message, - )) - }, CollationVersion::V1 => { let wire_message = protocol_v1::CollatorProtocolMessage::AdvertiseCollation(relay_parent); @@ -890,20 +864,15 @@ async fn handle_incoming_peer_message( runtime: &mut RuntimeInfo, state: &mut State, origin: PeerId, - msg: Versioned< - protocol_v1::CollatorProtocolMessage, - protocol_v2::CollatorProtocolMessage, - protocol_vstaging::CollatorProtocolMessage, - >, + msg: Versioned, ) -> Result<()> { use protocol_v1::CollatorProtocolMessage as V1; use protocol_v2::CollatorProtocolMessage as V2; - use protocol_vstaging::CollatorProtocolMessage as VStaging; match msg { Versioned::V1(V1::Declare(..)) | Versioned::V2(V2::Declare(..)) | - Versioned::VStaging(VStaging::Declare(..)) => { + Versioned::VStaging(V2::Declare(..)) => { gum::trace!( target: LOG_TARGET, ?origin, @@ -916,7 +885,7 @@ async fn handle_incoming_peer_message( }, Versioned::V1(V1::AdvertiseCollation(_)) | Versioned::V2(V2::AdvertiseCollation { .. }) | - Versioned::VStaging(VStaging::AdvertiseCollation { .. }) => { + Versioned::VStaging(V2::AdvertiseCollation { .. }) => { gum::trace!( target: LOG_TARGET, ?origin, @@ -932,7 +901,7 @@ async fn handle_incoming_peer_message( }, Versioned::V1(V1::CollationSeconded(relay_parent, statement)) | Versioned::V2(V2::CollationSeconded(relay_parent, statement)) | - Versioned::VStaging(VStaging::CollationSeconded(relay_parent, statement)) => { + Versioned::VStaging(V2::CollationSeconded(relay_parent, statement)) => { if !matches!(statement.unchecked_payload(), Statement::Seconded(_)) { gum::warn!( target: LOG_TARGET, diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 0567b33dfe85..56b2829b5e61 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -36,8 +36,8 @@ use polkadot_node_network_protocol::{ outgoing::{Recipient, RequestError}, v1 as request_v1, v2 as request_v2, OutgoingRequest, Requests, }, - v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, OurView, PeerId, - UnifiedReputationChange as Rep, Versioned, View, + v1 as protocol_v1, v2 as protocol_v2, OurView, PeerId, UnifiedReputationChange as Rep, + Versioned, View, }; use polkadot_node_primitives::{SignedFullStatement, Statement}; use polkadot_node_subsystem::{ @@ -627,13 +627,6 @@ async fn notify_collation_seconded( CollationVersion::V2 => Versioned::V2(protocol_v2::CollationProtocol::CollatorProtocol( protocol_v2::CollatorProtocolMessage::CollationSeconded(relay_parent, statement), )), - CollationVersion::VStaging => - Versioned::VStaging(protocol_vstaging::CollationProtocol::CollatorProtocol( - protocol_vstaging::CollatorProtocolMessage::CollationSeconded( - relay_parent, - statement, - ), - )), }; sender .send_message(NetworkBridgeTxMessage::SendCollationMessage(vec![peer_id], wire_message)) @@ -697,8 +690,7 @@ async fn request_collation( let requests = Requests::CollationFetchingV1(req); (requests, response_recv.boxed()) }, - (CollationVersion::V2, Some(ProspectiveCandidate { candidate_hash, .. })) | - (CollationVersion::VStaging, Some(ProspectiveCandidate { candidate_hash, .. })) => { + (CollationVersion::V2, Some(ProspectiveCandidate { candidate_hash, .. })) => { let (req, response_recv) = OutgoingRequest::new( Recipient::Peer(peer_id), request_v2::CollationFetchingRequest { relay_parent, para_id, candidate_hash }, @@ -758,21 +750,16 @@ async fn process_incoming_peer_message( ctx: &mut Context, state: &mut State, origin: PeerId, - msg: Versioned< - protocol_v1::CollatorProtocolMessage, - protocol_v2::CollatorProtocolMessage, - protocol_vstaging::CollatorProtocolMessage, - >, + msg: Versioned, ) { use protocol_v1::CollatorProtocolMessage as V1; use protocol_v2::CollatorProtocolMessage as V2; - use protocol_vstaging::CollatorProtocolMessage as VStaging; use sp_runtime::traits::AppVerify; match msg { Versioned::V1(V1::Declare(collator_id, para_id, signature)) | Versioned::V2(V2::Declare(collator_id, para_id, signature)) | - Versioned::VStaging(VStaging::Declare(collator_id, para_id, signature)) => { + Versioned::VStaging(V2::Declare(collator_id, para_id, signature)) => { if collator_peer_id(&state.peer_data, &collator_id).is_some() { modify_reputation( &mut state.reputation, @@ -889,7 +876,7 @@ async fn process_incoming_peer_message( candidate_hash, parent_head_data_hash, }) | - Versioned::VStaging(VStaging::AdvertiseCollation { + Versioned::VStaging(V2::AdvertiseCollation { relay_parent, candidate_hash, parent_head_data_hash, @@ -918,7 +905,7 @@ async fn process_incoming_peer_message( }, Versioned::V1(V1::CollationSeconded(..)) | Versioned::V2(V2::CollationSeconded(..)) | - Versioned::VStaging(VStaging::CollationSeconded(..)) => { + Versioned::VStaging(V2::CollationSeconded(..)) => { gum::warn!( target: LOG_TARGET, peer_id = ?origin, diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index f01de9862ce8..a7251f7bf8ed 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -253,7 +253,7 @@ impl View { /// A protocol-versioned type. #[derive(Debug, Clone, PartialEq, Eq)] -pub enum Versioned { +pub enum Versioned { /// V1 type. V1(V1), /// V2 type. @@ -296,8 +296,7 @@ impl From for VersionedValidationProtocol { } /// All supported versions of the collation protocol message. -pub type VersionedCollationProtocol = - Versioned; +pub type VersionedCollationProtocol = Versioned; impl From for VersionedCollationProtocol { fn from(v1: v1::CollationProtocol) -> Self { @@ -446,11 +445,8 @@ impl<'a> TryFrom<&'a VersionedValidationProtocol> for GossipSupportNetworkMessag } /// Version-annotated messages used by the bitfield distribution subsystem. -pub type CollatorProtocolMessage = Versioned< - v1::CollatorProtocolMessage, - v2::CollatorProtocolMessage, - vstaging::CollatorProtocolMessage, ->; +pub type CollatorProtocolMessage = + Versioned; impl_versioned_full_protocol_from!( CollatorProtocolMessage, VersionedCollationProtocol, @@ -461,7 +457,7 @@ impl_versioned_try_from!( CollatorProtocolMessage, v1::CollationProtocol::CollatorProtocol(x) => x, v2::CollationProtocol::CollatorProtocol(x) => x, - vstaging::CollationProtocol::CollatorProtocol(x) => x + v2::CollationProtocol::CollatorProtocol(x) => x ); /// v1 notification protocol types. @@ -890,8 +886,8 @@ pub mod vstaging { /// no reason why they can't be change untill vstaging becomes v3 and is released. pub use super::v2::{ declare_signature_payload, BackedCandidateAcknowledgement, BackedCandidateManifest, - BitfieldDistributionMessage, CollationProtocol, CollatorProtocolMessage, - GossipSupportNetworkMessage, StatementDistributionMessage, StatementFilter, + BitfieldDistributionMessage, GossipSupportNetworkMessage, StatementDistributionMessage, + StatementFilter, }; /// Network messages used by the approval distribution subsystem. diff --git a/polkadot/node/network/protocol/src/peer_set.rs b/polkadot/node/network/protocol/src/peer_set.rs index 0f1d9de2a565..eb483dec9709 100644 --- a/polkadot/node/network/protocol/src/peer_set.rs +++ b/polkadot/node/network/protocol/src/peer_set.rs @@ -164,8 +164,6 @@ impl PeerSet { Some("collation/1") } else if version == CollationVersion::V2.into() { Some("collation/2") - } else if version == CollationVersion::VStaging.into() { - Some("collation/3") } else { None }, @@ -241,8 +239,6 @@ pub enum CollationVersion { V1 = 1, /// The second version. V2 = 2, - /// Same format as V2, - VStaging = 3, } /// Marker indicating the version is unknown. From a8d234cab7174ecea0d49650899273fd21024d81 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 19 Oct 2023 16:13:34 +0300 Subject: [PATCH 045/192] Addressed review comments Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 19 ++++++---- polkadot/node/network/protocol/src/lib.rs | 2 +- .../statement-distribution/src/v2/mod.rs | 38 +++++++++++++++++-- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 46d687035f51..47482eef7640 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -1674,8 +1674,7 @@ impl State { .approval_entries(index) .into_iter() .filter_map(|approval_entry| approval_entry.approval(index)) - .map(|approval| (approval.validator, approval.signature)) - .collect::>(); + .map(|approval| (approval.validator, approval.signature)); all_sigs.extend(sigs); } all_sigs @@ -2304,10 +2303,9 @@ pub const MAX_APPROVAL_BATCH_SIZE: usize = ensure_size_not_zero( async fn send_assignments_batched_inner( sender: &mut impl overseer::ApprovalDistributionSenderTrait, batch: impl IntoIterator, - peers: &[PeerId], + peers: Vec, peer_version: ValidationVersion, ) { - let peers = peers.into_iter().cloned().collect::>(); if peer_version == ValidationVersion::VStaging { sender .send_message(NetworkBridgeTxMessage::SendValidationMessage( @@ -2384,15 +2382,20 @@ pub(crate) async fn send_assignments_batched( send_assignments_batched_inner( sender, batch.clone(), - &v1_peers, + v1_peers.clone(), ValidationVersion::V1, ) .await; } if !v2_peers.is_empty() { - send_assignments_batched_inner(sender, batch, &v2_peers, ValidationVersion::V2) - .await; + send_assignments_batched_inner( + sender, + batch, + v2_peers.clone(), + ValidationVersion::V2, + ) + .await; } } } @@ -2405,7 +2408,7 @@ pub(crate) async fn send_assignments_batched( send_assignments_batched_inner( sender, batch, - &vstaging_peers, + vstaging_peers.clone(), ValidationVersion::VStaging, ) .await; diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index a7251f7bf8ed..9aeeb98ea9d6 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -258,7 +258,7 @@ pub enum Versioned { V1(V1), /// V2 type. V2(V2), - /// VStaging type. + /// VStaging type VStaging(VStaging), } diff --git a/polkadot/node/network/statement-distribution/src/v2/mod.rs b/polkadot/node/network/statement-distribution/src/v2/mod.rs index 0f29e33676b5..2f3bf7b38b3e 100644 --- a/polkadot/node/network/statement-distribution/src/v2/mod.rs +++ b/polkadot/node/network/statement-distribution/src/v2/mod.rs @@ -752,13 +752,21 @@ fn pending_statement_network_message( protocol_v2::StatementDistributionMessage::Statement(relay_parent, signed) }) .map(|msg| (vec![peer.0], Versioned::V2(msg).into())), - _ => statement_store + ValidationVersion::VStaging => statement_store .validator_statement(originator, compact) .map(|s| s.as_unchecked().clone()) .map(|signed| { protocol_vstaging::StatementDistributionMessage::Statement(relay_parent, signed) }) .map(|msg| (vec![peer.0], Versioned::VStaging(msg).into())), + ValidationVersion::V1 => { + gum::error!( + target: LOG_TARGET, + "Bug ValidationVersion::V1 should not be used in statement-distribution v2, + legacy should have handled this" + ); + None + }, } } @@ -883,7 +891,7 @@ async fn send_pending_grid_messages( ) .into(), )), - _ => messages.push(( + ValidationVersion::VStaging => messages.push(( vec![peer_id.0], Versioned::VStaging( protocol_vstaging::StatementDistributionMessage::BackedCandidateManifest( @@ -892,6 +900,13 @@ async fn send_pending_grid_messages( ) .into(), )), + ValidationVersion::V1 => { + gum::error!( + target: LOG_TARGET, + "Bug ValidationVersion::V1 should not be used in statement-distribution v2, + legacy should have handled this" + ); + } }; }, grid::ManifestKind::Acknowledgement => { @@ -2183,13 +2198,20 @@ fn post_acknowledgement_statement_messages( ) .into(), )), - _ => messages.push(Versioned::VStaging( + ValidationVersion::VStaging => messages.push(Versioned::VStaging( protocol_vstaging::StatementDistributionMessage::Statement( relay_parent, statement.as_unchecked().clone(), ) .into(), )), + ValidationVersion::V1 => { + gum::error!( + target: LOG_TARGET, + "Bug ValidationVersion::V1 should not be used in statement-distribution v2, + legacy should have handled this" + ); + }, }; } @@ -2324,13 +2346,21 @@ fn acknowledgement_and_statement_messages( let mut messages = match peer.1 { ValidationVersion::V2 => vec![(vec![peer.0], msg_v2.into())], - _ => vec![( + ValidationVersion::VStaging => vec![( vec![peer.0], Versioned::VStaging(protocol_v2::StatementDistributionMessage::BackedCandidateKnown( acknowledgement, )) .into(), )], + ValidationVersion::V1 => { + gum::error!( + target: LOG_TARGET, + "Bug ValidationVersion::V1 should not be used in statement-distribution v2, + legacy should have handled this" + ); + return Vec::new() + }, }; local_validator.grid_tracker.manifest_sent_to( From d728cf26ad600a07700653d2213adbc1f458ad30 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 19 Oct 2023 17:02:38 +0300 Subject: [PATCH 046/192] Make clippy happy Signed-off-by: Alexandru Gheorghe --- .../collator-protocol/src/validator_side/tests/mod.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs index 55df1c6d18f4..9812998aab76 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs @@ -399,12 +399,6 @@ async fn connect_and_declare_collator( para_id, collator.sign(&protocol_v1::declare_signature_payload(&peer)), )), - CollationVersion::VStaging => - Versioned::VStaging(protocol_vstaging::CollatorProtocolMessage::Declare( - collator.public(), - para_id, - collator.sign(&protocol_v1::declare_signature_payload(&peer)), - )), }; overseer_send( From 05ea75d219d47f433cb640efd602f26250381010 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 19 Oct 2023 21:47:51 +0300 Subject: [PATCH 047/192] Build a test image with `network-protocol-staging` Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index fefa3739a9ff..1c1e3bb4b3c8 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -18,7 +18,7 @@ build-linux-stable: # Ensure we run the UI tests. RUN_UI_TESTS: 1 script: - - time cargo build --locked --profile testnet --features pyroscope,fast-runtime --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker + - time cargo build --locked --profile testnet --features pyroscope,network-protocol-staging --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker - time ROCOCO_EPOCH_DURATION=10 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-10/ - time ROCOCO_EPOCH_DURATION=100 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-100/ - time ROCOCO_EPOCH_DURATION=600 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-600/ From 261285f91a43560c1f7874a26f8789d033975db7 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 20 Oct 2023 09:54:51 +0300 Subject: [PATCH 048/192] Enable assignments Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/criteria.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 1f751e2bf140..db74302f0225 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -278,7 +278,7 @@ impl AssignmentCriteria for RealAssignmentCriteria { config: &Config, leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>, ) -> HashMap { - compute_assignments(keystore, relay_vrf_story, config, leaving_cores, false) + compute_assignments(keystore, relay_vrf_story, config, leaving_cores, true) } fn check_assignment_cert( From db9c3a94ab44ae2eef2e755e0e463811e91b049b Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 20 Oct 2023 12:05:10 +0300 Subject: [PATCH 049/192] Revert "Enable assignments v2 used for testing" This reverts commit 261285f91a43560c1f7874a26f8789d033975db7. --- .gitlab/pipeline/build.yml | 2 +- polkadot/node/core/approval-voting/src/criteria.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 1c1e3bb4b3c8..fefa3739a9ff 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -18,7 +18,7 @@ build-linux-stable: # Ensure we run the UI tests. RUN_UI_TESTS: 1 script: - - time cargo build --locked --profile testnet --features pyroscope,network-protocol-staging --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker + - time cargo build --locked --profile testnet --features pyroscope,fast-runtime --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker - time ROCOCO_EPOCH_DURATION=10 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-10/ - time ROCOCO_EPOCH_DURATION=100 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-100/ - time ROCOCO_EPOCH_DURATION=600 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-600/ diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index db74302f0225..1f751e2bf140 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -278,7 +278,7 @@ impl AssignmentCriteria for RealAssignmentCriteria { config: &Config, leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>, ) -> HashMap { - compute_assignments(keystore, relay_vrf_story, config, leaving_cores, true) + compute_assignments(keystore, relay_vrf_story, config, leaving_cores, false) } fn check_assignment_cert( From 01af6307b794cbf4a7d3bc02299414edd8ac3203 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 25 Oct 2023 11:50:45 +0300 Subject: [PATCH 050/192] skeleton Signed-off-by: Andrei Sandu --- Cargo.lock | 25 +++++++++++++ Cargo.toml | 1 + polkadot/node/subsystem-bench/Cargo.toml | 46 ++++++++++++++++++++++++ polkadot/node/subsystem-bench/README.md | 6 ++++ polkadot/node/subsystem-bench/build.rs | 22 ++++++++++++ 5 files changed, 100 insertions(+) create mode 100644 polkadot/node/subsystem-bench/Cargo.toml create mode 100644 polkadot/node/subsystem-bench/README.md create mode 100644 polkadot/node/subsystem-bench/build.rs diff --git a/Cargo.lock b/Cargo.lock index a8d679c6ce8b..e846ae53a543 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12990,6 +12990,31 @@ dependencies = [ "sp-core", ] +[[package]] +name = "polkadot-subsystem-bench" +version = "1.0.0" +dependencies = [ + "assert_matches", + "async-trait", + "clap 4.4.6", + "color-eyre", + "futures", + "futures-timer", + "polkadot-erasure-coding", + "polkadot-node-core-backing", + "polkadot-node-primitives", + "polkadot-node-subsystem", + "polkadot-node-subsystem-test-helpers", + "polkadot-node-subsystem-types", + "polkadot-node-subsystem-util", + "polkadot-primitives", + "rand 0.8.5", + "sp-core", + "sp-keystore", + "substrate-build-script-utils", + "tracing-gum", +] + [[package]] name = "polkadot-test-client" version = "1.0.0" diff --git a/Cargo.toml b/Cargo.toml index c98fe6d1a3ac..2c5acccd5cfb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -125,6 +125,7 @@ members = [ "polkadot/node/gum/proc-macro", "polkadot/node/jaeger", "polkadot/node/malus", + "polkadot/node/subsystem-bench", "polkadot/node/metrics", "polkadot/node/network/approval-distribution", "polkadot/node/network/availability-distribution", diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml new file mode 100644 index 000000000000..b2cc88ff057d --- /dev/null +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "polkadot-subsystem-bench" +description = "Subsystem performance benchmark client" +version = "1.0.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +readme = "README.md" +publish = false + +[[bin]] +name = "subsystem-bench" +path = "src/subsystem-bench.rs" + +# Prevent rustdoc error. Already documented from top-level Cargo.toml. +doc = false + +[dependencies] +polkadot-node-subsystem = { path = "../subsystem" } +polkadot-node-subsystem-util = { path = "../subsystem-util" } +polkadot-node-subsystem-types = { path = "../subsystem-types" } +polkadot-node-core-backing = { path = "../core/backing" } +polkadot-node-primitives = { path = "../primitives" } +polkadot-primitives = { path = "../../primitives" } +color-eyre = { version = "0.6.1", default-features = false } +assert_matches = "1.5" +async-trait = "0.1.57" +sp-keystore = { path = "../../../substrate/primitives/keystore" } +sp-core = { path = "../../../substrate/primitives/core" } +clap = { version = "4.4.6", features = ["derive"] } +futures = "0.3.21" +futures-timer = "3.0.2" +gum = { package = "tracing-gum", path = "../gum" } +erasure = { package = "polkadot-erasure-coding", path = "../../erasure-coding" } +rand = "0.8.5" + +[dev-dependencies] +polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" } +sp-core = { path = "../../../substrate/primitives/core" } +futures = { version = "0.3.21", features = ["thread-pool"] } + +[build-dependencies] +substrate-build-script-utils = { path = "../../../substrate/utils/build-script-utils" } + +[features] +default = [] diff --git a/polkadot/node/subsystem-bench/README.md b/polkadot/node/subsystem-bench/README.md new file mode 100644 index 000000000000..8843f9883116 --- /dev/null +++ b/polkadot/node/subsystem-bench/README.md @@ -0,0 +1,6 @@ +# Subsystem benchmark client + +Run subsystem performance tests in isolation. + +Currently implemented benchmarks: +* `availability-recovery` diff --git a/polkadot/node/subsystem-bench/build.rs b/polkadot/node/subsystem-bench/build.rs new file mode 100644 index 000000000000..84fe22e23ed6 --- /dev/null +++ b/polkadot/node/subsystem-bench/build.rs @@ -0,0 +1,22 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +fn main() { + substrate_build_script_utils::generate_cargo_keys(); + // For the node/worker version check, make sure we always rebuild the node and binary workers + // when the version changes. + substrate_build_script_utils::rerun_if_git_head_changed(); +} From 15eaa0a20f67d3d7569eda39e0ece1bfee8ca6ec Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 26 Oct 2023 11:01:32 +0300 Subject: [PATCH 051/192] Cleanup un-needed dependency to test-helpers Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 1 + polkadot/node/core/approval-voting/Cargo.toml | 2 -- .../src/approval_db/v2/migration_helpers.rs | 13 ++++++++----- polkadot/node/service/Cargo.toml | 1 + polkadot/node/service/src/parachains_db/upgrade.rs | 4 +++- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22ee0eff9e1c..a096faf19a3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12892,6 +12892,7 @@ dependencies = [ "polkadot-overseer", "polkadot-parachain-primitives", "polkadot-primitives", + "polkadot-primitives-test-helpers", "polkadot-rpc", "polkadot-runtime-parachains", "polkadot-statement-distribution", diff --git a/polkadot/node/core/approval-voting/Cargo.toml b/polkadot/node/core/approval-voting/Cargo.toml index 5dd8cc1714f0..b89d7d2cedc0 100644 --- a/polkadot/node/core/approval-voting/Cargo.toml +++ b/polkadot/node/core/approval-voting/Cargo.toml @@ -31,8 +31,6 @@ sp-consensus = { path = "../../../../substrate/primitives/consensus/common", def sp-consensus-slots = { path = "../../../../substrate/primitives/consensus/slots", default-features = false } sp-application-crypto = { path = "../../../../substrate/primitives/application-crypto", default-features = false, features = ["full_crypto"] } sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -# Needed for migration test helpers -test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" } rand_core = "0.5.1" [dev-dependencies] diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs index a70d766fc8c9..74e997c7af84 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs @@ -21,10 +21,9 @@ use polkadot_node_primitives::approval::v1::{ AssignmentCert, AssignmentCertKind, VrfOutput, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT, }; use polkadot_node_subsystem_util::database::Database; +use sp_application_crypto::sp_core::H256; use std::{collections::HashSet, sync::Arc}; -use ::test_helpers::dummy_candidate_receipt; - fn dummy_assignment_cert(kind: AssignmentCertKind) -> AssignmentCert { let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); let msg = b"test-garbage"; @@ -145,10 +144,14 @@ pub fn v1_to_v2_sanity_check( } // Fills the db with dummy data in v1 scheme. -pub fn v1_to_v2_fill_test_data( +pub fn v1_to_v2_fill_test_data( db: Arc, config: Config, -) -> Result> { + dummy_candidate_create: F, +) -> Result> +where + F: Fn(H256) -> CandidateReceipt, +{ let mut backend = crate::DbBackend::new(db.clone(), config); let mut overlay_db = crate::OverlayedBackend::new(&backend); let mut expected_candidates = HashSet::new(); @@ -161,7 +164,7 @@ pub fn v1_to_v2_fill_test_data( for relay_number in 1..=RELAY_BLOCK_COUNT { let relay_hash = Hash::repeat_byte(relay_number as u8); let assignment_core_index = CoreIndex(relay_number); - let candidate = dummy_candidate_receipt(relay_hash); + let candidate = dummy_candidate_create(relay_hash); let candidate_hash = candidate.hash(); let at_height = vec![relay_hash]; diff --git a/polkadot/node/service/Cargo.toml b/polkadot/node/service/Cargo.toml index 6d36c3674028..133a1094c4ca 100644 --- a/polkadot/node/service/Cargo.toml +++ b/polkadot/node/service/Cargo.toml @@ -139,6 +139,7 @@ polkadot-statement-distribution = { path = "../network/statement-distribution", [dev-dependencies] polkadot-test-client = { path = "../test/client" } polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" } +test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../primitives/test-helpers" } env_logger = "0.9.0" assert_matches = "1.5.0" serial_test = "2.0.0" diff --git a/polkadot/node/service/src/parachains_db/upgrade.rs b/polkadot/node/service/src/parachains_db/upgrade.rs index 78e53a251eed..1d76c79d3e32 100644 --- a/polkadot/node/service/src/parachains_db/upgrade.rs +++ b/polkadot/node/service/src/parachains_db/upgrade.rs @@ -442,6 +442,7 @@ mod tests { *, }; use polkadot_node_core_approval_voting::approval_db::v2::migration_helpers::v1_to_v2_fill_test_data; + use test_helpers::dummy_candidate_receipt; #[test] fn test_paritydb_migrate_0_to_1() { @@ -599,7 +600,8 @@ mod tests { assert_eq!(db.num_columns(), super::columns::v3::NUM_COLUMNS as u32); let db = DbAdapter::new(db, columns::v3::ORDERED_COL); // Fill the approval voting column with test data. - v1_to_v2_fill_test_data(std::sync::Arc::new(db), approval_cfg).unwrap() + v1_to_v2_fill_test_data(std::sync::Arc::new(db), approval_cfg, dummy_candidate_receipt) + .unwrap() }; try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB, 4).unwrap(); From be9dddd8676f9af77fe42d3a7ccf684679a2517a Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 26 Oct 2023 17:29:03 +0300 Subject: [PATCH 052/192] Address review findings Signed-off-by: Alexandru Gheorghe --- .../approval-voting/src/approval_checking.rs | 125 +++++++++++------- polkadot/node/core/approval-voting/src/lib.rs | 109 +++++++++++---- polkadot/node/core/runtime-api/src/cache.rs | 2 +- 3 files changed, 155 insertions(+), 81 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/approval_checking.rs b/polkadot/node/core/approval-voting/src/approval_checking.rs index 374b55aa0b4f..f1f3b2079220 100644 --- a/polkadot/node/core/approval-voting/src/approval_checking.rs +++ b/polkadot/node/core/approval-voting/src/approval_checking.rs @@ -25,6 +25,15 @@ use crate::{ time::Tick, }; +/// Result of counting the necessary tranches needed for approving a block. +#[derive(Debug, PartialEq, Clone)] +pub struct TranchesToApproveResult { + /// The required tranches for approving this block + pub required_tranches: RequiredTranches, + /// The total number of no_shows at the moment we are doing the counting. + pub total_observed_no_shows: usize, +} + /// The required tranches of assignments needed to determine whether a candidate is approved. #[derive(Debug, PartialEq, Clone)] pub enum RequiredTranches { @@ -178,6 +187,7 @@ struct State { next_no_show: Option, /// The last tick at which a considered assignment was received. last_assignment_tick: Option, + total_observed_no_shows: usize, } impl State { @@ -187,41 +197,53 @@ impl State { needed_approvals: usize, n_validators: usize, no_show_duration: Tick, - ) -> RequiredTranches { + ) -> TranchesToApproveResult { let covering = if self.depth == 0 { 0 } else { self.covering }; if self.depth != 0 && self.assignments + covering + self.uncovered >= n_validators { - return RequiredTranches::All + return TranchesToApproveResult { + required_tranches: RequiredTranches::All, + total_observed_no_shows: self.total_observed_no_shows, + } } // If we have enough assignments and all no-shows are covered, we have reached the number // of tranches that we need to have. if self.assignments >= needed_approvals && (covering + self.uncovered) == 0 { - return RequiredTranches::Exact { - needed: tranche, - tolerated_missing: self.covered, - next_no_show: self.next_no_show, - last_assignment_tick: self.last_assignment_tick, + return TranchesToApproveResult { + required_tranches: RequiredTranches::Exact { + needed: tranche, + tolerated_missing: self.covered, + next_no_show: self.next_no_show, + last_assignment_tick: self.last_assignment_tick, + }, + total_observed_no_shows: self.total_observed_no_shows, } } // We're pending more assignments and should look at more tranches. let clock_drift = self.clock_drift(no_show_duration); if self.depth == 0 { - RequiredTranches::Pending { - considered: tranche, - next_no_show: self.next_no_show, - // during the initial assignment-gathering phase, we want to accept assignments - // from any tranche. Note that honest validators will still not broadcast their - // assignment until it is time to do so, regardless of this value. - maximum_broadcast: DelayTranche::max_value(), - clock_drift, + TranchesToApproveResult { + required_tranches: RequiredTranches::Pending { + considered: tranche, + next_no_show: self.next_no_show, + // during the initial assignment-gathering phase, we want to accept assignments + // from any tranche. Note that honest validators will still not broadcast their + // assignment until it is time to do so, regardless of this value. + maximum_broadcast: DelayTranche::max_value(), + clock_drift, + }, + total_observed_no_shows: self.total_observed_no_shows, } } else { - RequiredTranches::Pending { - considered: tranche, - next_no_show: self.next_no_show, - maximum_broadcast: tranche + (covering + self.uncovered) as DelayTranche, - clock_drift, + TranchesToApproveResult { + required_tranches: RequiredTranches::Pending { + considered: tranche, + next_no_show: self.next_no_show, + maximum_broadcast: tranche + (covering + self.uncovered) as DelayTranche, + clock_drift, + }, + total_observed_no_shows: self.total_observed_no_shows, } } } @@ -276,6 +298,7 @@ impl State { uncovered, next_no_show, last_assignment_tick, + total_observed_no_shows: self.total_observed_no_shows + new_no_shows, } } } @@ -372,22 +395,20 @@ pub fn tranches_to_approve( block_tick: Tick, no_show_duration: Tick, needed_approvals: usize, -) -> (RequiredTranches, usize) { +) -> TranchesToApproveResult { let tick_now = tranche_now as Tick + block_tick; let n_validators = approval_entry.n_validators(); - let initial_state = ( - State { - assignments: 0, - depth: 0, - covered: 0, - covering: needed_approvals, - uncovered: 0, - next_no_show: None, - last_assignment_tick: None, - }, - 0usize, - ); + let initial_state = State { + assignments: 0, + depth: 0, + covered: 0, + covering: needed_approvals, + uncovered: 0, + next_no_show: None, + last_assignment_tick: None, + total_observed_no_shows: 0, + }; // The `ApprovalEntry` doesn't have any data for empty tranches. We still want to iterate over // these empty tranches, so we create an iterator to fill the gaps. @@ -398,7 +419,7 @@ pub fn tranches_to_approve( tranches_with_gaps_filled .scan(Some(initial_state), |state, (tranche, assignments)| { // The `Option` here is used for early exit. - let (s, prev_no_shows) = state.take()?; + let s = state.take()?; let clock_drift = s.clock_drift(no_show_duration); let drifted_tick_now = tick_now.saturating_sub(clock_drift); @@ -437,7 +458,7 @@ pub fn tranches_to_approve( let s = s.advance(n_assignments, no_shows, next_no_show, last_assignment_tick); let output = s.output(tranche, needed_approvals, n_validators, no_show_duration); - *state = match output { + *state = match output.required_tranches { RequiredTranches::Exact { .. } | RequiredTranches::All => { // Wipe the state clean so the next iteration of this closure will terminate // the iterator. This guarantees that we can call `last` further down to see @@ -447,11 +468,11 @@ pub fn tranches_to_approve( RequiredTranches::Pending { .. } => { // Pending results are only interesting when they are the last result of the iterator // i.e. we never achieve a satisfactory level of assignment. - Some((s, no_shows + prev_no_shows)) + Some(s) } }; - Some((output, no_shows + prev_no_shows)) + Some(output) }) .last() .expect("the underlying iterator is infinite, starts at 0, and never exits early before tranche 1; qed") @@ -685,7 +706,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Exact { needed: 1, tolerated_missing: 0, @@ -726,7 +747,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Pending { considered: 2, next_no_show: Some(block_tick + no_show_duration), @@ -771,7 +792,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Pending { considered: 11, next_no_show: None, @@ -820,7 +841,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Pending { considered: 1, next_no_show: None, @@ -840,7 +861,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Pending { considered: 1, next_no_show: None, @@ -894,7 +915,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Exact { needed: 1, tolerated_missing: 0, @@ -914,7 +935,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Exact { needed: 2, tolerated_missing: 1, @@ -934,7 +955,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Pending { considered: 2, next_no_show: None, @@ -988,7 +1009,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Exact { needed: 2, tolerated_missing: 1, @@ -1011,7 +1032,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Pending { considered: 2, next_no_show: None, @@ -1033,7 +1054,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Exact { needed: 3, tolerated_missing: 2, @@ -1091,7 +1112,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Pending { considered: 10, next_no_show: None, @@ -1368,10 +1389,11 @@ mod tests { uncovered: 0, next_no_show: None, last_assignment_tick: None, + total_observed_no_shows: 0, }; assert_eq!( - state.output(0, 10, 10, 20), + state.output(0, 10, 10, 20).required_tranches, RequiredTranches::Pending { considered: 0, next_no_show: None, @@ -1391,10 +1413,11 @@ mod tests { uncovered: 0, next_no_show: None, last_assignment_tick: None, + total_observed_no_shows: 0, }; assert_eq!( - state.output(0, 10, 10, 20), + state.output(0, 10, 10, 20).required_tranches, RequiredTranches::Exact { needed: 0, tolerated_missing: 0, diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index ffa7cb8b6a98..e21ad331431d 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -99,7 +99,7 @@ mod persisted_entries; mod time; use crate::{ - approval_checking::Check, + approval_checking::{Check, TranchesToApproveResult}, approval_db::v2::{Config as DatabaseConfig, DbBackend}, backend::{Backend, OverlayedBackend}, criteria::InvalidAssignmentReason, @@ -839,17 +839,22 @@ impl State { ); if let Some(approval_entry) = candidate_entry.approval_entry(&block_hash) { - let (required_tranches, last_no_shows) = approval_checking::tranches_to_approve( - approval_entry, - candidate_entry.approvals(), - tranche_now, - block_tick, - no_show_duration, - session_info.needed_approvals as _, - ); + let TranchesToApproveResult { required_tranches, total_observed_no_shows } = + approval_checking::tranches_to_approve( + approval_entry, + candidate_entry.approvals(), + tranche_now, + block_tick, + no_show_duration, + session_info.needed_approvals as _, + ); - let status = - ApprovalStatus { required_tranches, block_tick, tranche_now, last_no_shows }; + let status = ApprovalStatus { + required_tranches, + block_tick, + tranche_now, + last_no_shows: total_observed_no_shows, + }; Some((approval_entry, status)) } else { @@ -887,9 +892,18 @@ impl State { .map(|cache| cache.insert(block_hash, params)); params }, - _ => { + Ok(Err(err)) => { + gum::error!( + target: LOG_TARGET, + ?err, + "Could not request approval voting params from runtime using defaults" + ); + ApprovalVotingParams { max_approval_coalesce_count: 1 } + }, + Err(err) => { gum::error!( target: LOG_TARGET, + ?err, "Could not request approval voting params from runtime using defaults" ); ApprovalVotingParams { max_approval_coalesce_count: 1 } @@ -1764,13 +1778,34 @@ async fn get_approval_signatures_for_candidate( let votes = votes .into_iter() .map(|(validator_index, (hash, signed_candidates_indices, signature))| { - let candidates_hashes = - candidate_indices_to_candidate_hashes.get(&hash).expect("Can't fail because it is the same hash we sent to approval-distribution; qed"); + let candidates_hashes = candidate_indices_to_candidate_hashes.get(&hash); + + if candidates_hashes.is_none() { + gum::warn!( + target: LOG_TARGET, + ?hash, + "Possible bug! Could not find map of candidate_hashes for block hash received from approval-distribution" + ); + } + let signed_candidates_hashes: Vec = signed_candidates_indices .into_iter() - .map(|candidate_index| { - *(candidates_hashes.get(&candidate_index).expect("Can't fail because we already checked the signature was valid, so we should be able to find the hash; qed")) + .filter_map(|candidate_index| { + candidates_hashes.and_then(|candidate_hashes| { + if let Some(candidate_hash) = + candidate_hashes.get(&candidate_index) + { + Some(*candidate_hash) + } else { + gum::warn!( + target: LOG_TARGET, + ?candidate_index, + "Possible bug! Could not find candidate hash for candidate_index coming from approval-distribution" + ); + None + } + }) }) .collect(); (validator_index, (signed_candidates_hashes, signature)) @@ -2512,7 +2547,13 @@ where ) .check_signature( &pubkey, - *candidate_hashes.first().expect("Checked above this is not empty; qed"), + if let Some(candidate_hash) = candidate_hashes.first() { + *candidate_hash + } else { + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::InvalidValidatorIndex( + approval.validator + ),)) + }, block_entry.session(), &approval.signature, ) { @@ -2880,7 +2921,7 @@ async fn process_wakeup( None => return Ok(Vec::new()), }; - let (tranches_to_approve, _last_no_shows) = approval_checking::tranches_to_approve( + let tranches_to_approve = approval_checking::tranches_to_approve( &approval_entry, candidate_entry.approvals(), tranche_now, @@ -2892,7 +2933,7 @@ async fn process_wakeup( let should_trigger = should_trigger_assignment( &approval_entry, &candidate_entry, - tranches_to_approve, + tranches_to_approve.required_tranches, tranche_now, ); @@ -3471,17 +3512,27 @@ async fn maybe_create_signature( .map(|candidate_hash| db.load_candidate_entry(candidate_hash)) .collect::>>>()?; - for candidate_entry in candidate_entries { - let mut candidate_entry = candidate_entry - .expect("Candidate was scheduled to be signed entry in db should exist; qed"); - let approval_entry = candidate_entry - .approval_entry_mut(&block_entry.block_hash()) - .expect("Candidate was scheduled to be signed entry in db should exist; qed"); - approval_entry.import_approval_sig(OurApproval { - signature: signature.clone(), - signed_candidates_indices: candidates_indices.clone(), + for mut candidate_entry in candidate_entries { + // let mut candidate_entry = candidate_entry + // .expect("Candidate was scheduled to be signed entry in db should exist; qed"); + let approval_entry = candidate_entry.as_mut().and_then(|candidate_entry| { + candidate_entry.approval_entry_mut(&block_entry.block_hash()) }); - db.write_candidate_entry(candidate_entry); + + match approval_entry { + Some(approval_entry) => approval_entry.import_approval_sig(OurApproval { + signature: signature.clone(), + signed_candidates_indices: candidates_indices.clone(), + }), + None => { + gum::error!( + target: LOG_TARGET, + candidate_entry = ?candidate_entry, + "Candidate scheduled for signing approval entry should not be None" + ); + }, + }; + candidate_entry.map(|candidate_entry| db.write_candidate_entry(candidate_entry)); } metrics.on_approval_produced(); diff --git a/polkadot/node/core/runtime-api/src/cache.rs b/polkadot/node/core/runtime-api/src/cache.rs index 1996578dd4b6..83ebf6884890 100644 --- a/polkadot/node/core/runtime-api/src/cache.rs +++ b/polkadot/node/core/runtime-api/src/cache.rs @@ -61,13 +61,13 @@ pub(crate) struct RequestResultCache { LruMap<(Hash, ParaId, OccupiedCoreAssumption), Option>, version: LruMap, disputes: LruMap)>>, - approval_voting_params: LruMap, unapplied_slashes: LruMap>, key_ownership_proof: LruMap<(Hash, ValidatorId), Option>, minimum_backing_votes: LruMap, disabled_validators: LruMap>, para_backing_state: LruMap<(Hash, ParaId), Option>, async_backing_params: LruMap, + approval_voting_params: LruMap, } impl Default for RequestResultCache { From 2385fad585fca15357bc01c5f36cbc1f67d35bfa Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 26 Oct 2023 18:21:05 +0300 Subject: [PATCH 053/192] Address review findings Signed-off-by: Alexandru Gheorghe --- .../0007-approval-voting-coalescing.toml | 85 +------------------ .../0007-approval-voting-coalescing.zndsl | 42 ++------- 2 files changed, 10 insertions(+), 117 deletions(-) diff --git a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml index c7eeeb5383e0..872bf025de97 100644 --- a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml +++ b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml @@ -17,89 +17,10 @@ chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default limits = { memory = "4G", cpu = "2" } requests = { memory = "2G", cpu = "1" } - [[relaychain.nodes]] + [[relaychain.node_groups]] name = "alice" - args = [ "--alice", "-lparachain=trace,runtime=debug" ] - - [[relaychain.nodes]] - name = "bob" - args = [ "--bob", "-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "charlie" - args = [ "--charlie", "-lparachain=debug,runtime=debug" ] - - [[relaychain.nodes]] - name = "dave" - args = [ "--dave", "-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "ferdie" - args = [ "--ferdie", "-lparachain=debug,runtime=debug" ] - - [[relaychain.nodes]] - name = "eve" - args = [ "--eve", "-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "one" - args = [ "--one", "-lparachain=debug,runtime=debug" ] - - [[relaychain.nodes]] - name = "two" - args = [ "--two", "-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "nine" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "eleven" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "twelve" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "thirteen" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "fourteen" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "fifteen" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "sixteen" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "seventeen" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "eithteen" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "nineteen" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "twenty" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "twentyone" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "twentytwo" - args = ["-lparachain=debug,runtime=debug"] + args = [ "-lparachain=trace,runtime=debug" ] + count = 21 [[parachains]] id = 2000 diff --git a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl index a4cc454f38cc..07ef30d546a1 100644 --- a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl +++ b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl @@ -4,23 +4,16 @@ Creds: config # Check authority status. alice: reports node_roles is 4 -bob: reports node_roles is 4 -charlie: reports node_roles is 4 -dave: reports node_roles is 4 -eve: reports node_roles is 4 -ferdie: reports node_roles is 4 -one: reports node_roles is 4 -two: reports node_roles is 4 # Ensure parachains are registered. alice: parachain 2000 is registered within 60 seconds -bob: parachain 2001 is registered within 60 seconds -charlie: parachain 2002 is registered within 60 seconds -dave: parachain 2003 is registered within 60 seconds -ferdie: parachain 2004 is registered within 60 seconds -eve: parachain 2005 is registered within 60 seconds -one: parachain 2006 is registered within 60 seconds -two: parachain 2007 is registered within 60 seconds +alice: parachain 2001 is registered within 60 seconds +alice: parachain 2002 is registered within 60 seconds +alice: parachain 2003 is registered within 60 seconds +alice: parachain 2004 is registered within 60 seconds +alice: parachain 2005 is registered within 60 seconds +alice: parachain 2006 is registered within 60 seconds +alice: parachain 2007 is registered within 60 seconds # Ensure parachains made progress. alice: parachain 2000 block height is at least 10 within 300 seconds @@ -33,28 +26,7 @@ alice: parachain 2006 block height is at least 10 within 300 seconds alice: parachain 2007 block height is at least 10 within 300 seconds alice: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds -bob: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds -charlie: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds -dave: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds -eve: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds -ferdie: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds -one: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds -two: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds alice: reports polkadot_parachain_approval_checking_finality_lag < 3 -bob: reports polkadot_parachain_approval_checking_finality_lag < 3 -charlie: reports polkadot_parachain_approval_checking_finality_lag < 3 -dave: reports polkadot_parachain_approval_checking_finality_lag < 3 -ferdie: reports polkadot_parachain_approval_checking_finality_lag < 3 -eve: reports polkadot_parachain_approval_checking_finality_lag < 3 -one: reports polkadot_parachain_approval_checking_finality_lag < 3 -two: reports polkadot_parachain_approval_checking_finality_lag < 3 alice: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds -bob: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds -charlie: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds -dave: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds -ferdie: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds -eve: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds -one: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds -two: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds \ No newline at end of file From a2c032002502005fe3047e50560bfa235801e00d Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 27 Oct 2023 15:53:59 +0300 Subject: [PATCH 054/192] Add new approval_db version ... so that we can decouple this PR from https://github.com/paritytech/polkadot-sdk/pull/1178 and be able to merge them separately. Signed-off-by: Alexandru Gheorghe --- .../approval-voting/src/approval_checking.rs | 36 +- .../approval_db/common/migration_helpers.rs | 36 ++ .../src/approval_db/common/mod.rs | 293 +++++++++ .../approval-voting/src/approval_db/mod.rs | 2 + .../src/approval_db/v2/migration_helpers.rs | 64 +- .../approval-voting/src/approval_db/v2/mod.rs | 277 +-------- .../src/approval_db/v2/tests.rs | 66 +- .../src/approval_db/v3/migration_helpers.rs | 237 +++++++ .../approval-voting/src/approval_db/v3/mod.rs | 139 +++++ .../src/approval_db/v3/tests.rs | 580 ++++++++++++++++++ .../node/core/approval-voting/src/backend.rs | 15 +- .../node/core/approval-voting/src/import.rs | 22 +- polkadot/node/core/approval-voting/src/lib.rs | 6 +- polkadot/node/core/approval-voting/src/ops.rs | 2 +- .../approval-voting/src/persisted_entries.rs | 94 ++- .../node/core/approval-voting/src/tests.rs | 4 +- .../node/service/src/parachains_db/upgrade.rs | 99 ++- 17 files changed, 1543 insertions(+), 429 deletions(-) create mode 100644 polkadot/node/core/approval-voting/src/approval_db/common/migration_helpers.rs create mode 100644 polkadot/node/core/approval-voting/src/approval_db/common/mod.rs create mode 100644 polkadot/node/core/approval-voting/src/approval_db/v3/migration_helpers.rs create mode 100644 polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs create mode 100644 polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs diff --git a/polkadot/node/core/approval-voting/src/approval_checking.rs b/polkadot/node/core/approval-voting/src/approval_checking.rs index f1f3b2079220..0aa6102fbd6d 100644 --- a/polkadot/node/core/approval-voting/src/approval_checking.rs +++ b/polkadot/node/core/approval-voting/src/approval_checking.rs @@ -498,7 +498,7 @@ mod tests { 0, ); - let approval_entry = approval_db::v2::ApprovalEntry { + let approval_entry = approval_db::v3::ApprovalEntry { tranches: Vec::new(), assigned_validators: BitVec::default(), our_assignment: None, @@ -537,17 +537,17 @@ mod tests { candidate.mark_approval(ValidatorIndex(i)); } - let approval_entry = approval_db::v2::ApprovalEntry { + let approval_entry = approval_db::v3::ApprovalEntry { tranches: vec![ - approval_db::v2::TrancheEntry { + approval_db::v3::TrancheEntry { tranche: 0, assignments: (0..2).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, - approval_db::v2::TrancheEntry { + approval_db::v3::TrancheEntry { tranche: 1, assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(), }, - approval_db::v2::TrancheEntry { + approval_db::v3::TrancheEntry { tranche: 2, assignments: (5..10).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, @@ -611,17 +611,17 @@ mod tests { candidate.mark_approval(ValidatorIndex(i)); } - let approval_entry = approval_db::v2::ApprovalEntry { + let approval_entry = approval_db::v3::ApprovalEntry { tranches: vec![ - approval_db::v2::TrancheEntry { + approval_db::v3::TrancheEntry { tranche: 0, assignments: (0..4).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, - approval_db::v2::TrancheEntry { + approval_db::v3::TrancheEntry { tranche: 1, assignments: (4..6).map(|i| (ValidatorIndex(i), 1.into())).collect(), }, - approval_db::v2::TrancheEntry { + approval_db::v3::TrancheEntry { tranche: 2, assignments: (6..10).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, @@ -677,7 +677,7 @@ mod tests { let no_show_duration = 10; let needed_approvals = 4; - let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v3::ApprovalEntry { tranches: Vec::new(), assigned_validators: bitvec![u8, BitOrderLsb0; 0; 5], our_assignment: None, @@ -722,7 +722,7 @@ mod tests { let no_show_duration = 10; let needed_approvals = 4; - let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v3::ApprovalEntry { tranches: Vec::new(), assigned_validators: bitvec![u8, BitOrderLsb0; 0; 10], our_assignment: None, @@ -763,7 +763,7 @@ mod tests { let no_show_duration = 10; let needed_approvals = 4; - let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v3::ApprovalEntry { tranches: Vec::new(), assigned_validators: bitvec![u8, BitOrderLsb0; 0; 10], our_assignment: None, @@ -809,7 +809,7 @@ mod tests { let needed_approvals = 4; let n_validators = 8; - let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v3::ApprovalEntry { tranches: Vec::new(), assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators], our_assignment: None, @@ -878,7 +878,7 @@ mod tests { let needed_approvals = 4; let n_validators = 8; - let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v3::ApprovalEntry { tranches: Vec::new(), assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators], our_assignment: None, @@ -972,7 +972,7 @@ mod tests { let needed_approvals = 4; let n_validators = 8; - let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v3::ApprovalEntry { tranches: Vec::new(), assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators], our_assignment: None, @@ -1084,10 +1084,10 @@ mod tests { candidate.mark_approval(ValidatorIndex(i)); } - let approval_entry = approval_db::v2::ApprovalEntry { + let approval_entry = approval_db::v3::ApprovalEntry { tranches: vec![ // Assignments with invalid validator indexes. - approval_db::v2::TrancheEntry { + approval_db::v3::TrancheEntry { tranche: 1, assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(), }, @@ -1138,7 +1138,7 @@ mod tests { ]; for test_tranche in test_tranches { - let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v3::ApprovalEntry { tranches: Vec::new(), backing_group: GroupIndex(0), our_assignment: None, diff --git a/polkadot/node/core/approval-voting/src/approval_db/common/migration_helpers.rs b/polkadot/node/core/approval-voting/src/approval_db/common/migration_helpers.rs new file mode 100644 index 000000000000..32ddbaf81277 --- /dev/null +++ b/polkadot/node/core/approval-voting/src/approval_db/common/migration_helpers.rs @@ -0,0 +1,36 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; + +use polkadot_node_primitives::approval::v1::{ + AssignmentCert, AssignmentCertKind, VrfOutput, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT, +}; + +pub fn make_bitvec(len: usize) -> BitVec { + bitvec::bitvec![u8, BitOrderLsb0; 0; len] +} + +pub fn dummy_assignment_cert(kind: AssignmentCertKind) -> AssignmentCert { + let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); + let msg = b"test-garbage"; + let mut prng = rand_core::OsRng; + let keypair = schnorrkel::Keypair::generate_with(&mut prng); + let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); + let out = inout.to_output(); + + AssignmentCert { kind, vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) } } +} diff --git a/polkadot/node/core/approval-voting/src/approval_db/common/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/common/mod.rs new file mode 100644 index 000000000000..249dcf912df5 --- /dev/null +++ b/polkadot/node/core/approval-voting/src/approval_db/common/mod.rs @@ -0,0 +1,293 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Common helper functions for all versions of approval-voting database. +use std::sync::Arc; + +use parity_scale_codec::{Decode, Encode}; +use polkadot_node_subsystem::{SubsystemError, SubsystemResult}; +use polkadot_node_subsystem_util::database::{DBTransaction, Database}; +use polkadot_primitives::{BlockNumber, CandidateHash, CandidateIndex, Hash}; + +use crate::{ + backend::{Backend, BackendWriteOp, V1ReadBackend, V2ReadBackend}, + persisted_entries, +}; + +use super::{ + v2::{load_block_entry_v1, load_candidate_entry_v1}, + v3::{load_block_entry_v2, load_candidate_entry_v2, BlockEntry, CandidateEntry}, +}; + +pub mod migration_helpers; + +const STORED_BLOCKS_KEY: &[u8] = b"Approvals_StoredBlocks"; + +/// A range from earliest..last block number stored within the DB. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct StoredBlockRange(pub BlockNumber, pub BlockNumber); +/// The database config. +#[derive(Debug, Clone, Copy)] +pub struct Config { + /// The column family in the database where data is stored. + pub col_approval_data: u32, +} + +/// `DbBackend` is a concrete implementation of the higher-level Backend trait +pub struct DbBackend { + inner: Arc, + config: Config, +} + +impl DbBackend { + /// Create a new [`DbBackend`] with the supplied key-value store and + /// config. + pub fn new(db: Arc, config: Config) -> Self { + DbBackend { inner: db, config } + } +} + +/// Errors while accessing things from the DB. +#[derive(Debug, derive_more::From, derive_more::Display)] +pub enum Error { + Io(std::io::Error), + InvalidDecoding(parity_scale_codec::Error), + InternalError(SubsystemError), +} + +impl std::error::Error for Error {} + +/// Result alias for DB errors. +pub type Result = std::result::Result; + +impl Backend for DbBackend { + fn load_block_entry( + &self, + block_hash: &Hash, + ) -> SubsystemResult> { + load_block_entry(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) + } + + fn load_candidate_entry( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult> { + load_candidate_entry(&*self.inner, &self.config, candidate_hash).map(|e| e.map(Into::into)) + } + + fn load_blocks_at_height(&self, block_height: &BlockNumber) -> SubsystemResult> { + load_blocks_at_height(&*self.inner, &self.config, block_height) + } + + fn load_all_blocks(&self) -> SubsystemResult> { + load_all_blocks(&*self.inner, &self.config) + } + + fn load_stored_blocks(&self) -> SubsystemResult> { + load_stored_blocks(&*self.inner, &self.config) + } + + /// Atomically write the list of operations, with later operations taking precedence over prior. + fn write(&mut self, ops: I) -> SubsystemResult<()> + where + I: IntoIterator, + { + let mut tx = DBTransaction::new(); + for op in ops { + match op { + BackendWriteOp::WriteStoredBlockRange(stored_block_range) => { + tx.put_vec( + self.config.col_approval_data, + &STORED_BLOCKS_KEY, + stored_block_range.encode(), + ); + }, + BackendWriteOp::DeleteStoredBlockRange => { + tx.delete(self.config.col_approval_data, &STORED_BLOCKS_KEY); + }, + BackendWriteOp::WriteBlocksAtHeight(h, blocks) => { + tx.put_vec( + self.config.col_approval_data, + &blocks_at_height_key(h), + blocks.encode(), + ); + }, + BackendWriteOp::DeleteBlocksAtHeight(h) => { + tx.delete(self.config.col_approval_data, &blocks_at_height_key(h)); + }, + BackendWriteOp::WriteBlockEntry(block_entry) => { + let block_entry: BlockEntry = block_entry.into(); + tx.put_vec( + self.config.col_approval_data, + &block_entry_key(&block_entry.block_hash), + block_entry.encode(), + ); + }, + BackendWriteOp::DeleteBlockEntry(hash) => { + tx.delete(self.config.col_approval_data, &block_entry_key(&hash)); + }, + BackendWriteOp::WriteCandidateEntry(candidate_entry) => { + let candidate_entry: CandidateEntry = candidate_entry.into(); + tx.put_vec( + self.config.col_approval_data, + &candidate_entry_key(&candidate_entry.candidate.hash()), + candidate_entry.encode(), + ); + }, + BackendWriteOp::DeleteCandidateEntry(candidate_hash) => { + tx.delete(self.config.col_approval_data, &candidate_entry_key(&candidate_hash)); + }, + } + } + + self.inner.write(tx).map_err(|e| e.into()) + } +} + +impl V1ReadBackend for DbBackend { + fn load_candidate_entry_v1( + &self, + candidate_hash: &CandidateHash, + candidate_index: CandidateIndex, + ) -> SubsystemResult> { + load_candidate_entry_v1(&*self.inner, &self.config, candidate_hash) + .map(|e| e.map(|e| persisted_entries::CandidateEntry::from_v1(e, candidate_index))) + } + + fn load_block_entry_v1( + &self, + block_hash: &Hash, + ) -> SubsystemResult> { + load_block_entry_v1(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) + } +} + +impl V2ReadBackend for DbBackend { + fn load_candidate_entry_v2( + &self, + candidate_hash: &CandidateHash, + candidate_index: CandidateIndex, + ) -> SubsystemResult> { + load_candidate_entry_v2(&*self.inner, &self.config, candidate_hash) + .map(|e| e.map(|e| persisted_entries::CandidateEntry::from_v2(e, candidate_index))) + } + + fn load_block_entry_v2( + &self, + block_hash: &Hash, + ) -> SubsystemResult> { + load_block_entry_v2(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) + } +} + +pub(crate) fn load_decode( + store: &dyn Database, + col_approval_data: u32, + key: &[u8], +) -> Result> { + match store.get(col_approval_data, key)? { + None => Ok(None), + Some(raw) => D::decode(&mut &raw[..]).map(Some).map_err(Into::into), + } +} + +/// The key a given block entry is stored under. +pub(crate) fn block_entry_key(block_hash: &Hash) -> [u8; 46] { + const BLOCK_ENTRY_PREFIX: [u8; 14] = *b"Approvals_blck"; + + let mut key = [0u8; 14 + 32]; + key[0..14].copy_from_slice(&BLOCK_ENTRY_PREFIX); + key[14..][..32].copy_from_slice(block_hash.as_ref()); + + key +} + +/// The key a given candidate entry is stored under. +pub(crate) fn candidate_entry_key(candidate_hash: &CandidateHash) -> [u8; 46] { + const CANDIDATE_ENTRY_PREFIX: [u8; 14] = *b"Approvals_cand"; + + let mut key = [0u8; 14 + 32]; + key[0..14].copy_from_slice(&CANDIDATE_ENTRY_PREFIX); + key[14..][..32].copy_from_slice(candidate_hash.0.as_ref()); + + key +} + +/// The key a set of block hashes corresponding to a block number is stored under. +pub(crate) fn blocks_at_height_key(block_number: BlockNumber) -> [u8; 16] { + const BLOCKS_AT_HEIGHT_PREFIX: [u8; 12] = *b"Approvals_at"; + + let mut key = [0u8; 12 + 4]; + key[0..12].copy_from_slice(&BLOCKS_AT_HEIGHT_PREFIX); + block_number.using_encoded(|s| key[12..16].copy_from_slice(s)); + + key +} + +/// Return all blocks which have entries in the DB, ascending, by height. +pub fn load_all_blocks(store: &dyn Database, config: &Config) -> SubsystemResult> { + let mut hashes = Vec::new(); + if let Some(stored_blocks) = load_stored_blocks(store, config)? { + for height in stored_blocks.0..stored_blocks.1 { + let blocks = load_blocks_at_height(store, config, &height)?; + hashes.extend(blocks); + } + } + + Ok(hashes) +} + +/// Load the stored-blocks key from the state. +pub fn load_stored_blocks( + store: &dyn Database, + config: &Config, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, STORED_BLOCKS_KEY) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a blocks-at-height entry for a given block number. +pub fn load_blocks_at_height( + store: &dyn Database, + config: &Config, + block_number: &BlockNumber, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &blocks_at_height_key(*block_number)) + .map(|x| x.unwrap_or_default()) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a block entry from the aux store. +pub fn load_block_entry( + store: &dyn Database, + config: &Config, + block_hash: &Hash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &block_entry_key(block_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a candidate entry from the aux store in current version format. +pub fn load_candidate_entry( + store: &dyn Database, + config: &Config, + candidate_hash: &CandidateHash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} diff --git a/polkadot/node/core/approval-voting/src/approval_db/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/mod.rs index 20fb6aa82d8d..78942a507f4b 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/mod.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/mod.rs @@ -30,5 +30,7 @@ //! In the future, we may use a temporary DB which doesn't need to be wiped, but for the //! time being we share the same DB with the rest of Substrate. +pub mod common; pub mod v1; pub mod v2; +pub mod v3; diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs index e123abf59419..df6e4754dbd6 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs @@ -16,25 +16,19 @@ //! Approval DB migration helpers. use super::*; -use crate::backend::Backend; -use polkadot_node_primitives::approval::v1::{ - AssignmentCert, AssignmentCertKind, VrfOutput, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT, +use crate::{ + approval_db::common::{ + migration_helpers::{dummy_assignment_cert, make_bitvec}, + Error, Result, StoredBlockRange, + }, + backend::Backend, }; + +use polkadot_node_primitives::approval::v1::AssignmentCertKind; use polkadot_node_subsystem_util::database::Database; use sp_application_crypto::sp_core::H256; use std::{collections::HashSet, sync::Arc}; -fn dummy_assignment_cert(kind: AssignmentCertKind) -> AssignmentCert { - let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); - let msg = b"test-garbage"; - let mut prng = rand_core::OsRng; - let keypair = schnorrkel::Keypair::generate_with(&mut prng); - let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); - let out = inout.to_output(); - - AssignmentCert { kind, vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) } } -} - fn make_block_entry_v1( block_hash: Hash, parent_hash: Hash, @@ -54,14 +48,10 @@ fn make_block_entry_v1( } } -fn make_bitvec(len: usize) -> BitVec { - bitvec::bitvec![u8, BitOrderLsb0; 0; len] -} - /// Migrates `OurAssignment`, `CandidateEntry` and `ApprovalEntry` to version 2. /// Returns on any error. /// Must only be used in parachains DB migration code - `polkadot-service` crate. -pub fn v1_to_v2(db: Arc, config: Config) -> Result<()> { +pub fn v1_to_latest(db: Arc, config: Config) -> Result<()> { let mut backend = crate::DbBackend::new(db, config); let all_blocks = backend .load_all_blocks() @@ -111,42 +101,8 @@ pub fn v1_to_v2(db: Arc, config: Config) -> Result<()> { Ok(()) } -// Checks if the migration doesn't leave the DB in an unsane state. -// This function is to be used in tests. -pub fn v1_to_v2_sanity_check( - db: Arc, - config: Config, - expected_candidates: HashSet, -) -> Result<()> { - let backend = crate::DbBackend::new(db, config); - - let all_blocks = backend - .load_all_blocks() - .unwrap() - .iter() - .map(|block_hash| backend.load_block_entry(block_hash).unwrap().unwrap()) - .collect::>(); - - let mut candidates = HashSet::new(); - - // Iterate all blocks and approval entries. - for block in all_blocks { - for (_core_index, candidate_hash) in block.candidates() { - // Loading the candidate will also perform the conversion to the updated format and - // return that represantation. - if let Some(candidate_entry) = backend.load_candidate_entry(&candidate_hash).unwrap() { - candidates.insert(candidate_entry.candidate.hash()); - } - } - } - - assert_eq!(candidates, expected_candidates); - - Ok(()) -} - // Fills the db with dummy data in v1 scheme. -pub fn v1_to_v2_fill_test_data( +pub fn v1_fill_test_data( db: Arc, config: Config, dummy_candidate_create: F, diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs index 2846b0ca499b..da42fc5be485 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs @@ -17,10 +17,7 @@ //! Version 2 of the DB schema. use parity_scale_codec::{Decode, Encode}; -use polkadot_node_primitives::approval::{ - v1::DelayTranche, - v2::{AssignmentCertV2, CandidateBitfield}, -}; +use polkadot_node_primitives::approval::{v1::DelayTranche, v2::AssignmentCertV2}; use polkadot_node_subsystem::{SubsystemError, SubsystemResult}; use polkadot_node_subsystem_util::database::{DBTransaction, Database}; use polkadot_primitives::{ @@ -31,139 +28,16 @@ use polkadot_primitives::{ use sp_consensus_slots::Slot; use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; -use std::{collections::BTreeMap, sync::Arc}; +use std::collections::BTreeMap; -use crate::{ - backend::{Backend, BackendWriteOp, V1ReadBackend}, - persisted_entries, -}; +use crate::backend::V1ReadBackend; -const STORED_BLOCKS_KEY: &[u8] = b"Approvals_StoredBlocks"; +use super::common::{block_entry_key, candidate_entry_key, load_decode, Config}; pub mod migration_helpers; #[cfg(test)] pub mod tests; -/// `DbBackend` is a concrete implementation of the higher-level Backend trait -pub struct DbBackend { - inner: Arc, - config: Config, -} - -impl DbBackend { - /// Create a new [`DbBackend`] with the supplied key-value store and - /// config. - pub fn new(db: Arc, config: Config) -> Self { - DbBackend { inner: db, config } - } -} - -impl V1ReadBackend for DbBackend { - fn load_candidate_entry_v1( - &self, - candidate_hash: &CandidateHash, - candidate_index: CandidateIndex, - ) -> SubsystemResult> { - load_candidate_entry_v1(&*self.inner, &self.config, candidate_hash) - .map(|e| e.map(|e| persisted_entries::CandidateEntry::from_v1(e, candidate_index))) - } - - fn load_block_entry_v1( - &self, - block_hash: &Hash, - ) -> SubsystemResult> { - load_block_entry_v1(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) - } -} - -impl Backend for DbBackend { - fn load_block_entry( - &self, - block_hash: &Hash, - ) -> SubsystemResult> { - load_block_entry(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) - } - - fn load_candidate_entry( - &self, - candidate_hash: &CandidateHash, - ) -> SubsystemResult> { - load_candidate_entry(&*self.inner, &self.config, candidate_hash).map(|e| e.map(Into::into)) - } - - fn load_blocks_at_height(&self, block_height: &BlockNumber) -> SubsystemResult> { - load_blocks_at_height(&*self.inner, &self.config, block_height) - } - - fn load_all_blocks(&self) -> SubsystemResult> { - load_all_blocks(&*self.inner, &self.config) - } - - fn load_stored_blocks(&self) -> SubsystemResult> { - load_stored_blocks(&*self.inner, &self.config) - } - - /// Atomically write the list of operations, with later operations taking precedence over prior. - fn write(&mut self, ops: I) -> SubsystemResult<()> - where - I: IntoIterator, - { - let mut tx = DBTransaction::new(); - for op in ops { - match op { - BackendWriteOp::WriteStoredBlockRange(stored_block_range) => { - tx.put_vec( - self.config.col_approval_data, - &STORED_BLOCKS_KEY, - stored_block_range.encode(), - ); - }, - BackendWriteOp::DeleteStoredBlockRange => { - tx.delete(self.config.col_approval_data, &STORED_BLOCKS_KEY); - }, - BackendWriteOp::WriteBlocksAtHeight(h, blocks) => { - tx.put_vec( - self.config.col_approval_data, - &blocks_at_height_key(h), - blocks.encode(), - ); - }, - BackendWriteOp::DeleteBlocksAtHeight(h) => { - tx.delete(self.config.col_approval_data, &blocks_at_height_key(h)); - }, - BackendWriteOp::WriteBlockEntry(block_entry) => { - let block_entry: BlockEntry = block_entry.into(); - tx.put_vec( - self.config.col_approval_data, - &block_entry_key(&block_entry.block_hash), - block_entry.encode(), - ); - }, - BackendWriteOp::DeleteBlockEntry(hash) => { - tx.delete(self.config.col_approval_data, &block_entry_key(&hash)); - }, - BackendWriteOp::WriteCandidateEntry(candidate_entry) => { - let candidate_entry: CandidateEntry = candidate_entry.into(); - tx.put_vec( - self.config.col_approval_data, - &candidate_entry_key(&candidate_entry.candidate.hash()), - candidate_entry.encode(), - ); - }, - BackendWriteOp::DeleteCandidateEntry(candidate_hash) => { - tx.delete(self.config.col_approval_data, &candidate_entry_key(&candidate_hash)); - }, - } - } - - self.inner.write(tx).map_err(|e| e.into()) - } -} - -/// A range from earliest..last block number stored within the DB. -#[derive(Encode, Decode, Debug, Clone, PartialEq)] -pub struct StoredBlockRange(pub BlockNumber, pub BlockNumber); - // slot_duration * 2 + DelayTranche gives the number of delay tranches since the // unix epoch. #[derive(Encode, Decode, Clone, Copy, Debug, PartialEq)] @@ -172,13 +46,6 @@ pub struct Tick(u64); /// Convenience type definition pub type Bitfield = BitVec; -/// The database config. -#[derive(Debug, Clone, Copy)] -pub struct Config { - /// The column family in the database where data is stored. - pub col_approval_data: u32, -} - /// Details pertaining to our assignment on a block. #[derive(Encode, Decode, Debug, Clone, PartialEq)] pub struct OurAssignment { @@ -201,15 +68,6 @@ pub struct TrancheEntry { pub assignments: Vec<(ValidatorIndex, Tick)>, } -/// Metadata about our approval signature -#[derive(Encode, Decode, Debug, Clone, PartialEq)] -pub struct OurApproval { - /// The signature for the candidates hashes pointed by indices. - pub signature: ValidatorSignature, - /// The indices of the candidates signed in this approval. - pub signed_candidates_indices: CandidateBitfield, -} - /// Metadata regarding approval of a particular candidate within the context of some /// particular block. #[derive(Encode, Decode, Debug, Clone, PartialEq)] @@ -217,7 +75,7 @@ pub struct ApprovalEntry { pub tranches: Vec, pub backing_group: GroupIndex, pub our_assignment: Option, - pub our_approval_sig: Option, + pub our_approval_sig: Option, // `n_validators` bits. pub assigned_validators: Bitfield, pub approved: bool, @@ -254,25 +112,12 @@ pub struct BlockEntry { // block. The block can be considered approved if the bitfield has all bits set to `true`. pub approved_bitfield: Bitfield, pub children: Vec, - // A list of candidates that has been approved, but we didn't not sign and - // advertise the vote yet. - pub candidates_pending_signature: BTreeMap, // Assignments we already distributed. A 1 bit means the candidate index for which // we already have sent out an assignment. We need this to avoid distributing // multiple core assignments more than once. pub distributed_assignments: Bitfield, } -#[derive(Encode, Decode, Debug, Clone, PartialEq)] - -/// Context needed for creating an approval signature for a given candidate. -pub struct CandidateSigningContext { - /// The candidate hash, to be included in the signature. - pub candidate_hash: CandidateHash, - /// The latest tick we have to create and send the approval. - pub sign_no_later_than_tick: Tick, -} - impl From for Tick { fn from(tick: crate::Tick) -> Tick { Tick(tick) @@ -285,118 +130,6 @@ impl From for crate::Tick { } } -/// Errors while accessing things from the DB. -#[derive(Debug, derive_more::From, derive_more::Display)] -pub enum Error { - Io(std::io::Error), - InvalidDecoding(parity_scale_codec::Error), - InternalError(SubsystemError), -} - -impl std::error::Error for Error {} - -/// Result alias for DB errors. -pub type Result = std::result::Result; - -pub(crate) fn load_decode( - store: &dyn Database, - col_approval_data: u32, - key: &[u8], -) -> Result> { - match store.get(col_approval_data, key)? { - None => Ok(None), - Some(raw) => D::decode(&mut &raw[..]).map(Some).map_err(Into::into), - } -} - -/// The key a given block entry is stored under. -pub(crate) fn block_entry_key(block_hash: &Hash) -> [u8; 46] { - const BLOCK_ENTRY_PREFIX: [u8; 14] = *b"Approvals_blck"; - - let mut key = [0u8; 14 + 32]; - key[0..14].copy_from_slice(&BLOCK_ENTRY_PREFIX); - key[14..][..32].copy_from_slice(block_hash.as_ref()); - - key -} - -/// The key a given candidate entry is stored under. -pub(crate) fn candidate_entry_key(candidate_hash: &CandidateHash) -> [u8; 46] { - const CANDIDATE_ENTRY_PREFIX: [u8; 14] = *b"Approvals_cand"; - - let mut key = [0u8; 14 + 32]; - key[0..14].copy_from_slice(&CANDIDATE_ENTRY_PREFIX); - key[14..][..32].copy_from_slice(candidate_hash.0.as_ref()); - - key -} - -/// The key a set of block hashes corresponding to a block number is stored under. -pub(crate) fn blocks_at_height_key(block_number: BlockNumber) -> [u8; 16] { - const BLOCKS_AT_HEIGHT_PREFIX: [u8; 12] = *b"Approvals_at"; - - let mut key = [0u8; 12 + 4]; - key[0..12].copy_from_slice(&BLOCKS_AT_HEIGHT_PREFIX); - block_number.using_encoded(|s| key[12..16].copy_from_slice(s)); - - key -} - -/// Return all blocks which have entries in the DB, ascending, by height. -pub fn load_all_blocks(store: &dyn Database, config: &Config) -> SubsystemResult> { - let mut hashes = Vec::new(); - if let Some(stored_blocks) = load_stored_blocks(store, config)? { - for height in stored_blocks.0..stored_blocks.1 { - let blocks = load_blocks_at_height(store, config, &height)?; - hashes.extend(blocks); - } - } - - Ok(hashes) -} - -/// Load the stored-blocks key from the state. -pub fn load_stored_blocks( - store: &dyn Database, - config: &Config, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, STORED_BLOCKS_KEY) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - -/// Load a blocks-at-height entry for a given block number. -pub fn load_blocks_at_height( - store: &dyn Database, - config: &Config, - block_number: &BlockNumber, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, &blocks_at_height_key(*block_number)) - .map(|x| x.unwrap_or_default()) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - -/// Load a block entry from the aux store. -pub fn load_block_entry( - store: &dyn Database, - config: &Config, - block_hash: &Hash, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, &block_entry_key(block_hash)) - .map(|u: Option| u.map(|v| v.into())) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - -/// Load a candidate entry from the aux store in current version format. -pub fn load_candidate_entry( - store: &dyn Database, - config: &Config, - candidate_hash: &CandidateHash, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash)) - .map(|u: Option| u.map(|v| v.into())) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - /// Load a candidate entry from the aux store in v1 format. pub fn load_candidate_entry_v1( store: &dyn Database, diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs index 31b66d88577f..1121660fe658 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs @@ -16,13 +16,23 @@ //! Tests for the aux-schema of approval voting. -use super::{DbBackend, StoredBlockRange, *}; use crate::{ + approval_db::{ + common::{DbBackend, StoredBlockRange, *}, + v2::*, + v3::{load_block_entry_v2, load_candidate_entry_v2}, + }, backend::{Backend, OverlayedBackend}, ops::{add_block_entry, canonicalize, force_approve, NewCandidateInfo}, }; +use polkadot_primitives::{ + BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, +}; + +use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; use polkadot_node_subsystem_util::database::Database; use polkadot_primitives::Id as ParaId; +use sp_consensus_slots::Slot; use std::{collections::HashMap, sync::Arc}; use ::test_helpers::{dummy_candidate_receipt, dummy_candidate_receipt_bad_sig, dummy_hash}; @@ -56,7 +66,6 @@ fn make_block_entry( approved_bitfield: make_bitvec(candidates.len()), candidates, children: Vec::new(), - candidates_pending_signature: Default::default(), distributed_assignments: Default::default(), } } @@ -111,7 +120,10 @@ fn read_write() { overlay_db.write_stored_block_range(range.clone()); overlay_db.write_blocks_at_height(1, at_height.clone()); overlay_db.write_block_entry(block_entry.clone().into()); - overlay_db.write_candidate_entry(candidate_entry.clone().into()); + overlay_db.write_candidate_entry(crate::persisted_entries::CandidateEntry::from_v2( + candidate_entry.clone(), + 0, + )); let write_ops = overlay_db.into_write_ops(); db.write(write_ops).unwrap(); @@ -119,11 +131,11 @@ fn read_write() { assert_eq!(load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap(), Some(range)); assert_eq!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap(), at_height); assert_eq!( - load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap(), + load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap(), Some(block_entry.into()) ); assert_eq!( - load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash).unwrap(), + load_candidate_entry_v2(store.as_ref(), &TEST_CONFIG, &candidate_hash).unwrap(), Some(candidate_entry.into()), ); @@ -135,8 +147,8 @@ fn read_write() { db.write(write_ops).unwrap(); assert!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap().is_empty()); - assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap().is_none()); - assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash) + assert!(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap().is_none()); + assert!(load_candidate_entry_v2(store.as_ref(), &TEST_CONFIG, &candidate_hash) .unwrap() .is_none()); } @@ -197,25 +209,27 @@ fn add_block_entry_works() { db.write(write_ops).unwrap(); assert_eq!( - load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(), + load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(), Some(block_entry_a.into()) ); assert_eq!( - load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(), + load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(), Some(block_entry_b.into()) ); - let candidate_entry_a = load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash_a) - .unwrap() - .unwrap(); + let candidate_entry_a = + load_candidate_entry_v2(store.as_ref(), &TEST_CONFIG, &candidate_hash_a) + .unwrap() + .unwrap(); assert_eq!( candidate_entry_a.block_assignments.keys().collect::>(), vec![&block_hash_a, &block_hash_b] ); - let candidate_entry_b = load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash_b) - .unwrap() - .unwrap(); + let candidate_entry_b = + load_candidate_entry_v2(store.as_ref(), &TEST_CONFIG, &candidate_hash_b) + .unwrap() + .unwrap(); assert_eq!(candidate_entry_b.block_assignments.keys().collect::>(), vec![&block_hash_b]); } @@ -244,11 +258,11 @@ fn add_block_entry_adds_child() { block_entry_a.children.push(block_hash_b); assert_eq!( - load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(), + load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(), Some(block_entry_a.into()) ); assert_eq!( - load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(), + load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(), Some(block_entry_b.into()) ); } @@ -366,13 +380,15 @@ fn canonicalize_works() { for (c_hash, in_blocks) in expected { let (entry, in_blocks) = match in_blocks { None => { - assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash) + assert!(load_candidate_entry_v2(store.as_ref(), &TEST_CONFIG, &c_hash) .unwrap() .is_none()); continue }, Some(i) => ( - load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash).unwrap().unwrap(), + load_candidate_entry_v2(store.as_ref(), &TEST_CONFIG, &c_hash) + .unwrap() + .unwrap(), i, ), }; @@ -389,13 +405,13 @@ fn canonicalize_works() { for (hash, with_candidates) in expected { let (entry, with_candidates) = match with_candidates { None => { - assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash) + assert!(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &hash) .unwrap() .is_none()); continue }, Some(i) => - (load_block_entry(store.as_ref(), &TEST_CONFIG, &hash).unwrap().unwrap(), i), + (load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &hash).unwrap().unwrap(), i), }; assert_eq!(entry.candidates.len(), with_candidates.len()); @@ -511,22 +527,22 @@ fn force_approve_works() { let write_ops = overlay_db.into_write_ops(); db.write(write_ops).unwrap(); - assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a,) + assert!(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_a,) .unwrap() .unwrap() .approved_bitfield .all()); - assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b,) + assert!(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_b,) .unwrap() .unwrap() .approved_bitfield .all()); - assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_c,) + assert!(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_c,) .unwrap() .unwrap() .approved_bitfield .not_any()); - assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_d,) + assert!(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_d,) .unwrap() .unwrap() .approved_bitfield diff --git a/polkadot/node/core/approval-voting/src/approval_db/v3/migration_helpers.rs b/polkadot/node/core/approval-voting/src/approval_db/v3/migration_helpers.rs new file mode 100644 index 000000000000..3ffb66015215 --- /dev/null +++ b/polkadot/node/core/approval-voting/src/approval_db/v3/migration_helpers.rs @@ -0,0 +1,237 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Approval DB migration helpers. +use super::*; +use crate::{ + approval_db::common::{ + block_entry_key, candidate_entry_key, + migration_helpers::{dummy_assignment_cert, make_bitvec}, + Config, Error, Result, StoredBlockRange, + }, + backend::{Backend, V2ReadBackend}, +}; +use polkadot_node_primitives::approval::v1::AssignmentCertKind; +use polkadot_node_subsystem_util::database::Database; +use sp_application_crypto::sp_core::H256; +use std::{collections::HashSet, sync::Arc}; + +/// Migrates `BlockEntry`, `CandidateEntry`, `ApprovalEntry` and `OurApproval` to version 3. +/// Returns on any error. +/// Must only be used in parachains DB migration code - `polkadot-service` crate. +pub fn v2_to_latest(db: Arc, config: Config) -> Result<()> { + let mut backend = crate::DbBackend::new(db, config); + let all_blocks = backend + .load_all_blocks() + .map_err(|e| Error::InternalError(e))? + .iter() + .filter_map(|block_hash| { + backend + .load_block_entry_v2(block_hash) + .map_err(|e| Error::InternalError(e)) + .ok()? + }) + .collect::>(); + + gum::info!( + target: crate::LOG_TARGET, + "Migrating candidate entries on top of {} blocks", + all_blocks.len() + ); + + let mut overlay = crate::OverlayedBackend::new(&backend); + let mut counter = 0; + // Get all candidate entries, approval entries and convert each of them. + for block in all_blocks { + for (candidate_index, (_core_index, candidate_hash)) in + block.candidates().iter().enumerate() + { + // Loading the candidate will also perform the conversion to the updated format and + // return that represantation. + if let Some(candidate_entry) = backend + .load_candidate_entry_v2(&candidate_hash, candidate_index as CandidateIndex) + .map_err(|e| Error::InternalError(e))? + { + // Write the updated representation. + overlay.write_candidate_entry(candidate_entry); + counter += 1; + } + } + overlay.write_block_entry(block); + } + + gum::info!(target: crate::LOG_TARGET, "Migrated {} entries", counter); + + // Commit all changes to DB. + let write_ops = overlay.into_write_ops(); + backend.write(write_ops).unwrap(); + + Ok(()) +} + +// Checks if the migration doesn't leave the DB in an unsane state. +// This function is to be used in tests. +pub fn v1_to_latest_sanity_check( + db: Arc, + config: Config, + expected_candidates: HashSet, +) -> Result<()> { + let backend = crate::DbBackend::new(db, config); + + let all_blocks = backend + .load_all_blocks() + .unwrap() + .iter() + .map(|block_hash| backend.load_block_entry(block_hash).unwrap().unwrap()) + .collect::>(); + + let mut candidates = HashSet::new(); + + // Iterate all blocks and approval entries. + for block in all_blocks { + for (_core_index, candidate_hash) in block.candidates() { + // Loading the candidate will also perform the conversion to the updated format and + // return that represantation. + if let Some(candidate_entry) = backend.load_candidate_entry(&candidate_hash).unwrap() { + candidates.insert(candidate_entry.candidate.hash()); + } + } + } + + assert_eq!(candidates, expected_candidates); + + Ok(()) +} + +// Fills the db with dummy data in v1 scheme. +pub fn v2_fill_test_data( + db: Arc, + config: Config, + dummy_candidate_create: F, +) -> Result> +where + F: Fn(H256) -> CandidateReceipt, +{ + let mut backend = crate::DbBackend::new(db.clone(), config); + let mut overlay_db = crate::OverlayedBackend::new(&backend); + let mut expected_candidates = HashSet::new(); + + const RELAY_BLOCK_COUNT: u32 = 10; + + let range = StoredBlockRange(1, 11); + overlay_db.write_stored_block_range(range.clone()); + + for relay_number in 1..=RELAY_BLOCK_COUNT { + let relay_hash = Hash::repeat_byte(relay_number as u8); + let assignment_core_index = CoreIndex(relay_number); + let candidate = dummy_candidate_create(relay_hash); + let candidate_hash = candidate.hash(); + + let at_height = vec![relay_hash]; + + let block_entry = make_block_entry_v2( + relay_hash, + Default::default(), + relay_number, + vec![(assignment_core_index, candidate_hash)], + ); + + let dummy_assignment = crate::approval_db::v2::OurAssignment { + cert: dummy_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }).into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + }; + + let candidate_entry = crate::approval_db::v2::CandidateEntry { + candidate, + session: 123, + block_assignments: vec![( + relay_hash, + crate::approval_db::v2::ApprovalEntry { + tranches: Vec::new(), + backing_group: GroupIndex(1), + our_assignment: Some(dummy_assignment), + our_approval_sig: None, + approved: false, + assigned_validators: make_bitvec(1), + }, + )] + .into_iter() + .collect(), + approvals: Default::default(), + }; + + overlay_db.write_blocks_at_height(relay_number, at_height.clone()); + expected_candidates.insert(candidate_entry.candidate.hash()); + + db.write(write_candidate_entry_v2(candidate_entry, config)).unwrap(); + db.write(write_block_entry_v2(block_entry, config)).unwrap(); + } + + let write_ops = overlay_db.into_write_ops(); + backend.write(write_ops).unwrap(); + + Ok(expected_candidates) +} + +fn make_block_entry_v2( + block_hash: Hash, + parent_hash: Hash, + block_number: BlockNumber, + candidates: Vec<(CoreIndex, CandidateHash)>, +) -> crate::approval_db::v2::BlockEntry { + crate::approval_db::v2::BlockEntry { + block_hash, + parent_hash, + block_number, + session: 1, + slot: Slot::from(1), + relay_vrf_story: [0u8; 32], + approved_bitfield: make_bitvec(candidates.len()), + distributed_assignments: make_bitvec(candidates.len()), + candidates, + children: Vec::new(), + } +} + +// Low level DB helper to write a candidate entry in v1 scheme. +fn write_candidate_entry_v2( + candidate_entry: crate::approval_db::v2::CandidateEntry, + config: Config, +) -> DBTransaction { + let mut tx = DBTransaction::new(); + tx.put_vec( + config.col_approval_data, + &candidate_entry_key(&candidate_entry.candidate.hash()), + candidate_entry.encode(), + ); + tx +} + +// Low level DB helper to write a block entry in v1 scheme. +fn write_block_entry_v2( + block_entry: crate::approval_db::v2::BlockEntry, + config: Config, +) -> DBTransaction { + let mut tx = DBTransaction::new(); + tx.put_vec( + config.col_approval_data, + &block_entry_key(&block_entry.block_hash), + block_entry.encode(), + ); + tx +} diff --git a/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs new file mode 100644 index 000000000000..4483a4740a1f --- /dev/null +++ b/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs @@ -0,0 +1,139 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Version 3 of the DB schema. +//! +//! Version 3 modifies the our_approval format of `ApprovalEntry` +//! and adds a new field `pending_signatures` for `BlockEntry` +//! and adds a new field `pending_signatures` for `BlockEntry` + +use parity_scale_codec::{Decode, Encode}; +use polkadot_node_primitives::approval::v2::CandidateBitfield; +use polkadot_node_subsystem::SubsystemResult; +use polkadot_node_subsystem_util::database::{DBTransaction, Database}; +use polkadot_overseer::SubsystemError; +use polkadot_primitives::{ + BlockNumber, CandidateHash, CandidateIndex, CandidateReceipt, CoreIndex, GroupIndex, Hash, + SessionIndex, ValidatorIndex, ValidatorSignature, +}; + +use sp_consensus_slots::Slot; + +use std::collections::BTreeMap; + +use super::common::{block_entry_key, candidate_entry_key, load_decode, Config}; + +/// Re-export this structs as v3 since they did not change between v2 and v3. +pub use super::v2::{Bitfield, OurAssignment, Tick, TrancheEntry}; + +pub mod migration_helpers; + +#[cfg(test)] +pub mod tests; + +/// Metadata about our approval signature +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct OurApproval { + /// The signature for the candidates hashes pointed by indices. + pub signature: ValidatorSignature, + /// The indices of the candidates signed in this approval. + pub signed_candidates_indices: CandidateBitfield, +} + +/// Metadata regarding approval of a particular candidate within the context of some +/// particular block. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct ApprovalEntry { + pub tranches: Vec, + pub backing_group: GroupIndex, + pub our_assignment: Option, + pub our_approval_sig: Option, + // `n_validators` bits. + pub assigned_validators: Bitfield, + pub approved: bool, +} + +/// Metadata regarding approval of a particular candidate. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct CandidateEntry { + pub candidate: CandidateReceipt, + pub session: SessionIndex, + // Assignments are based on blocks, so we need to track assignments separately + // based on the block we are looking at. + pub block_assignments: BTreeMap, + pub approvals: Bitfield, +} + +/// Metadata regarding approval of a particular block, by way of approval of the +/// candidates contained within it. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct BlockEntry { + pub block_hash: Hash, + pub block_number: BlockNumber, + pub parent_hash: Hash, + pub session: SessionIndex, + pub slot: Slot, + /// Random bytes derived from the VRF submitted within the block by the block + /// author as a credential and used as input to approval assignment criteria. + pub relay_vrf_story: [u8; 32], + // The candidates included as-of this block and the index of the core they are + // leaving. Sorted ascending by core index. + pub candidates: Vec<(CoreIndex, CandidateHash)>, + // A bitfield where the i'th bit corresponds to the i'th candidate in `candidates`. + // The i'th bit is `true` iff the candidate has been approved in the context of this + // block. The block can be considered approved if the bitfield has all bits set to `true`. + pub approved_bitfield: Bitfield, + pub children: Vec, + // A list of candidates that has been approved, but we didn't not sign and + // advertise the vote yet. + pub candidates_pending_signature: BTreeMap, + // Assignments we already distributed. A 1 bit means the candidate index for which + // we already have sent out an assignment. We need this to avoid distributing + // multiple core assignments more than once. + pub distributed_assignments: Bitfield, +} + +#[derive(Encode, Decode, Debug, Clone, PartialEq)] + +/// Context needed for creating an approval signature for a given candidate. +pub struct CandidateSigningContext { + /// The candidate hash, to be included in the signature. + pub candidate_hash: CandidateHash, + /// The latest tick we have to create and send the approval. + pub sign_no_later_than_tick: Tick, +} + +/// Load a candidate entry from the aux store in v2 format. +pub fn load_candidate_entry_v2( + store: &dyn Database, + config: &Config, + candidate_hash: &CandidateHash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a block entry from the aux store in v2 format. +pub fn load_block_entry_v2( + store: &dyn Database, + config: &Config, + block_hash: &Hash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &block_entry_key(block_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} diff --git a/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs b/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs new file mode 100644 index 000000000000..7ff6433b5365 --- /dev/null +++ b/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs @@ -0,0 +1,580 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Tests for the aux-schema of approval voting. + +use crate::{ + approval_db::{ + common::{DbBackend, StoredBlockRange, *}, + v3::*, + }, + backend::{Backend, OverlayedBackend}, + ops::{add_block_entry, canonicalize, force_approve, NewCandidateInfo}, +}; +use polkadot_primitives::{ + BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, +}; + +use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; +use polkadot_node_subsystem_util::database::Database; +use polkadot_primitives::Id as ParaId; +use sp_consensus_slots::Slot; +use std::{collections::HashMap, sync::Arc}; + +use ::test_helpers::{dummy_candidate_receipt, dummy_candidate_receipt_bad_sig, dummy_hash}; + +const DATA_COL: u32 = 0; + +const NUM_COLUMNS: u32 = 1; + +const TEST_CONFIG: Config = Config { col_approval_data: DATA_COL }; + +fn make_db() -> (DbBackend, Arc) { + let db = kvdb_memorydb::create(NUM_COLUMNS); + let db = polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[]); + let db_writer: Arc = Arc::new(db); + (DbBackend::new(db_writer.clone(), TEST_CONFIG), db_writer) +} + +fn make_block_entry( + block_hash: Hash, + parent_hash: Hash, + block_number: BlockNumber, + candidates: Vec<(CoreIndex, CandidateHash)>, +) -> BlockEntry { + BlockEntry { + block_hash, + parent_hash, + block_number, + session: 1, + slot: Slot::from(1), + relay_vrf_story: [0u8; 32], + approved_bitfield: make_bitvec(candidates.len()), + candidates, + children: Vec::new(), + candidates_pending_signature: Default::default(), + distributed_assignments: Default::default(), + } +} + +fn make_bitvec(len: usize) -> BitVec { + bitvec::bitvec![u8, BitOrderLsb0; 0; len] +} + +fn make_candidate(para_id: ParaId, relay_parent: Hash) -> CandidateReceipt { + let mut c = dummy_candidate_receipt(dummy_hash()); + + c.descriptor.para_id = para_id; + c.descriptor.relay_parent = relay_parent; + + c +} + +#[test] +fn read_write() { + let (mut db, store) = make_db(); + + let hash_a = Hash::repeat_byte(1); + let hash_b = Hash::repeat_byte(2); + let candidate_hash = dummy_candidate_receipt_bad_sig(dummy_hash(), None).hash(); + + let range = StoredBlockRange(10, 20); + let at_height = vec![hash_a, hash_b]; + + let block_entry = + make_block_entry(hash_a, Default::default(), 1, vec![(CoreIndex(0), candidate_hash)]); + + let candidate_entry = CandidateEntry { + candidate: dummy_candidate_receipt_bad_sig(dummy_hash(), None), + session: 5, + block_assignments: vec![( + hash_a, + ApprovalEntry { + tranches: Vec::new(), + backing_group: GroupIndex(1), + our_assignment: None, + our_approval_sig: None, + assigned_validators: Default::default(), + approved: false, + }, + )] + .into_iter() + .collect(), + approvals: Default::default(), + }; + + let mut overlay_db = OverlayedBackend::new(&db); + overlay_db.write_stored_block_range(range.clone()); + overlay_db.write_blocks_at_height(1, at_height.clone()); + overlay_db.write_block_entry(block_entry.clone().into()); + overlay_db.write_candidate_entry(candidate_entry.clone().into()); + + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert_eq!(load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap(), Some(range)); + assert_eq!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap(), at_height); + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap(), + Some(block_entry.into()) + ); + assert_eq!( + load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash).unwrap(), + Some(candidate_entry.into()), + ); + + let mut overlay_db = OverlayedBackend::new(&db); + overlay_db.delete_blocks_at_height(1); + overlay_db.delete_block_entry(&hash_a); + overlay_db.delete_candidate_entry(&candidate_hash); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap().is_empty()); + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap().is_none()); + assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash) + .unwrap() + .is_none()); +} + +#[test] +fn add_block_entry_works() { + let (mut db, store) = make_db(); + + let parent_hash = Hash::repeat_byte(1); + let block_hash_a = Hash::repeat_byte(2); + let block_hash_b = Hash::repeat_byte(69); + + let candidate_receipt_a = make_candidate(ParaId::from(1_u32), parent_hash); + let candidate_receipt_b = make_candidate(ParaId::from(2_u32), parent_hash); + + let candidate_hash_a = candidate_receipt_a.hash(); + let candidate_hash_b = candidate_receipt_b.hash(); + + let block_number = 10; + + let block_entry_a = make_block_entry( + block_hash_a, + parent_hash, + block_number, + vec![(CoreIndex(0), candidate_hash_a)], + ); + + let block_entry_b = make_block_entry( + block_hash_b, + parent_hash, + block_number, + vec![(CoreIndex(0), candidate_hash_a), (CoreIndex(1), candidate_hash_b)], + ); + + let n_validators = 10; + + let mut new_candidate_info = HashMap::new(); + new_candidate_info + .insert(candidate_hash_a, NewCandidateInfo::new(candidate_receipt_a, GroupIndex(0), None)); + + let mut overlay_db = OverlayedBackend::new(&db); + add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |h| { + new_candidate_info.get(h).map(|x| x.clone()) + }) + .unwrap(); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + new_candidate_info + .insert(candidate_hash_b, NewCandidateInfo::new(candidate_receipt_b, GroupIndex(1), None)); + + let mut overlay_db = OverlayedBackend::new(&db); + add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |h| { + new_candidate_info.get(h).map(|x| x.clone()) + }) + .unwrap(); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(), + Some(block_entry_a.into()) + ); + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(), + Some(block_entry_b.into()) + ); + + let candidate_entry_a = load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash_a) + .unwrap() + .unwrap(); + assert_eq!( + candidate_entry_a.block_assignments.keys().collect::>(), + vec![&block_hash_a, &block_hash_b] + ); + + let candidate_entry_b = load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash_b) + .unwrap() + .unwrap(); + assert_eq!(candidate_entry_b.block_assignments.keys().collect::>(), vec![&block_hash_b]); +} + +#[test] +fn add_block_entry_adds_child() { + let (mut db, store) = make_db(); + + let parent_hash = Hash::repeat_byte(1); + let block_hash_a = Hash::repeat_byte(2); + let block_hash_b = Hash::repeat_byte(69); + + let mut block_entry_a = make_block_entry(block_hash_a, parent_hash, 1, Vec::new()); + + let block_entry_b = make_block_entry(block_hash_b, block_hash_a, 2, Vec::new()); + + let n_validators = 10; + + let mut overlay_db = OverlayedBackend::new(&db); + add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |_| None).unwrap(); + + add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |_| None).unwrap(); + + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + block_entry_a.children.push(block_hash_b); + + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(), + Some(block_entry_a.into()) + ); + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(), + Some(block_entry_b.into()) + ); +} + +#[test] +fn canonicalize_works() { + let (mut db, store) = make_db(); + + // -> B1 -> C1 -> D1 + // A -> B2 -> C2 -> D2 + // + // We'll canonicalize C1. Everytning except D1 should disappear. + // + // Candidates: + // Cand1 in B2 + // Cand2 in C2 + // Cand3 in C2 and D1 + // Cand4 in D1 + // Cand5 in D2 + // Only Cand3 and Cand4 should remain after canonicalize. + + let n_validators = 10; + + let mut overlay_db = OverlayedBackend::new(&db); + overlay_db.write_stored_block_range(StoredBlockRange(1, 5)); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + let genesis = Hash::repeat_byte(0); + + let block_hash_a = Hash::repeat_byte(1); + let block_hash_b1 = Hash::repeat_byte(2); + let block_hash_b2 = Hash::repeat_byte(3); + let block_hash_c1 = Hash::repeat_byte(4); + let block_hash_c2 = Hash::repeat_byte(5); + let block_hash_d1 = Hash::repeat_byte(6); + let block_hash_d2 = Hash::repeat_byte(7); + + let candidate_receipt_genesis = make_candidate(ParaId::from(1_u32), genesis); + let candidate_receipt_a = make_candidate(ParaId::from(2_u32), block_hash_a); + let candidate_receipt_b = make_candidate(ParaId::from(3_u32), block_hash_a); + let candidate_receipt_b1 = make_candidate(ParaId::from(4_u32), block_hash_b1); + let candidate_receipt_c1 = make_candidate(ParaId::from(5_u32), block_hash_c1); + + let cand_hash_1 = candidate_receipt_genesis.hash(); + let cand_hash_2 = candidate_receipt_a.hash(); + let cand_hash_3 = candidate_receipt_b.hash(); + let cand_hash_4 = candidate_receipt_b1.hash(); + let cand_hash_5 = candidate_receipt_c1.hash(); + + let block_entry_a = make_block_entry(block_hash_a, genesis, 1, Vec::new()); + let block_entry_b1 = make_block_entry(block_hash_b1, block_hash_a, 2, Vec::new()); + let block_entry_b2 = + make_block_entry(block_hash_b2, block_hash_a, 2, vec![(CoreIndex(0), cand_hash_1)]); + let block_entry_c1 = make_block_entry(block_hash_c1, block_hash_b1, 3, Vec::new()); + let block_entry_c2 = make_block_entry( + block_hash_c2, + block_hash_b2, + 3, + vec![(CoreIndex(0), cand_hash_2), (CoreIndex(1), cand_hash_3)], + ); + let block_entry_d1 = make_block_entry( + block_hash_d1, + block_hash_c1, + 4, + vec![(CoreIndex(0), cand_hash_3), (CoreIndex(1), cand_hash_4)], + ); + let block_entry_d2 = + make_block_entry(block_hash_d2, block_hash_c2, 4, vec![(CoreIndex(0), cand_hash_5)]); + + let candidate_info = { + let mut candidate_info = HashMap::new(); + candidate_info.insert( + cand_hash_1, + NewCandidateInfo::new(candidate_receipt_genesis, GroupIndex(1), None), + ); + + candidate_info + .insert(cand_hash_2, NewCandidateInfo::new(candidate_receipt_a, GroupIndex(2), None)); + + candidate_info + .insert(cand_hash_3, NewCandidateInfo::new(candidate_receipt_b, GroupIndex(3), None)); + + candidate_info + .insert(cand_hash_4, NewCandidateInfo::new(candidate_receipt_b1, GroupIndex(4), None)); + + candidate_info + .insert(cand_hash_5, NewCandidateInfo::new(candidate_receipt_c1, GroupIndex(5), None)); + + candidate_info + }; + + // now insert all the blocks. + let blocks = vec![ + block_entry_a.clone(), + block_entry_b1.clone(), + block_entry_b2.clone(), + block_entry_c1.clone(), + block_entry_c2.clone(), + block_entry_d1.clone(), + block_entry_d2.clone(), + ]; + + let mut overlay_db = OverlayedBackend::new(&db); + for block_entry in blocks { + add_block_entry(&mut overlay_db, block_entry.into(), n_validators, |h| { + candidate_info.get(h).map(|x| x.clone()) + }) + .unwrap(); + } + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + let check_candidates_in_store = |expected: Vec<(CandidateHash, Option>)>| { + for (c_hash, in_blocks) in expected { + let (entry, in_blocks) = match in_blocks { + None => { + assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash) + .unwrap() + .is_none()); + continue + }, + Some(i) => ( + load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash).unwrap().unwrap(), + i, + ), + }; + + assert_eq!(entry.block_assignments.len(), in_blocks.len()); + + for x in in_blocks { + assert!(entry.block_assignments.contains_key(&x)); + } + } + }; + + let check_blocks_in_store = |expected: Vec<(Hash, Option>)>| { + for (hash, with_candidates) in expected { + let (entry, with_candidates) = match with_candidates { + None => { + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash) + .unwrap() + .is_none()); + continue + }, + Some(i) => + (load_block_entry(store.as_ref(), &TEST_CONFIG, &hash).unwrap().unwrap(), i), + }; + + assert_eq!(entry.candidates.len(), with_candidates.len()); + + for x in with_candidates { + assert!(entry.candidates.iter().any(|(_, c)| c == &x)); + } + } + }; + + check_candidates_in_store(vec![ + (cand_hash_1, Some(vec![block_hash_b2])), + (cand_hash_2, Some(vec![block_hash_c2])), + (cand_hash_3, Some(vec![block_hash_c2, block_hash_d1])), + (cand_hash_4, Some(vec![block_hash_d1])), + (cand_hash_5, Some(vec![block_hash_d2])), + ]); + + check_blocks_in_store(vec![ + (block_hash_a, Some(vec![])), + (block_hash_b1, Some(vec![])), + (block_hash_b2, Some(vec![cand_hash_1])), + (block_hash_c1, Some(vec![])), + (block_hash_c2, Some(vec![cand_hash_2, cand_hash_3])), + (block_hash_d1, Some(vec![cand_hash_3, cand_hash_4])), + (block_hash_d2, Some(vec![cand_hash_5])), + ]); + + let mut overlay_db = OverlayedBackend::new(&db); + canonicalize(&mut overlay_db, 3, block_hash_c1).unwrap(); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert_eq!( + load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap().unwrap(), + StoredBlockRange(4, 5) + ); + + check_candidates_in_store(vec![ + (cand_hash_1, None), + (cand_hash_2, None), + (cand_hash_3, Some(vec![block_hash_d1])), + (cand_hash_4, Some(vec![block_hash_d1])), + (cand_hash_5, None), + ]); + + check_blocks_in_store(vec![ + (block_hash_a, None), + (block_hash_b1, None), + (block_hash_b2, None), + (block_hash_c1, None), + (block_hash_c2, None), + (block_hash_d1, Some(vec![cand_hash_3, cand_hash_4])), + (block_hash_d2, None), + ]); +} + +#[test] +fn force_approve_works() { + let (mut db, store) = make_db(); + let n_validators = 10; + + let mut overlay_db = OverlayedBackend::new(&db); + overlay_db.write_stored_block_range(StoredBlockRange(1, 4)); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + let single_candidate_vec = vec![(CoreIndex(0), candidate_hash)]; + let candidate_info = { + let mut candidate_info = HashMap::new(); + candidate_info.insert( + candidate_hash, + NewCandidateInfo::new( + make_candidate(ParaId::from(1_u32), Default::default()), + GroupIndex(1), + None, + ), + ); + + candidate_info + }; + + let block_hash_a = Hash::repeat_byte(1); // 1 + let block_hash_b = Hash::repeat_byte(2); + let block_hash_c = Hash::repeat_byte(3); + let block_hash_d = Hash::repeat_byte(4); // 4 + + let block_entry_a = + make_block_entry(block_hash_a, Default::default(), 1, single_candidate_vec.clone()); + let block_entry_b = + make_block_entry(block_hash_b, block_hash_a, 2, single_candidate_vec.clone()); + let block_entry_c = + make_block_entry(block_hash_c, block_hash_b, 3, single_candidate_vec.clone()); + let block_entry_d = + make_block_entry(block_hash_d, block_hash_c, 4, single_candidate_vec.clone()); + + let blocks = vec![ + block_entry_a.clone(), + block_entry_b.clone(), + block_entry_c.clone(), + block_entry_d.clone(), + ]; + + let mut overlay_db = OverlayedBackend::new(&db); + for block_entry in blocks { + add_block_entry(&mut overlay_db, block_entry.into(), n_validators, |h| { + candidate_info.get(h).map(|x| x.clone()) + }) + .unwrap(); + } + let approved_hashes = force_approve(&mut overlay_db, block_hash_d, 2).unwrap(); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a,) + .unwrap() + .unwrap() + .approved_bitfield + .all()); + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b,) + .unwrap() + .unwrap() + .approved_bitfield + .all()); + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_c,) + .unwrap() + .unwrap() + .approved_bitfield + .not_any()); + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_d,) + .unwrap() + .unwrap() + .approved_bitfield + .not_any()); + assert_eq!(approved_hashes, vec![block_hash_b, block_hash_a]); +} + +#[test] +fn load_all_blocks_works() { + let (mut db, store) = make_db(); + + let parent_hash = Hash::repeat_byte(1); + let block_hash_a = Hash::repeat_byte(2); + let block_hash_b = Hash::repeat_byte(69); + let block_hash_c = Hash::repeat_byte(42); + + let block_number = 10; + + let block_entry_a = make_block_entry(block_hash_a, parent_hash, block_number, vec![]); + + let block_entry_b = make_block_entry(block_hash_b, parent_hash, block_number, vec![]); + + let block_entry_c = make_block_entry(block_hash_c, block_hash_a, block_number + 1, vec![]); + + let n_validators = 10; + + let mut overlay_db = OverlayedBackend::new(&db); + add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |_| None).unwrap(); + + // add C before B to test sorting. + add_block_entry(&mut overlay_db, block_entry_c.clone().into(), n_validators, |_| None).unwrap(); + + add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |_| None).unwrap(); + + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert_eq!( + load_all_blocks(store.as_ref(), &TEST_CONFIG).unwrap(), + vec![block_hash_a, block_hash_b, block_hash_c], + ) +} diff --git a/polkadot/node/core/approval-voting/src/backend.rs b/polkadot/node/core/approval-voting/src/backend.rs index 801120b210b5..9ce25334c0fa 100644 --- a/polkadot/node/core/approval-voting/src/backend.rs +++ b/polkadot/node/core/approval-voting/src/backend.rs @@ -27,7 +27,7 @@ use polkadot_primitives::{BlockNumber, CandidateHash, CandidateIndex, Hash}; use std::collections::HashMap; use super::{ - approval_db::v2::StoredBlockRange, + approval_db::common::StoredBlockRange, persisted_entries::{BlockEntry, CandidateEntry}, }; @@ -79,6 +79,19 @@ pub trait V1ReadBackend: Backend { fn load_block_entry_v1(&self, block_hash: &Hash) -> SubsystemResult>; } +/// A read only backend to enable db migration from version 2 of DB. +pub trait V2ReadBackend: Backend { + /// Load a candidate entry from the DB with scheme version 1. + fn load_candidate_entry_v2( + &self, + candidate_hash: &CandidateHash, + candidate_index: CandidateIndex, + ) -> SubsystemResult>; + + /// Load a block entry from the DB with scheme version 1. + fn load_block_entry_v2(&self, block_hash: &Hash) -> SubsystemResult>; +} + // Status of block range in the `OverlayedBackend`. #[derive(PartialEq)] enum BlockRangeStatus { diff --git a/polkadot/node/core/approval-voting/src/import.rs b/polkadot/node/core/approval-voting/src/import.rs index 47587eb27ef0..ba894c2a30ff 100644 --- a/polkadot/node/core/approval-voting/src/import.rs +++ b/polkadot/node/core/approval-voting/src/import.rs @@ -56,7 +56,7 @@ use futures::{channel::oneshot, prelude::*}; use std::collections::HashMap; -use super::approval_db::v2; +use super::approval_db::v3; use crate::{ backend::{Backend, OverlayedBackend}, criteria::{AssignmentCriteria, OurAssignment}, @@ -500,7 +500,7 @@ pub(crate) async fn handle_new_head( ctx.send_message(ChainSelectionMessage::Approved(block_hash)).await; } - let block_entry = v2::BlockEntry { + let block_entry = v3::BlockEntry { block_hash, parent_hash: block_header.parent_hash, block_number: block_header.number, @@ -593,7 +593,10 @@ pub(crate) async fn handle_new_head( #[cfg(test)] pub(crate) mod tests { use super::*; - use crate::{approval_db::v2::DbBackend, RuntimeInfo, RuntimeInfoConfig}; + use crate::{ + approval_db::common::{load_block_entry, DbBackend}, + RuntimeInfo, RuntimeInfoConfig, + }; use ::test_helpers::{dummy_candidate_receipt, dummy_hash}; use assert_matches::assert_matches; use polkadot_node_primitives::{ @@ -615,7 +618,7 @@ pub(crate) mod tests { pub(crate) use sp_runtime::{Digest, DigestItem}; use std::{pin::Pin, sync::Arc}; - use crate::{approval_db::v2::Config as DatabaseConfig, criteria, BlockEntry}; + use crate::{approval_db::common::Config as DatabaseConfig, criteria, BlockEntry}; const DATA_COL: u32 = 0; @@ -1303,7 +1306,7 @@ pub(crate) mod tests { let (state, mut session_info_provider) = single_session_state(); overlay_db.write_block_entry( - v2::BlockEntry { + v3::BlockEntry { block_hash: parent_hash, parent_hash: Default::default(), block_number: 4, @@ -1346,11 +1349,10 @@ pub(crate) mod tests { assert_eq!(candidates[1].1.approvals().len(), 6); // the first candidate should be insta-approved // the second should not - let entry: BlockEntry = - v2::load_block_entry(db_writer.as_ref(), &TEST_CONFIG, &hash) - .unwrap() - .unwrap() - .into(); + let entry: BlockEntry = load_block_entry(db_writer.as_ref(), &TEST_CONFIG, &hash) + .unwrap() + .unwrap() + .into(); assert!(entry.is_candidate_approved(&candidates[0].0)); assert!(!entry.is_candidate_approved(&candidates[1].0)); }) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index e21ad331431d..c12ba99fc319 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -100,7 +100,7 @@ mod time; use crate::{ approval_checking::{Check, TranchesToApproveResult}, - approval_db::v2::{Config as DatabaseConfig, DbBackend}, + approval_db::common::{Config as DatabaseConfig, DbBackend}, backend::{Backend, OverlayedBackend}, criteria::InvalidAssignmentReason, persisted_entries::OurApproval, @@ -485,8 +485,8 @@ impl ApprovalVotingSubsystem { /// The operation is not allowed for blocks older than the last finalized one. pub fn revert_to(&self, hash: Hash) -> Result<(), SubsystemError> { let config = - approval_db::v2::Config { col_approval_data: self.db_config.col_approval_data }; - let mut backend = approval_db::v2::DbBackend::new(self.db.clone(), config); + approval_db::common::Config { col_approval_data: self.db_config.col_approval_data }; + let mut backend = approval_db::common::DbBackend::new(self.db.clone(), config); let mut overlay = OverlayedBackend::new(&backend); ops::revert_to(&mut overlay, hash)?; diff --git a/polkadot/node/core/approval-voting/src/ops.rs b/polkadot/node/core/approval-voting/src/ops.rs index a6f0ecf9d1f0..2a8fdba5aa36 100644 --- a/polkadot/node/core/approval-voting/src/ops.rs +++ b/polkadot/node/core/approval-voting/src/ops.rs @@ -25,7 +25,7 @@ use polkadot_primitives::{BlockNumber, CandidateHash, CandidateReceipt, GroupInd use std::collections::{hash_map::Entry, BTreeMap, HashMap}; use super::{ - approval_db::v2::{OurAssignment, StoredBlockRange}, + approval_db::{common::StoredBlockRange, v2::OurAssignment}, backend::{Backend, OverlayedBackend}, persisted_entries::{ApprovalEntry, BlockEntry, CandidateEntry}, LOG_TARGET, diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index d9906c45ecc4..05da6f844063 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -77,15 +77,15 @@ impl From for crate::approval_db::v2::TrancheEntry { } } -impl From for OurApproval { - fn from(approval: crate::approval_db::v2::OurApproval) -> Self { +impl From for OurApproval { + fn from(approval: crate::approval_db::v3::OurApproval) -> Self { Self { signature: approval.signature, signed_candidates_indices: approval.signed_candidates_indices, } } } -impl From for crate::approval_db::v2::OurApproval { +impl From for crate::approval_db::v3::OurApproval { fn from(approval: OurApproval) -> Self { Self { signature: approval.signature, @@ -105,10 +105,16 @@ pub struct OurApproval { impl OurApproval { /// Converts a ValidatorSignature to an OurApproval. - /// It used in converting the database from v1 to v2. + /// It used in converting the database from v1 to latest. pub fn from_v1(value: ValidatorSignature, candidate_index: CandidateIndex) -> Self { Self { signature: value, signed_candidates_indices: candidate_index.into() } } + + /// Converts a ValidatorSignature to an OurApproval. + /// It used in converting the database from v2 to latest. + pub fn from_v2(value: ValidatorSignature, candidate_index: CandidateIndex) -> Self { + Self::from_v1(value, candidate_index) + } } /// Metadata regarding approval of a particular candidate within the context of some /// particular block. @@ -267,7 +273,7 @@ impl ApprovalEntry { } } - // Convert an ApprovalEntry from v1 version to a v2 version + // Convert an ApprovalEntry from v1 version to latest version pub fn from_v1( value: crate::approval_db::v1::ApprovalEntry, candidate_index: CandidateIndex, @@ -283,10 +289,27 @@ impl ApprovalEntry { approved: value.approved, } } + + // Convert an ApprovalEntry from v1 version to latest version + pub fn from_v2( + value: crate::approval_db::v2::ApprovalEntry, + candidate_index: CandidateIndex, + ) -> Self { + ApprovalEntry { + tranches: value.tranches.into_iter().map(|tranche| tranche.into()).collect(), + backing_group: value.backing_group, + our_assignment: value.our_assignment.map(|assignment| assignment.into()), + our_approval_sig: value + .our_approval_sig + .map(|sig| OurApproval::from_v2(sig, candidate_index)), + assigned_validators: value.assigned_validators, + approved: value.approved, + } + } } -impl From for ApprovalEntry { - fn from(entry: crate::approval_db::v2::ApprovalEntry) -> Self { +impl From for ApprovalEntry { + fn from(entry: crate::approval_db::v3::ApprovalEntry) -> Self { ApprovalEntry { tranches: entry.tranches.into_iter().map(Into::into).collect(), backing_group: entry.backing_group, @@ -298,7 +321,7 @@ impl From for ApprovalEntry { } } -impl From for crate::approval_db::v2::ApprovalEntry { +impl From for crate::approval_db::v3::ApprovalEntry { fn from(entry: ApprovalEntry) -> Self { Self { tranches: entry.tranches.into_iter().map(Into::into).collect(), @@ -355,7 +378,7 @@ impl CandidateEntry { self.block_assignments.get(block_hash) } - /// Convert a CandidateEntry from a v1 to its v2 equivalent. + /// Convert a CandidateEntry from a v1 to its latest equivalent. pub fn from_v1( value: crate::approval_db::v1::CandidateEntry, candidate_index: CandidateIndex, @@ -371,10 +394,27 @@ impl CandidateEntry { session: value.session, } } + + /// Convert a CandidateEntry from a v2 to its latest equivalent. + pub fn from_v2( + value: crate::approval_db::v2::CandidateEntry, + candidate_index: CandidateIndex, + ) -> Self { + Self { + approvals: value.approvals, + block_assignments: value + .block_assignments + .into_iter() + .map(|(h, ae)| (h, ApprovalEntry::from_v2(ae, candidate_index))) + .collect(), + candidate: value.candidate, + session: value.session, + } + } } -impl From for CandidateEntry { - fn from(entry: crate::approval_db::v2::CandidateEntry) -> Self { +impl From for CandidateEntry { + fn from(entry: crate::approval_db::v3::CandidateEntry) -> Self { CandidateEntry { candidate: entry.candidate, session: entry.session, @@ -388,7 +428,7 @@ impl From for CandidateEntry { } } -impl From for crate::approval_db::v2::CandidateEntry { +impl From for crate::approval_db::v3::CandidateEntry { fn from(entry: CandidateEntry) -> Self { Self { candidate: entry.candidate, @@ -614,8 +654,8 @@ impl BlockEntry { } } -impl From for BlockEntry { - fn from(entry: crate::approval_db::v2::BlockEntry) -> Self { +impl From for BlockEntry { + fn from(entry: crate::approval_db::v3::BlockEntry) -> Self { BlockEntry { block_hash: entry.block_hash, parent_hash: entry.parent_hash, @@ -654,7 +694,25 @@ impl From for BlockEntry { } } -impl From for crate::approval_db::v2::BlockEntry { +impl From for BlockEntry { + fn from(entry: crate::approval_db::v2::BlockEntry) -> Self { + BlockEntry { + block_hash: entry.block_hash, + parent_hash: entry.parent_hash, + block_number: entry.block_number, + session: entry.session, + slot: entry.slot, + relay_vrf_story: RelayVRFStory(entry.relay_vrf_story), + candidates: entry.candidates, + approved_bitfield: entry.approved_bitfield, + children: entry.children, + distributed_assignments: entry.distributed_assignments, + candidates_pending_signature: Default::default(), + } + } +} + +impl From for crate::approval_db::v3::BlockEntry { fn from(entry: BlockEntry) -> Self { Self { block_hash: entry.block_hash, @@ -676,8 +734,8 @@ impl From for crate::approval_db::v2::BlockEntry { } } -impl From for CandidateSigningContext { - fn from(signing_context: crate::approval_db::v2::CandidateSigningContext) -> Self { +impl From for CandidateSigningContext { + fn from(signing_context: crate::approval_db::v3::CandidateSigningContext) -> Self { Self { candidate_hash: signing_context.candidate_hash, sign_no_later_than_tick: signing_context.sign_no_later_than_tick.into(), @@ -685,7 +743,7 @@ impl From for CandidateSigningC } } -impl From for crate::approval_db::v2::CandidateSigningContext { +impl From for crate::approval_db::v3::CandidateSigningContext { fn from(signing_context: CandidateSigningContext) -> Self { Self { candidate_hash: signing_context.candidate_hash, diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 04aef90aae97..4b78e990e660 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -56,7 +56,7 @@ use std::{ }; use super::{ - approval_db::v2::StoredBlockRange, + approval_db::common::StoredBlockRange, backend::BackendWriteOp, import::tests::{ garbage_vrf_signature, AllowedSlots, BabeEpoch, BabeEpochConfiguration, @@ -116,7 +116,7 @@ fn make_sync_oracle(val: bool) -> (Box, TestSyncOracleHan #[cfg(test)] pub mod test_constants { - use crate::approval_db::v2::Config as DatabaseConfig; + use crate::approval_db::common::Config as DatabaseConfig; const DATA_COL: u32 = 0; pub(crate) const NUM_COLUMNS: u32 = 1; diff --git a/polkadot/node/service/src/parachains_db/upgrade.rs b/polkadot/node/service/src/parachains_db/upgrade.rs index 1d76c79d3e32..d22eebb5c8d4 100644 --- a/polkadot/node/service/src/parachains_db/upgrade.rs +++ b/polkadot/node/service/src/parachains_db/upgrade.rs @@ -20,10 +20,16 @@ use std::{ fs, io, path::{Path, PathBuf}, str::FromStr, + sync::Arc, }; -use polkadot_node_core_approval_voting::approval_db::v2::{ - migration_helpers::v1_to_v2, Config as ApprovalDbConfig, +use polkadot_node_core_approval_voting::approval_db::{ + common::{Config as ApprovalDbConfig, Result as ApprovalDbResult}, + v2::migration_helpers::v1_to_latest, + v3::migration_helpers::v2_to_latest, +}; +use polkadot_node_subsystem_util::database::{ + kvdb_impl::DbAdapter as RocksDbAdapter, paritydb_impl::DbAdapter as ParityDbAdapter, Database, }; type Version = u32; @@ -32,7 +38,9 @@ const VERSION_FILE_NAME: &'static str = "parachain_db_version"; /// Current db version. /// Version 4 changes approval db format for `OurAssignment`. -pub(crate) const CURRENT_VERSION: Version = 4; +/// Version 5 changes approval db format to hold some additional +/// information about delayed approvals. +pub(crate) const CURRENT_VERSION: Version = 5; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -101,7 +109,8 @@ pub(crate) fn try_upgrade_db_to_next_version( // 2 -> 3 migration Some(2) => migrate_from_version_2_to_3(db_path, db_kind)?, // 3 -> 4 migration - Some(3) => migrate_from_version_3_to_4(db_path, db_kind)?, + Some(3) => migrate_from_version_3_or_4_to_5(db_path, db_kind, v1_to_latest)?, + Some(4) => migrate_from_version_3_or_4_to_5(db_path, db_kind, v2_to_latest)?, // Already at current version, do nothing. Some(CURRENT_VERSION) => CURRENT_VERSION, // This is an arbitrary future version, we don't handle it. @@ -174,14 +183,19 @@ fn migrate_from_version_1_to_2(path: &Path, db_kind: DatabaseKind) -> Result Result { +fn migrate_from_version_3_or_4_to_5( + path: &Path, + db_kind: DatabaseKind, + migration_function: F, +) -> Result +where + F: Fn(Arc, ApprovalDbConfig) -> ApprovalDbResult<()>, +{ gum::info!(target: LOG_TARGET, "Migrating parachains db from version 3 to version 4 ..."); - use polkadot_node_subsystem_util::database::{ - kvdb_impl::DbAdapter as RocksDbAdapter, paritydb_impl::DbAdapter as ParityDbAdapter, - }; - use std::sync::Arc; let approval_db_config = ApprovalDbConfig { col_approval_data: super::REAL_COLUMNS.col_approval_data }; @@ -194,7 +208,8 @@ fn migrate_from_version_3_to_4(path: &Path, db_kind: DatabaseKind) -> Result { let db_path = path @@ -207,7 +222,8 @@ fn migrate_from_version_3_to_4(path: &Path, db_kind: DatabaseKind) -> Result Date: Tue, 31 Oct 2023 11:02:07 +0200 Subject: [PATCH 055/192] Modify the way we are doing the sampling See https://github.com/paritytech/polkadot-sdk/pull/1178#discussion_r1375786733 for more details Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 2 + polkadot/node/core/approval-voting/Cargo.toml | 2 + .../node/core/approval-voting/src/criteria.rs | 63 +++++++++++++------ 3 files changed, 47 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a096faf19a3b..43f72366593f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11940,6 +11940,8 @@ dependencies = [ "polkadot-overseer", "polkadot-primitives", "polkadot-primitives-test-helpers", + "rand 0.8.5", + "rand_chacha 0.3.1", "rand_core 0.5.1", "sc-keystore", "schnellru", diff --git a/polkadot/node/core/approval-voting/Cargo.toml b/polkadot/node/core/approval-voting/Cargo.toml index b89d7d2cedc0..f7ea1d210777 100644 --- a/polkadot/node/core/approval-voting/Cargo.toml +++ b/polkadot/node/core/approval-voting/Cargo.toml @@ -32,6 +32,8 @@ sp-consensus-slots = { path = "../../../../substrate/primitives/consensus/slots" sp-application-crypto = { path = "../../../../substrate/primitives/application-crypto", default-features = false, features = ["full_crypto"] } sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } rand_core = "0.5.1" +rand_chacha = { version = "0.3.1" } +rand = "0.8.5" [dev-dependencies] async-trait = "0.1.57" diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 1f751e2bf140..0b69afc53280 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -16,6 +16,7 @@ //! Assignment criteria VRF generation and checking. +use itertools::Itertools; use parity_scale_codec::{Decode, Encode}; use polkadot_node_primitives::approval::{ self as approval_types, @@ -26,14 +27,18 @@ use polkadot_primitives::{ AssignmentId, AssignmentPair, CandidateHash, CoreIndex, GroupIndex, IndexedVec, SessionInfo, ValidatorIndex, }; +use rand::Rng; +use rand_chacha::{rand_core::SeedableRng, ChaCha20Rng}; use sc_keystore::LocalKeystore; use sp_application_crypto::ByteArray; use merlin::Transcript; use schnorrkel::vrf::VRFInOut; -use itertools::Itertools; -use std::collections::{hash_map::Entry, HashMap}; +use std::{ + cmp::min, + collections::{hash_map::Entry, HashMap, HashSet}, +}; use super::LOG_TARGET; @@ -125,17 +130,11 @@ fn relay_vrf_modulo_transcript_v2(relay_vrf_story: RelayVRFStory) -> Transcript /// A hard upper bound on num_cores * target_checkers / num_validators const MAX_MODULO_SAMPLES: usize = 40; -use std::convert::AsMut; +/// The maximum number of samples we take to try to obtain an +/// a distinct core_index. +const MAX_SAMPLING_ITERATIONS: usize = 20; -fn clone_into_array(slice: &[T]) -> A -where - A: Default + AsMut<[T]>, - T: Clone, -{ - let mut a = A::default(); - >::as_mut(&mut a).clone_from_slice(slice); - a -} +use std::convert::AsMut; struct BigArray(pub [u8; MAX_MODULO_SAMPLES * 4]); @@ -170,14 +169,38 @@ fn relay_vrf_modulo_cores( ); } - vrf_in_out - .make_bytes::(approval_types::v2::CORE_RANDOMNESS_CONTEXT) - .0 - .chunks_exact(4) - .take(num_samples as usize) - .map(move |sample| CoreIndex(u32::from_le_bytes(clone_into_array(&sample)) % max_cores)) - .unique() - .collect::>() + if 2 * num_samples > max_cores { + gum::error!( + target: LOG_TARGET, + n_cores = max_cores, + num_samples, + max_modulo_samples = MAX_MODULO_SAMPLES, + "Suboptimal configuration `num_samples` should be less than `n_cores` / 2", + ); + } + + let mut rand_chacha = + ChaCha20Rng::from_seed(vrf_in_out.make_bytes::<::Seed>( + approval_types::v2::CORE_RANDOMNESS_CONTEXT, + )); + + let mut res = HashSet::with_capacity(num_samples as usize); + while res.len() < min(num_samples, max_cores) as usize { + // Production networks should run with num_samples no larger than `n_cores/2`, so on worst + // case on each iteration this loop has >1/2 chance of finishing. + // With our current production parameters the chances for each iteration not hitting + // a duplicate are around 90%. + // See: https://github.com/paritytech/polkadot-sdk/pull/1178#discussion_r1376501206 + // for more details. + for _ in 0..MAX_SAMPLING_ITERATIONS { + let random_core = rand_chacha.gen_range(0..max_cores).into(); + if !res.contains(&random_core) { + res.insert(random_core); + break + } + } + } + res.into_iter().collect_vec() } fn relay_vrf_modulo_core(vrf_in_out: &VRFInOut, n_cores: u32) -> CoreIndex { From 64f2195f45736c3718d824a4068e4894513dbdc6 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 31 Oct 2023 12:51:09 +0200 Subject: [PATCH 056/192] Fixup check_rejects_delay_bad_vrf Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/criteria.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 0b69afc53280..01fff31a4be0 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -1135,7 +1135,7 @@ mod tests { #[test] fn check_rejects_delay_bad_vrf() { - check_mutated_assignments(40, 10, 8, |m| { + check_mutated_assignments(40, 100, 8, |m| { let vrf_signature = garbage_vrf_signature(); match m.cert.kind.clone() { AssignmentCertKindV2::RelayVRFDelay { .. } => { From 04798fa7b8edd76ae4431b468f8ab1dc6cf4e81e Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 31 Oct 2023 16:01:44 +0200 Subject: [PATCH 057/192] Address some trivial review feedback Signed-off-by: Alexandru Gheorghe --- .../node/core/approval-voting/src/approval_db/v3/mod.rs | 2 +- .../node/core/approval-voting/src/persisted_entries.rs | 2 +- polkadot/node/network/approval-distribution/src/lib.rs | 2 +- polkadot/primitives/src/v6/mod.rs | 5 ++++- .../src/node/approval/approval-voting.md | 6 +++--- .../roadmap/implementers-guide/src/protocol-approval.md | 8 ++++---- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs index 4483a4740a1f..031a51edd64b 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs @@ -97,7 +97,7 @@ pub struct BlockEntry { // block. The block can be considered approved if the bitfield has all bits set to `true`. pub approved_bitfield: Bitfield, pub children: Vec, - // A list of candidates that has been approved, but we didn't not sign and + // A list of candidates we have checked, but didn't not sign and // advertise the vote yet. pub candidates_pending_signature: BTreeMap, // Assignments we already distributed. A 1 bit means the candidate index for which diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index 05da6f844063..ac3b704c31c2 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -461,7 +461,7 @@ pub struct BlockEntry { // block. The block can be considered approved if the bitfield has all bits set to `true`. pub approved_bitfield: Bitfield, pub children: Vec, - // A list of candidates that has been approved, but we didn't not sign and + // A list of candidates we have checked, but didn't not sign and // advertise the vote yet. candidates_pending_signature: BTreeMap, // A list of assignments for which we already distributed the assignment. diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index d5188ef52ec7..91e850ad5b31 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -2606,7 +2606,7 @@ pub(crate) async fn send_approvals_batched( .clone() .into_iter() .filter(|approval| approval.candidate_indices.count_ones() == 1) - .map(|val| val.try_into().expect("We checked conversion should succeed; qed")) + .filter_map(|val| val.try_into().ok()) .peekable(); while batches.peek().is_some() { diff --git a/polkadot/primitives/src/v6/mod.rs b/polkadot/primitives/src/v6/mod.rs index 0d9a85e1fd81..8f11092b8455 100644 --- a/polkadot/primitives/src/v6/mod.rs +++ b/polkadot/primitives/src/v6/mod.rs @@ -1070,7 +1070,7 @@ impl ApprovalVote { } } -/// A vote of approvalf for multiple candidates. +/// A vote of approval for multiple candidates. #[derive(Clone, RuntimeDebug)] pub struct ApprovalVoteMultipleCandidates<'a>(pub &'a Vec); @@ -1266,6 +1266,9 @@ pub enum DisputeStatement { impl DisputeStatement { /// Get the payload data for this type of dispute statement. + /// + /// Returns Error if the candidate_hash is not included in the list of signed + /// candidate from ApprovalCheckingMultipleCandidate. pub fn payload_data( &self, candidate_hash: CandidateHash, diff --git a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md index 8d7daa173abc..345b3d2e6970 100644 --- a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md +++ b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md @@ -5,8 +5,8 @@ aims of this subsystem. Approval votes are split into two parts: Assignments and Approvals. Validators first broadcast their assignment to indicate intent to check a candidate. Upon successfully checking, they don't immediately send the vote instead -they queue the check for a short period of time `MAX_APPROVALS_COALESCE_TICKS` to give the opportunity of the -validator to vote for more than one candidate. Once MAX_APPROVALS_COALESCE_TICKS have passed or at least +they queue the check for a short period of time `MAX_APPROVAL_COALESCE_WAIT_TICKS` to give the opportunity of the +validator to vote for more than one candidate. Once MAX_APPROVAL_COALESCE_WAIT_TICKS have passed or at least `MAX_APPROVAL_COALESCE_COUNT` are ready they broadcast an approval vote for all candidates. If a validator doesn't broadcast their approval vote shortly after issuing an assignment, this is an indication that they are being prevented from recovering or validating the block data and that more validators should self-select to @@ -123,7 +123,7 @@ struct BlockEntry { // this block. The block can be considered approved has all bits set to 1 approved_bitfield: Bitfield, children: Vec, - // A list of candidates that has been approved, but we didn't not sign and + // A list of candidates we have checked, but didn't not sign and // advertise the vote yet. candidates_pending_signature: BTreeMap, // Assignments we already distributed. A 1 bit means the candidate index for which diff --git a/polkadot/roadmap/implementers-guide/src/protocol-approval.md b/polkadot/roadmap/implementers-guide/src/protocol-approval.md index 1a217afb9b71..a3427831753d 100644 --- a/polkadot/roadmap/implementers-guide/src/protocol-approval.md +++ b/polkadot/roadmap/implementers-guide/src/protocol-approval.md @@ -300,11 +300,11 @@ TODO: When? Is this optimal for the network? etc. To reduce the necessary network bandwidth and cpu time when a validator has more than one candidate to approve we are doing our best effort to send a single message that approves all available candidates with a single signature. The implemented heuristic, is that each time we are ready to create a signature and send a vote for a candidate we -delay the sending of it untill one of three things happen: +delay sending it until one of three things happen: - We gathered a maximum of `MAX_APPROVAL_COALESCE_COUNT` candidates that we are ready to vote for. -- `MAX_APPROVALS_COALESCE_TICKS` have passed since the we were ready to approve the candidate. -- We are already in the last third of the now-show period in order to avoid creating accidental no shows, which in - turn my trigger other assignments. +- `MAX_APPROVAL_COALESCE_WAIT_TICKS` have passed since the we were ready to approve the candidate. +- We are already in the last third of the no-show period in order to avoid creating accidental no-shows, which in + turn might trigger other assignments. ## On-chain verification From 440c38be89816566fa45ba03abcf60d23738fe6c Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 31 Oct 2023 17:10:27 +0200 Subject: [PATCH 058/192] Use VersionedMigration Signed-off-by: Alexandru Gheorghe --- .../src/configuration/migration/v10.rs | 45 ++++++++++--------- polkadot/runtime/rococo/src/lib.rs | 2 +- polkadot/runtime/westend/src/lib.rs | 2 +- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/polkadot/runtime/parachains/src/configuration/migration/v10.rs b/polkadot/runtime/parachains/src/configuration/migration/v10.rs index 53c5f4d2c52c..ea6624e99743 100644 --- a/polkadot/runtime/parachains/src/configuration/migration/v10.rs +++ b/polkadot/runtime/parachains/src/configuration/migration/v10.rs @@ -18,13 +18,10 @@ use crate::configuration::{self, Config, Pallet}; use frame_support::{ - pallet_prelude::*, - traits::{Defensive, StorageVersion}, - weights::Weight, + migrations::VersionedMigration, pallet_prelude::*, traits::Defensive, weights::Weight, }; use frame_system::pallet_prelude::BlockNumberFor; -use primitives::{vstaging::ApprovalVotingParams, SessionIndex, LEGACY_MIN_BACKING_VOTES}; -use sp_runtime::Perbill; +use primitives::{vstaging::ApprovalVotingParams, SessionIndex}; use sp_std::vec::Vec; use frame_support::traits::OnRuntimeUpgrade; @@ -62,8 +59,16 @@ mod v10 { >; } -pub struct MigrateToV10(sp_std::marker::PhantomData); -impl OnRuntimeUpgrade for MigrateToV10 { +pub type MigrateV9ToV10 = VersionedMigration< + 9, + 10, + UncheckedMigrateToV10, + Pallet, + ::DbWeight, +>; + +pub struct UncheckedMigrateToV10(sp_std::marker::PhantomData); +impl OnRuntimeUpgrade for UncheckedMigrateToV10 { #[cfg(feature = "try-runtime")] fn pre_upgrade() -> Result, sp_runtime::TryRuntimeError> { log::trace!(target: crate::configuration::LOG_TARGET, "Running pre_upgrade() for HostConfiguration MigrateToV10"); @@ -72,17 +77,11 @@ impl OnRuntimeUpgrade for MigrateToV10 { fn on_runtime_upgrade() -> Weight { log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV10 started"); - if StorageVersion::get::>() == 9 { - let weight_consumed = migrate_to_v10::(); + let weight_consumed = migrate_to_v10::(); - log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV10 executed successfully"); - StorageVersion::new(10).put::>(); + log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV10 executed successfully"); - weight_consumed - } else { - log::warn!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV10 should be removed."); - T::DbWeight::get().reads(1) - } + weight_consumed } #[cfg(feature = "try-runtime")] @@ -145,12 +144,12 @@ pvf_voting_ttl : pre.pvf_voting_ttl, minimum_validation_upgrade_delay : pre.minimum_validation_upgrade_delay, async_backing_params : pre.async_backing_params, executor_params : pre.executor_params, -on_demand_queue_max_size : 10_000u32, -on_demand_base_fee : 10_000_000u128, -on_demand_fee_variability : Perbill::from_percent(3), -on_demand_target_queue_utilization : Perbill::from_percent(25), -on_demand_ttl : 5u32.into(), -minimum_backing_votes : LEGACY_MIN_BACKING_VOTES, +on_demand_queue_max_size : pre.on_demand_queue_max_size, +on_demand_base_fee : pre.on_demand_base_fee, +on_demand_fee_variability : pre.on_demand_fee_variability, +on_demand_target_queue_utilization : pre.on_demand_target_queue_utilization, +on_demand_ttl : pre.on_demand_ttl, +minimum_backing_votes : pre.minimum_backing_votes, approval_voting_params : ApprovalVotingParams { max_approval_coalesce_count: 1, } @@ -179,6 +178,8 @@ approval_voting_params : ApprovalVotingParams { #[cfg(test)] mod tests { + use primitives::LEGACY_MIN_BACKING_VOTES; + use super::*; use crate::mock::{new_test_ext, Test}; diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 0a7322e0c058..de66fa0098c1 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -1484,7 +1484,7 @@ pub mod migrations { frame_support::migrations::RemovePallet::DbWeight>, frame_support::migrations::RemovePallet::DbWeight>, frame_support::migrations::RemovePallet::DbWeight>, - parachains_configuration::migration::v10::MigrateToV10, + parachains_configuration::migration::v10::MigrateV9ToV10, ); } diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index db3cf2b14e00..30fbda5d9c4f 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -1549,7 +1549,7 @@ pub mod migrations { pallet_nomination_pools::migration::versioned_migrations::V5toV6, pallet_referenda::migration::v1::MigrateV0ToV1, pallet_nomination_pools::migration::versioned_migrations::V6ToV7, - parachains_configuration::migration::v10::MigrateToV10, + parachains_configuration::migration::v10::MigrateV9ToV10, ); } From 569ebb7ee5a9110d347fb47ca2336468d47de3b0 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 31 Oct 2023 18:38:13 +0200 Subject: [PATCH 059/192] Do not accept ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates until coalescing is not enabled Signed-off-by: Alexandru Gheorghe --- polkadot/runtime/parachains/src/disputes.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/polkadot/runtime/parachains/src/disputes.rs b/polkadot/runtime/parachains/src/disputes.rs index e0ff68f79063..fce5dc24d28d 100644 --- a/polkadot/runtime/parachains/src/disputes.rs +++ b/polkadot/runtime/parachains/src/disputes.rs @@ -952,6 +952,8 @@ impl Pallet { None => return StatementSetFilter::RemoveAll, }; + let config = >::config(); + let n_validators = session_info.validators.len(); // Check for ancient. @@ -1015,6 +1017,7 @@ impl Pallet { set.session, statement, signature, + config.approval_voting_params.max_approval_coalesce_count > 1, ) { log::warn!("Failed to check dispute signature"); @@ -1262,6 +1265,7 @@ fn check_signature( session: SessionIndex, statement: &DisputeStatement, validator_signature: &ValidatorSignature, + approval_multiple_candidates_enabled: bool, ) -> Result<(), ()> { let payload = match statement { DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) => @@ -1281,7 +1285,7 @@ fn check_signature( DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates( candidates, )) => - if candidates.contains(&candidate_hash) { + if approval_multiple_candidates_enabled && candidates.contains(&candidate_hash) { ApprovalVoteMultipleCandidates(candidates).signing_payload(session) } else { return Err(()) From ac2271b43ea2638c69468be3884ead95a7852410 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 1 Nov 2023 10:02:28 +0200 Subject: [PATCH 060/192] Make approval_voting_params session buferred Signed-off-by: Alexandru Gheorghe --- .../src/blockchain_rpc_client.rs | 11 +- .../src/rpc_client.rs | 1 + .../node/core/approval-voting/src/import.rs | 1 - polkadot/node/core/approval-voting/src/lib.rs | 116 ++++++------------ .../node/core/approval-voting/src/tests.rs | 17 ++- polkadot/node/core/runtime-api/src/cache.rs | 34 ++--- polkadot/node/core/runtime-api/src/lib.rs | 13 +- polkadot/node/subsystem-types/src/messages.rs | 5 +- .../subsystem-types/src/runtime_client.rs | 20 ++- 9 files changed, 99 insertions(+), 119 deletions(-) diff --git a/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs b/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs index 43d2adf8171a..cabf9ef73c28 100644 --- a/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs +++ b/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs @@ -367,8 +367,15 @@ impl RuntimeApiSubsystemClient for BlockChainRpcClient { } /// Approval voting configuration parameters - async fn approval_voting_params(&self, at: Hash) -> Result { - Ok(self.rpc_client.parachain_host_staging_approval_voting_params(at).await?) + async fn approval_voting_params( + &self, + at: Hash, + session_index: polkadot_primitives::SessionIndex, + ) -> Result { + Ok(self + .rpc_client + .parachain_host_staging_approval_voting_params(at, session_index) + .await?) } } diff --git a/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs b/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs index 4418f4b53002..e927c37e559a 100644 --- a/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs +++ b/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs @@ -620,6 +620,7 @@ impl RelayChainRpcClient { pub async fn parachain_host_staging_approval_voting_params( &self, at: RelayHash, + _session_index: SessionIndex, ) -> Result { self.call_remote_runtime_function( "ParachainHost_staging_approval_voting_params", diff --git a/polkadot/node/core/approval-voting/src/import.rs b/polkadot/node/core/approval-voting/src/import.rs index ba894c2a30ff..0379b5ad8c15 100644 --- a/polkadot/node/core/approval-voting/src/import.rs +++ b/polkadot/node/core/approval-voting/src/import.rs @@ -645,7 +645,6 @@ pub(crate) mod tests { clock: Box::new(MockClock::default()), assignment_criteria: Box::new(MockAssignmentCriteria), spans: HashMap::new(), - approval_voting_params_cache: None, } } diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index c12ba99fc319..ac32aa957068 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -124,10 +124,6 @@ pub(crate) const LOG_TARGET: &str = "parachain::approval-voting"; // The max number of ticks we delay sending the approval after we are ready to issue the approval const MAX_APPROVAL_COALESCE_WAIT_TICKS: Tick = 12; -// The maximum approval params we cache locally in the subsytem, so that we don't have -// to do the back and forth to the runtime subsystem api. -const APPROVAL_PARAMS_CACHE_SIZE: u32 = 128; - /// Configuration for the approval voting subsystem #[derive(Debug, Clone)] pub struct Config { @@ -163,9 +159,6 @@ pub struct ApprovalVotingSubsystem { db: Arc, mode: Mode, metrics: Metrics, - // Store approval-voting params, so that we don't to always go to RuntimeAPI - // to ask this information which requires a task context switch. - approval_voting_params_cache: Option>, } #[derive(Clone)] @@ -451,24 +444,6 @@ impl ApprovalVotingSubsystem { keystore: Arc, sync_oracle: Box, metrics: Metrics, - ) -> Self { - Self::with_config_and_cache( - config, - db, - keystore, - sync_oracle, - metrics, - Some(LruMap::new(ByLength::new(APPROVAL_PARAMS_CACHE_SIZE))), - ) - } - - pub fn with_config_and_cache( - config: Config, - db: Arc, - keystore: Arc, - sync_oracle: Box, - metrics: Metrics, - approval_voting_params_cache: Option>, ) -> Self { ApprovalVotingSubsystem { keystore, @@ -477,7 +452,6 @@ impl ApprovalVotingSubsystem { db_config: DatabaseConfig { col_approval_data: config.col_approval_data }, mode: Mode::Syncing(sync_oracle), metrics, - approval_voting_params_cache, } } @@ -798,9 +772,6 @@ struct State { clock: Box, assignment_criteria: Box, spans: HashMap, - // Store approval-voting params, so that we don't to always go to RuntimeAPI - // to ask this information which requires a task context switch. - approval_voting_params_cache: Option>, } #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] @@ -866,49 +837,45 @@ impl State { // To avoid crossing the subsystem boundary every-time we are caching locally the values. #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn get_approval_voting_params_or_default( - &mut self, + &self, ctx: &mut Context, + session_index: SessionIndex, block_hash: Hash, ) -> ApprovalVotingParams { - if let Some(params) = self - .approval_voting_params_cache - .as_mut() - .and_then(|cache| cache.get(&block_hash)) - { - *params - } else { - let (s_tx, s_rx) = oneshot::channel(); + let (s_tx, s_rx) = oneshot::channel(); - ctx.send_message(RuntimeApiMessage::Request( - block_hash, - RuntimeApiRequest::ApprovalVotingParams(s_tx), - )) - .await; + ctx.send_message(RuntimeApiMessage::Request( + block_hash, + RuntimeApiRequest::ApprovalVotingParams(session_index, s_tx), + )) + .await; - match s_rx.await { - Ok(Ok(params)) => { - self.approval_voting_params_cache - .as_mut() - .map(|cache| cache.insert(block_hash, params)); - params - }, - Ok(Err(err)) => { - gum::error!( - target: LOG_TARGET, - ?err, - "Could not request approval voting params from runtime using defaults" - ); - ApprovalVotingParams { max_approval_coalesce_count: 1 } - }, - Err(err) => { - gum::error!( - target: LOG_TARGET, - ?err, - "Could not request approval voting params from runtime using defaults" - ); - ApprovalVotingParams { max_approval_coalesce_count: 1 } - }, - } + match s_rx.await { + Ok(Ok(params)) => { + gum::trace!( + target: LOG_TARGET, + approval_voting_params = ?params, + session = ?session_index, + "Using the following subsystem params" + ); + params + }, + Ok(Err(err)) => { + gum::error!( + target: LOG_TARGET, + ?err, + "Could not request approval voting params from runtime using defaults" + ); + ApprovalVotingParams { max_approval_coalesce_count: 1 } + }, + Err(err) => { + gum::error!( + target: LOG_TARGET, + ?err, + "Could not request approval voting params from runtime using defaults" + ); + ApprovalVotingParams { max_approval_coalesce_count: 1 } + }, } } } @@ -960,7 +927,6 @@ where clock, assignment_criteria, spans: HashMap::new(), - approval_voting_params_cache: subsystem.approval_voting_params_cache.take(), }; // `None` on start-up. Gets initialized/updated on leaf update @@ -1056,13 +1022,11 @@ where "Sign approval for multiple candidates", ); - let approval_params = state.get_approval_voting_params_or_default(&mut ctx, block_hash).await; - match maybe_create_signature( &mut overlayed_db, &mut session_info_provider, - approval_params, - &state, &mut ctx, + &state, + &mut ctx, block_hash, validator_index, &subsystem.metrics, @@ -3376,8 +3340,6 @@ async fn issue_approval( "Ready to issue approval vote", ); - let approval_params = state.get_approval_voting_params_or_default(ctx, block_hash).await; - let actions = advance_approval_state( ctx.sender(), state, @@ -3394,7 +3356,6 @@ async fn issue_approval( if let Some(next_wakeup) = maybe_create_signature( db, session_info_provider, - approval_params, state, ctx, block_hash, @@ -3418,7 +3379,6 @@ async fn issue_approval( async fn maybe_create_signature( db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, - approval_params: ApprovalVotingParams, state: &State, ctx: &mut Context, block_hash: Hash, @@ -3438,6 +3398,10 @@ async fn maybe_create_signature( }, }; + let approval_params = state + .get_approval_voting_params_or_default(ctx, block_entry.session(), block_hash) + .await; + gum::trace!( target: LOG_TARGET, "Candidates pending signatures {:}", block_entry.num_candidates_pending_signature() diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 4b78e990e660..a27315d416e3 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -542,7 +542,7 @@ fn test_harness>( let subsystem = run( context, - ApprovalVotingSubsystem::with_config_and_cache( + ApprovalVotingSubsystem::with_config( Config { col_approval_data: test_constants::TEST_CONFIG.col_approval_data, slot_duration_millis: SLOT_DURATION_MILLIS, @@ -551,7 +551,6 @@ fn test_harness>( Arc::new(keystore), sync_oracle, Metrics::default(), - None, ), clock.clone(), assignment_criteria, @@ -2814,7 +2813,7 @@ async fn handle_double_assignment_import( assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { let _ = sender.send(Ok(ApprovalVotingParams { max_approval_coalesce_count: 1, })); @@ -2828,7 +2827,7 @@ async fn handle_double_assignment_import( assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { let _ = sender.send(Ok(ApprovalVotingParams { max_approval_coalesce_count: 1, })); @@ -3751,7 +3750,7 @@ async fn handle_approval_on_max_coalesce_count( assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { let _ = sender.send(Ok(ApprovalVotingParams { max_approval_coalesce_count: 2, })); @@ -3760,7 +3759,7 @@ async fn handle_approval_on_max_coalesce_count( assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { let _ = sender.send(Ok(ApprovalVotingParams { max_approval_coalesce_count: 2, })); @@ -3816,7 +3815,7 @@ async fn handle_approval_on_max_wait_time( // First time we fetch the configuration when we are ready to approve the first candidate assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { let _ = sender.send(Ok(ApprovalVotingParams { max_approval_coalesce_count: MAX_COALESCE_COUNT, })); @@ -3826,7 +3825,7 @@ async fn handle_approval_on_max_wait_time( // Second time we fetch the configuration when we are ready to approve the second candidate assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { let _ = sender.send(Ok(ApprovalVotingParams { max_approval_coalesce_count: MAX_COALESCE_COUNT, })); @@ -3853,7 +3852,7 @@ async fn handle_approval_on_max_wait_time( // approval assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { let _ = sender.send(Ok(ApprovalVotingParams { max_approval_coalesce_count: 3, })); diff --git a/polkadot/node/core/runtime-api/src/cache.rs b/polkadot/node/core/runtime-api/src/cache.rs index 83ebf6884890..0b21da919bf5 100644 --- a/polkadot/node/core/runtime-api/src/cache.rs +++ b/polkadot/node/core/runtime-api/src/cache.rs @@ -67,7 +67,7 @@ pub(crate) struct RequestResultCache { disabled_validators: LruMap>, para_backing_state: LruMap<(Hash, ParaId), Option>, async_backing_params: LruMap, - approval_voting_params: LruMap, + approval_voting_params: LruMap, } impl Default for RequestResultCache { @@ -398,21 +398,6 @@ impl RequestResultCache { self.disputes.insert(relay_parent, value); } - pub(crate) fn approval_voting_params( - &mut self, - relay_parent: &Hash, - ) -> Option<&ApprovalVotingParams> { - self.approval_voting_params.get(relay_parent).map(|v| &*v) - } - - pub(crate) fn cache_approval_voting_params( - &mut self, - relay_parent: Hash, - value: ApprovalVotingParams, - ) { - self.approval_voting_params.insert(relay_parent, value); - } - pub(crate) fn unapplied_slashes( &mut self, relay_parent: &Hash, @@ -507,6 +492,21 @@ impl RequestResultCache { ) { self.async_backing_params.insert(key, value); } + + pub(crate) fn approval_voting_params( + &mut self, + key: (Hash, SessionIndex), + ) -> Option<&ApprovalVotingParams> { + self.approval_voting_params.get(&key.1).map(|v| &*v) + } + + pub(crate) fn cache_approval_voting_params( + &mut self, + session_index: SessionIndex, + value: ApprovalVotingParams, + ) { + self.approval_voting_params.insert(session_index, value); + } } pub(crate) enum RequestResult { @@ -554,7 +554,7 @@ pub(crate) enum RequestResult { slashing::OpaqueKeyOwnershipProof, Option<()>, ), - ApprovalVotingParams(Hash, ApprovalVotingParams), + ApprovalVotingParams(Hash, SessionIndex, ApprovalVotingParams), DisabledValidators(Hash, Vec), ParaBackingState(Hash, ParaId, Option), AsyncBackingParams(Hash, async_backing::AsyncBackingParams), diff --git a/polkadot/node/core/runtime-api/src/lib.rs b/polkadot/node/core/runtime-api/src/lib.rs index bc6dc8722876..36ec8325f8aa 100644 --- a/polkadot/node/core/runtime-api/src/lib.rs +++ b/polkadot/node/core/runtime-api/src/lib.rs @@ -165,8 +165,8 @@ where KeyOwnershipProof(relay_parent, validator_id, key_ownership_proof) => self .requests_cache .cache_key_ownership_proof((relay_parent, validator_id), key_ownership_proof), - RequestResult::ApprovalVotingParams(relay_parent, params) => - self.requests_cache.cache_approval_voting_params(relay_parent, params), + RequestResult::ApprovalVotingParams(_relay_parent, session_index, params) => + self.requests_cache.cache_approval_voting_params(session_index, params), SubmitReportDisputeLost(_, _, _, _) => {}, DisabledValidators(relay_parent, disabled_validators) => self.requests_cache.cache_disabled_validators(relay_parent, disabled_validators), @@ -300,8 +300,9 @@ where Request::SubmitReportDisputeLost(dispute_proof, key_ownership_proof, sender) }, ), - Request::ApprovalVotingParams(sender) => query!(approval_voting_params(), sender) - .map(|sender| Request::ApprovalVotingParams(sender)), + Request::ApprovalVotingParams(session_index, sender) => + query!(approval_voting_params(session_index), sender) + .map(|sender| Request::ApprovalVotingParams(session_index, sender)), Request::DisabledValidators(sender) => query!(disabled_validators(), sender) .map(|sender| Request::DisabledValidators(sender)), Request::ParaBackingState(para, sender) => query!(para_backing_state(para), sender) @@ -561,10 +562,10 @@ where ver = Request::KEY_OWNERSHIP_PROOF_RUNTIME_REQUIREMENT, sender ), - Request::ApprovalVotingParams(sender) => { + Request::ApprovalVotingParams(session_index, sender) => { query!( ApprovalVotingParams, - approval_voting_params(), + approval_voting_params(session_index), ver = Request::APPROVAL_VOTING_PARAMS_REQUIREMENT, sender ) diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index c5be711ea301..6b18e3e48209 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -696,8 +696,6 @@ pub enum RuntimeApiRequest { slashing::OpaqueKeyOwnershipProof, RuntimeApiSender>, ), - /// Approval voting params - ApprovalVotingParams(RuntimeApiSender), /// Get the minimum required backing votes. MinimumBackingVotes(SessionIndex, RuntimeApiSender), /// Returns all disabled validators at a given block height. @@ -708,6 +706,9 @@ pub enum RuntimeApiRequest { /// /// If it's not supported by the Runtime, the async backing is said to be disabled. AsyncBackingParams(RuntimeApiSender), + /// Approval voting params + /// `V9` + ApprovalVotingParams(SessionIndex, RuntimeApiSender), } impl RuntimeApiRequest { diff --git a/polkadot/node/subsystem-types/src/runtime_client.rs b/polkadot/node/subsystem-types/src/runtime_client.rs index e48dc046896d..4bb383f4107d 100644 --- a/polkadot/node/subsystem-types/src/runtime_client.rs +++ b/polkadot/node/subsystem-types/src/runtime_client.rs @@ -262,7 +262,11 @@ pub trait RuntimeApiSubsystemClient { // == v9: Approval voting params == /// Approval voting configuration parameters - async fn approval_voting_params(&self, at: Hash) -> Result; + async fn approval_voting_params( + &self, + at: Hash, + session_index: SessionIndex, + ) -> Result; } /// Default implementation of [`RuntimeApiSubsystemClient`] using the client. @@ -489,11 +493,6 @@ where runtime_api.submit_report_dispute_lost(at, dispute_proof, key_ownership_proof) } - /// Approval voting configuration parameters - async fn approval_voting_params(&self, at: Hash) -> Result { - self.client.runtime_api().approval_voting_params(at) - } - async fn minimum_backing_votes( &self, at: Hash, @@ -520,4 +519,13 @@ where async fn disabled_validators(&self, at: Hash) -> Result, ApiError> { self.client.runtime_api().disabled_validators(at) } + + /// Approval voting configuration parameters + async fn approval_voting_params( + &self, + at: Hash, + _session_index: SessionIndex, + ) -> Result { + self.client.runtime_api().approval_voting_params(at) + } } From 35cbc0ec465d314439bfbc21ab4431960bfad062 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 1 Nov 2023 12:11:31 +0200 Subject: [PATCH 061/192] Make clippy happy Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/runtime-api/src/tests.rs | 6 +- .../runtime/parachains/src/disputes/tests.rs | 126 ++++++++++++------ 2 files changed, 89 insertions(+), 43 deletions(-) diff --git a/polkadot/node/core/runtime-api/src/tests.rs b/polkadot/node/core/runtime-api/src/tests.rs index 5207d03a412d..79bdbf4f0080 100644 --- a/polkadot/node/core/runtime-api/src/tests.rs +++ b/polkadot/node/core/runtime-api/src/tests.rs @@ -243,7 +243,11 @@ impl RuntimeApiSubsystemClient for MockSubsystemClient { } /// Approval voting configuration parameters - async fn approval_voting_params(&self, _: Hash) -> Result { + async fn approval_voting_params( + &self, + _: Hash, + _: SessionIndex, + ) -> Result { todo!("Not required for tests") } diff --git a/polkadot/runtime/parachains/src/disputes/tests.rs b/polkadot/runtime/parachains/src/disputes/tests.rs index 0757084084f6..1f3f00132d68 100644 --- a/polkadot/runtime/parachains/src/disputes/tests.rs +++ b/polkadot/runtime/parachains/src/disputes/tests.rs @@ -1500,7 +1500,8 @@ fn test_check_signature() { candidate_hash, session, &statement_1, - &signed_1 + &signed_1, + true, ) .is_ok()); assert!(check_signature( @@ -1508,7 +1509,8 @@ fn test_check_signature() { candidate_hash, session, &statement_1, - &signed_1 + &signed_1, + true ) .is_err()); assert!(check_signature( @@ -1516,7 +1518,8 @@ fn test_check_signature() { wrong_candidate_hash, session, &statement_1, - &signed_1 + &signed_1, + true, ) .is_err()); assert!(check_signature( @@ -1524,7 +1527,8 @@ fn test_check_signature() { candidate_hash, wrong_session, &statement_1, - &signed_1 + &signed_1, + true ) .is_err()); assert!(check_signature( @@ -1532,7 +1536,8 @@ fn test_check_signature() { candidate_hash, session, &statement_2, - &signed_1 + &signed_1, + true, ) .is_err()); assert!(check_signature( @@ -1540,7 +1545,8 @@ fn test_check_signature() { candidate_hash, session, &statement_3, - &signed_1 + &signed_1, + true ) .is_err()); assert!(check_signature( @@ -1548,7 +1554,8 @@ fn test_check_signature() { candidate_hash, session, &statement_4, - &signed_1 + &signed_1, + true ) .is_err()); assert!(check_signature( @@ -1556,7 +1563,8 @@ fn test_check_signature() { candidate_hash, session, &statement_5, - &signed_1 + &signed_1, + true, ) .is_err()); @@ -1565,7 +1573,8 @@ fn test_check_signature() { candidate_hash, session, &statement_2, - &signed_2 + &signed_2, + true, ) .is_ok()); assert!(check_signature( @@ -1573,7 +1582,8 @@ fn test_check_signature() { candidate_hash, session, &statement_2, - &signed_2 + &signed_2, + true, ) .is_err()); assert!(check_signature( @@ -1581,7 +1591,8 @@ fn test_check_signature() { wrong_candidate_hash, session, &statement_2, - &signed_2 + &signed_2, + true, ) .is_err()); assert!(check_signature( @@ -1589,7 +1600,8 @@ fn test_check_signature() { candidate_hash, wrong_session, &statement_2, - &signed_2 + &signed_2, + true ) .is_err()); assert!(check_signature( @@ -1597,7 +1609,8 @@ fn test_check_signature() { candidate_hash, session, &wrong_statement_2, - &signed_2 + &signed_2, + true, ) .is_err()); assert!(check_signature( @@ -1605,7 +1618,8 @@ fn test_check_signature() { candidate_hash, session, &statement_1, - &signed_2 + &signed_2, + true, ) .is_err()); assert!(check_signature( @@ -1613,7 +1627,8 @@ fn test_check_signature() { candidate_hash, session, &statement_3, - &signed_2 + &signed_2, + true, ) .is_err()); assert!(check_signature( @@ -1621,7 +1636,8 @@ fn test_check_signature() { candidate_hash, session, &statement_4, - &signed_2 + &signed_2, + true, ) .is_err()); assert!(check_signature( @@ -1629,7 +1645,8 @@ fn test_check_signature() { candidate_hash, session, &statement_5, - &signed_2 + &signed_2, + true, ) .is_err()); @@ -1638,7 +1655,8 @@ fn test_check_signature() { candidate_hash, session, &statement_3, - &signed_3 + &signed_3, + true, ) .is_ok()); assert!(check_signature( @@ -1646,7 +1664,8 @@ fn test_check_signature() { candidate_hash, session, &statement_3, - &signed_3 + &signed_3, + true, ) .is_err()); assert!(check_signature( @@ -1654,7 +1673,8 @@ fn test_check_signature() { wrong_candidate_hash, session, &statement_3, - &signed_3 + &signed_3, + true, ) .is_err()); assert!(check_signature( @@ -1662,7 +1682,8 @@ fn test_check_signature() { candidate_hash, wrong_session, &statement_3, - &signed_3 + &signed_3, + true, ) .is_err()); assert!(check_signature( @@ -1670,7 +1691,8 @@ fn test_check_signature() { candidate_hash, session, &wrong_statement_3, - &signed_3 + &signed_3, + true, ) .is_err()); assert!(check_signature( @@ -1678,7 +1700,8 @@ fn test_check_signature() { candidate_hash, session, &statement_1, - &signed_3 + &signed_3, + true, ) .is_err()); assert!(check_signature( @@ -1686,7 +1709,8 @@ fn test_check_signature() { candidate_hash, session, &statement_2, - &signed_3 + &signed_3, + true ) .is_err()); assert!(check_signature( @@ -1694,7 +1718,8 @@ fn test_check_signature() { candidate_hash, session, &statement_4, - &signed_3 + &signed_3, + true, ) .is_err()); assert!(check_signature( @@ -1702,7 +1727,8 @@ fn test_check_signature() { candidate_hash, session, &statement_5, - &signed_3 + &signed_3, + true, ) .is_err()); @@ -1711,7 +1737,8 @@ fn test_check_signature() { candidate_hash, session, &statement_4, - &signed_4 + &signed_4, + true, ) .is_ok()); assert!(check_signature( @@ -1719,7 +1746,8 @@ fn test_check_signature() { candidate_hash, session, &statement_4, - &signed_4 + &signed_4, + true, ) .is_err()); assert!(check_signature( @@ -1727,7 +1755,8 @@ fn test_check_signature() { wrong_candidate_hash, session, &statement_4, - &signed_4 + &signed_4, + true, ) .is_err()); assert!(check_signature( @@ -1735,7 +1764,8 @@ fn test_check_signature() { candidate_hash, wrong_session, &statement_4, - &signed_4 + &signed_4, + true, ) .is_err()); assert!(check_signature( @@ -1743,7 +1773,8 @@ fn test_check_signature() { candidate_hash, session, &statement_1, - &signed_4 + &signed_4, + true, ) .is_err()); assert!(check_signature( @@ -1751,7 +1782,8 @@ fn test_check_signature() { candidate_hash, session, &statement_2, - &signed_4 + &signed_4, + true, ) .is_err()); assert!(check_signature( @@ -1759,7 +1791,8 @@ fn test_check_signature() { candidate_hash, session, &statement_3, - &signed_4 + &signed_4, + true, ) .is_err()); assert!(check_signature( @@ -1767,7 +1800,8 @@ fn test_check_signature() { candidate_hash, session, &statement_5, - &signed_4 + &signed_4, + true, ) .is_err()); @@ -1776,7 +1810,8 @@ fn test_check_signature() { candidate_hash, session, &statement_5, - &signed_5 + &signed_5, + true, ) .is_ok()); assert!(check_signature( @@ -1784,7 +1819,8 @@ fn test_check_signature() { candidate_hash, session, &statement_5, - &signed_5 + &signed_5, + true, ) .is_err()); assert!(check_signature( @@ -1792,7 +1828,8 @@ fn test_check_signature() { wrong_candidate_hash, session, &statement_5, - &signed_5 + &signed_5, + true, ) .is_err()); assert!(check_signature( @@ -1800,7 +1837,8 @@ fn test_check_signature() { candidate_hash, wrong_session, &statement_5, - &signed_5 + &signed_5, + true, ) .is_err()); assert!(check_signature( @@ -1808,7 +1846,8 @@ fn test_check_signature() { candidate_hash, session, &statement_1, - &signed_5 + &signed_5, + true, ) .is_err()); assert!(check_signature( @@ -1816,7 +1855,8 @@ fn test_check_signature() { candidate_hash, session, &statement_2, - &signed_5 + &signed_5, + true, ) .is_err()); assert!(check_signature( @@ -1824,7 +1864,8 @@ fn test_check_signature() { candidate_hash, session, &statement_3, - &signed_5 + &signed_5, + true, ) .is_err()); assert!(check_signature( @@ -1832,7 +1873,8 @@ fn test_check_signature() { candidate_hash, session, &statement_4, - &signed_5 + &signed_5, + true, ) .is_err()); } From 3af951747d1a570ea9654fb513a5c7bf6e256a9a Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 3 Nov 2023 10:47:58 +0200 Subject: [PATCH 062/192] Address review feedback in relay_vrf_modulo_cores Signed-off-by: Alexandru Gheorghe --- .../node/core/approval-voting/src/criteria.rs | 81 ++++++++++++------- 1 file changed, 53 insertions(+), 28 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 01fff31a4be0..b5f4ad5c6194 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -27,8 +27,8 @@ use polkadot_primitives::{ AssignmentId, AssignmentPair, CandidateHash, CoreIndex, GroupIndex, IndexedVec, SessionInfo, ValidatorIndex, }; -use rand::Rng; -use rand_chacha::{rand_core::SeedableRng, ChaCha20Rng}; +use rand::{seq::SliceRandom, SeedableRng}; +use rand_chacha::ChaCha20Rng; use sc_keystore::LocalKeystore; use sp_application_crypto::ByteArray; @@ -36,8 +36,8 @@ use merlin::Transcript; use schnorrkel::vrf::VRFInOut; use std::{ - cmp::min, - collections::{hash_map::Entry, HashMap, HashSet}, + cmp::{max, min}, + collections::{hash_map::Entry, HashMap}, }; use super::LOG_TARGET; @@ -130,10 +130,6 @@ fn relay_vrf_modulo_transcript_v2(relay_vrf_story: RelayVRFStory) -> Transcript /// A hard upper bound on num_cores * target_checkers / num_validators const MAX_MODULO_SAMPLES: usize = 40; -/// The maximum number of samples we take to try to obtain an -/// a distinct core_index. -const MAX_SAMPLING_ITERATIONS: usize = 20; - use std::convert::AsMut; struct BigArray(pub [u8; MAX_MODULO_SAMPLES * 4]); @@ -165,7 +161,7 @@ fn relay_vrf_modulo_cores( n_cores = max_cores, num_samples, max_modulo_samples = MAX_MODULO_SAMPLES, - "`num_samples` is greater than `MAX_MODULO_SAMPLES`", + "`num_samples` is greater than `MAX_MODU`LO_SAMPLES`", ); } @@ -178,29 +174,30 @@ fn relay_vrf_modulo_cores( "Suboptimal configuration `num_samples` should be less than `n_cores` / 2", ); } - - let mut rand_chacha = + let rand_chacha = ChaCha20Rng::from_seed(vrf_in_out.make_bytes::<::Seed>( approval_types::v2::CORE_RANDOMNESS_CONTEXT, )); + generate_samples(rand_chacha, num_samples as usize, max_cores as usize) +} - let mut res = HashSet::with_capacity(num_samples as usize); - while res.len() < min(num_samples, max_cores) as usize { - // Production networks should run with num_samples no larger than `n_cores/2`, so on worst - // case on each iteration this loop has >1/2 chance of finishing. - // With our current production parameters the chances for each iteration not hitting - // a duplicate are around 90%. - // See: https://github.com/paritytech/polkadot-sdk/pull/1178#discussion_r1376501206 - // for more details. - for _ in 0..MAX_SAMPLING_ITERATIONS { - let random_core = rand_chacha.gen_range(0..max_cores).into(); - if !res.contains(&random_core) { - res.insert(random_core); - break - } - } - } - res.into_iter().collect_vec() +/// Generates `num_sumples` randomly from (0..max_cores) range +/// +/// Note! The algorithm can't change because validators on the other +/// side won't be able to check the assignments until they update. +/// This invariant is tested with `generate_samples_invariant`, so the +/// tests will catch any subtle changes in the implementation of this function +/// and its dependencies. +fn generate_samples( + mut rand_chacha: ChaCha20Rng, + num_samples: usize, + max_cores: usize, +) -> Vec { + let num_samples = max(1, min(num_samples, max_cores / 2)); + + let mut random_cores = (0..max_cores as u32).map(|val| val.into()).collect::>(); + let (samples, _) = random_cores.partial_shuffle(&mut rand_chacha, num_samples as usize); + samples.into_iter().map(|val| *val).collect_vec() } fn relay_vrf_modulo_core(vrf_in_out: &VRFInOut, n_cores: u32) -> CoreIndex { @@ -1209,4 +1206,32 @@ mod tests { } }); } + + #[test] + fn generate_samples_invariant() { + let seed = [ + 1, 0, 52, 0, 0, 0, 0, 0, 1, 0, 10, 0, 22, 32, 0, 0, 2, 0, 55, 49, 0, 11, 0, 0, 3, 0, 0, + 0, 0, 0, 2, 92, + ]; + let rand_chacha = ChaCha20Rng::from_seed(seed); + + let samples = generate_samples(rand_chacha.clone(), 6, 100); + let expected = vec![19, 79, 17, 75, 66, 30].into_iter().map(Into::into).collect_vec(); + assert_eq!(samples, expected); + + let samples = generate_samples(rand_chacha.clone(), 6, 7); + let expected = vec![5, 4, 2].into_iter().map(Into::into).collect_vec(); + assert_eq!(samples, expected); + + let samples = generate_samples(rand_chacha.clone(), 6, 12); + let expected = vec![2, 4, 7, 5, 11, 3].into_iter().map(Into::into).collect_vec(); + assert_eq!(samples, expected); + + let samples = generate_samples(rand_chacha.clone(), 1, 100); + let expected = vec![30].into_iter().map(Into::into).collect_vec(); + assert_eq!(samples, expected); + + let samples = generate_samples(rand_chacha, 0, 100); + assert_eq!(samples, expected); + } } From 1b84bb0d6e36aa453442156614186efc4e9191e0 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 3 Nov 2023 15:31:31 +0200 Subject: [PATCH 063/192] Address more review feedback Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 11 +++++------ .../node/network/approval-distribution/src/lib.rs | 11 +++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index ac32aa957068..f6d843a908e7 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -833,8 +833,7 @@ impl State { } } - // Returns the approval voting from the RuntimeApi. - // To avoid crossing the subsystem boundary every-time we are caching locally the values. + // Returns the approval voting params from the RuntimeApi. #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn get_approval_voting_params_or_default( &self, @@ -861,7 +860,7 @@ impl State { params }, Ok(Err(err)) => { - gum::error!( + gum::debug!( target: LOG_TARGET, ?err, "Could not request approval voting params from runtime using defaults" @@ -869,7 +868,7 @@ impl State { ApprovalVotingParams { max_approval_coalesce_count: 1 } }, Err(err) => { - gum::error!( + gum::debug!( target: LOG_TARGET, ?err, "Could not request approval voting params from runtime using defaults" @@ -3586,8 +3585,8 @@ fn compute_delayed_approval_sending_tick( let sign_no_later_than = min( tick_now + MAX_APPROVAL_COALESCE_WAIT_TICKS as Tick, - // We don't want to accidentally cause, no-shows so if we are past - // the seconnd half of the no show time, force the sending of the + // We don't want to accidentally cause no-shows, so if we are past + // the second half of the no show time, force the sending of the // approval immediately. assignment_triggered_tick + no_show_duration_ticks / 2, ); diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 91e850ad5b31..f30170b0f116 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -415,9 +415,8 @@ impl Knowledge { // Tells if all keys are contained by this peer_knowledge pub fn contains_all_keys(&self, keys: &Vec<(MessageSubject, MessageKind)>) -> bool { - keys.iter().fold(true, |accumulator, assignment_key| { - accumulator && self.contains(&assignment_key.0, assignment_key.1) - }) + keys.iter() + .all(|assignment_key| self.contains(&assignment_key.0, assignment_key.1)) } } @@ -539,9 +538,9 @@ impl BlockEntry { // Tels if all candidate_indices are valid candidates pub fn contains_candidates(&self, candidate_indices: &CandidateBitfield) -> bool { - candidate_indices.iter_ones().fold(true, |accumulator, candidate_index| { - self.candidates.get(candidate_index as usize).is_some() && accumulator - }) + candidate_indices + .iter_ones() + .all(|candidate_index| self.candidates.get(candidate_index as usize).is_some()) } // Saves the given approval in all ApprovalEntries that contain an assignment for any of the // candidates in the approval. From 7a1c88c3ca3b29ed2c0cfcefc7689e5d48cc3a42 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 3 Nov 2023 16:40:58 +0200 Subject: [PATCH 064/192] Add few more tests Signed-off-by: Alexandru Gheorghe --- .../node/core/approval-voting/src/criteria.rs | 56 +++++++++++-------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index b5f4ad5c6194..5e7718edb246 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -36,7 +36,7 @@ use merlin::Transcript; use schnorrkel::vrf::VRFInOut; use std::{ - cmp::{max, min}, + cmp::min, collections::{hash_map::Entry, HashMap}, }; @@ -155,25 +155,6 @@ fn relay_vrf_modulo_cores( // Configuration - `n_cores`. max_cores: u32, ) -> Vec { - if num_samples as usize > MAX_MODULO_SAMPLES { - gum::warn!( - target: LOG_TARGET, - n_cores = max_cores, - num_samples, - max_modulo_samples = MAX_MODULO_SAMPLES, - "`num_samples` is greater than `MAX_MODU`LO_SAMPLES`", - ); - } - - if 2 * num_samples > max_cores { - gum::error!( - target: LOG_TARGET, - n_cores = max_cores, - num_samples, - max_modulo_samples = MAX_MODULO_SAMPLES, - "Suboptimal configuration `num_samples` should be less than `n_cores` / 2", - ); - } let rand_chacha = ChaCha20Rng::from_seed(vrf_in_out.make_bytes::<::Seed>( approval_types::v2::CORE_RANDOMNESS_CONTEXT, @@ -193,7 +174,27 @@ fn generate_samples( num_samples: usize, max_cores: usize, ) -> Vec { - let num_samples = max(1, min(num_samples, max_cores / 2)); + if num_samples as usize > MAX_MODULO_SAMPLES { + gum::warn!( + target: LOG_TARGET, + n_cores = max_cores, + num_samples, + max_modulo_samples = MAX_MODULO_SAMPLES, + "`num_samples` is greater than `MAX_MODULO_SAMPLES`", + ); + } + + if 2 * num_samples > max_cores { + gum::error!( + target: LOG_TARGET, + n_cores = max_cores, + num_samples, + max_modulo_samples = MAX_MODULO_SAMPLES, + "Suboptimal configuration `num_samples` should be less than `n_cores` / 2", + ); + } + + let num_samples = min(MAX_MODULO_SAMPLES, min(num_samples, max_cores / 2)); let mut random_cores = (0..max_cores as u32).map(|val| val.into()).collect::>(); let (samples, _) = random_cores.partial_shuffle(&mut rand_chacha, num_samples as usize); @@ -1231,7 +1232,18 @@ mod tests { let expected = vec![30].into_iter().map(Into::into).collect_vec(); assert_eq!(samples, expected); - let samples = generate_samples(rand_chacha, 0, 100); + let samples = generate_samples(rand_chacha.clone(), 0, 100); + let expected = vec![]; + assert_eq!(samples, expected); + + let samples = generate_samples(rand_chacha, MAX_MODULO_SAMPLES + 1, 100); + let expected = vec![ + 42, 54, 55, 93, 64, 27, 49, 15, 83, 71, 62, 1, 43, 77, 97, 41, 7, 69, 0, 88, 59, 14, + 82, 19, 79, 17, 75, 66, 30, + ] + .into_iter() + .map(Into::into) + .collect_vec(); assert_eq!(samples, expected); } } From ec712151a6168aeebd6f40736cde3539353237a0 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Sun, 5 Nov 2023 13:23:22 +0200 Subject: [PATCH 065/192] Fix unittest Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/criteria.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 5e7718edb246..ae0127f6ff23 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -1239,7 +1239,7 @@ mod tests { let samples = generate_samples(rand_chacha, MAX_MODULO_SAMPLES + 1, 100); let expected = vec![ 42, 54, 55, 93, 64, 27, 49, 15, 83, 71, 62, 1, 43, 77, 97, 41, 7, 69, 0, 88, 59, 14, - 82, 19, 79, 17, 75, 66, 30, + 23, 87, 47, 4, 51, 12, 74, 56, 50, 44, 9, 82, 19, 79, 17, 75, 66, 30, ] .into_iter() .map(Into::into) From 552e6fabd18e85d7341cb8b7f8021b8213f94bda Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Sun, 5 Nov 2023 14:15:46 +0200 Subject: [PATCH 066/192] Add pr_doc Signed-off-by: Alexandru Gheorghe --- prdoc/pr_1178.prdoc | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 prdoc/pr_1178.prdoc diff --git a/prdoc/pr_1178.prdoc b/prdoc/pr_1178.prdoc new file mode 100644 index 000000000000..36c3b05c7a3f --- /dev/null +++ b/prdoc/pr_1178.prdoc @@ -0,0 +1,23 @@ +title: tranche0 assignments in one certificate part1 + +doc: + - audience: Node Operator + description: | + Changed approval-voting, approval-distribution to send all messages tranche0 assignments in one message. + This required: + * A new parachains_db version. + * A new validation protocol to support the new message types. + The new logic will be disabled and will be enabled at a later date after all validators have upgraded. + +migrations: + db: + - name: Parachains database change from v3 to v4. + description: | + Approval-voting column format has been updated with several new fields. All existing data will be automatically + be migrated to the new values. + +crates: + - name: "polkadot" + semver: patch + +host_functions: [] From 2625deffb92cd708d834c9cca6cfa5377efa7830 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 19 Oct 2023 21:47:51 +0300 Subject: [PATCH 067/192] Build a test image with `network-protocol-staging` Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 5c13045706c4..7e673e3fb652 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -18,7 +18,7 @@ build-linux-stable: # Ensure we run the UI tests. RUN_UI_TESTS: 1 script: - - time cargo build --locked --profile testnet --features pyroscope,fast-runtime --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker + - time cargo build --locked --profile testnet --features pyroscope,network-protocol-staging --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker - time ROCOCO_EPOCH_DURATION=10 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-10/ - time ROCOCO_EPOCH_DURATION=100 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-100/ - time ROCOCO_EPOCH_DURATION=600 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-600/ From 43f5529473147285264a656487adf5181dad977b Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 20 Oct 2023 09:54:51 +0300 Subject: [PATCH 068/192] Build a test image with assignments enabled Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/criteria.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index ae0127f6ff23..d398f361eb6b 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -299,7 +299,7 @@ impl AssignmentCriteria for RealAssignmentCriteria { config: &Config, leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>, ) -> HashMap { - compute_assignments(keystore, relay_vrf_story, config, leaving_cores, false) + compute_assignments(keystore, relay_vrf_story, config, leaving_cores, true) } fn check_assignment_cert( From 590b59e7b5f755394e990e892abbfdd1a53be420 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Sun, 5 Nov 2023 15:47:49 +0200 Subject: [PATCH 069/192] Cleanup un-needed structures Signed-off-by: Alexandru Gheorghe --- .../node/core/approval-voting/src/criteria.rs | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index d398f361eb6b..194a7c1781e1 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -130,22 +130,6 @@ fn relay_vrf_modulo_transcript_v2(relay_vrf_story: RelayVRFStory) -> Transcript /// A hard upper bound on num_cores * target_checkers / num_validators const MAX_MODULO_SAMPLES: usize = 40; -use std::convert::AsMut; - -struct BigArray(pub [u8; MAX_MODULO_SAMPLES * 4]); - -impl Default for BigArray { - fn default() -> Self { - BigArray([0u8; MAX_MODULO_SAMPLES * 4]) - } -} - -impl AsMut<[u8]> for BigArray { - fn as_mut(&mut self) -> &mut [u8] { - self.0.as_mut() - } -} - /// Takes the VRF output as input and returns a Vec of cores the validator is assigned /// to as a tranche0 checker. fn relay_vrf_modulo_cores( @@ -194,7 +178,7 @@ fn generate_samples( ); } - let num_samples = min(MAX_MODULO_SAMPLES, min(num_samples, max_cores / 2)); + let num_samples = min(MAX_MODULO_SAMPLES, min(num_samples, max_cores)); let mut random_cores = (0..max_cores as u32).map(|val| val.into()).collect::>(); let (samples, _) = random_cores.partial_shuffle(&mut rand_chacha, num_samples as usize); @@ -1221,7 +1205,7 @@ mod tests { assert_eq!(samples, expected); let samples = generate_samples(rand_chacha.clone(), 6, 7); - let expected = vec![5, 4, 2].into_iter().map(Into::into).collect_vec(); + let expected = vec![0, 3, 6, 5, 4, 2].into_iter().map(Into::into).collect_vec(); assert_eq!(samples, expected); let samples = generate_samples(rand_chacha.clone(), 6, 12); From bd76cf64582acffdd6b9be266be3d8ffcd9e4149 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Sun, 5 Nov 2023 16:28:27 +0200 Subject: [PATCH 070/192] Revert test configurations This reverts commit 43f5529473147285264a656487adf5181dad977b. --- .gitlab/pipeline/build.yml | 2 +- polkadot/node/core/approval-voting/src/criteria.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 7e673e3fb652..5c13045706c4 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -18,7 +18,7 @@ build-linux-stable: # Ensure we run the UI tests. RUN_UI_TESTS: 1 script: - - time cargo build --locked --profile testnet --features pyroscope,network-protocol-staging --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker + - time cargo build --locked --profile testnet --features pyroscope,fast-runtime --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker - time ROCOCO_EPOCH_DURATION=10 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-10/ - time ROCOCO_EPOCH_DURATION=100 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-100/ - time ROCOCO_EPOCH_DURATION=600 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-600/ diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 194a7c1781e1..1870f2a6158d 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -283,7 +283,7 @@ impl AssignmentCriteria for RealAssignmentCriteria { config: &Config, leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>, ) -> HashMap { - compute_assignments(keystore, relay_vrf_story, config, leaving_cores, true) + compute_assignments(keystore, relay_vrf_story, config, leaving_cores, false) } fn check_assignment_cert( From 20cb37a8443b29582df994943b90bd0292db11b1 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 6 Nov 2023 09:07:58 +0200 Subject: [PATCH 071/192] Fix upgrade to latest zombienet version Signed-off-by: Alexandru Gheorghe --- .../zombienet_tests/functional/0006-parachains-max-tranche0.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml b/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml index 0b75b1a48743..68ac90f5a7e1 100644 --- a/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml +++ b/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml @@ -10,7 +10,6 @@ bootnode = true [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" -chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default-bootnode" default_command = "polkadot" [relaychain.default_resources] From 311b5732932d982cd49ef8c3f06a8c5555b7941f Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 6 Nov 2023 10:19:21 +0200 Subject: [PATCH 072/192] Set correct log level Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/criteria.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 1870f2a6158d..2bb5a151fe23 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -169,7 +169,7 @@ fn generate_samples( } if 2 * num_samples > max_cores { - gum::error!( + gum::debug!( target: LOG_TARGET, n_cores = max_cores, num_samples, From 46f7f4d60a8e892ed37285e68bd1d252378ba0c8 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 6 Nov 2023 18:38:24 +0200 Subject: [PATCH 073/192] Make zombient behave with latest cli Signed-off-by: Alexandru Gheorghe --- .../functional/0007-approval-voting-coalescing.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml index 872bf025de97..20dbb3577849 100644 --- a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml +++ b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml @@ -4,7 +4,6 @@ timeout = 1000 [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" -chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default-bootnode" [relaychain.genesis.runtime.configuration.config] needed_approvals = 5 From a2e602444662bd0e2443219b438d609da62a995d Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 6 Nov 2023 18:44:24 +0200 Subject: [PATCH 074/192] Fix ocasional no-shows in zombienets ... Because we coalesce approvals they might cover more than one assignments so in the case of the assignments sent randomly we need to make sure we send them all to the peer before we can send the approval Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 274 ++++++++++++++---- 1 file changed, 218 insertions(+), 56 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index f30170b0f116..ed7708b7f0cd 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -418,6 +418,12 @@ impl Knowledge { keys.iter() .all(|assignment_key| self.contains(&assignment_key.0, assignment_key.1)) } + + // Tells if all keys are contained by this peer_knowledge + pub fn contains_any(&self, keys: &Vec<(MessageSubject, MessageKind)>) -> bool { + keys.iter() + .any(|assignment_key| self.contains(&assignment_key.0, assignment_key.1)) + } } /// Information that has been circulated to and from a peer. @@ -434,6 +440,62 @@ impl PeerKnowledge { self.sent.contains(message, kind) || self.received.contains(message, kind) } + /// Partition a list of assignments into two lists. + /// + /// The the first one contains the list of assignments that we had + /// sent to the peer and the second one the list of assignments that + /// we have no knowledge if the peer has it or not. + fn partition_sent_notknown<'a>( + &self, + assignments: &'a Vec<(IndirectAssignmentCertV2, CandidateBitfield)>, + ) -> ( + Vec<&'a (IndirectAssignmentCertV2, CandidateBitfield)>, + Vec<&'a (IndirectAssignmentCertV2, CandidateBitfield)>, + ) { + let (sent, not_sent): ( + Vec<&'a (IndirectAssignmentCertV2, CandidateBitfield)>, + Vec<&'a (IndirectAssignmentCertV2, CandidateBitfield)>, + ) = assignments.iter().partition(|assignment| { + self.sent.contains( + &MessageSubject( + assignment.0.block_hash, + assignment.1.clone(), + assignment.0.validator, + ), + MessageKind::Assignment, + ) + }); + + let notknown_by_peer = not_sent + .into_iter() + .filter(|(assignment, candidate_bitfield)| { + !self.contains( + &MessageSubject( + assignment.block_hash, + candidate_bitfield.clone(), + assignment.validator, + ), + MessageKind::Assignment, + ) + }) + .collect_vec(); + + (sent, notknown_by_peer) + } + + fn mark_sent(&mut self, assignments: &Vec<(IndirectAssignmentCertV2, CandidateBitfield)>) { + for assignment in assignments { + self.sent.insert( + MessageSubject( + assignment.0.block_hash, + assignment.1.clone(), + assignment.0.validator, + ), + MessageKind::Assignment, + ); + } + } + // Tells if all assignments for a given approval are included in the knowledge of the peer fn contains_assignments_for_approval(&self, approval: &IndirectSignedApprovalVoteV2) -> bool { self.contains_all_keys(&Self::generate_assignments_keys(&approval)) @@ -542,6 +604,40 @@ impl BlockEntry { .iter_ones() .all(|candidate_index| self.candidates.get(candidate_index as usize).is_some()) } + + // Returns all assignments covering the candidates in a given `approval` + pub fn assignments_for_approval( + &self, + approval: &IndirectSignedApprovalVoteV2, + ) -> Result, ApprovalEntryError> { + let mut assignments = Vec::new(); + + if self.candidates.len() < approval.candidate_indices.len() as usize { + return Err(ApprovalEntryError::CandidateIndexOutOfBounds) + } + + // First determine all assignments bitfields that might be covered by this approval + let covered_assignments_bitfields: HashSet = approval + .candidate_indices + .iter_ones() + .filter_map(|candidate_index| { + self.candidates.get(candidate_index).map_or(None, |candidate_entry| { + candidate_entry.assignments.get(&approval.validator).cloned() + }) + }) + .collect(); + + for assignment_bitfield in covered_assignments_bitfields { + if let Some(approval_entry) = + self.approval_entries.get(&(approval.validator, assignment_bitfield.clone())) + { + assignments.push((approval_entry.assignment.clone(), assignment_bitfield.clone())) + } + } + + Ok(assignments) + } + // Saves the given approval in all ApprovalEntries that contain an assignment for any of the // candidates in the approval. // @@ -1425,6 +1521,7 @@ impl State { if !topology.map(|topology| topology.is_validator(&peer)).unwrap_or(false) { continue } + // Note: at this point, we haven't received the message from any peers // other than the source peer, and we just got it, so we haven't sent it // to any peers either. @@ -1715,6 +1812,8 @@ impl State { }, }; + let assignments = entry.assignments_for_approval(&vote).unwrap_or_default(); + // Invariant: to our knowledge, none of the peers except for the `source` know about the // approval. metrics.on_approval_imported(); @@ -1740,7 +1839,7 @@ impl State { // 3. Any randomly selected peers have been sent the assignment already. let in_topology = topology .map_or(false, |t| t.local_grid_neighbors().route_to_peer(required_routing, peer)); - in_topology || knowledge.sent.contains_all_keys(&assignments_knowledge_keys) + in_topology || knowledge.sent.contains_any(&assignments_knowledge_keys) }; let peers = entry @@ -1754,6 +1853,27 @@ impl State { for peer in peers.iter() { // we already filtered peers above, so this should always be Some if let Some(entry) = entry.known_by.get_mut(&peer.0) { + // A random assignment could have been sent to the peer, so we need to make sure + // we send all the other assignments, before we can send the corresponding approval. + let (sent_to_peer, notknown_by_peer) = entry.partition_sent_notknown(&assignments); + if !notknown_by_peer.is_empty() { + let notknown_by_peer = notknown_by_peer + .into_iter() + .map(|(assignment, bitfield)| (assignment.clone(), bitfield.clone())) + .collect_vec(); + gum::trace!( + target: LOG_TARGET, + ?block_hash, + ?peer, + missing = notknown_by_peer.len(), + part1 = sent_to_peer.len(), + "Sending missing assignments", + ); + + entry.mark_sent(¬known_by_peer); + send_assignments_batched(ctx.sender(), notknown_by_peer, &[(peer.0, peer.1)]).await; + } + entry.sent.insert(approval_knwowledge_key.0.clone(), approval_knwowledge_key.1); } } @@ -1851,75 +1971,99 @@ impl State { if entry.known_by.contains_key(&peer_id) { break } - - let peer_knowledge = entry.known_by.entry(peer_id).or_default(); - let topology = topologies.get_topology(entry.session); - - let mut approvals_to_send_this_block = HashMap::new(); - // We want to iterate the `approval_entries` of the block entry as these contain all - // assignments that also link all approval votes. - for approval_entry in entry.approval_entries.values_mut() { - // Propagate the message to all peers in the required routing set OR - // randomly sample peers. - { - let required_routing = approval_entry.routing_info().required_routing; - let random_routing = &mut approval_entry.routing_info_mut().random_routing; - let rng = &mut *rng; - let mut peer_filter = move |peer_id| { - let in_topology = topology.as_ref().map_or(false, |t| { - t.local_grid_neighbors().route_to_peer(required_routing, peer_id) - }); - in_topology || { - if !topology - .map(|topology| topology.is_validator(peer_id)) - .unwrap_or(false) - { - return false - } - - let route_random = random_routing.sample(total_peers, rng); - if route_random { - random_routing.inc_sent(); + let approvals_to_send_this_block = { + let peer_knowledge = entry.known_by.entry(peer_id).or_default(); + let topology = topologies.get_topology(entry.session); + + let mut approvals_to_send_this_block = HashMap::new(); + // We want to iterate the `approval_entries` of the block entry as these contain + // all assignments that also link all approval votes. + for approval_entry in entry.approval_entries.values_mut() { + // Propagate the message to all peers in the required routing set OR + // randomly sample peers. + { + let required_routing = approval_entry.routing_info().required_routing; + let random_routing = + &mut approval_entry.routing_info_mut().random_routing; + let rng = &mut *rng; + let mut peer_filter = move |peer_id| { + let in_topology = topology.as_ref().map_or(false, |t| { + t.local_grid_neighbors() + .route_to_peer(required_routing, peer_id) + }); + in_topology || { + if !topology + .map(|topology| topology.is_validator(peer_id)) + .unwrap_or(false) + { + return false + } + + let route_random = random_routing.sample(total_peers, rng); + if route_random { + random_routing.inc_sent(); + } + + route_random } + }; - route_random + if !peer_filter(&peer_id) { + continue } - }; - - if !peer_filter(&peer_id) { - continue } - } - let assignment_message = approval_entry.assignment(); - let approval_messages = approval_entry.approvals(); - let (assignment_knowledge, message_kind) = - approval_entry.create_assignment_knowledge(block); + let assignment_message = approval_entry.assignment(); + let approval_messages = approval_entry.approvals(); + let (assignment_knowledge, message_kind) = + approval_entry.create_assignment_knowledge(block); - // Only send stuff a peer doesn't know in the context of a relay chain block. - if !peer_knowledge.contains(&assignment_knowledge, message_kind) { - peer_knowledge.sent.insert(assignment_knowledge, message_kind); - assignments_to_send.push(assignment_message); - } + // Only send stuff a peer doesn't know in the context of a relay chain + // block. + if !peer_knowledge.contains(&assignment_knowledge, message_kind) { + peer_knowledge.sent.insert(assignment_knowledge, message_kind); + assignments_to_send.push(assignment_message); + } - // Filter approval votes. - for approval_message in approval_messages { - let approval_knowledge = - PeerKnowledge::generate_approval_key(&approval_message); + // Filter approval votes. + for approval_message in approval_messages { + let approval_knowledge = + PeerKnowledge::generate_approval_key(&approval_message); - if !peer_knowledge.contains(&approval_knowledge.0, approval_knowledge.1) { - if !approvals_to_send_this_block.contains_key(&approval_knowledge.0) { - approvals_to_send_this_block - .insert(approval_knowledge.0.clone(), approval_message); + if !peer_knowledge.contains(&approval_knowledge.0, approval_knowledge.1) + { + if !approvals_to_send_this_block.contains_key(&approval_knowledge.0) + { + approvals_to_send_this_block + .insert(approval_knowledge.0.clone(), approval_message); + } } } } - } + approvals_to_send_this_block + }; // An approval can span multiple assignments/ApprovalEntries, so after we processed // all of the assignments decide which of the approvals we can safely send, because // all of assignments are already sent or about to be sent. for (approval_key, approval) in approvals_to_send_this_block { + let assignments = entry.assignments_for_approval(&approval).unwrap_or_default(); + let peer_knowledge = entry.known_by.entry(peer_id).or_default(); + let (sent_to_peer, notknown_by_peer) = peer_knowledge.partition_sent_notknown(&assignments); + if !sent_to_peer.is_empty() { + let notknown_by_peer = notknown_by_peer + .into_iter() + .map(|(assignment, bitfield)| (assignment.clone(), bitfield.clone())) + .collect_vec(); + gum::trace!( + target: LOG_TARGET, + ?notknown_by_peer, + part1 = sent_to_peer.len(), + "Sending missing assignment unify_with_peer", + ); + peer_knowledge.mark_sent(¬known_by_peer); + assignments_to_send.extend(notknown_by_peer); + } if peer_knowledge.contains_assignments_for_approval(&approval) { approvals_to_send.push(approval); peer_knowledge.sent.insert(approval_key, MessageKind::Approval); @@ -2243,8 +2387,26 @@ async fn adjust_required_routing_and_propagate Date: Mon, 6 Nov 2023 19:15:38 +0200 Subject: [PATCH 075/192] wip Signed-off-by: Andrei Sandu --- Cargo.lock | 18 +- .../network/availability-recovery/src/lib.rs | 13 +- polkadot/node/subsystem-bench/Cargo.toml | 21 +- .../node/subsystem-bench/src/availability.rs | 501 ++++++++++++++++++ .../subsystem-bench/src/subsystem-bench.rs | 133 +++++ 5 files changed, 674 insertions(+), 12 deletions(-) create mode 100644 polkadot/node/subsystem-bench/src/availability.rs create mode 100644 polkadot/node/subsystem-bench/src/subsystem-bench.rs diff --git a/Cargo.lock b/Cargo.lock index e846ae53a543..d113fd7e43cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12998,20 +12998,32 @@ dependencies = [ "async-trait", "clap 4.4.6", "color-eyre", + "env_logger 0.9.3", "futures", "futures-timer", + "log", + "parity-scale-codec", + "polkadot-availability-recovery", "polkadot-erasure-coding", - "polkadot-node-core-backing", + "polkadot-node-metrics", + "polkadot-node-network-protocol", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-types", "polkadot-node-subsystem-util", "polkadot-primitives", + "polkadot-primitives-test-helpers", + "prometheus", "rand 0.8.5", + "sc-network", + "sc-service", + "sp-application-crypto", "sp-core", + "sp-keyring", "sp-keystore", "substrate-build-script-utils", + "tokio", "tracing-gum", ] @@ -18675,9 +18687,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.32.0" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ "backtrace", "bytes", diff --git a/polkadot/node/network/availability-recovery/src/lib.rs b/polkadot/node/network/availability-recovery/src/lib.rs index e2146981da92..156a8cbbc82e 100644 --- a/polkadot/node/network/availability-recovery/src/lib.rs +++ b/polkadot/node/network/availability-recovery/src/lib.rs @@ -65,7 +65,7 @@ mod error; mod futures_undead; mod metrics; mod task; -use metrics::Metrics; +pub use metrics::Metrics; #[cfg(test)] mod tests; @@ -582,7 +582,7 @@ impl AvailabilityRecoverySubsystem { } } - async fn run(self, mut ctx: Context) -> SubsystemResult<()> { + pub async fn run(self, mut ctx: Context) -> SubsystemResult<()> { let mut state = State::default(); let Self { mut req_receiver, metrics, recovery_strategy_kind, bypass_availability_store } = self; @@ -617,9 +617,12 @@ impl AvailabilityRecoverySubsystem { .into_iter() .cycle(); + gum::debug!("Subsystem running"); loop { let recv_req = req_receiver.recv(|| vec![COST_INVALID_REQUEST]).fuse(); pin_mut!(recv_req); + gum::debug!("waiting for message"); + futures::select! { erasure_task = erasure_task_rx.next() => { match erasure_task { @@ -640,7 +643,7 @@ impl AvailabilityRecoverySubsystem { } }, None => { - gum::debug!( + gum::trace!( target: LOG_TARGET, "Erasure task channel closed", ); @@ -655,6 +658,7 @@ impl AvailabilityRecoverySubsystem { &mut state, signal, ).await? { + gum::debug!(target: LOG_TARGET, "subsystem concluded"); return Ok(()); } FromOrchestra::Communication { msg } => { @@ -818,10 +822,11 @@ async fn erasure_task_thread( let _ = sender.send(maybe_data); }, None => { - gum::debug!( + gum::trace!( target: LOG_TARGET, "Erasure task channel closed. Node shutting down ?", ); + break }, } } diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index b2cc88ff057d..729749ab153b 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -19,9 +19,10 @@ doc = false polkadot-node-subsystem = { path = "../subsystem" } polkadot-node-subsystem-util = { path = "../subsystem-util" } polkadot-node-subsystem-types = { path = "../subsystem-types" } -polkadot-node-core-backing = { path = "../core/backing" } polkadot-node-primitives = { path = "../primitives" } polkadot-primitives = { path = "../../primitives" } +polkadot-node-network-protocol = { path = "../network/protocol" } +polkadot-availability-recovery = { path = "../network/availability-recovery" } color-eyre = { version = "0.6.1", default-features = false } assert_matches = "1.5" async-trait = "0.1.57" @@ -31,13 +32,23 @@ clap = { version = "4.4.6", features = ["derive"] } futures = "0.3.21" futures-timer = "3.0.2" gum = { package = "tracing-gum", path = "../gum" } -erasure = { package = "polkadot-erasure-coding", path = "../../erasure-coding" } +polkadot-erasure-coding = { package = "polkadot-erasure-coding", path = "../../erasure-coding" } +log = "0.4.17" +env_logger = "0.9.0" rand = "0.8.5" +parity-scale-codec = { version = "3.6.1", features = ["std", "derive"] } +tokio = "1.24.2" -[dev-dependencies] polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" } -sp-core = { path = "../../../substrate/primitives/core" } -futures = { version = "0.3.21", features = ["thread-pool"] } +sp-keyring = { path = "../../../substrate/primitives/keyring" } +sp-application-crypto = { path = "../../../substrate/primitives/application-crypto" } +sc-network = { path = "../../../substrate/client/network" } +sc-service = { path = "../../../substrate/client/service" } +polkadot-node-metrics = { path = "../metrics" } + +polkadot-primitives-test-helpers = { path = "../../primitives/test-helpers" } +# prometheus = { package = "substrate-prometheus-endpoint", path = "../../../substrate/utils/prometheus" } +prometheus = { version = "0.13.0", default-features = false } [build-dependencies] substrate-build-script-utils = { path = "../../../substrate/utils/build-script-utils" } diff --git a/polkadot/node/subsystem-bench/src/availability.rs b/polkadot/node/subsystem-bench/src/availability.rs new file mode 100644 index 000000000000..d5cb9515ca68 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/availability.rs @@ -0,0 +1,501 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use std::{sync::Arc, time::Duration}; + +use assert_matches::assert_matches; +use env_logger::Env; +use futures::{ + channel::{mpsc, oneshot}, + executor, future, Future, FutureExt, SinkExt, +}; +use futures_timer::Delay; +use polkadot_node_metrics::metrics::Metrics; + +use polkadot_availability_recovery::{AvailabilityRecoverySubsystem, Metrics as SubsystemMetrics}; + +use parity_scale_codec::Encode; +use polkadot_node_network_protocol::request_response::{ + self as req_res, v1::ChunkResponse, IncomingRequest, Recipient, ReqProtocolNames, Requests, +}; + +use prometheus::Registry; +use sc_network::{config::RequestResponseConfig, IfDisconnected, OutboundFailure, RequestFailure}; + +use polkadot_erasure_coding::{branches, obtain_chunks_v1 as obtain_chunks}; +use polkadot_node_primitives::{BlockData, PoV, Proof}; +use polkadot_node_subsystem::{ + errors::RecoveryError, + jaeger, + messages::{ + AllMessages, AvailabilityRecoveryMessage, AvailabilityStoreMessage, NetworkBridgeTxMessage, + RuntimeApiMessage, RuntimeApiRequest, + }, + overseer, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, SpawnedSubsystem, Subsystem, + SubsystemContext, SubsystemError, SubsystemResult, +}; + +const LOG_TARGET: &str = "subsystem-bench::availability"; + +use polkadot_erasure_coding::recovery_threshold; +use polkadot_node_primitives::{AvailableData, ErasureChunk}; +// use polkadot_node_subsystem::{ +// errors::RecoveryError, +// jaeger, +// messages::{AvailabilityRecoveryMessage, AvailabilityStoreMessage}, +// overseer, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, SpawnedSubsystem, +// SubsystemContext, SubsystemError, SubsystemResult, +// }; +use polkadot_node_subsystem_test_helpers::{ + make_subsystem_context, mock::new_leaf, TestSubsystemContextHandle, +}; +use polkadot_node_subsystem_util::TimeoutExt; +use polkadot_primitives::{ + AuthorityDiscoveryId, CandidateHash, CandidateReceipt, GroupIndex, Hash, HeadData, IndexedVec, + PersistedValidationData, SessionIndex, SessionInfo, ValidatorId, ValidatorIndex, +}; +use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; +use sc_service::{SpawnTaskHandle, TaskManager}; + +type VirtualOverseer = TestSubsystemContextHandle; + +// Deterministic genesis hash for protocol names +const GENESIS_HASH: Hash = Hash::repeat_byte(0xff); + +struct AvailabilityRecoverySubsystemInstance { + protocol_config: RequestResponseConfig, +} + +pub struct EnvParams { + // The candidate we will recover in the benchmark. + candidate: CandidateReceipt, +} + +// Implements a mockup of NetworkBridge and AvilabilityStore to support provide state for +// `AvailabilityRecoverySubsystemInstance` +pub struct TestEnvironment { + // A tokio runtime to use in the test + runtime: tokio::runtime::Handle, + // A task manager that tracks task poll durations. + task_manager: TaskManager, + // The Prometheus metrics registry + registry: Registry, + // A test overseer. + to_subsystem: mpsc::Sender>, + // Parameters + params: EnvParams, + // Subsystem instance, currently keeps req/response protocol channel senders. + instance: AvailabilityRecoverySubsystemInstance, +} + +impl TestEnvironment { + pub fn new(runtime: tokio::runtime::Handle, mut params: EnvParams, registry: Registry) -> Self { + let task_manager: TaskManager = TaskManager::new(runtime.clone(), Some(®istry)).unwrap(); + let (instance, virtual_overseer) = AvailabilityRecoverySubsystemInstance::new( + ®istry, + task_manager.spawn_handle(), + runtime.clone(), + ); + + // TODO: support parametrization of initial test state + // n_validator, n_cores. + let state = TestState::new(params.candidate.clone()); + // Override candidate after computing erasure in `TestState::new` + params.candidate = state.candidate(); + + // Create channel to inject messages int the subsystem. + let to_subsystem = virtual_overseer.tx.clone(); + + // We need to start a receiver to process messages from the subsystem. + task_manager.spawn_handle().spawn_blocking( + "test-environment", + "test-environment", + async move { Self::env_task(virtual_overseer, state).await }, + ); + + TestEnvironment { runtime, task_manager, registry, to_subsystem, params, instance } + } + + pub fn params(&self) -> &EnvParams { + &self.params + } + + async fn respond_to_send_request(state: &mut TestState, request: Requests) { + match request { + Requests::ChunkFetchingV1(outgoing_request) => { + let validator_index = outgoing_request.payload.index.0 as usize; + let chunk: ChunkResponse = state.chunks[validator_index].clone().into(); + + let _ = outgoing_request + .pending_response + .send(Ok(req_res::v1::ChunkFetchingResponse::from(Some(chunk)).encode())); + }, + _ => panic!("received an unexpected request"), + } + } + + // A task that mocks dependent subsystems based on environment configuration. + // TODO: Spawn real subsystems, user overseer builder. + async fn env_task( + mut ctx: TestSubsystemContextHandle, + mut state: TestState, + ) { + loop { + futures::select! { + message = ctx.recv().fuse() => { + gum::debug!(target: LOG_TARGET, ?message, "Env task received message"); + + match message { + AllMessages::NetworkBridgeTx( + NetworkBridgeTxMessage::SendRequests( + requests, + _if_disconnected, + ) + ) => { + for request in requests { + // TODO: add latency variance when answering requests. This should be an env parameter. + Self::respond_to_send_request(&mut state, request).await; + } + }, + AllMessages::AvailabilityStore(AvailabilityStoreMessage::QueryAvailableData(_candidate_hash, tx)) => { + // TODO: Simulate av store load by delaying the response. + state.respond_none_to_available_data_query(tx).await; + }, + AllMessages::AvailabilityStore(AvailabilityStoreMessage::QueryAllChunks(_candidate_hash, tx)) => { + // Test env: We always have our own chunk. + state.respond_to_query_all_request(|index| index == state.validator_index.0 as usize, tx).await; + }, + AllMessages::AvailabilityStore( + AvailabilityStoreMessage::QueryChunkSize(_, tx) + ) => { + let chunk_size = state.chunks[0].encoded_size(); + let _ = tx.send(Some(chunk_size)); + } + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + relay_parent, + RuntimeApiRequest::SessionInfo( + session_index, + tx, + ) + )) => { + tx.send(Ok(Some(state.session_info()))).unwrap(); + } + _ => panic!("Unexpected input") + } + } + } + } + } + + // Send a message to the subsystem under test environment. + pub async fn send_message(&mut self, msg: AvailabilityRecoveryMessage) { + gum::trace!(msg = ?msg, "sending message"); + self.to_subsystem + .send(FromOrchestra::Communication { msg }) + .timeout(MAX_TIME_OF_FLIGHT) + .await + .unwrap_or_else(|| { + panic!("{}ms maximum time of flight breached", MAX_TIME_OF_FLIGHT.as_millis()) + }) + .unwrap(); + } + + // Send a signal to the subsystem under test environment. + pub async fn send_signal(&mut self, signal: OverseerSignal) { + self.to_subsystem + .send(FromOrchestra::Signal(signal)) + .timeout(MAX_TIME_OF_FLIGHT) + .await + .unwrap_or_else(|| { + panic!("{}ms is more than enough for sending signals.", TIMEOUT.as_millis()) + }) + .unwrap(); + } +} + +/// Implementation for chunks only +/// TODO: all recovery methods. +impl AvailabilityRecoverySubsystemInstance { + pub fn new( + registry: &Registry, + spawn_task_handle: SpawnTaskHandle, + runtime: tokio::runtime::Handle, + ) -> (Self, TestSubsystemContextHandle) { + let (context, virtual_overseer) = make_subsystem_context(spawn_task_handle.clone()); + let (collation_req_receiver, req_cfg) = + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); + let subsystem = AvailabilityRecoverySubsystem::with_chunks_only( + collation_req_receiver, + Metrics::try_register(®istry).unwrap(), + ); + + let spawned_subsystem = subsystem.start(context); + let subsystem_future = async move { + spawned_subsystem.future.await.unwrap(); + }; + + spawn_task_handle.spawn_blocking( + spawned_subsystem.name, + spawned_subsystem.name, + subsystem_future, + ); + + (Self { protocol_config: req_cfg }, virtual_overseer) + } +} + +const TIMEOUT: Duration = Duration::from_millis(300); + +// We use this to bail out sending messages to the subsystem if it is overloaded such that +// the time of flight is breaches 5s. +// This should eventually be a test parameter. +const MAX_TIME_OF_FLIGHT: Duration = Duration::from_millis(5000); + +macro_rules! delay { + ($delay:expr) => { + Delay::new(Duration::from_millis($delay)).await; + }; +} + +use sp_keyring::Sr25519Keyring; + +#[derive(Debug)] +enum Has { + No, + Yes, + NetworkError(RequestFailure), + /// Make request not return at all, instead the sender is returned from the function. + /// + /// Note, if you use `DoesNotReturn` you have to keep the returned senders alive, otherwise the + /// subsystem will receive a cancel event and the request actually does return. + DoesNotReturn, +} + +impl Has { + fn timeout() -> Self { + Has::NetworkError(RequestFailure::Network(OutboundFailure::Timeout)) + } +} + +#[derive(Clone)] +struct TestState { + validators: Vec, + validator_public: IndexedVec, + validator_authority_id: Vec, + // The test node validator index. + validator_index: ValidatorIndex, + candidate: CandidateReceipt, + session_index: SessionIndex, + + persisted_validation_data: PersistedValidationData, + + available_data: AvailableData, + chunks: Vec, + invalid_chunks: Vec, +} + +impl TestState { + fn candidate(&self) -> CandidateReceipt { + self.candidate.clone() + } + + fn threshold(&self) -> usize { + recovery_threshold(self.validators.len()).unwrap() + } + + fn impossibility_threshold(&self) -> usize { + self.validators.len() - self.threshold() + 1 + } + + async fn respond_to_available_data_query(&self, tx: oneshot::Sender>) { + let _ = tx.send(Some(self.available_data.clone())); + } + + async fn respond_none_to_available_data_query( + &self, + tx: oneshot::Sender>, + ) { + let _ = tx.send(None); + } + + fn session_info(&self) -> SessionInfo { + SessionInfo { + validators: self.validator_public.clone(), + discovery_keys: self.validator_authority_id.clone(), + // all validators in the same group. + validator_groups: IndexedVec::>::from(vec![(0..self + .validators + .len()) + .map(|i| ValidatorIndex(i as _)) + .collect()]), + assignment_keys: vec![], + n_cores: 0, + zeroth_delay_tranche_width: 0, + relay_vrf_modulo_samples: 0, + n_delay_tranches: 0, + no_show_slots: 0, + needed_approvals: 0, + active_validator_indices: vec![], + dispute_period: 6, + random_seed: [0u8; 32], + } + } + async fn respond_to_query_all_request( + &self, + send_chunk: impl Fn(usize) -> bool, + tx: oneshot::Sender>, + ) { + let v = self.chunks.iter().filter(|c| send_chunk(c.index.0 as usize)).cloned().collect(); + + let _ = tx.send(v); + } +} + +fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> IndexedVec { + val_ids.iter().map(|v| v.public().into()).collect() +} + +fn validator_authority_id(val_ids: &[Sr25519Keyring]) -> Vec { + val_ids.iter().map(|v| v.public().into()).collect() +} + +fn derive_erasure_chunks_with_proofs_and_root( + n_validators: usize, + available_data: &AvailableData, + alter_chunk: impl Fn(usize, &mut Vec), +) -> (Vec, Hash) { + let mut chunks: Vec> = obtain_chunks(n_validators, available_data).unwrap(); + + for (i, chunk) in chunks.iter_mut().enumerate() { + alter_chunk(i, chunk) + } + + // create proofs for each erasure chunk + let branches = branches(chunks.as_ref()); + + let root = branches.root(); + let erasure_chunks = branches + .enumerate() + .map(|(index, (proof, chunk))| ErasureChunk { + chunk: chunk.to_vec(), + index: ValidatorIndex(index as _), + proof: Proof::try_from(proof).unwrap(), + }) + .collect::>(); + + (erasure_chunks, root) +} + +impl TestState { + fn new(mut candidate: CandidateReceipt) -> Self { + let validators = vec![ + Sr25519Keyring::Ferdie, // <- this node, role: validator + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + ]; + + let validator_public = validator_pubkeys(&validators); + let validator_authority_id = validator_authority_id(&validators); + let validator_index = ValidatorIndex(0); + + let session_index = 10; + + let persisted_validation_data = PersistedValidationData { + parent_head: HeadData(vec![7, 8, 9]), + relay_parent_number: Default::default(), + max_pov_size: 1024, + relay_parent_storage_root: Default::default(), + }; + + /// A 5MB PoV. + let pov = PoV { block_data: BlockData(vec![42; 1024 * 1024 * 5]) }; + + let available_data = AvailableData { + validation_data: persisted_validation_data.clone(), + pov: Arc::new(pov), + }; + + let (chunks, erasure_root) = derive_erasure_chunks_with_proofs_and_root( + validators.len(), + &available_data, + |_, _| {}, + ); + // Mess around: + let invalid_chunks = chunks + .iter() + .cloned() + .map(|mut chunk| { + if chunk.chunk.len() >= 2 && chunk.chunk[0] != chunk.chunk[1] { + chunk.chunk[0] = chunk.chunk[1]; + } else if chunk.chunk.len() >= 1 { + chunk.chunk[0] = !chunk.chunk[0]; + } else { + chunk.proof = Proof::dummy_proof(); + } + chunk + }) + .collect(); + debug_assert_ne!(chunks, invalid_chunks); + + candidate.descriptor.erasure_root = erasure_root; + + Self { + validators, + validator_public, + validator_authority_id, + validator_index, + candidate, + session_index, + persisted_validation_data, + available_data, + chunks, + invalid_chunks, + } + } +} + +pub fn bench_chunk_recovery_params() -> EnvParams { + let mut candidate = dummy_candidate_receipt(dummy_hash()); + EnvParams { candidate } +} +pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { + env.send_signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + Hash::repeat_byte(1), + 1, + )))) + .await; + + let mut candidate = env.params().candidate.clone(); + + for candidate_num in 0..10u64 { + let (tx, rx) = oneshot::channel(); + + candidate.descriptor.relay_parent = Hash::from_low_u64_be(candidate_num); + + env.send_message(AvailabilityRecoveryMessage::RecoverAvailableData( + candidate.clone(), + 1, + Some(GroupIndex(0)), + tx, + )) + .await; + + let available_data = rx.await.unwrap().unwrap(); + } + env.send_signal(OverseerSignal::Conclude).await; +} diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs new file mode 100644 index 000000000000..3acf561e0daf --- /dev/null +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -0,0 +1,133 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! A tool for running subsystem benchmark tests designed for development and +//! CI regression testing. + +use clap::Parser; +use color_eyre::eyre; +use prometheus::proto::LabelPair; +use sc_service::TaskManager; + +pub(crate) mod availability; + +use availability::{EnvParams, TestEnvironment}; +const LOG_TARGET: &str = "subsystem-bench"; + +/// Define the supported benchmarks targets +#[derive(Debug, Parser)] +#[command(about = "Target subsystems", version, rename_all = "kebab-case")] +enum BenchmarkTarget { + /// Benchmark availability recovery strategies. + AvailabilityRecovery, +} + +#[derive(Debug, Parser)] +#[allow(missing_docs)] +struct BenchCli { + #[command(subcommand)] + pub target: BenchmarkTarget, +} + +fn new_runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_multi_thread() + .thread_name("subsystem-bench") + .enable_all() + .thread_stack_size(3 * 1024 * 1024) + .build() + .unwrap() +} + +impl BenchCli { + /// Launch a malus node. + fn launch(self) -> eyre::Result<()> { + use prometheus::{proto::MetricType, Counter, Encoder, Opts, Registry, TextEncoder}; + + let encoder = TextEncoder::new(); + + println!("Preparing {:?} benchmarks", self.target); + + let runtime = new_runtime(); + let registry = Registry::new(); + + let params = availability::bench_chunk_recovery_params(); + let mut env = TestEnvironment::new(runtime.handle().clone(), params, registry.clone()); + + runtime.block_on(availability::bench_chunk_recovery(&mut env)); + + let metric_families = registry.gather(); + let total_subsystem_cpu = 0; + + for familiy in metric_families { + let metric_type = familiy.get_field_type(); + + for metric in familiy.get_metric() { + match metric_type { + MetricType::HISTOGRAM => { + let h = metric.get_histogram(); + + let mut inf_seen = false; + + let labels = metric.get_label(); + // Skip test env usage. + let mut env_label = LabelPair::default(); + env_label.set_name("task_group".into()); + env_label.set_value("test-environment".into()); + + let mut is_env_metric = false; + for label_pair in labels { + if &env_label == label_pair { + is_env_metric = true; + break + } + } + + if !is_env_metric { + println!( + "{:?} CPU seconds used: {:?}", + familiy.get_name(), + h.get_sample_sum() + ); + } + }, + _ => {}, + } + } + } + // encoder.encode(&metric_families, &mut buffer).unwrap(); + + // Output to the standard output. + // println!("Metrics: {}", String::from_utf8(buffer).unwrap()); + Ok(()) + } +} + +fn main() -> eyre::Result<()> { + color_eyre::install()?; + let _ = env_logger::builder() + .is_test(true) + .filter(Some(LOG_TARGET), log::LevelFilter::Debug) + .try_init(); + + let cli: BenchCli = BenchCli::parse(); + cli.launch()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; +} From c3adc77f2920363d0df458c26f1b9a2a70e8ad2b Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 6 Nov 2023 22:54:39 +0200 Subject: [PATCH 076/192] measure tput and fixes Signed-off-by: Andrei Sandu --- .../node/subsystem-bench/src/availability.rs | 183 +++++++++++------- .../subsystem-bench/src/subsystem-bench.rs | 10 +- 2 files changed, 118 insertions(+), 75 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability.rs b/polkadot/node/subsystem-bench/src/availability.rs index d5cb9515ca68..72c8a736217d 100644 --- a/polkadot/node/subsystem-bench/src/availability.rs +++ b/polkadot/node/subsystem-bench/src/availability.rs @@ -14,10 +14,13 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use std::{sync::Arc, time::Duration}; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; use assert_matches::assert_matches; -use env_logger::Env; +use color_eyre::owo_colors::colors::xterm; use futures::{ channel::{mpsc, oneshot}, executor, future, Future, FutureExt, SinkExt, @@ -52,20 +55,14 @@ const LOG_TARGET: &str = "subsystem-bench::availability"; use polkadot_erasure_coding::recovery_threshold; use polkadot_node_primitives::{AvailableData, ErasureChunk}; -// use polkadot_node_subsystem::{ -// errors::RecoveryError, -// jaeger, -// messages::{AvailabilityRecoveryMessage, AvailabilityStoreMessage}, -// overseer, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, SpawnedSubsystem, -// SubsystemContext, SubsystemError, SubsystemResult, -// }; + use polkadot_node_subsystem_test_helpers::{ - make_subsystem_context, mock::new_leaf, TestSubsystemContextHandle, + make_buffered_subsystem_context, mock::new_leaf, TestSubsystemContextHandle, }; use polkadot_node_subsystem_util::TimeoutExt; use polkadot_primitives::{ - AuthorityDiscoveryId, CandidateHash, CandidateReceipt, GroupIndex, Hash, HeadData, IndexedVec, - PersistedValidationData, SessionIndex, SessionInfo, ValidatorId, ValidatorIndex, + AuthorityDiscoveryId, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, HeadData, + IndexedVec, PersistedValidationData, SessionIndex, SessionInfo, ValidatorId, ValidatorIndex, }; use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; use sc_service::{SpawnTaskHandle, TaskManager}; @@ -97,12 +94,18 @@ pub struct TestEnvironment { to_subsystem: mpsc::Sender>, // Parameters params: EnvParams, - // Subsystem instance, currently keeps req/response protocol channel senders. + // Subsystem instance, currently keeps req/response protocol channel senders + // for the whole duration of the test. instance: AvailabilityRecoverySubsystemInstance, + // The test intial state. The current state is owned by the task doing the overseer/subsystem + // mockings. + state: TestState, } impl TestEnvironment { - pub fn new(runtime: tokio::runtime::Handle, mut params: EnvParams, registry: Registry) -> Self { + // Create a new test environment with specified initial state and prometheus registry. + // We use prometheus metrics to collect per job task poll time and subsystem metrics. + pub fn new(runtime: tokio::runtime::Handle, state: TestState, registry: Registry) -> Self { let task_manager: TaskManager = TaskManager::new(runtime.clone(), Some(®istry)).unwrap(); let (instance, virtual_overseer) = AvailabilityRecoverySubsystemInstance::new( ®istry, @@ -112,26 +115,29 @@ impl TestEnvironment { // TODO: support parametrization of initial test state // n_validator, n_cores. - let state = TestState::new(params.candidate.clone()); - // Override candidate after computing erasure in `TestState::new` - params.candidate = state.candidate(); + let params = EnvParams { candidate: state.candidate() }; - // Create channel to inject messages int the subsystem. + // Copy sender for later when we need to inject messages in to the subsystem. let to_subsystem = virtual_overseer.tx.clone(); + let task_state = state.clone(); // We need to start a receiver to process messages from the subsystem. + // This mocks an overseer and all dependent subsystems task_manager.spawn_handle().spawn_blocking( "test-environment", "test-environment", - async move { Self::env_task(virtual_overseer, state).await }, + async move { Self::env_task(virtual_overseer, task_state).await }, ); - TestEnvironment { runtime, task_manager, registry, to_subsystem, params, instance } + TestEnvironment { runtime, task_manager, registry, to_subsystem, params, instance, state } } pub fn params(&self) -> &EnvParams { &self.params } + pub fn input(&self) -> &TestInput { + self.state.input() + } async fn respond_to_send_request(state: &mut TestState, request: Requests) { match request { @@ -234,7 +240,8 @@ impl AvailabilityRecoverySubsystemInstance { spawn_task_handle: SpawnTaskHandle, runtime: tokio::runtime::Handle, ) -> (Self, TestSubsystemContextHandle) { - let (context, virtual_overseer) = make_subsystem_context(spawn_task_handle.clone()); + let (context, virtual_overseer) = + make_buffered_subsystem_context(spawn_task_handle.clone(), 4096); let (collation_req_receiver, req_cfg) = IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); let subsystem = AvailabilityRecoverySubsystem::with_chunks_only( @@ -291,7 +298,7 @@ impl Has { } #[derive(Clone)] -struct TestState { +pub struct TestState { validators: Vec, validator_public: IndexedVec, validator_authority_id: Vec, @@ -305,9 +312,14 @@ struct TestState { available_data: AvailableData, chunks: Vec, invalid_chunks: Vec, + input: TestInput, } impl TestState { + fn input(&self) -> &TestInput { + &self.input + } + fn candidate(&self) -> CandidateReceipt { self.candidate.clone() } @@ -362,53 +374,14 @@ impl TestState { let _ = tx.send(v); } -} -fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> IndexedVec { - val_ids.iter().map(|v| v.public().into()).collect() -} - -fn validator_authority_id(val_ids: &[Sr25519Keyring]) -> Vec { - val_ids.iter().map(|v| v.public().into()).collect() -} - -fn derive_erasure_chunks_with_proofs_and_root( - n_validators: usize, - available_data: &AvailableData, - alter_chunk: impl Fn(usize, &mut Vec), -) -> (Vec, Hash) { - let mut chunks: Vec> = obtain_chunks(n_validators, available_data).unwrap(); - - for (i, chunk) in chunks.iter_mut().enumerate() { - alter_chunk(i, chunk) - } - - // create proofs for each erasure chunk - let branches = branches(chunks.as_ref()); - - let root = branches.root(); - let erasure_chunks = branches - .enumerate() - .map(|(index, (proof, chunk))| ErasureChunk { - chunk: chunk.to_vec(), - index: ValidatorIndex(index as _), - proof: Proof::try_from(proof).unwrap(), - }) - .collect::>(); - - (erasure_chunks, root) -} - -impl TestState { - fn new(mut candidate: CandidateReceipt) -> Self { - let validators = vec![ - Sr25519Keyring::Ferdie, // <- this node, role: validator - Sr25519Keyring::Alice, - Sr25519Keyring::Bob, - Sr25519Keyring::Charlie, - Sr25519Keyring::Dave, - ]; + pub fn new(input: TestInput) -> Self { + let validators = (0..input.n_validators as u64) + .into_iter() + .map(|v| Sr25519Keyring::Alice) + .collect::>(); + let mut candidate = dummy_candidate_receipt(dummy_hash()); let validator_public = validator_pubkeys(&validators); let validator_authority_id = validator_authority_id(&validators); let validator_index = ValidatorIndex(0); @@ -465,15 +438,66 @@ impl TestState { available_data, chunks, invalid_chunks, + input, } } } -pub fn bench_chunk_recovery_params() -> EnvParams { - let mut candidate = dummy_candidate_receipt(dummy_hash()); - EnvParams { candidate } +fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> IndexedVec { + val_ids.iter().map(|v| v.public().into()).collect() } + +fn validator_authority_id(val_ids: &[Sr25519Keyring]) -> Vec { + val_ids.iter().map(|v| v.public().into()).collect() +} + +fn derive_erasure_chunks_with_proofs_and_root( + n_validators: usize, + available_data: &AvailableData, + alter_chunk: impl Fn(usize, &mut Vec), +) -> (Vec, Hash) { + let mut chunks: Vec> = obtain_chunks(n_validators, available_data).unwrap(); + + for (i, chunk) in chunks.iter_mut().enumerate() { + alter_chunk(i, chunk) + } + + // create proofs for each erasure chunk + let branches = branches(chunks.as_ref()); + + let root = branches.root(); + let erasure_chunks = branches + .enumerate() + .map(|(index, (proof, chunk))| ErasureChunk { + chunk: chunk.to_vec(), + index: ValidatorIndex(index as _), + proof: Proof::try_from(proof).unwrap(), + }) + .collect::>(); + + (erasure_chunks, root) +} + +/// The test input parameters +#[derive(Clone)] +pub struct TestInput { + pub n_validators: usize, + pub n_cores: usize, + pub pov_size: usize, + // This parameter is used to determine how many recoveries we batch in parallel + // similarly to how in practice tranche0 assignments work. + pub vrf_modulo_samples: usize, +} + +impl Default for TestInput { + fn default() -> Self { + Self { n_validators: 300, n_cores: 50, pov_size: 5 * 1024 * 1024, vrf_modulo_samples: 6 } + } +} + pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { + let input = env.input().clone(); + env.send_signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( Hash::repeat_byte(1), 1, @@ -482,8 +506,12 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { let mut candidate = env.params().candidate.clone(); - for candidate_num in 0..10u64 { + let start_marker = Instant::now(); + + let mut batch = Vec::new(); + for candidate_num in 0..input.n_cores as u64 { let (tx, rx) = oneshot::channel(); + batch.push(rx); candidate.descriptor.relay_parent = Hash::from_low_u64_be(candidate_num); @@ -495,7 +523,20 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { )) .await; + if batch.len() >= input.vrf_modulo_samples { + for rx in std::mem::take(&mut batch) { + let available_data = rx.await.unwrap().unwrap(); + } + } + } + + for rx in std::mem::take(&mut batch) { let available_data = rx.await.unwrap().unwrap(); } + env.send_signal(OverseerSignal::Conclude).await; + let duration = start_marker.elapsed().as_millis(); + let tput = ((input.n_cores * input.pov_size) as u128) / duration * 1000; + println!("Benchmark completed in {:?}ms", duration); + println!("Throughput: {}KiB/s", tput / 1024); } diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 3acf561e0daf..bfc0b63e86d3 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -24,7 +24,7 @@ use sc_service::TaskManager; pub(crate) mod availability; -use availability::{EnvParams, TestEnvironment}; +use availability::{EnvParams, TestEnvironment, TestInput, TestState}; const LOG_TARGET: &str = "subsystem-bench"; /// Define the supported benchmarks targets @@ -45,6 +45,7 @@ struct BenchCli { fn new_runtime() -> tokio::runtime::Runtime { tokio::runtime::Builder::new_multi_thread() .thread_name("subsystem-bench") + .max_blocking_threads(32) .enable_all() .thread_stack_size(3 * 1024 * 1024) .build() @@ -63,8 +64,9 @@ impl BenchCli { let runtime = new_runtime(); let registry = Registry::new(); - let params = availability::bench_chunk_recovery_params(); - let mut env = TestEnvironment::new(runtime.handle().clone(), params, registry.clone()); + let state = TestState::new(TestInput::default()); + + let mut env = TestEnvironment::new(runtime.handle().clone(), state, registry.clone()); runtime.block_on(availability::bench_chunk_recovery(&mut env)); @@ -119,7 +121,7 @@ fn main() -> eyre::Result<()> { color_eyre::install()?; let _ = env_logger::builder() .is_test(true) - .filter(Some(LOG_TARGET), log::LevelFilter::Debug) + .filter(Some(LOG_TARGET), log::LevelFilter::Info) .try_init(); let cli: BenchCli = BenchCli::parse(); From cd81d38b3d060bea8d73d2a796fbea43a954d85a Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 7 Nov 2023 10:31:35 +0200 Subject: [PATCH 077/192] Fixup comment Signed-off-by: Alexandru Gheorghe --- polkadot/primitives/src/v6/mod.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/polkadot/primitives/src/v6/mod.rs b/polkadot/primitives/src/v6/mod.rs index 8f11092b8455..fe571236109e 100644 --- a/polkadot/primitives/src/v6/mod.rs +++ b/polkadot/primitives/src/v6/mod.rs @@ -1367,8 +1367,9 @@ pub enum ValidDisputeStatementKind { #[codec(index = 3)] ApprovalChecking, /// An approval vote from the new version. - /// TODO: Fixme this probably means we can't create this version - /// untill all nodes have been updated to support it. + /// We can't create this version untill all nodes + /// have been updated to support it and max_approval_coalesce_count + /// is set to more than 1. #[codec(index = 4)] ApprovalCheckingMultipleCandidates(Vec), } From 624bb5f9e66eed9fad5d9319285580b6ccc226c6 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 7 Nov 2023 11:15:14 +0200 Subject: [PATCH 078/192] Fix genesis error in zombienet Signed-off-by: Alexandru Gheorghe --- .../functional/0007-approval-voting-coalescing.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml index 20dbb3577849..f84a87f918bd 100644 --- a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml +++ b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml @@ -5,11 +5,11 @@ timeout = 1000 default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" -[relaychain.genesis.runtime.configuration.config] +[relaychain.genesis.runtimeGenesis.patch.configuration.config] needed_approvals = 5 relay_vrf_modulo_samples = 6 -[relaychain.genesis.runtime.configuration.config.approval_voting_params] +[relaychain.genesis.runtimeGenesis.patch.configuration.config.approval_voting_params] max_approval_coalesce_count = 5 [relaychain.default_resources] From 31b0351eaea643f181fe3216e03aae5d4da12ed6 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 7 Nov 2023 15:37:33 +0200 Subject: [PATCH 079/192] add network emulation Signed-off-by: Andrei Sandu --- .../{availability.rs => availability/mod.rs} | 50 ++++- .../src/availability/network.rs | 212 ++++++++++++++++++ .../subsystem-bench/src/subsystem-bench.rs | 3 +- 3 files changed, 255 insertions(+), 10 deletions(-) rename polkadot/node/subsystem-bench/src/{availability.rs => availability/mod.rs} (92%) create mode 100644 polkadot/node/subsystem-bench/src/availability/network.rs diff --git a/polkadot/node/subsystem-bench/src/availability.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs similarity index 92% rename from polkadot/node/subsystem-bench/src/availability.rs rename to polkadot/node/subsystem-bench/src/availability/mod.rs index 72c8a736217d..cdc2bf5ce644 100644 --- a/polkadot/node/subsystem-bench/src/availability.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -16,6 +16,7 @@ use std::{ sync::Arc, + thread::sleep, time::{Duration, Instant}, }; @@ -67,6 +68,8 @@ use polkadot_primitives::{ use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; use sc_service::{SpawnTaskHandle, TaskManager}; +mod network; + type VirtualOverseer = TestSubsystemContextHandle; // Deterministic genesis hash for protocol names @@ -121,12 +124,13 @@ impl TestEnvironment { let to_subsystem = virtual_overseer.tx.clone(); let task_state = state.clone(); + let spawn_task_handle = task_manager.spawn_handle(); // We need to start a receiver to process messages from the subsystem. // This mocks an overseer and all dependent subsystems task_manager.spawn_handle().spawn_blocking( "test-environment", "test-environment", - async move { Self::env_task(virtual_overseer, task_state).await }, + async move { Self::env_task(virtual_overseer, task_state, spawn_task_handle).await }, ); TestEnvironment { runtime, task_manager, registry, to_subsystem, params, instance, state } @@ -139,15 +143,20 @@ impl TestEnvironment { self.state.input() } - async fn respond_to_send_request(state: &mut TestState, request: Requests) { + pub fn respond_to_send_request(state: &mut TestState, request: Requests) -> NetworkAction { match request { Requests::ChunkFetchingV1(outgoing_request) => { let validator_index = outgoing_request.payload.index.0 as usize; let chunk: ChunkResponse = state.chunks[validator_index].clone().into(); + let size = chunk.encoded_size(); + let future = async move { + let _ = outgoing_request + .pending_response + .send(Ok(req_res::v1::ChunkFetchingResponse::from(Some(chunk)).encode())); + } + .boxed(); - let _ = outgoing_request - .pending_response - .send(Ok(req_res::v1::ChunkFetchingResponse::from(Some(chunk)).encode())); + NetworkAction::new(validator_index, future, size) }, _ => panic!("received an unexpected request"), } @@ -158,7 +167,15 @@ impl TestEnvironment { async fn env_task( mut ctx: TestSubsystemContextHandle, mut state: TestState, + spawn_task_handle: SpawnTaskHandle, ) { + // Emulate `n_validators` each with 1MiB of bandwidth available. + let mut network = NetworkEmulator::new( + state.input().n_validators, + state.input().bandwidth, + spawn_task_handle, + ); + loop { futures::select! { message = ctx.recv().fuse() => { @@ -173,7 +190,9 @@ impl TestEnvironment { ) => { for request in requests { // TODO: add latency variance when answering requests. This should be an env parameter. - Self::respond_to_send_request(&mut state, request).await; + let action = Self::respond_to_send_request(&mut state, request); + // action.run().await; + network.submit_peer_action(action.index(), action); } }, AllMessages::AvailabilityStore(AvailabilityStoreMessage::QueryAvailableData(_candidate_hash, tx)) => { @@ -241,7 +260,7 @@ impl AvailabilityRecoverySubsystemInstance { runtime: tokio::runtime::Handle, ) -> (Self, TestSubsystemContextHandle) { let (context, virtual_overseer) = - make_buffered_subsystem_context(spawn_task_handle.clone(), 4096); + make_buffered_subsystem_context(spawn_task_handle.clone(), 4096 * 4); let (collation_req_receiver, req_cfg) = IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); let subsystem = AvailabilityRecoverySubsystem::with_chunks_only( @@ -279,6 +298,10 @@ macro_rules! delay { use sp_keyring::Sr25519Keyring; +use crate::availability::network::NetworkAction; + +use self::network::NetworkEmulator; + #[derive(Debug)] enum Has { No, @@ -479,7 +502,7 @@ fn derive_erasure_chunks_with_proofs_and_root( } /// The test input parameters -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct TestInput { pub n_validators: usize, pub n_cores: usize, @@ -487,11 +510,19 @@ pub struct TestInput { // This parameter is used to determine how many recoveries we batch in parallel // similarly to how in practice tranche0 assignments work. pub vrf_modulo_samples: usize, + // The amount of bandiwdht remote validators have. + pub bandwidth: usize, } impl Default for TestInput { fn default() -> Self { - Self { n_validators: 300, n_cores: 50, pov_size: 5 * 1024 * 1024, vrf_modulo_samples: 6 } + Self { + n_validators: 10, + n_cores: 10, + pov_size: 5 * 1024 * 1024, + vrf_modulo_samples: 6, + bandwidth: 15 * 1024 * 1024, + } } } @@ -535,6 +566,7 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { } env.send_signal(OverseerSignal::Conclude).await; + delay!(5); let duration = start_marker.elapsed().as_millis(); let tput = ((input.n_cores * input.pov_size) as u128) / duration * 1000; println!("Benchmark completed in {:?}ms", duration); diff --git a/polkadot/node/subsystem-bench/src/availability/network.rs b/polkadot/node/subsystem-bench/src/availability/network.rs new file mode 100644 index 000000000000..268de5d828eb --- /dev/null +++ b/polkadot/node/subsystem-bench/src/availability/network.rs @@ -0,0 +1,212 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use super::*; +use futures::stream::FuturesOrdered; +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; + +// An emulated node egress traffic rate_limiter. +#[derive(Debug)] +struct RateLimit { + // How often we refill credits in buckets + tick_rate: usize, + // Total ticks + total_ticks: usize, + // Max refill per tick + max_refill: usize, + // Available credit. We allow for bursts over 1/tick_rate of `cps` budget, but we + // account it by negative credit. + credits: isize, + // When last refilled. + last_refill: Instant, +} + +impl RateLimit { + // Create a new `RateLimit` from a `cps` (credits per second) budget and + // `tick_rate`. + pub fn new(tick_rate: usize, cps: usize) -> Self { + // Compute how much refill for each tick + let max_refill = cps / tick_rate; + RateLimit { + tick_rate, + total_ticks: 0, + max_refill, + // A fresh start + credits: max_refill as isize, + last_refill: Instant::now(), + } + } + + pub async fn refill(&mut self) { + // If this is called to early, we need to sleep until next tick. + let now = Instant::now(); + let next_tick_delta = + (self.last_refill + Duration::from_millis(1000 / self.tick_rate as u64)) - now; + + // Sleep until next tick. + if !next_tick_delta.is_zero() { + gum::trace!(target: LOG_TARGET, "need to sleep {}ms", next_tick_delta.as_millis()); + tokio::time::sleep(next_tick_delta).await; + } + + self.total_ticks += 1; + self.credits += self.max_refill as isize; + self.last_refill = Instant::now(); + } + + // Reap credits from the bucket. + // Blocks if credits budged goes negative during call. + pub async fn reap(&mut self, amount: usize) { + self.credits -= amount as isize; + + if self.credits >= 0 { + return + } + + while self.credits < 0 { + gum::trace!(target: LOG_TARGET, "Before refill: {:?}", &self); + self.refill().await; + gum::trace!(target: LOG_TARGET, "After refill: {:?}", &self); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use polkadot_node_metrics::metered::CoarseDuration; + use std::time::Instant; + + use super::RateLimit; + + #[tokio::test] + async fn test_expected_rate() { + let tick_rate = 200; + let budget = 1_000_000; + // rate must not exceeed 100 credits per second + let mut rate_limiter = RateLimit::new(tick_rate, budget); + let mut total_sent = 0usize; + let start = Instant::now(); + + let mut reap_amount = 0; + while rate_limiter.total_ticks < tick_rate { + reap_amount += 1; + reap_amount = reap_amount % 100; + + rate_limiter.reap(reap_amount).await; + total_sent += reap_amount; + } + + let end = Instant::now(); + + // assert_eq!(end - start, Duration::from_secs(1)); + println!("duration: {}", (end - start).as_millis()); + + // Allow up to `budget/max_refill` error tolerance + let lower_bound = budget as u128 * ((end - start).as_millis() / 1000u128); + let upper_bound = budget as u128 * + ((end - start).as_millis() / 1000u128 + rate_limiter.max_refill as u128); + assert!(total_sent as u128 >= lower_bound); + assert!(total_sent as u128 <= upper_bound); + } +} +// A network peer emulator +struct PeerEmulator { + // The queue of requests waiting to be served by the emulator + actions_tx: UnboundedSender, +} + +impl PeerEmulator { + pub fn new(bandwidth: usize, spawn_task_handle: SpawnTaskHandle) -> Self { + let (actions_tx, mut actions_rx) = tokio::sync::mpsc::unbounded_channel(); + + spawn_task_handle.spawn("peer-emulator", "test-environment", async move { + let mut rate_limiter = RateLimit::new(20, bandwidth); + loop { + let maybe_action: Option = actions_rx.recv().await; + if let Some(action) = maybe_action { + let size = action.size(); + rate_limiter.reap(size).await; + action.run().await; + } else { + break + } + } + }); + + Self { actions_tx } + } + + // Queue a send request from the emulated peer. + pub fn send(&mut self, action: NetworkAction) { + self.actions_tx.send(action).expect("peer emulator task lives"); + } +} + +pub type ActionFuture = std::pin::Pin + std::marker::Send>>; +// An network action to be completed by the emulator task. +pub struct NetworkAction { + // The function that performs the action + run: ActionFuture, + // The payload size that we simulate sending from a peer + size: usize, + // Peer index + index: usize, +} + +impl NetworkAction { + pub fn new(index: usize, run: ActionFuture, size: usize) -> Self { + Self { run, size, index } + } + pub fn size(&self) -> usize { + self.size + } + + pub async fn run(self) { + self.run.await; + } + + pub fn index(&self) -> usize { + self.index + } +} + +// Mocks the network bridge and an arbitrary number of connected peer nodes. +// Implements network latency, bandwidth and error. +pub struct NetworkEmulator { + // Number of peers connected on validation protocol + n_peers: usize, + // The maximum Rx/Tx bandwidth in bytes per second. + bandwidth: usize, + // Per peer network emulation + peers: Vec, +} + +impl NetworkEmulator { + pub fn new(n_peers: usize, bandwidth: usize, spawn_task_handle: SpawnTaskHandle) -> Self { + Self { + n_peers, + bandwidth, + peers: (0..n_peers) + .map(|index| PeerEmulator::new(bandwidth, spawn_task_handle.clone())) + .collect::>(), + } + } + + pub fn submit_peer_action(&mut self, index: usize, action: NetworkAction) { + let _ = self.peers[index].send(action); + } +} diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index bfc0b63e86d3..d58f0bccba9b 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -45,7 +45,6 @@ struct BenchCli { fn new_runtime() -> tokio::runtime::Runtime { tokio::runtime::Builder::new_multi_thread() .thread_name("subsystem-bench") - .max_blocking_threads(32) .enable_all() .thread_stack_size(3 * 1024 * 1024) .build() @@ -68,6 +67,8 @@ impl BenchCli { let mut env = TestEnvironment::new(runtime.handle().clone(), state, registry.clone()); + println!("{:?}", env.input()); + runtime.block_on(availability::bench_chunk_recovery(&mut env)); let metric_families = registry.gather(); From e4bb037260e1fee1f06dee1d5581f9d7763e2548 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 7 Nov 2023 15:51:50 +0200 Subject: [PATCH 080/192] cleanup Signed-off-by: Andrei Sandu --- Cargo.lock | 1 - polkadot/node/subsystem-bench/Cargo.toml | 3 - polkadot/node/subsystem-bench/build.rs | 22 ----- .../subsystem-bench/src/availability/mod.rs | 90 +++++-------------- .../src/availability/network.rs | 5 +- .../subsystem-bench/src/subsystem-bench.rs | 10 +-- 6 files changed, 24 insertions(+), 107 deletions(-) delete mode 100644 polkadot/node/subsystem-bench/build.rs diff --git a/Cargo.lock b/Cargo.lock index d113fd7e43cd..4645aeee6aab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13022,7 +13022,6 @@ dependencies = [ "sp-core", "sp-keyring", "sp-keystore", - "substrate-build-script-utils", "tokio", "tracing-gum", ] diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index 729749ab153b..7408397f930c 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -50,8 +50,5 @@ polkadot-primitives-test-helpers = { path = "../../primitives/test-helpers" } # prometheus = { package = "substrate-prometheus-endpoint", path = "../../../substrate/utils/prometheus" } prometheus = { version = "0.13.0", default-features = false } -[build-dependencies] -substrate-build-script-utils = { path = "../../../substrate/utils/build-script-utils" } - [features] default = [] diff --git a/polkadot/node/subsystem-bench/build.rs b/polkadot/node/subsystem-bench/build.rs deleted file mode 100644 index 84fe22e23ed6..000000000000 --- a/polkadot/node/subsystem-bench/build.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -fn main() { - substrate_build_script_utils::generate_cargo_keys(); - // For the node/worker version check, make sure we always rebuild the node and binary workers - // when the version changes. - substrate_build_script_utils::rerun_if_git_head_changed(); -} diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index cdc2bf5ce644..c6e9dead09c1 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -16,40 +16,34 @@ use std::{ sync::Arc, - thread::sleep, time::{Duration, Instant}, }; -use assert_matches::assert_matches; -use color_eyre::owo_colors::colors::xterm; use futures::{ channel::{mpsc, oneshot}, - executor, future, Future, FutureExt, SinkExt, + FutureExt, SinkExt, }; use futures_timer::Delay; use polkadot_node_metrics::metrics::Metrics; -use polkadot_availability_recovery::{AvailabilityRecoverySubsystem, Metrics as SubsystemMetrics}; +use polkadot_availability_recovery::AvailabilityRecoverySubsystem; use parity_scale_codec::Encode; use polkadot_node_network_protocol::request_response::{ - self as req_res, v1::ChunkResponse, IncomingRequest, Recipient, ReqProtocolNames, Requests, + self as req_res, v1::ChunkResponse, IncomingRequest, ReqProtocolNames, Requests, }; use prometheus::Registry; -use sc_network::{config::RequestResponseConfig, IfDisconnected, OutboundFailure, RequestFailure}; +use sc_network::{config::RequestResponseConfig, OutboundFailure, RequestFailure}; use polkadot_erasure_coding::{branches, obtain_chunks_v1 as obtain_chunks}; use polkadot_node_primitives::{BlockData, PoV, Proof}; use polkadot_node_subsystem::{ - errors::RecoveryError, - jaeger, messages::{ AllMessages, AvailabilityRecoveryMessage, AvailabilityStoreMessage, NetworkBridgeTxMessage, RuntimeApiMessage, RuntimeApiRequest, }, - overseer, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, SpawnedSubsystem, Subsystem, - SubsystemContext, SubsystemError, SubsystemResult, + ActiveLeavesUpdate, FromOrchestra, OverseerSignal, Subsystem, }; const LOG_TARGET: &str = "subsystem-bench::availability"; @@ -62,41 +56,30 @@ use polkadot_node_subsystem_test_helpers::{ }; use polkadot_node_subsystem_util::TimeoutExt; use polkadot_primitives::{ - AuthorityDiscoveryId, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, HeadData, - IndexedVec, PersistedValidationData, SessionIndex, SessionInfo, ValidatorId, ValidatorIndex, + AuthorityDiscoveryId, CandidateReceipt, GroupIndex, Hash, HeadData, IndexedVec, + PersistedValidationData, SessionIndex, SessionInfo, ValidatorId, ValidatorIndex, }; use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; use sc_service::{SpawnTaskHandle, TaskManager}; mod network; -type VirtualOverseer = TestSubsystemContextHandle; - // Deterministic genesis hash for protocol names const GENESIS_HASH: Hash = Hash::repeat_byte(0xff); struct AvailabilityRecoverySubsystemInstance { - protocol_config: RequestResponseConfig, -} - -pub struct EnvParams { - // The candidate we will recover in the benchmark. - candidate: CandidateReceipt, + _protocol_config: RequestResponseConfig, } // Implements a mockup of NetworkBridge and AvilabilityStore to support provide state for // `AvailabilityRecoverySubsystemInstance` pub struct TestEnvironment { - // A tokio runtime to use in the test - runtime: tokio::runtime::Handle, // A task manager that tracks task poll durations. task_manager: TaskManager, // The Prometheus metrics registry registry: Registry, // A test overseer. to_subsystem: mpsc::Sender>, - // Parameters - params: EnvParams, // Subsystem instance, currently keeps req/response protocol channel senders // for the whole duration of the test. instance: AvailabilityRecoverySubsystemInstance, @@ -110,15 +93,8 @@ impl TestEnvironment { // We use prometheus metrics to collect per job task poll time and subsystem metrics. pub fn new(runtime: tokio::runtime::Handle, state: TestState, registry: Registry) -> Self { let task_manager: TaskManager = TaskManager::new(runtime.clone(), Some(®istry)).unwrap(); - let (instance, virtual_overseer) = AvailabilityRecoverySubsystemInstance::new( - ®istry, - task_manager.spawn_handle(), - runtime.clone(), - ); - - // TODO: support parametrization of initial test state - // n_validator, n_cores. - let params = EnvParams { candidate: state.candidate() }; + let (instance, virtual_overseer) = + AvailabilityRecoverySubsystemInstance::new(®istry, task_manager.spawn_handle()); // Copy sender for later when we need to inject messages in to the subsystem. let to_subsystem = virtual_overseer.tx.clone(); @@ -133,12 +109,9 @@ impl TestEnvironment { async move { Self::env_task(virtual_overseer, task_state, spawn_task_handle).await }, ); - TestEnvironment { runtime, task_manager, registry, to_subsystem, params, instance, state } + TestEnvironment { task_manager, registry, to_subsystem, instance, state } } - pub fn params(&self) -> &EnvParams { - &self.params - } pub fn input(&self) -> &TestInput { self.state.input() } @@ -189,9 +162,7 @@ impl TestEnvironment { ) ) => { for request in requests { - // TODO: add latency variance when answering requests. This should be an env parameter. let action = Self::respond_to_send_request(&mut state, request); - // action.run().await; network.submit_peer_action(action.index(), action); } }, @@ -210,9 +181,9 @@ impl TestEnvironment { let _ = tx.send(Some(chunk_size)); } AllMessages::RuntimeApi(RuntimeApiMessage::Request( - relay_parent, + _relay_parent, RuntimeApiRequest::SessionInfo( - session_index, + _session_index, tx, ) )) => { @@ -257,7 +228,6 @@ impl AvailabilityRecoverySubsystemInstance { pub fn new( registry: &Registry, spawn_task_handle: SpawnTaskHandle, - runtime: tokio::runtime::Handle, ) -> (Self, TestSubsystemContextHandle) { let (context, virtual_overseer) = make_buffered_subsystem_context(spawn_task_handle.clone(), 4096 * 4); @@ -279,7 +249,7 @@ impl AvailabilityRecoverySubsystemInstance { subsystem_future, ); - (Self { protocol_config: req_cfg }, virtual_overseer) + (Self { _protocol_config: req_cfg }, virtual_overseer) } } @@ -302,24 +272,6 @@ use crate::availability::network::NetworkAction; use self::network::NetworkEmulator; -#[derive(Debug)] -enum Has { - No, - Yes, - NetworkError(RequestFailure), - /// Make request not return at all, instead the sender is returned from the function. - /// - /// Note, if you use `DoesNotReturn` you have to keep the returned senders alive, otherwise the - /// subsystem will receive a cancel event and the request actually does return. - DoesNotReturn, -} - -impl Has { - fn timeout() -> Self { - Has::NetworkError(RequestFailure::Network(OutboundFailure::Timeout)) - } -} - #[derive(Clone)] pub struct TestState { validators: Vec, @@ -401,7 +353,7 @@ impl TestState { pub fn new(input: TestInput) -> Self { let validators = (0..input.n_validators as u64) .into_iter() - .map(|v| Sr25519Keyring::Alice) + .map(|_v| Sr25519Keyring::Alice) .collect::>(); let mut candidate = dummy_candidate_receipt(dummy_hash()); @@ -418,8 +370,8 @@ impl TestState { relay_parent_storage_root: Default::default(), }; - /// A 5MB PoV. - let pov = PoV { block_data: BlockData(vec![42; 1024 * 1024 * 5]) }; + // A 5MB PoV. + let pov = PoV { block_data: BlockData(vec![42; input.pov_size]) }; let available_data = AvailableData { validation_data: persisted_validation_data.clone(), @@ -535,10 +487,8 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { )))) .await; - let mut candidate = env.params().candidate.clone(); - let start_marker = Instant::now(); - + let mut candidate = env.state.candidate(); let mut batch = Vec::new(); for candidate_num in 0..input.n_cores as u64 { let (tx, rx) = oneshot::channel(); @@ -556,13 +506,13 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { if batch.len() >= input.vrf_modulo_samples { for rx in std::mem::take(&mut batch) { - let available_data = rx.await.unwrap().unwrap(); + let _available_data = rx.await.unwrap().unwrap(); } } } for rx in std::mem::take(&mut batch) { - let available_data = rx.await.unwrap().unwrap(); + let _available_data = rx.await.unwrap().unwrap(); } env.send_signal(OverseerSignal::Conclude).await; diff --git a/polkadot/node/subsystem-bench/src/availability/network.rs b/polkadot/node/subsystem-bench/src/availability/network.rs index 268de5d828eb..1889e971cc1e 100644 --- a/polkadot/node/subsystem-bench/src/availability/network.rs +++ b/polkadot/node/subsystem-bench/src/availability/network.rs @@ -15,8 +15,7 @@ // along with Polkadot. If not, see . use super::*; -use futures::stream::FuturesOrdered; -use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; +use tokio::sync::mpsc::UnboundedSender; // An emulated node egress traffic rate_limiter. #[derive(Debug)] @@ -201,7 +200,7 @@ impl NetworkEmulator { n_peers, bandwidth, peers: (0..n_peers) - .map(|index| PeerEmulator::new(bandwidth, spawn_task_handle.clone())) + .map(|_index| PeerEmulator::new(bandwidth, spawn_task_handle.clone())) .collect::>(), } } diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index d58f0bccba9b..30a9dff02757 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -20,11 +20,10 @@ use clap::Parser; use color_eyre::eyre; use prometheus::proto::LabelPair; -use sc_service::TaskManager; pub(crate) mod availability; -use availability::{EnvParams, TestEnvironment, TestInput, TestState}; +use availability::{TestEnvironment, TestInput, TestState}; const LOG_TARGET: &str = "subsystem-bench"; /// Define the supported benchmarks targets @@ -54,9 +53,7 @@ fn new_runtime() -> tokio::runtime::Runtime { impl BenchCli { /// Launch a malus node. fn launch(self) -> eyre::Result<()> { - use prometheus::{proto::MetricType, Counter, Encoder, Opts, Registry, TextEncoder}; - - let encoder = TextEncoder::new(); + use prometheus::{proto::MetricType, Registry, TextEncoder}; println!("Preparing {:?} benchmarks", self.target); @@ -72,7 +69,6 @@ impl BenchCli { runtime.block_on(availability::bench_chunk_recovery(&mut env)); let metric_families = registry.gather(); - let total_subsystem_cpu = 0; for familiy in metric_families { let metric_type = familiy.get_field_type(); @@ -82,8 +78,6 @@ impl BenchCli { MetricType::HISTOGRAM => { let h = metric.get_histogram(); - let mut inf_seen = false; - let labels = metric.get_label(); // Skip test env usage. let mut env_label = LabelPair::default(); From a69492481061bebdfbcfe9bb834a5c24a1160a33 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 7 Nov 2023 18:31:15 +0200 Subject: [PATCH 081/192] Add latency emulation Signed-off-by: Andrei Sandu --- .../src/availability/configuration.rs | 107 ++++++++++++++++++ .../subsystem-bench/src/availability/mod.rs | 89 +++++++-------- .../src/availability/network.rs | 50 ++++---- .../subsystem-bench/src/subsystem-bench.rs | 9 +- 4 files changed, 182 insertions(+), 73 deletions(-) create mode 100644 polkadot/node/subsystem-bench/src/availability/configuration.rs diff --git a/polkadot/node/subsystem-bench/src/availability/configuration.rs b/polkadot/node/subsystem-bench/src/availability/configuration.rs new file mode 100644 index 000000000000..14e8f55128d9 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/availability/configuration.rs @@ -0,0 +1,107 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use super::*; + +/// Peer response latency configuration. +#[derive(Clone, Debug)] +pub struct PeerLatency { + /// Min latency for `NetworkAction` completion. + pub min_latency: Duration, + /// Max latency or `NetworkAction` completion. + pub max_latency: Duration, +} + +/// The test input parameters +#[derive(Clone, Debug)] +pub struct TestConfiguration { + /// Number of validators + pub n_validators: usize, + /// Number of cores + pub n_cores: usize, + /// The PoV size + pub pov_size: usize, + /// This parameter is used to determine how many recoveries we batch in parallel + /// similarly to how in practice tranche0 assignments work. + pub vrf_modulo_samples: usize, + /// The amount of bandiwdht remote validators have. + pub bandwidth: usize, + /// Optional peer emulation latency + pub latency: Option, +} + +impl Default for TestConfiguration { + fn default() -> Self { + Self { + n_validators: 10, + n_cores: 10, + pov_size: 5 * 1024 * 1024, + vrf_modulo_samples: 6, + bandwidth: 15 * 1024 * 1024, + latency: None, + } + } +} + +impl TestConfiguration { + /// An unconstrained standard configuration matching Polkadot/Kusama + pub fn unconstrained_300_validators_60_cores(pov_size: usize) -> TestConfiguration { + Self { + n_validators: 300, + n_cores: 60, + pov_size, + vrf_modulo_samples: 6, + // HW specs node bandwidth + bandwidth: 60 * 1024 * 1024, + // No latency + latency: None, + } + } + + /// Polkadot/Kusama configuration with typical latency constraints. + pub fn healthy_network_300_validators_60_cores(pov_size: usize) -> TestConfiguration { + Self { + n_validators: 300, + n_cores: 60, + pov_size, + vrf_modulo_samples: 6, + // HW specs node bandwidth + bandwidth: 60 * 1024 * 1024, + latency: Some(PeerLatency { + min_latency: Duration::from_millis(1), + max_latency: Duration::from_millis(50), + }), + } + } + + /// Polkadot/Kusama configuration with degraded due to latencies. + /// TODO: implement errors. + pub fn degraded_network_300_validators_60_cores(pov_size: usize) -> TestConfiguration { + Self { + n_validators: 300, + n_cores: 60, + pov_size, + vrf_modulo_samples: 6, + // HW specs node bandwidth + bandwidth: 60 * 1024 * 1024, + // A range of latencies to expect in a degraded network + latency: Some(PeerLatency { + min_latency: Duration::from_millis(1), + max_latency: Duration::from_millis(1000), + }), + } + } +} diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index c6e9dead09c1..6c0c41c86c0f 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -32,6 +32,7 @@ use parity_scale_codec::Encode; use polkadot_node_network_protocol::request_response::{ self as req_res, v1::ChunkResponse, IncomingRequest, ReqProtocolNames, Requests, }; +use rand::{distributions::Uniform, prelude::Distribution, thread_rng}; use prometheus::Registry; use sc_network::{config::RequestResponseConfig, OutboundFailure, RequestFailure}; @@ -62,8 +63,11 @@ use polkadot_primitives::{ use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; use sc_service::{SpawnTaskHandle, TaskManager}; +mod configuration; mod network; +pub use configuration::TestConfiguration; + // Deterministic genesis hash for protocol names const GENESIS_HASH: Hash = Hash::repeat_byte(0xff); @@ -112,8 +116,20 @@ impl TestEnvironment { TestEnvironment { task_manager, registry, to_subsystem, instance, state } } - pub fn input(&self) -> &TestInput { - self.state.input() + pub fn config(&self) -> &TestConfiguration { + self.state.config() + } + + /// Produce a randomized duration between `min` and `max`. + fn random_latency(maybe_peer_latency: Option<&PeerLatency>) -> Option { + if let Some(peer_latency) = maybe_peer_latency { + Some( + Uniform::from(peer_latency.min_latency..=peer_latency.max_latency) + .sample(&mut thread_rng()), + ) + } else { + None + } } pub fn respond_to_send_request(state: &mut TestState, request: Requests) -> NetworkAction { @@ -129,7 +145,13 @@ impl TestEnvironment { } .boxed(); - NetworkAction::new(validator_index, future, size) + NetworkAction::new( + validator_index, + future, + size, + // Generate a random latency based on configuration. + Self::random_latency(state.config().latency.as_ref()), + ) }, _ => panic!("received an unexpected request"), } @@ -144,8 +166,8 @@ impl TestEnvironment { ) { // Emulate `n_validators` each with 1MiB of bandwidth available. let mut network = NetworkEmulator::new( - state.input().n_validators, - state.input().bandwidth, + state.config().n_validators, + state.config().bandwidth, spawn_task_handle, ); @@ -270,7 +292,7 @@ use sp_keyring::Sr25519Keyring; use crate::availability::network::NetworkAction; -use self::network::NetworkEmulator; +use self::{configuration::PeerLatency, network::NetworkEmulator}; #[derive(Clone)] pub struct TestState { @@ -287,26 +309,18 @@ pub struct TestState { available_data: AvailableData, chunks: Vec, invalid_chunks: Vec, - input: TestInput, + config: TestConfiguration, } impl TestState { - fn input(&self) -> &TestInput { - &self.input + fn config(&self) -> &TestConfiguration { + &self.config } fn candidate(&self) -> CandidateReceipt { self.candidate.clone() } - fn threshold(&self) -> usize { - recovery_threshold(self.validators.len()).unwrap() - } - - fn impossibility_threshold(&self) -> usize { - self.validators.len() - self.threshold() + 1 - } - async fn respond_to_available_data_query(&self, tx: oneshot::Sender>) { let _ = tx.send(Some(self.available_data.clone())); } @@ -350,8 +364,8 @@ impl TestState { let _ = tx.send(v); } - pub fn new(input: TestInput) -> Self { - let validators = (0..input.n_validators as u64) + pub fn new(config: TestConfiguration) -> Self { + let validators = (0..config.n_validators as u64) .into_iter() .map(|_v| Sr25519Keyring::Alice) .collect::>(); @@ -371,7 +385,7 @@ impl TestState { }; // A 5MB PoV. - let pov = PoV { block_data: BlockData(vec![42; input.pov_size]) }; + let pov = PoV { block_data: BlockData(vec![42; config.pov_size]) }; let available_data = AvailableData { validation_data: persisted_validation_data.clone(), @@ -413,7 +427,7 @@ impl TestState { available_data, chunks, invalid_chunks, - input, + config, } } } @@ -453,33 +467,8 @@ fn derive_erasure_chunks_with_proofs_and_root( (erasure_chunks, root) } -/// The test input parameters -#[derive(Clone, Debug)] -pub struct TestInput { - pub n_validators: usize, - pub n_cores: usize, - pub pov_size: usize, - // This parameter is used to determine how many recoveries we batch in parallel - // similarly to how in practice tranche0 assignments work. - pub vrf_modulo_samples: usize, - // The amount of bandiwdht remote validators have. - pub bandwidth: usize, -} - -impl Default for TestInput { - fn default() -> Self { - Self { - n_validators: 10, - n_cores: 10, - pov_size: 5 * 1024 * 1024, - vrf_modulo_samples: 6, - bandwidth: 15 * 1024 * 1024, - } - } -} - pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { - let input = env.input().clone(); + let config = env.config().clone(); env.send_signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( Hash::repeat_byte(1), @@ -490,7 +479,7 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { let start_marker = Instant::now(); let mut candidate = env.state.candidate(); let mut batch = Vec::new(); - for candidate_num in 0..input.n_cores as u64 { + for candidate_num in 0..config.n_cores as u64 { let (tx, rx) = oneshot::channel(); batch.push(rx); @@ -504,7 +493,7 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { )) .await; - if batch.len() >= input.vrf_modulo_samples { + if batch.len() >= config.vrf_modulo_samples { for rx in std::mem::take(&mut batch) { let _available_data = rx.await.unwrap().unwrap(); } @@ -518,7 +507,7 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { env.send_signal(OverseerSignal::Conclude).await; delay!(5); let duration = start_marker.elapsed().as_millis(); - let tput = ((input.n_cores * input.pov_size) as u128) / duration * 1000; + let tput = ((config.n_cores * config.pov_size) as u128) / duration * 1000; println!("Benchmark completed in {:?}ms", duration); println!("Throughput: {}KiB/s", tput / 1024); } diff --git a/polkadot/node/subsystem-bench/src/availability/network.rs b/polkadot/node/subsystem-bench/src/availability/network.rs index 1889e971cc1e..d6fc175c859b 100644 --- a/polkadot/node/subsystem-bench/src/availability/network.rs +++ b/polkadot/node/subsystem-bench/src/availability/network.rs @@ -122,6 +122,7 @@ mod tests { assert!(total_sent as u128 <= upper_bound); } } + // A network peer emulator struct PeerEmulator { // The queue of requests waiting to be served by the emulator @@ -132,19 +133,32 @@ impl PeerEmulator { pub fn new(bandwidth: usize, spawn_task_handle: SpawnTaskHandle) -> Self { let (actions_tx, mut actions_rx) = tokio::sync::mpsc::unbounded_channel(); - spawn_task_handle.spawn("peer-emulator", "test-environment", async move { - let mut rate_limiter = RateLimit::new(20, bandwidth); - loop { - let maybe_action: Option = actions_rx.recv().await; - if let Some(action) = maybe_action { - let size = action.size(); - rate_limiter.reap(size).await; - action.run().await; - } else { - break + spawn_task_handle + .clone() + .spawn("peer-emulator", "test-environment", async move { + let mut rate_limiter = RateLimit::new(20, bandwidth); + loop { + let maybe_action: Option = actions_rx.recv().await; + if let Some(action) = maybe_action { + let size = action.size(); + rate_limiter.reap(size).await; + if let Some(latency) = action.latency { + spawn_task_handle.spawn( + "peer-emulator-latency", + "test-environment", + async move { + tokio::time::sleep(latency).await; + action.run().await; + }, + ) + } else { + action.run().await; + } + } else { + break + } } - } - }); + }); Self { actions_tx } } @@ -164,11 +178,13 @@ pub struct NetworkAction { size: usize, // Peer index index: usize, + // The amount of time to delay the polling `run` + latency: Option, } impl NetworkAction { - pub fn new(index: usize, run: ActionFuture, size: usize) -> Self { - Self { run, size, index } + pub fn new(index: usize, run: ActionFuture, size: usize, latency: Option) -> Self { + Self { run, size, index, latency } } pub fn size(&self) -> usize { self.size @@ -186,10 +202,6 @@ impl NetworkAction { // Mocks the network bridge and an arbitrary number of connected peer nodes. // Implements network latency, bandwidth and error. pub struct NetworkEmulator { - // Number of peers connected on validation protocol - n_peers: usize, - // The maximum Rx/Tx bandwidth in bytes per second. - bandwidth: usize, // Per peer network emulation peers: Vec, } @@ -197,8 +209,6 @@ pub struct NetworkEmulator { impl NetworkEmulator { pub fn new(n_peers: usize, bandwidth: usize, spawn_task_handle: SpawnTaskHandle) -> Self { Self { - n_peers, - bandwidth, peers: (0..n_peers) .map(|_index| PeerEmulator::new(bandwidth, spawn_task_handle.clone())) .collect::>(), diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 30a9dff02757..52c522726799 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -23,7 +23,7 @@ use prometheus::proto::LabelPair; pub(crate) mod availability; -use availability::{TestEnvironment, TestInput, TestState}; +use availability::{TestConfiguration, TestEnvironment, TestState}; const LOG_TARGET: &str = "subsystem-bench"; /// Define the supported benchmarks targets @@ -60,11 +60,14 @@ impl BenchCli { let runtime = new_runtime(); let registry = Registry::new(); - let state = TestState::new(TestInput::default()); + let test_config = + TestConfiguration::degraded_network_300_validators_60_cores(1024 * 1024); + + let state = TestState::new(test_config); let mut env = TestEnvironment::new(runtime.handle().clone(), state, registry.clone()); - println!("{:?}", env.input()); + println!("{:?}", env.config()); runtime.block_on(availability::bench_chunk_recovery(&mut env)); From b0faa095c88f2084fbacf0ce645c29cadeb872a4 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 8 Nov 2023 12:23:33 +0200 Subject: [PATCH 082/192] Minor fixes Signed-off-by: Alexandru Gheorghe --- .../src/approval_db/v1/tests.rs | 4 ---- .../src/approval_db/v2/tests.rs | 7 +------ .../src/approval_db/v3/migration_helpers.rs | 2 +- .../approval-voting/src/approval_db/v3/mod.rs | 6 ++---- .../src/approval_db/v3/tests.rs | 7 +------ polkadot/node/core/approval-voting/src/lib.rs | 18 ++++++++++++++---- .../approval-voting/src/persisted_entries.rs | 10 +++++----- .../node/core/approval-voting/src/tests.rs | 2 +- .../network/approval-distribution/src/lib.rs | 1 + polkadot/runtime/parachains/src/disputes.rs | 2 ++ .../0007-approval-voting-coalescing.toml | 4 ++-- 11 files changed, 30 insertions(+), 33 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/approval_db/v1/tests.rs b/polkadot/node/core/approval-voting/src/approval_db/v1/tests.rs index 07d8242b772e..b979cb7ef45f 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v1/tests.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v1/tests.rs @@ -40,10 +40,6 @@ fn make_db() -> (DbBackend, Arc) { (DbBackend::new(db_writer.clone(), TEST_CONFIG), db_writer) } -fn make_bitvec(len: usize) -> BitVec { - bitvec::bitvec![u8, BitOrderLsb0; 0; len] -} - fn make_block_entry( block_hash: Hash, parent_hash: Hash, diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs index 1121660fe658..3dbda3639d28 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs @@ -18,7 +18,7 @@ use crate::{ approval_db::{ - common::{DbBackend, StoredBlockRange, *}, + common::{DbBackend, StoredBlockRange, *, migration_helpers::make_bitvec}, v2::*, v3::{load_block_entry_v2, load_candidate_entry_v2}, }, @@ -29,7 +29,6 @@ use polkadot_primitives::{ BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, }; -use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; use polkadot_node_subsystem_util::database::Database; use polkadot_primitives::Id as ParaId; use sp_consensus_slots::Slot; @@ -70,10 +69,6 @@ fn make_block_entry( } } -fn make_bitvec(len: usize) -> BitVec { - bitvec::bitvec![u8, BitOrderLsb0; 0; len] -} - fn make_candidate(para_id: ParaId, relay_parent: Hash) -> CandidateReceipt { let mut c = dummy_candidate_receipt(dummy_hash()); diff --git a/polkadot/node/core/approval-voting/src/approval_db/v3/migration_helpers.rs b/polkadot/node/core/approval-voting/src/approval_db/v3/migration_helpers.rs index 3ffb66015215..ad5e89ef3de8 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v3/migration_helpers.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v3/migration_helpers.rs @@ -116,7 +116,7 @@ pub fn v1_to_latest_sanity_check( Ok(()) } -// Fills the db with dummy data in v1 scheme. +// Fills the db with dummy data in v2 scheme. pub fn v2_fill_test_data( db: Arc, config: Config, diff --git a/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs index 031a51edd64b..3e4f43021952 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs @@ -16,8 +16,7 @@ //! Version 3 of the DB schema. //! -//! Version 3 modifies the our_approval format of `ApprovalEntry` -//! and adds a new field `pending_signatures` for `BlockEntry` +//! Version 3 modifies the `our_approval` format of `ApprovalEntry` //! and adds a new field `pending_signatures` for `BlockEntry` use parity_scale_codec::{Decode, Encode}; @@ -107,8 +106,7 @@ pub struct BlockEntry { } #[derive(Encode, Decode, Debug, Clone, PartialEq)] - -/// Context needed for creating an approval signature for a given candidate. +/// Context needed for creating an approval signature for a given candidate. pub struct CandidateSigningContext { /// The candidate hash, to be included in the signature. pub candidate_hash: CandidateHash, diff --git a/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs b/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs index 7ff6433b5365..29b3373e3f25 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs @@ -18,7 +18,7 @@ use crate::{ approval_db::{ - common::{DbBackend, StoredBlockRange, *}, + common::{DbBackend, StoredBlockRange, *, migration_helpers::make_bitvec}, v3::*, }, backend::{Backend, OverlayedBackend}, @@ -28,7 +28,6 @@ use polkadot_primitives::{ BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, }; -use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; use polkadot_node_subsystem_util::database::Database; use polkadot_primitives::Id as ParaId; use sp_consensus_slots::Slot; @@ -70,10 +69,6 @@ fn make_block_entry( } } -fn make_bitvec(len: usize) -> BitVec { - bitvec::bitvec![u8, BitOrderLsb0; 0; len] -} - fn make_candidate(para_id: ParaId, relay_parent: Hash) -> CandidateReceipt { let mut c = dummy_candidate_receipt(dummy_hash()); diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index f6d843a908e7..4909a4d6ad9a 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -1018,6 +1018,8 @@ where (block_hash, validator_index) = delayed_approvals_timers.select_next_some() => { gum::debug!( target: LOG_TARGET, + ?block_hash, + ?validator_index, "Sign approval for multiple candidates", ); @@ -1740,7 +1742,7 @@ async fn get_approval_signatures_for_candidate( Some(Ok(votes)) => { let votes = votes .into_iter() - .map(|(validator_index, (hash, signed_candidates_indices, signature))| { + .filter_map(|(validator_index, (hash, signed_candidates_indices, signature))| { let candidates_hashes = candidate_indices_to_candidate_hashes.get(&hash); if candidates_hashes.is_none() { @@ -1751,6 +1753,8 @@ async fn get_approval_signatures_for_candidate( ); } + let num_signed_candidates = signed_candidates_indices.len(); + let signed_candidates_hashes: Vec = signed_candidates_indices .into_iter() @@ -1771,7 +1775,15 @@ async fn get_approval_signatures_for_candidate( }) }) .collect(); - (validator_index, (signed_candidates_hashes, signature)) + if num_signed_candidates == signed_candidates_hashes.len() { + Some((validator_index, (signed_candidates_hashes, signature))) + } else { + gum::warn!( + target: LOG_TARGET, + "Possible bug! Could not find all hashes for candidates coming from approval-distribution" + ); + None + } }) .collect(); send_votes(votes) @@ -3476,8 +3488,6 @@ async fn maybe_create_signature( .collect::>>>()?; for mut candidate_entry in candidate_entries { - // let mut candidate_entry = candidate_entry - // .expect("Candidate was scheduled to be signed entry in db should exist; qed"); let approval_entry = candidate_entry.as_mut().and_then(|candidate_entry| { candidate_entry.approval_entry_mut(&block_entry.block_hash()) }); diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index ac3b704c31c2..39c1fffc2ee7 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -597,13 +597,12 @@ impl BlockEntry { } /// Candidate indices for candidates pending signature - fn candidate_indices_pending_signature(&self) -> CandidateBitfield { + fn candidate_indices_pending_signature(&self) -> Option { self.candidates_pending_signature .keys() .map(|val| *val) .collect_vec() - .try_into() - .expect("Fails only of array empty, it can't be, qed") + .try_into().ok() } /// Returns a list of candidates hashes that need need signature created at the current tick: @@ -629,10 +628,11 @@ impl BlockEntry { self.num_candidates_pending_signature() >= max_approval_coalesce_count as usize { ( + self.candidate_indices_pending_signature().and_then(|candidate_indices| Some(( self.candidate_hashes_pending_signature(), - self.candidate_indices_pending_signature(), - )), + candidate_indices, + ))), Some(sign_no_later_than_tick), ) } else { diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index a27315d416e3..941c2ec5a32b 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -2028,7 +2028,7 @@ fn test_signing_a_single_candidate_is_backwards_compatible() { .map(|candidate_descriptor| candidate_descriptor.hash()) .collect_vec(); - let first_descriptor = candidate_descriptors.first().expect("TODO"); + let first_descriptor = candidate_descriptors.first().unwrap(); let candidate_hash = first_descriptor.hash(); diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index cf70d253d48b..2ebf050849f1 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -483,6 +483,7 @@ impl PeerKnowledge { (sent, notknown_by_peer) } + /// Marks a list of assignments as sent to the peer fn mark_sent(&mut self, assignments: &Vec<(IndirectAssignmentCertV2, CandidateBitfield)>) { for assignment in assignments { self.sent.insert( diff --git a/polkadot/runtime/parachains/src/disputes.rs b/polkadot/runtime/parachains/src/disputes.rs index fce5dc24d28d..ca275cf6442c 100644 --- a/polkadot/runtime/parachains/src/disputes.rs +++ b/polkadot/runtime/parachains/src/disputes.rs @@ -1017,6 +1017,8 @@ impl Pallet { set.session, statement, signature, + // This is here to prevent malicious nodes of generating `ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates` + // before that is enabled, via setting `max_approval_coalesce_count` in the parachain host config. config.approval_voting_params.max_approval_coalesce_count > 1, ) { log::warn!("Failed to check dispute signature"); diff --git a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml index f84a87f918bd..19c7015403d7 100644 --- a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml +++ b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml @@ -6,7 +6,7 @@ default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" [relaychain.genesis.runtimeGenesis.patch.configuration.config] - needed_approvals = 5 + needed_approvals = 4 relay_vrf_modulo_samples = 6 [relaychain.genesis.runtimeGenesis.patch.configuration.config.approval_voting_params] @@ -19,7 +19,7 @@ requests = { memory = "2G", cpu = "1" } [[relaychain.node_groups]] name = "alice" args = [ "-lparachain=trace,runtime=debug" ] - count = 21 + count = 13 [[parachains]] id = 2000 From 7ca4dbadf6d24bd94abb0a06bf25d3aaacea8e9f Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 8 Nov 2023 13:15:02 +0200 Subject: [PATCH 083/192] support multiple pov sizes Signed-off-by: Andrei Sandu --- .../src/availability/configuration.rs | 44 ++-- .../subsystem-bench/src/availability/mod.rs | 222 ++++++++++++------ .../subsystem-bench/src/subsystem-bench.rs | 6 +- 3 files changed, 176 insertions(+), 96 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/configuration.rs b/polkadot/node/subsystem-bench/src/availability/configuration.rs index 14e8f55128d9..3df496ad0428 100644 --- a/polkadot/node/subsystem-bench/src/availability/configuration.rs +++ b/polkadot/node/subsystem-bench/src/availability/configuration.rs @@ -28,80 +28,92 @@ pub struct PeerLatency { /// The test input parameters #[derive(Clone, Debug)] pub struct TestConfiguration { + /// Configuration for the `availability-recovery` subsystem. + pub use_fast_path: bool, /// Number of validators pub n_validators: usize, /// Number of cores pub n_cores: usize, /// The PoV size - pub pov_size: usize, + pub pov_sizes: Vec, /// This parameter is used to determine how many recoveries we batch in parallel - /// similarly to how in practice tranche0 assignments work. - pub vrf_modulo_samples: usize, + /// to simulate tranche0 recoveries. + pub max_parallel_recoveries: usize, /// The amount of bandiwdht remote validators have. pub bandwidth: usize, /// Optional peer emulation latency pub latency: Option, + /// Error probability + pub error: usize, } impl Default for TestConfiguration { fn default() -> Self { Self { + use_fast_path: false, n_validators: 10, n_cores: 10, - pov_size: 5 * 1024 * 1024, - vrf_modulo_samples: 6, - bandwidth: 15 * 1024 * 1024, + pov_sizes: vec![5 * 1024 * 1024], + max_parallel_recoveries: 6, + bandwidth: 60 * 1024 * 1024, latency: None, + error: 0, } } } impl TestConfiguration { /// An unconstrained standard configuration matching Polkadot/Kusama - pub fn unconstrained_300_validators_60_cores(pov_size: usize) -> TestConfiguration { + pub fn unconstrained_300_validators_60_cores(pov_sizes: Vec) -> TestConfiguration { Self { + use_fast_path: false, n_validators: 300, n_cores: 60, - pov_size, - vrf_modulo_samples: 6, + pov_sizes, + max_parallel_recoveries: 20, // HW specs node bandwidth bandwidth: 60 * 1024 * 1024, // No latency latency: None, + error: 0, } } /// Polkadot/Kusama configuration with typical latency constraints. - pub fn healthy_network_300_validators_60_cores(pov_size: usize) -> TestConfiguration { + pub fn healthy_network_300_validators_60_cores(pov_sizes: Vec) -> TestConfiguration { Self { + use_fast_path: true, n_validators: 300, n_cores: 60, - pov_size, - vrf_modulo_samples: 6, + pov_sizes, + max_parallel_recoveries: 6, // HW specs node bandwidth bandwidth: 60 * 1024 * 1024, latency: Some(PeerLatency { min_latency: Duration::from_millis(1), max_latency: Duration::from_millis(50), }), + error: 5, } } /// Polkadot/Kusama configuration with degraded due to latencies. /// TODO: implement errors. - pub fn degraded_network_300_validators_60_cores(pov_size: usize) -> TestConfiguration { + pub fn degraded_network_300_validators_60_cores(pov_sizes: Vec) -> TestConfiguration { Self { + use_fast_path: true, n_validators: 300, n_cores: 60, - pov_size, - vrf_modulo_samples: 6, + pov_sizes, + max_parallel_recoveries: 6, // HW specs node bandwidth bandwidth: 60 * 1024 * 1024, // A range of latencies to expect in a degraded network latency: Some(PeerLatency { min_latency: Duration::from_millis(1), - max_latency: Duration::from_millis(1000), + max_latency: Duration::from_millis(500), }), + error: 30, } } } diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 6c0c41c86c0f..dcfeb4287780 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -15,6 +15,7 @@ // along with Polkadot. If not, see . use std::{ + collections::HashMap, sync::Arc, time::{Duration, Instant}, }; @@ -23,7 +24,6 @@ use futures::{ channel::{mpsc, oneshot}, FutureExt, SinkExt, }; -use futures_timer::Delay; use polkadot_node_metrics::metrics::Metrics; use polkadot_availability_recovery::AvailabilityRecoverySubsystem; @@ -49,7 +49,6 @@ use polkadot_node_subsystem::{ const LOG_TARGET: &str = "subsystem-bench::availability"; -use polkadot_erasure_coding::recovery_threshold; use polkadot_node_primitives::{AvailableData, ErasureChunk}; use polkadot_node_subsystem_test_helpers::{ @@ -57,7 +56,7 @@ use polkadot_node_subsystem_test_helpers::{ }; use polkadot_node_subsystem_util::TimeoutExt; use polkadot_primitives::{ - AuthorityDiscoveryId, CandidateReceipt, GroupIndex, Hash, HeadData, IndexedVec, + AuthorityDiscoveryId, CandidateHash, CandidateReceipt, GroupIndex, Hash, HeadData, IndexedVec, PersistedValidationData, SessionIndex, SessionInfo, ValidatorId, ValidatorIndex, }; use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; @@ -97,8 +96,11 @@ impl TestEnvironment { // We use prometheus metrics to collect per job task poll time and subsystem metrics. pub fn new(runtime: tokio::runtime::Handle, state: TestState, registry: Registry) -> Self { let task_manager: TaskManager = TaskManager::new(runtime.clone(), Some(®istry)).unwrap(); - let (instance, virtual_overseer) = - AvailabilityRecoverySubsystemInstance::new(®istry, task_manager.spawn_handle()); + let (instance, virtual_overseer) = AvailabilityRecoverySubsystemInstance::new( + ®istry, + task_manager.spawn_handle(), + state.config().use_fast_path, + ); // Copy sender for later when we need to inject messages in to the subsystem. let to_subsystem = virtual_overseer.tx.clone(); @@ -132,16 +134,60 @@ impl TestEnvironment { } } + /// Generate a random error based on `probability`. + /// `probability` should be a number between 0 and 100. + fn random_error(probability: usize) -> bool { + Uniform::from(0..=99).sample(&mut thread_rng()) < probability + } + pub fn respond_to_send_request(state: &mut TestState, request: Requests) -> NetworkAction { match request { Requests::ChunkFetchingV1(outgoing_request) => { let validator_index = outgoing_request.payload.index.0 as usize; - let chunk: ChunkResponse = state.chunks[validator_index].clone().into(); + let chunk: ChunkResponse = + state.chunks.get(&outgoing_request.payload.candidate_hash).unwrap() + [validator_index] + .clone() + .into(); let size = chunk.encoded_size(); + + let response = if Self::random_error(state.config().error) { + Err(RequestFailure::Network(OutboundFailure::ConnectionClosed)) + } else { + Ok(req_res::v1::ChunkFetchingResponse::from(Some(chunk)).encode()) + }; + let future = async move { - let _ = outgoing_request - .pending_response - .send(Ok(req_res::v1::ChunkFetchingResponse::from(Some(chunk)).encode())); + let _ = outgoing_request.pending_response.send(response); + } + .boxed(); + + NetworkAction::new( + validator_index, + future, + size, + // Generate a random latency based on configuration. + Self::random_latency(state.config().latency.as_ref()), + ) + }, + Requests::AvailableDataFetchingV1(outgoing_request) => { + // TODO: do better, by implementing diff authority ids and mapping network actions + // to authority id, + let validator_index = + Uniform::from(0..state.config().n_validators).sample(&mut thread_rng()); + let available_data = + state.candidates.get(&outgoing_request.payload.candidate_hash).unwrap().clone(); + let size = available_data.encoded_size(); + + let response = if Self::random_error(state.config().error) { + Err(RequestFailure::Network(OutboundFailure::ConnectionClosed)) + } else { + Ok(req_res::v1::AvailableDataFetchingResponse::from(Some(available_data)) + .encode()) + }; + + let future = async move { + let _ = outgoing_request.pending_response.send(response); } .boxed(); @@ -192,14 +238,14 @@ impl TestEnvironment { // TODO: Simulate av store load by delaying the response. state.respond_none_to_available_data_query(tx).await; }, - AllMessages::AvailabilityStore(AvailabilityStoreMessage::QueryAllChunks(_candidate_hash, tx)) => { + AllMessages::AvailabilityStore(AvailabilityStoreMessage::QueryAllChunks(candidate_hash, tx)) => { // Test env: We always have our own chunk. - state.respond_to_query_all_request(|index| index == state.validator_index.0 as usize, tx).await; + state.respond_to_query_all_request(candidate_hash, |index| index == state.validator_index.0 as usize, tx).await; }, AllMessages::AvailabilityStore( - AvailabilityStoreMessage::QueryChunkSize(_, tx) + AvailabilityStoreMessage::QueryChunkSize(candidate_hash, tx) ) => { - let chunk_size = state.chunks[0].encoded_size(); + let chunk_size = state.chunks.get(&candidate_hash).unwrap()[0].encoded_size(); let _ = tx.send(Some(chunk_size)); } AllMessages::RuntimeApi(RuntimeApiMessage::Request( @@ -250,15 +296,24 @@ impl AvailabilityRecoverySubsystemInstance { pub fn new( registry: &Registry, spawn_task_handle: SpawnTaskHandle, + use_fast_path: bool, ) -> (Self, TestSubsystemContextHandle) { let (context, virtual_overseer) = make_buffered_subsystem_context(spawn_task_handle.clone(), 4096 * 4); let (collation_req_receiver, req_cfg) = IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); - let subsystem = AvailabilityRecoverySubsystem::with_chunks_only( - collation_req_receiver, - Metrics::try_register(®istry).unwrap(), - ); + + let subsystem = if use_fast_path { + AvailabilityRecoverySubsystem::with_fast_path( + collation_req_receiver, + Metrics::try_register(®istry).unwrap(), + ) + } else { + AvailabilityRecoverySubsystem::with_chunks_only( + collation_req_receiver, + Metrics::try_register(®istry).unwrap(), + ) + }; let spawned_subsystem = subsystem.start(context); let subsystem_future = async move { @@ -282,12 +337,6 @@ const TIMEOUT: Duration = Duration::from_millis(300); // This should eventually be a test parameter. const MAX_TIME_OF_FLIGHT: Duration = Duration::from_millis(5000); -macro_rules! delay { - ($delay:expr) => { - Delay::new(Duration::from_millis($delay)).await; - }; -} - use sp_keyring::Sr25519Keyring; use crate::availability::network::NetworkAction; @@ -301,14 +350,15 @@ pub struct TestState { validator_authority_id: Vec, // The test node validator index. validator_index: ValidatorIndex, - candidate: CandidateReceipt, + // Per core candidates receipts. + candidate_receipts: Vec, session_index: SessionIndex, persisted_validation_data: PersistedValidationData, + /// A per size pov mapping to available data. + candidates: HashMap, - available_data: AvailableData, - chunks: Vec, - invalid_chunks: Vec, + chunks: HashMap>, config: TestConfiguration, } @@ -317,12 +367,8 @@ impl TestState { &self.config } - fn candidate(&self) -> CandidateReceipt { - self.candidate.clone() - } - - async fn respond_to_available_data_query(&self, tx: oneshot::Sender>) { - let _ = tx.send(Some(self.available_data.clone())); + fn candidate(&self, candidate_index: usize) -> CandidateReceipt { + self.candidate_receipts.get(candidate_index).unwrap().clone() } async fn respond_none_to_available_data_query( @@ -337,9 +383,7 @@ impl TestState { validators: self.validator_public.clone(), discovery_keys: self.validator_authority_id.clone(), // all validators in the same group. - validator_groups: IndexedVec::>::from(vec![(0..self - .validators - .len()) + validator_groups: IndexedVec::>::from(vec![(0..5) .map(|i| ValidatorIndex(i as _)) .collect()]), assignment_keys: vec![], @@ -356,10 +400,18 @@ impl TestState { } async fn respond_to_query_all_request( &self, + candidate_hash: CandidateHash, send_chunk: impl Fn(usize) -> bool, tx: oneshot::Sender>, ) { - let v = self.chunks.iter().filter(|c| send_chunk(c.index.0 as usize)).cloned().collect(); + let v = self + .chunks + .get(&candidate_hash) + .unwrap() + .iter() + .filter(|c| send_chunk(c.index.0 as usize)) + .cloned() + .collect(); let _ = tx.send(v); } @@ -370,13 +422,15 @@ impl TestState { .map(|_v| Sr25519Keyring::Alice) .collect::>(); - let mut candidate = dummy_candidate_receipt(dummy_hash()); let validator_public = validator_pubkeys(&validators); let validator_authority_id = validator_authority_id(&validators); let validator_index = ValidatorIndex(0); - + let mut pov_size_to_candidate = HashMap::new(); + let mut chunks = HashMap::new(); + let mut candidates = HashMap::new(); let session_index = 10; + // we use it for all candidates. let persisted_validation_data = PersistedValidationData { parent_head: HeadData(vec![7, 8, 9]), relay_parent_number: Default::default(), @@ -384,49 +438,57 @@ impl TestState { relay_parent_storage_root: Default::default(), }; - // A 5MB PoV. - let pov = PoV { block_data: BlockData(vec![42; config.pov_size]) }; + // Create initial candidate receipts + let mut candidate_receipts = config + .pov_sizes + .iter() + .map(|_index| dummy_candidate_receipt(dummy_hash())) + .collect::>(); - let available_data = AvailableData { - validation_data: persisted_validation_data.clone(), - pov: Arc::new(pov), - }; + for (index, pov_size) in config.pov_sizes.iter().enumerate() { + let mut candidate = &mut candidate_receipts[index]; + // a hack to make candidate unique. + candidate.descriptor.relay_parent = Hash::from_low_u64_be(index as u64); - let (chunks, erasure_root) = derive_erasure_chunks_with_proofs_and_root( - validators.len(), - &available_data, - |_, _| {}, - ); - // Mess around: - let invalid_chunks = chunks - .iter() - .cloned() - .map(|mut chunk| { - if chunk.chunk.len() >= 2 && chunk.chunk[0] != chunk.chunk[1] { - chunk.chunk[0] = chunk.chunk[1]; - } else if chunk.chunk.len() >= 1 { - chunk.chunk[0] = !chunk.chunk[0]; - } else { - chunk.proof = Proof::dummy_proof(); - } - chunk - }) - .collect(); - debug_assert_ne!(chunks, invalid_chunks); + // We reuse candidates of same size, to speed up the test startup. + let (erasure_root, available_data, new_chunks) = + pov_size_to_candidate.entry(pov_size).or_insert_with(|| { + let pov = PoV { block_data: BlockData(vec![index as u8; *pov_size]) }; + + let available_data = AvailableData { + validation_data: persisted_validation_data.clone(), + pov: Arc::new(pov), + }; + + let (new_chunks, erasure_root) = derive_erasure_chunks_with_proofs_and_root( + validators.len(), + &available_data, + |_, _| {}, + ); + + candidate.descriptor.erasure_root = erasure_root; - candidate.descriptor.erasure_root = erasure_root; + chunks.insert(candidate.hash(), new_chunks.clone()); + candidates.insert(candidate.hash(), available_data.clone()); + + (erasure_root, available_data, new_chunks) + }); + + candidate.descriptor.erasure_root = *erasure_root; + candidates.insert(candidate.hash(), available_data.clone()); + chunks.insert(candidate.hash(), new_chunks.clone()); + } Self { validators, validator_public, validator_authority_id, validator_index, - candidate, + candidate_receipts, session_index, persisted_validation_data, - available_data, + candidates, chunks, - invalid_chunks, config, } } @@ -467,6 +529,9 @@ fn derive_erasure_chunks_with_proofs_and_root( (erasure_chunks, root) } +pub async fn bench_with_chunks_if_pov_large(env: &mut TestEnvironment) {} + +pub async fn bench_inner(env: &mut TestEnvironment) {} pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { let config = env.config().clone(); @@ -477,14 +542,14 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { .await; let start_marker = Instant::now(); - let mut candidate = env.state.candidate(); let mut batch = Vec::new(); + let mut availability_bytes = 0; for candidate_num in 0..config.n_cores as u64 { + let candidate = env.state.candidate_receipts[candidate_num as usize].clone(); + let (tx, rx) = oneshot::channel(); batch.push(rx); - candidate.descriptor.relay_parent = Hash::from_low_u64_be(candidate_num); - env.send_message(AvailabilityRecoveryMessage::RecoverAvailableData( candidate.clone(), 1, @@ -493,21 +558,22 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { )) .await; - if batch.len() >= config.vrf_modulo_samples { + if batch.len() >= config.max_parallel_recoveries { for rx in std::mem::take(&mut batch) { - let _available_data = rx.await.unwrap().unwrap(); + let available_data = rx.await.unwrap().unwrap(); + availability_bytes += available_data.encoded_size(); } } } for rx in std::mem::take(&mut batch) { - let _available_data = rx.await.unwrap().unwrap(); + let available_data = rx.await.unwrap().unwrap(); + availability_bytes += available_data.encoded_size(); } env.send_signal(OverseerSignal::Conclude).await; - delay!(5); let duration = start_marker.elapsed().as_millis(); - let tput = ((config.n_cores * config.pov_size) as u128) / duration * 1000; + let tput = (availability_bytes as u128) / duration * 1000; println!("Benchmark completed in {:?}ms", duration); println!("Throughput: {}KiB/s", tput / 1024); } diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 52c522726799..2a5edf7cf197 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -60,8 +60,10 @@ impl BenchCli { let runtime = new_runtime(); let registry = Registry::new(); - let test_config = - TestConfiguration::degraded_network_300_validators_60_cores(1024 * 1024); + let mut pov_sizes = Vec::new(); + pov_sizes.append(&mut vec![1024 * 1024 * 5; 60]); + + let test_config = TestConfiguration::unconstrained_300_validators_60_cores(pov_sizes); let state = TestState::new(test_config); From 0430b5b909b84abc5bb4c078924ce46e944dc18a Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 8 Nov 2023 15:07:22 +0200 Subject: [PATCH 084/192] new metric in recovery and more testing Signed-off-by: Andrei Sandu --- Cargo.lock | 1 + .../availability-recovery/src/metrics.rs | 17 +++++++++++--- .../network/availability-recovery/src/task.rs | 3 ++- polkadot/node/subsystem-bench/Cargo.toml | 2 +- .../src/availability/configuration.rs | 22 +++++++++++++++---- .../subsystem-bench/src/availability/mod.rs | 10 +++++++-- .../subsystem-bench/src/subsystem-bench.rs | 14 ++++++++++-- .../node/subsystem-test-helpers/src/lib.rs | 12 ++++++---- 8 files changed, 64 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4645aeee6aab..5b54745cdcc3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13022,6 +13022,7 @@ dependencies = [ "sp-core", "sp-keyring", "sp-keystore", + "substrate-prometheus-endpoint", "tokio", "tracing-gum", ] diff --git a/polkadot/node/network/availability-recovery/src/metrics.rs b/polkadot/node/network/availability-recovery/src/metrics.rs index aa7216739507..d82a8f9ae5fa 100644 --- a/polkadot/node/network/availability-recovery/src/metrics.rs +++ b/polkadot/node/network/availability-recovery/src/metrics.rs @@ -29,7 +29,10 @@ struct MetricsInner { /// /// Gets incremented on each sent chunk requests. chunk_requests_issued: Counter, - + /// Total number of bytes recovered + /// + /// Gets incremented on each succesful recovery + recovered_bytes_total: Counter, /// A counter for finished chunk requests. /// /// Split by result: @@ -133,9 +136,10 @@ impl Metrics { } /// A full recovery succeeded. - pub fn on_recovery_succeeded(&self) { + pub fn on_recovery_succeeded(&self, bytes: usize) { if let Some(metrics) = &self.0 { - metrics.full_recoveries_finished.with_label_values(&["success"]).inc() + metrics.full_recoveries_finished.with_label_values(&["success"]).inc(); + metrics.recovered_bytes_total.inc_by(bytes as u64) } } @@ -171,6 +175,13 @@ impl metrics::Metrics for Metrics { )?, registry, )?, + recovered_bytes_total: prometheus::register( + Counter::new( + "polkadot_parachain_availability_recovery_bytes_total", + "Total number of bytes recovered", + )?, + registry, + )?, chunk_requests_finished: prometheus::register( CounterVec::new( Opts::new( diff --git a/polkadot/node/network/availability-recovery/src/task.rs b/polkadot/node/network/availability-recovery/src/task.rs index d5bc2da84944..9ed911f3b5a7 100644 --- a/polkadot/node/network/availability-recovery/src/task.rs +++ b/polkadot/node/network/availability-recovery/src/task.rs @@ -23,6 +23,7 @@ use crate::{ LOG_TARGET, }; use futures::{channel::oneshot, SinkExt}; +use parity_scale_codec::Encode; #[cfg(not(test))] use polkadot_node_network_protocol::request_response::CHUNK_REQUEST_TIMEOUT; use polkadot_node_network_protocol::request_response::{ @@ -426,7 +427,7 @@ where return Err(err) }, Ok(data) => { - self.params.metrics.on_recovery_succeeded(); + self.params.metrics.on_recovery_succeeded(data.encoded_size()); return Ok(data) }, } diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index 7408397f930c..2de978234a63 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -47,7 +47,7 @@ sc-service = { path = "../../../substrate/client/service" } polkadot-node-metrics = { path = "../metrics" } polkadot-primitives-test-helpers = { path = "../../primitives/test-helpers" } -# prometheus = { package = "substrate-prometheus-endpoint", path = "../../../substrate/utils/prometheus" } +prometheus_endpoint = { package = "substrate-prometheus-endpoint", path = "../../../substrate/utils/prometheus" } prometheus = { version = "0.13.0", default-features = false } [features] diff --git a/polkadot/node/subsystem-bench/src/availability/configuration.rs b/polkadot/node/subsystem-bench/src/availability/configuration.rs index 3df496ad0428..9a93bf12e114 100644 --- a/polkadot/node/subsystem-bench/src/availability/configuration.rs +++ b/polkadot/node/subsystem-bench/src/availability/configuration.rs @@ -65,12 +65,26 @@ impl Default for TestConfiguration { impl TestConfiguration { /// An unconstrained standard configuration matching Polkadot/Kusama pub fn unconstrained_300_validators_60_cores(pov_sizes: Vec) -> TestConfiguration { + Self { + use_fast_path: false, + n_validators: 300, + n_cores: 100, + pov_sizes, + max_parallel_recoveries: 100, + // HW specs node bandwidth + bandwidth: 60 * 1024 * 1024, + // No latency + latency: None, + error: 0, + } + } + pub fn unconstrained_1000_validators_60_cores(pov_sizes: Vec) -> TestConfiguration { Self { use_fast_path: false, n_validators: 300, n_cores: 60, pov_sizes, - max_parallel_recoveries: 20, + max_parallel_recoveries: 30, // HW specs node bandwidth bandwidth: 60 * 1024 * 1024, // No latency @@ -101,11 +115,11 @@ impl TestConfiguration { /// TODO: implement errors. pub fn degraded_network_300_validators_60_cores(pov_sizes: Vec) -> TestConfiguration { Self { - use_fast_path: true, + use_fast_path: false, n_validators: 300, - n_cores: 60, + n_cores: 100, pov_sizes, - max_parallel_recoveries: 6, + max_parallel_recoveries: 20, // HW specs node bandwidth bandwidth: 60 * 1024 * 1024, // A range of latencies to expect in a degraded network diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index dcfeb4287780..8f8eca104385 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -298,8 +298,11 @@ impl AvailabilityRecoverySubsystemInstance { spawn_task_handle: SpawnTaskHandle, use_fast_path: bool, ) -> (Self, TestSubsystemContextHandle) { - let (context, virtual_overseer) = - make_buffered_subsystem_context(spawn_task_handle.clone(), 4096 * 4); + let (context, virtual_overseer) = make_buffered_subsystem_context( + spawn_task_handle.clone(), + 4096 * 4, + "availability-recovery", + ); let (collation_req_receiver, req_cfg) = IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); @@ -558,6 +561,7 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { )) .await; + // TODO: select between futures unordered of rx await and timer to send next request. if batch.len() >= config.max_parallel_recoveries { for rx in std::mem::take(&mut batch) { let available_data = rx.await.unwrap().unwrap(); @@ -576,4 +580,6 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { let tput = (availability_bytes as u128) / duration * 1000; println!("Benchmark completed in {:?}ms", duration); println!("Throughput: {}KiB/s", tput / 1024); + + tokio::time::sleep(Duration::from_secs(1)).await; } diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 2a5edf7cf197..4dc0936291b1 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -20,6 +20,7 @@ use clap::Parser; use color_eyre::eyre; use prometheus::proto::LabelPair; +use std::net::{Ipv4Addr, SocketAddr}; pub(crate) mod availability; @@ -59,16 +60,25 @@ impl BenchCli { let runtime = new_runtime(); let registry = Registry::new(); + let registry_clone = registry.clone(); let mut pov_sizes = Vec::new(); - pov_sizes.append(&mut vec![1024 * 1024 * 5; 60]); + pov_sizes.append(&mut vec![1024 * 1024; 100]); - let test_config = TestConfiguration::unconstrained_300_validators_60_cores(pov_sizes); + let test_config = TestConfiguration::unconstrained_1000_validators_60_cores(pov_sizes); let state = TestState::new(test_config); let mut env = TestEnvironment::new(runtime.handle().clone(), state, registry.clone()); + let handle = runtime.spawn(async move { + prometheus_endpoint::init_prometheus( + SocketAddr::new(std::net::IpAddr::V4(Ipv4Addr::LOCALHOST), 9999), + registry_clone, + ) + .await + }); + println!("{:?}", env.config()); runtime.block_on(availability::bench_chunk_recovery(&mut env)); diff --git a/polkadot/node/subsystem-test-helpers/src/lib.rs b/polkadot/node/subsystem-test-helpers/src/lib.rs index 3f92513498c4..5393ccafa6f3 100644 --- a/polkadot/node/subsystem-test-helpers/src/lib.rs +++ b/polkadot/node/subsystem-test-helpers/src/lib.rs @@ -187,6 +187,7 @@ pub struct TestSubsystemContext { tx: TestSubsystemSender, rx: mpsc::Receiver>, spawn: S, + name: &'static str, } #[async_trait::async_trait] @@ -223,7 +224,7 @@ where name: &'static str, s: Pin + Send>>, ) -> SubsystemResult<()> { - self.spawn.spawn(name, None, s); + self.spawn.spawn(name, Some(self.name), s); Ok(()) } @@ -232,7 +233,7 @@ where name: &'static str, s: Pin + Send>>, ) -> SubsystemResult<()> { - self.spawn.spawn_blocking(name, None, s); + self.spawn.spawn_blocking(name, Some(self.name), s); Ok(()) } @@ -292,8 +293,9 @@ impl TestSubsystemContextHandle { /// of the tests. pub fn make_subsystem_context( spawner: S, + name: &'static str, ) -> (TestSubsystemContext>, TestSubsystemContextHandle) { - make_buffered_subsystem_context(spawner, 0) + make_buffered_subsystem_context(spawner, 0, name) } /// Make a test subsystem context with buffered overseer channel. Some tests (e.g. @@ -302,6 +304,7 @@ pub fn make_subsystem_context( pub fn make_buffered_subsystem_context( spawner: S, buffer_size: usize, + name: &'static str, ) -> (TestSubsystemContext>, TestSubsystemContextHandle) { let (overseer_tx, overseer_rx) = mpsc::channel(buffer_size); let (all_messages_tx, all_messages_rx) = mpsc::unbounded(); @@ -311,6 +314,7 @@ pub fn make_buffered_subsystem_context( tx: TestSubsystemSender { tx: all_messages_tx }, rx: overseer_rx, spawn: SpawnGlue(spawner), + name, }, TestSubsystemContextHandle { tx: overseer_tx, rx: all_messages_rx }, ) @@ -332,7 +336,7 @@ pub fn subsystem_test_harness( Test: Future, { let pool = TaskExecutor::new(); - let (context, handle) = make_subsystem_context(pool); + let (context, handle) = make_subsystem_context(pool, "default"); let overseer = overseer_factory(handle); let test = test_factory(context); From 027bcd862eef7d2f776ceb0d2bf3dc11ef490b5c Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 9 Nov 2023 23:15:01 +0200 Subject: [PATCH 085/192] CLI update and fixes Signed-off-by: Andrei Sandu --- Cargo.lock | 11 ++ cumulus/pallets/xcmp-queue/src/tests.rs | 22 ++- .../network/availability-recovery/Cargo.toml | 4 + .../network/availability-recovery/src/lib.rs | 10 +- polkadot/node/subsystem-bench/Cargo.toml | 4 +- .../src/availability/configuration.rs | 94 ++++++------ .../subsystem-bench/src/availability/mod.rs | 140 ++++++++++++------ .../subsystem-bench/src/subsystem-bench.rs | 125 ++++++++++++++-- 8 files changed, 292 insertions(+), 118 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b54745cdcc3..05355cad0e2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2561,6 +2561,15 @@ dependencies = [ "clap_derive 4.4.2", ] +[[package]] +name = "clap-num" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488557e97528174edaa2ee268b23a809e0c598213a4bbcb4f34575a46fda147e" +dependencies = [ + "num-traits", +] + [[package]] name = "clap_builder" version = "4.4.6" @@ -11715,6 +11724,7 @@ dependencies = [ "sp-core", "sp-keyring", "thiserror", + "tokio", "tracing-gum", ] @@ -12997,6 +13007,7 @@ dependencies = [ "assert_matches", "async-trait", "clap 4.4.6", + "clap-num", "color-eyre", "env_logger 0.9.3", "futures", diff --git a/cumulus/pallets/xcmp-queue/src/tests.rs b/cumulus/pallets/xcmp-queue/src/tests.rs index cf6d947609d2..bab7e92ca2de 100644 --- a/cumulus/pallets/xcmp-queue/src/tests.rs +++ b/cumulus/pallets/xcmp-queue/src/tests.rs @@ -410,9 +410,11 @@ fn verify_fee_factor_increase_and_decrease() { assert_eq!(DeliveryFeeFactor::::get(sibling_para_id), initial); // Sending the message right now is cheap - let (_, delivery_fees) = validate_send::(destination, xcm.clone()) - .expect("message can be sent; qed"); - let Fungible(delivery_fee_amount) = delivery_fees.inner()[0].fun else { unreachable!("asset is fungible; qed"); }; + let (_, delivery_fees) = + validate_send::(destination, xcm.clone()).expect("message can be sent; qed"); + let Fungible(delivery_fee_amount) = delivery_fees.inner()[0].fun else { + unreachable!("asset is fungible; qed"); + }; assert_eq!(delivery_fee_amount, 402_000_000); let smaller_xcm = Xcm(vec![ClearOrigin; 30]); @@ -422,19 +424,23 @@ fn verify_fee_factor_increase_and_decrease() { assert_ok!(send_xcm::(destination, xcm.clone())); // Size 520 assert_eq!(DeliveryFeeFactor::::get(sibling_para_id), FixedU128::from_float(1.05)); - for _ in 0..12 { // We finish at size 929 + for _ in 0..12 { + // We finish at size 929 assert_ok!(send_xcm::(destination, smaller_xcm.clone())); } assert!(DeliveryFeeFactor::::get(sibling_para_id) > FixedU128::from_float(1.88)); // Sending the message right now is expensive - let (_, delivery_fees) = validate_send::(destination, xcm.clone()) - .expect("message can be sent; qed"); - let Fungible(delivery_fee_amount) = delivery_fees.inner()[0].fun else { unreachable!("asset is fungible; qed"); }; + let (_, delivery_fees) = + validate_send::(destination, xcm.clone()).expect("message can be sent; qed"); + let Fungible(delivery_fee_amount) = delivery_fees.inner()[0].fun else { + unreachable!("asset is fungible; qed"); + }; assert_eq!(delivery_fee_amount, 758_030_955); // Fee factor only decreases in `take_outbound_messages` - for _ in 0..5 { // We take 5 100 byte pages + for _ in 0..5 { + // We take 5 100 byte pages XcmpQueue::take_outbound_messages(1); } assert!(DeliveryFeeFactor::::get(sibling_para_id) < FixedU128::from_float(1.72)); diff --git a/polkadot/node/network/availability-recovery/Cargo.toml b/polkadot/node/network/availability-recovery/Cargo.toml index 42c3abef547b..5f3df09c2bd9 100644 --- a/polkadot/node/network/availability-recovery/Cargo.toml +++ b/polkadot/node/network/availability-recovery/Cargo.toml @@ -7,6 +7,7 @@ license.workspace = true [dependencies] futures = "0.3.21" +tokio = "1.24.2" schnellru = "0.2.1" rand = "0.8.5" fatality = "0.0.6" @@ -36,3 +37,6 @@ sc-network = { path = "../../../../substrate/client/network" } polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } polkadot-primitives-test-helpers = { path = "../../../primitives/test-helpers" } + +[features] +subsystem-benchmarks = [] \ No newline at end of file diff --git a/polkadot/node/network/availability-recovery/src/lib.rs b/polkadot/node/network/availability-recovery/src/lib.rs index 156a8cbbc82e..ffb634ad76e2 100644 --- a/polkadot/node/network/availability-recovery/src/lib.rs +++ b/polkadot/node/network/availability-recovery/src/lib.rs @@ -617,12 +617,9 @@ impl AvailabilityRecoverySubsystem { .into_iter() .cycle(); - gum::debug!("Subsystem running"); loop { let recv_req = req_receiver.recv(|| vec![COST_INVALID_REQUEST]).fuse(); pin_mut!(recv_req); - gum::debug!("waiting for message"); - futures::select! { erasure_task = erasure_task_rx.next() => { match erasure_task { @@ -729,6 +726,8 @@ impl AvailabilityRecoverySubsystem { } } output = state.ongoing_recoveries.select_next_some() => { + // No caching for benchmark. + #[cfg(not(feature = "subsystem-benchmarks"))] if let Some((candidate_hash, result)) = output { if let Ok(recovery) = CachedRecovery::try_from(result) { state.availability_lru.insert(candidate_hash, recovery); @@ -829,5 +828,10 @@ async fn erasure_task_thread( break }, } + + // In benchmarks this is a very hot loop not yielding at all. + // To update promehteus metrics for the task we need to yield. + #[cfg(feature = "subsystem-benchmarks")] + tokio::task::yield_now().await; } } diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index 2de978234a63..01b992d15fc6 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -22,7 +22,7 @@ polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-primitives = { path = "../primitives" } polkadot-primitives = { path = "../../primitives" } polkadot-node-network-protocol = { path = "../network/protocol" } -polkadot-availability-recovery = { path = "../network/availability-recovery" } +polkadot-availability-recovery = { path = "../network/availability-recovery", features=["subsystem-benchmarks"]} color-eyre = { version = "0.6.1", default-features = false } assert_matches = "1.5" async-trait = "0.1.57" @@ -38,7 +38,7 @@ env_logger = "0.9.0" rand = "0.8.5" parity-scale-codec = { version = "3.6.1", features = ["std", "derive"] } tokio = "1.24.2" - +clap-num = "1.0.2" polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" } sp-keyring = { path = "../../../substrate/primitives/keyring" } sp-application-crypto = { path = "../../../substrate/primitives/application-crypto" } diff --git a/polkadot/node/subsystem-bench/src/availability/configuration.rs b/polkadot/node/subsystem-bench/src/availability/configuration.rs index 9a93bf12e114..1355c67edea0 100644 --- a/polkadot/node/subsystem-bench/src/availability/configuration.rs +++ b/polkadot/node/subsystem-bench/src/availability/configuration.rs @@ -17,7 +17,7 @@ use super::*; /// Peer response latency configuration. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct PeerLatency { /// Min latency for `NetworkAction` completion. pub min_latency: Duration, @@ -36,15 +36,15 @@ pub struct TestConfiguration { pub n_cores: usize, /// The PoV size pub pov_sizes: Vec, - /// This parameter is used to determine how many recoveries we batch in parallel - /// to simulate tranche0 recoveries. - pub max_parallel_recoveries: usize, - /// The amount of bandiwdht remote validators have. + /// The amount of bandiwdth remote validators have. pub bandwidth: usize, /// Optional peer emulation latency pub latency: Option, /// Error probability pub error: usize, + /// Number of loops + /// In one loop `n_cores` candidates are recovered + pub num_loops: usize, } impl Default for TestConfiguration { @@ -54,80 +54,78 @@ impl Default for TestConfiguration { n_validators: 10, n_cores: 10, pov_sizes: vec![5 * 1024 * 1024], - max_parallel_recoveries: 6, bandwidth: 60 * 1024 * 1024, latency: None, error: 0, + num_loops: 1, } } } impl TestConfiguration { /// An unconstrained standard configuration matching Polkadot/Kusama - pub fn unconstrained_300_validators_60_cores(pov_sizes: Vec) -> TestConfiguration { + pub fn ideal_network( + num_loops: usize, + use_fast_path: bool, + n_validators: usize, + n_cores: usize, + pov_sizes: Vec, + ) -> TestConfiguration { Self { - use_fast_path: false, - n_validators: 300, - n_cores: 100, + use_fast_path, + n_cores, + n_validators, pov_sizes, - max_parallel_recoveries: 100, // HW specs node bandwidth - bandwidth: 60 * 1024 * 1024, - // No latency - latency: None, - error: 0, - } - } - pub fn unconstrained_1000_validators_60_cores(pov_sizes: Vec) -> TestConfiguration { - Self { - use_fast_path: false, - n_validators: 300, - n_cores: 60, - pov_sizes, - max_parallel_recoveries: 30, - // HW specs node bandwidth - bandwidth: 60 * 1024 * 1024, + bandwidth: 50 * 1024 * 1024, // No latency latency: None, error: 0, + num_loops, } } - /// Polkadot/Kusama configuration with typical latency constraints. - pub fn healthy_network_300_validators_60_cores(pov_sizes: Vec) -> TestConfiguration { + pub fn healthy_network( + num_loops: usize, + use_fast_path: bool, + n_validators: usize, + n_cores: usize, + pov_sizes: Vec, + ) -> TestConfiguration { Self { - use_fast_path: true, - n_validators: 300, - n_cores: 60, + use_fast_path, + n_cores, + n_validators, pov_sizes, - max_parallel_recoveries: 6, - // HW specs node bandwidth - bandwidth: 60 * 1024 * 1024, + bandwidth: 50 * 1024 * 1024, latency: Some(PeerLatency { min_latency: Duration::from_millis(1), - max_latency: Duration::from_millis(50), + max_latency: Duration::from_millis(100), }), - error: 5, + error: 3, + num_loops, } } - /// Polkadot/Kusama configuration with degraded due to latencies. - /// TODO: implement errors. - pub fn degraded_network_300_validators_60_cores(pov_sizes: Vec) -> TestConfiguration { + pub fn degraded_network( + num_loops: usize, + use_fast_path: bool, + n_validators: usize, + n_cores: usize, + pov_sizes: Vec, + ) -> TestConfiguration { Self { - use_fast_path: false, - n_validators: 300, - n_cores: 100, + use_fast_path, + n_cores, + n_validators, pov_sizes, - max_parallel_recoveries: 20, - // HW specs node bandwidth - bandwidth: 60 * 1024 * 1024, - // A range of latencies to expect in a degraded network + bandwidth: 50 * 1024 * 1024, latency: Some(PeerLatency { - min_latency: Duration::from_millis(1), + min_latency: Duration::from_millis(10), max_latency: Duration::from_millis(500), }), - error: 30, + error: 33, + num_loops, } } } diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 8f8eca104385..7b9b64c07096 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -22,7 +22,8 @@ use std::{ use futures::{ channel::{mpsc, oneshot}, - FutureExt, SinkExt, + stream::FuturesUnordered, + FutureExt, SinkExt, StreamExt, }; use polkadot_node_metrics::metrics::Metrics; @@ -32,7 +33,7 @@ use parity_scale_codec::Encode; use polkadot_node_network_protocol::request_response::{ self as req_res, v1::ChunkResponse, IncomingRequest, ReqProtocolNames, Requests, }; -use rand::{distributions::Uniform, prelude::Distribution, thread_rng}; +use rand::{distributions::Uniform, prelude::Distribution, seq::IteratorRandom, thread_rng}; use prometheus::Registry; use sc_network::{config::RequestResponseConfig, OutboundFailure, RequestFailure}; @@ -46,6 +47,7 @@ use polkadot_node_subsystem::{ }, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, Subsystem, }; +use std::net::{Ipv4Addr, SocketAddr}; const LOG_TARGET: &str = "subsystem-bench::availability"; @@ -74,20 +76,47 @@ struct AvailabilityRecoverySubsystemInstance { _protocol_config: RequestResponseConfig, } -// Implements a mockup of NetworkBridge and AvilabilityStore to support provide state for -// `AvailabilityRecoverySubsystemInstance` +/// The test environment is responsible for creating an instance of the availability recovery +/// subsystem and connecting it to an emulated overseer. +/// +/// ## Mockups +/// We emulate the following subsystems: +/// - runtime api +/// - network bridge +/// - availability store +/// +/// As the subsystem's performance depends on network connectivity, the test environment +/// emulates validator nodes on the network, see `NetworkEmulator`. The network emulation +/// is configurable in terms of peer bandwidth, latency and connection error rate using +/// uniform distribution sampling. +/// +/// The mockup logic is implemented in `env_task` which owns and advances the `TestState`. +/// +/// ## Usage +/// `TestEnvironment` is used in tests to send `Overseer` messages or signals to the subsystem +/// under test. +/// +/// ## Collecting test metrics +/// +/// ### Prometheus +/// A prometheus endpoint is exposed while the test is running. A local Prometheus instance +/// can scrape it every 1s and a Grafana dashboard is the preferred way of visualizing +/// the performance characteristics of the subsystem. +/// +/// ### CLI +/// A subset of the Prometheus metrics are printed at the end of the test. pub struct TestEnvironment { - // A task manager that tracks task poll durations. + // A task manager that tracks task poll durations allows us to measure + // per task CPU usage as we do in the Polkadot node. task_manager: TaskManager, // The Prometheus metrics registry registry: Registry, - // A test overseer. + // A channel to the availability recovery subsystem to_subsystem: mpsc::Sender>, // Subsystem instance, currently keeps req/response protocol channel senders // for the whole duration of the test. instance: AvailabilityRecoverySubsystemInstance, - // The test intial state. The current state is owned by the task doing the overseer/subsystem - // mockings. + // The test intial state. The current state is owned by `env_task`. state: TestState, } @@ -115,6 +144,18 @@ impl TestEnvironment { async move { Self::env_task(virtual_overseer, task_state, spawn_task_handle).await }, ); + let registry_clone = registry.clone(); + task_manager + .spawn_handle() + .spawn_blocking("prometheus", "test-environment", async move { + prometheus_endpoint::init_prometheus( + SocketAddr::new(std::net::IpAddr::V4(Ipv4Addr::LOCALHOST), 9999), + registry_clone, + ) + .await + .unwrap(); + }); + TestEnvironment { task_manager, registry, to_subsystem, instance, state } } @@ -284,7 +325,10 @@ impl TestEnvironment { .timeout(MAX_TIME_OF_FLIGHT) .await .unwrap_or_else(|| { - panic!("{}ms is more than enough for sending signals.", TIMEOUT.as_millis()) + panic!( + "{}ms is more than enough for sending signals.", + MAX_TIME_OF_FLIGHT.as_millis() + ) }) .unwrap(); } @@ -382,15 +426,18 @@ impl TestState { } fn session_info(&self) -> SessionInfo { + let my_vec = (0..self.config().n_validators) + .map(|i| ValidatorIndex(i as _)) + .collect::>(); + + let validator_groups = my_vec.chunks(5).map(|x| Vec::from(x)).collect::>(); + SessionInfo { validators: self.validator_public.clone(), discovery_keys: self.validator_authority_id.clone(), - // all validators in the same group. - validator_groups: IndexedVec::>::from(vec![(0..5) - .map(|i| ValidatorIndex(i as _)) - .collect()]), + validator_groups: IndexedVec::>::from(validator_groups), assignment_keys: vec![], - n_cores: 0, + n_cores: self.config().n_cores as u32, zeroth_delay_tranche_width: 0, relay_vrf_modulo_samples: 0, n_delay_tranches: 0, @@ -449,7 +496,7 @@ impl TestState { .collect::>(); for (index, pov_size) in config.pov_sizes.iter().enumerate() { - let mut candidate = &mut candidate_receipts[index]; + let candidate = &mut candidate_receipts[index]; // a hack to make candidate unique. candidate.descriptor.relay_parent = Hash::from_low_u64_be(index as u64); @@ -532,9 +579,6 @@ fn derive_erasure_chunks_with_proofs_and_root( (erasure_chunks, root) } -pub async fn bench_with_chunks_if_pov_large(env: &mut TestEnvironment) {} - -pub async fn bench_inner(env: &mut TestEnvironment) {} pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { let config = env.config().clone(); @@ -545,39 +589,45 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { .await; let start_marker = Instant::now(); - let mut batch = Vec::new(); - let mut availability_bytes = 0; - for candidate_num in 0..config.n_cores as u64 { - let candidate = env.state.candidate_receipts[candidate_num as usize].clone(); - - let (tx, rx) = oneshot::channel(); - batch.push(rx); - - env.send_message(AvailabilityRecoveryMessage::RecoverAvailableData( - candidate.clone(), - 1, - Some(GroupIndex(0)), - tx, - )) - .await; - - // TODO: select between futures unordered of rx await and timer to send next request. - if batch.len() >= config.max_parallel_recoveries { - for rx in std::mem::take(&mut batch) { - let available_data = rx.await.unwrap().unwrap(); - availability_bytes += available_data.encoded_size(); - } + let mut batch = FuturesUnordered::new(); + let mut availability_bytes = 0u128; + + for loop_num in 0..env.config().num_loops { + gum::info!(target: LOG_TARGET, loop_num, "Starting loop"); + + for candidate_num in 0..config.n_cores as u64 { + let candidate = env.state.candidate(candidate_num as usize); + + let (tx, rx) = oneshot::channel(); + batch.push(rx); + + env.send_message(AvailabilityRecoveryMessage::RecoverAvailableData( + candidate.clone(), + 1, + Some(GroupIndex(candidate_num as u32 % (config.n_cores / 5) as u32)), + tx, + )) + .await; + + // // TODO: select between futures unordered of rx await and timer to send next request. + // if batch.len() >= config.max_parallel_recoveries { + // for rx in std::mem::take(&mut batch) { + // let available_data = rx.await.unwrap().unwrap(); + // availability_bytes += available_data.encoded_size() as u128; + // } + // } } - } - for rx in std::mem::take(&mut batch) { - let available_data = rx.await.unwrap().unwrap(); - availability_bytes += available_data.encoded_size(); + while let Some(completed) = batch.next().await { + let available_data = completed.unwrap().unwrap(); + availability_bytes += available_data.encoded_size() as u128; + } } + println!("Waiting for subsystem to complete work... {} requests ", batch.len()); env.send_signal(OverseerSignal::Conclude).await; let duration = start_marker.elapsed().as_millis(); - let tput = (availability_bytes as u128) / duration * 1000; + let tput = ((availability_bytes) / duration) * 1000; println!("Benchmark completed in {:?}ms", duration); println!("Throughput: {}KiB/s", tput / 1024); diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 4dc0936291b1..f5180004840c 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -20,24 +20,89 @@ use clap::Parser; use color_eyre::eyre; use prometheus::proto::LabelPair; -use std::net::{Ipv4Addr, SocketAddr}; +use std::time::Duration; pub(crate) mod availability; use availability::{TestConfiguration, TestEnvironment, TestState}; const LOG_TARGET: &str = "subsystem-bench"; +use clap_num::number_range; + +fn le_100(s: &str) -> Result { + number_range(s, 0, 100) +} + +fn le_5000(s: &str) -> Result { + number_range(s, 0, 5000) +} + +#[derive(Debug, clap::Parser, Clone)] +#[clap(rename_all = "kebab-case")] +#[allow(missing_docs)] +pub struct NetworkOptions {} + +#[derive(clap::ValueEnum, Clone, Copy, Debug, PartialEq)] +#[value(rename_all = "kebab-case")] +#[non_exhaustive] +pub enum NetworkEmulation { + Ideal, + Healthy, + Degraded, +} + +#[derive(Debug, clap::Parser)] +#[clap(rename_all = "kebab-case")] +#[allow(missing_docs)] +pub struct DataAvailabilityReadOptions { + #[clap(long, ignore_case = true, default_value_t = 100)] + /// Number of cores to fetch availability for. + pub n_cores: usize, + + #[clap(long, ignore_case = true, default_value_t = 500)] + /// Number of validators to fetch chunks from. + pub n_validators: usize, + + #[clap(short, long, default_value_t = false)] + /// Turbo boost AD Read by fetching from backers first. Tipically this is only faster if nodes + /// have enough bandwidth. + pub fetch_from_backers: bool, + + #[clap(short, long, ignore_case = true, default_value_t = 1)] + /// Number of times to loop fetching for each core. + pub num_loops: usize, +} /// Define the supported benchmarks targets #[derive(Debug, Parser)] #[command(about = "Target subsystems", version, rename_all = "kebab-case")] enum BenchmarkTarget { /// Benchmark availability recovery strategies. - AvailabilityRecovery, + DataAvailabilityRead(DataAvailabilityReadOptions), } #[derive(Debug, Parser)] #[allow(missing_docs)] struct BenchCli { + #[arg(long, value_enum, ignore_case = true, default_value_t = NetworkEmulation::Ideal)] + /// The type of network to be emulated + pub network: NetworkEmulation, + + #[clap(short, long)] + /// The bandwidth of simulated remote peers in KiB + pub peer_bandwidth: Option, + + #[clap(long, value_parser=le_100)] + /// Simulated connection error rate [0-100]. + pub peer_error: Option, + + #[clap(long, value_parser=le_5000)] + /// Minimum remote peer latency in milliseconds [0-5000]. + pub peer_min_latency: Option, + + #[clap(long, value_parser=le_5000)] + /// Maximum remote peer latency in milliseconds [0-5000]. + pub peer_max_latency: Option, + #[command(subcommand)] pub target: BenchmarkTarget, } @@ -63,21 +128,57 @@ impl BenchCli { let registry_clone = registry.clone(); let mut pov_sizes = Vec::new(); - pov_sizes.append(&mut vec![1024 * 1024; 100]); + pov_sizes.append(&mut vec![5 * 1024 * 1024; 200]); + + let mut test_config = match self.target { + BenchmarkTarget::DataAvailabilityRead(options) => match self.network { + NetworkEmulation::Healthy => TestConfiguration::healthy_network( + options.num_loops, + options.fetch_from_backers, + options.n_validators, + options.n_cores, + pov_sizes, + ), + NetworkEmulation::Degraded => TestConfiguration::degraded_network( + options.num_loops, + options.fetch_from_backers, + options.n_validators, + options.n_cores, + pov_sizes, + ), + NetworkEmulation::Ideal => TestConfiguration::ideal_network( + options.num_loops, + options.fetch_from_backers, + options.n_validators, + options.n_cores, + pov_sizes, + ), + }, + }; + + let mut latency_config = test_config.latency.clone().unwrap_or_default(); + + if let Some(latency) = self.peer_min_latency { + latency_config.min_latency = Duration::from_millis(latency); + } - let test_config = TestConfiguration::unconstrained_1000_validators_60_cores(pov_sizes); + if let Some(latency) = self.peer_max_latency { + latency_config.max_latency = Duration::from_millis(latency); + } - let state = TestState::new(test_config); + if let Some(error) = self.peer_error { + test_config.error = error; + } + if let Some(bandwidth) = self.peer_bandwidth { + // CLI expects bw in KiB + test_config.bandwidth = bandwidth * 1024; + } + + let state = TestState::new(test_config); let mut env = TestEnvironment::new(runtime.handle().clone(), state, registry.clone()); - let handle = runtime.spawn(async move { - prometheus_endpoint::init_prometheus( - SocketAddr::new(std::net::IpAddr::V4(Ipv4Addr::LOCALHOST), 9999), - registry_clone, - ) - .await - }); + let runtime_handle = runtime.handle().clone(); println!("{:?}", env.config()); From 5a05da0f6c87e7e19ff1940d4b9f035cbb4cf7e9 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 9 Nov 2023 23:51:58 +0200 Subject: [PATCH 086/192] peer stats Signed-off-by: Andrei Sandu --- .../subsystem-bench/src/availability/mod.rs | 28 ++++++----- .../src/availability/network.rs | 49 +++++++++++++++++-- 2 files changed, 63 insertions(+), 14 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 7b9b64c07096..a4980ffc5fdd 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -118,6 +118,8 @@ pub struct TestEnvironment { instance: AvailabilityRecoverySubsystemInstance, // The test intial state. The current state is owned by `env_task`. state: TestState, + // A handle to the network emulator. + network: NetworkEmulator, } impl TestEnvironment { @@ -131,17 +133,24 @@ impl TestEnvironment { state.config().use_fast_path, ); + let mut network = NetworkEmulator::new( + state.config().n_validators, + state.config().bandwidth, + task_manager.spawn_handle(), + ); + // Copy sender for later when we need to inject messages in to the subsystem. let to_subsystem = virtual_overseer.tx.clone(); let task_state = state.clone(); - let spawn_task_handle = task_manager.spawn_handle(); + let task_network = network.clone(); + // We need to start a receiver to process messages from the subsystem. // This mocks an overseer and all dependent subsystems task_manager.spawn_handle().spawn_blocking( "test-environment", "test-environment", - async move { Self::env_task(virtual_overseer, task_state, spawn_task_handle).await }, + async move { Self::env_task(virtual_overseer, task_state, task_network).await }, ); let registry_clone = registry.clone(); @@ -156,7 +165,7 @@ impl TestEnvironment { .unwrap(); }); - TestEnvironment { task_manager, registry, to_subsystem, instance, state } + TestEnvironment { task_manager, registry, to_subsystem, instance, state, network } } pub fn config(&self) -> &TestConfiguration { @@ -249,15 +258,8 @@ impl TestEnvironment { async fn env_task( mut ctx: TestSubsystemContextHandle, mut state: TestState, - spawn_task_handle: SpawnTaskHandle, + mut network: NetworkEmulator, ) { - // Emulate `n_validators` each with 1MiB of bandwidth available. - let mut network = NetworkEmulator::new( - state.config().n_validators, - state.config().bandwidth, - spawn_task_handle, - ); - loop { futures::select! { message = ctx.recv().fuse() => { @@ -631,5 +633,9 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { println!("Benchmark completed in {:?}ms", duration); println!("Throughput: {}KiB/s", tput / 1024); + let stats = env.network.stats().await; + for (index, stat) in stats.iter().enumerate() { + println!("Validator #{} : {:?}", index, stat); + } tokio::time::sleep(Duration::from_secs(1)).await; } diff --git a/polkadot/node/subsystem-bench/src/availability/network.rs b/polkadot/node/subsystem-bench/src/availability/network.rs index d6fc175c859b..544ecf06372a 100644 --- a/polkadot/node/subsystem-bench/src/availability/network.rs +++ b/polkadot/node/subsystem-bench/src/availability/network.rs @@ -124,6 +124,7 @@ mod tests { } // A network peer emulator +#[derive(Clone)] struct PeerEmulator { // The queue of requests waiting to be served by the emulator actions_tx: UnboundedSender, @@ -137,11 +138,15 @@ impl PeerEmulator { .clone() .spawn("peer-emulator", "test-environment", async move { let mut rate_limiter = RateLimit::new(20, bandwidth); + let rx_bytes_total = 0; + let mut tx_bytes_total = 0u128; + loop { let maybe_action: Option = actions_rx.recv().await; if let Some(action) = maybe_action { let size = action.size(); rate_limiter.reap(size).await; + tx_bytes_total += size as u128; if let Some(latency) = action.latency { spawn_task_handle.spawn( "peer-emulator-latency", @@ -152,7 +157,12 @@ impl PeerEmulator { }, ) } else { - action.run().await; + // Send stats if requested + if let Some(stats_sender) = action.stats { + stats_sender.send(PeerEmulatorStats { rx_bytes_total, tx_bytes_total }).unwrap(); + } else { + action.run().await; + } } } else { break @@ -170,7 +180,7 @@ impl PeerEmulator { } pub type ActionFuture = std::pin::Pin + std::marker::Send>>; -// An network action to be completed by the emulator task. +/// An network action to be completed by the emulator task. pub struct NetworkAction { // The function that performs the action run: ActionFuture, @@ -180,12 +190,28 @@ pub struct NetworkAction { index: usize, // The amount of time to delay the polling `run` latency: Option, + // An optional request of rx/tx statistics for the peer at `index` + stats: Option>, +} + +/// Book keeping of sent and received bytes. +#[derive(Debug, Clone)] +pub struct PeerEmulatorStats { + pub rx_bytes_total: u128, + pub tx_bytes_total: u128, } impl NetworkAction { pub fn new(index: usize, run: ActionFuture, size: usize, latency: Option) -> Self { - Self { run, size, index, latency } + Self { run, size, index, latency, stats: None } + } + + pub fn stats(index: usize, stats_sender:oneshot::Sender) -> Self { + let run = async move {}.boxed(); + + Self { run, size: 0, index, latency: None, stats: Some(stats_sender) } } + pub fn size(&self) -> usize { self.size } @@ -201,6 +227,7 @@ impl NetworkAction { // Mocks the network bridge and an arbitrary number of connected peer nodes. // Implements network latency, bandwidth and error. +#[derive(Clone)] pub struct NetworkEmulator { // Per peer network emulation peers: Vec, @@ -218,4 +245,20 @@ impl NetworkEmulator { pub fn submit_peer_action(&mut self, index: usize, action: NetworkAction) { let _ = self.peers[index].send(action); } + + // Returns the sent/received stats for all peers. + pub async fn stats(&mut self) -> Vec { + let receivers = (0..self.peers.len()).map(|peer_index| { + let (stats_tx, stats_rx) = oneshot::channel(); + self.submit_peer_action(peer_index, NetworkAction::stats(peer_index, stats_tx)); + stats_rx + }).collect::>(); + + let mut stats = Vec::new(); + for receiver in receivers { + stats.push(receiver.await.unwrap()); + } + + stats + } } From 895e8d6a627334b46025a212242e31684eb8a9fc Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 10 Nov 2023 12:42:51 +0200 Subject: [PATCH 087/192] Switch stats to atomics Signed-off-by: Andrei Sandu --- .../subsystem-bench/src/availability/mod.rs | 1 - .../src/availability/network.rs | 83 ++++++++++--------- .../subsystem-bench/src/subsystem-bench.rs | 1 - 3 files changed, 42 insertions(+), 43 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index a4980ffc5fdd..7903ba08b616 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -13,7 +13,6 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . - use std::{ collections::HashMap, sync::Arc, diff --git a/polkadot/node/subsystem-bench/src/availability/network.rs b/polkadot/node/subsystem-bench/src/availability/network.rs index 544ecf06372a..02af817e691f 100644 --- a/polkadot/node/subsystem-bench/src/availability/network.rs +++ b/polkadot/node/subsystem-bench/src/availability/network.rs @@ -13,10 +13,11 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . - use super::*; +use prometheus_endpoint::U64; +use sc_network::network_state::Peer; +use std::sync::atomic::{AtomicU64, Ordering}; use tokio::sync::mpsc::UnboundedSender; - // An emulated node egress traffic rate_limiter. #[derive(Debug)] struct RateLimit { @@ -131,7 +132,11 @@ struct PeerEmulator { } impl PeerEmulator { - pub fn new(bandwidth: usize, spawn_task_handle: SpawnTaskHandle) -> Self { + pub fn new( + bandwidth: usize, + spawn_task_handle: SpawnTaskHandle, + stats: Arc, + ) -> Self { let (actions_tx, mut actions_rx) = tokio::sync::mpsc::unbounded_channel(); spawn_task_handle @@ -140,13 +145,12 @@ impl PeerEmulator { let mut rate_limiter = RateLimit::new(20, bandwidth); let rx_bytes_total = 0; let mut tx_bytes_total = 0u128; - loop { + let stats_clone = stats.clone(); let maybe_action: Option = actions_rx.recv().await; if let Some(action) = maybe_action { let size = action.size(); rate_limiter.reap(size).await; - tx_bytes_total += size as u128; if let Some(latency) = action.latency { spawn_task_handle.spawn( "peer-emulator-latency", @@ -154,15 +158,14 @@ impl PeerEmulator { async move { tokio::time::sleep(latency).await; action.run().await; + stats_clone + .tx_bytes_total + .fetch_add(size as u64, Ordering::Relaxed); }, ) } else { - // Send stats if requested - if let Some(stats_sender) = action.stats { - stats_sender.send(PeerEmulatorStats { rx_bytes_total, tx_bytes_total }).unwrap(); - } else { - action.run().await; - } + action.run().await; + stats_clone.tx_bytes_total.fetch_add(size as u64, Ordering::Relaxed); } } else { break @@ -190,26 +193,23 @@ pub struct NetworkAction { index: usize, // The amount of time to delay the polling `run` latency: Option, - // An optional request of rx/tx statistics for the peer at `index` - stats: Option>, } /// Book keeping of sent and received bytes. -#[derive(Debug, Clone)] +#[derive(Debug, Default)] pub struct PeerEmulatorStats { - pub rx_bytes_total: u128, - pub tx_bytes_total: u128, + pub rx_bytes_total: AtomicU64, + pub tx_bytes_total: AtomicU64, } +#[derive(Debug, Default)] +pub struct PeerStats { + pub rx_bytes_total: u64, + pub tx_bytes_total: u64, +} impl NetworkAction { pub fn new(index: usize, run: ActionFuture, size: usize, latency: Option) -> Self { - Self { run, size, index, latency, stats: None } - } - - pub fn stats(index: usize, stats_sender:oneshot::Sender) -> Self { - let run = async move {}.boxed(); - - Self { run, size: 0, index, latency: None, stats: Some(stats_sender) } + Self { run, size, index, latency } } pub fn size(&self) -> usize { @@ -231,15 +231,19 @@ impl NetworkAction { pub struct NetworkEmulator { // Per peer network emulation peers: Vec, + stats: Vec>, } impl NetworkEmulator { pub fn new(n_peers: usize, bandwidth: usize, spawn_task_handle: SpawnTaskHandle) -> Self { - Self { - peers: (0..n_peers) - .map(|_index| PeerEmulator::new(bandwidth, spawn_task_handle.clone())) - .collect::>(), - } + let (stats, peers) = (0..n_peers) + .map(|_index| { + let stats = Arc::new(PeerEmulatorStats::default()); + (stats.clone(), PeerEmulator::new(bandwidth, spawn_task_handle.clone(), stats)) + }) + .unzip(); + + Self { peers, stats } } pub fn submit_peer_action(&mut self, index: usize, action: NetworkAction) { @@ -247,18 +251,15 @@ impl NetworkEmulator { } // Returns the sent/received stats for all peers. - pub async fn stats(&mut self) -> Vec { - let receivers = (0..self.peers.len()).map(|peer_index| { - let (stats_tx, stats_rx) = oneshot::channel(); - self.submit_peer_action(peer_index, NetworkAction::stats(peer_index, stats_tx)); - stats_rx - }).collect::>(); - - let mut stats = Vec::new(); - for receiver in receivers { - stats.push(receiver.await.unwrap()); - } - - stats + pub async fn stats(&mut self) -> Vec { + let r = self + .stats + .iter() + .map(|stats| PeerStats { + rx_bytes_total: stats.rx_bytes_total.load(Ordering::Relaxed), + tx_bytes_total: stats.tx_bytes_total.load(Ordering::Relaxed), + }) + .collect::>(); + r } } diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index f5180004840c..ba66d06fe320 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -16,7 +16,6 @@ //! A tool for running subsystem benchmark tests designed for development and //! CI regression testing. - use clap::Parser; use color_eyre::eyre; use prometheus::proto::LabelPair; From a2fb0c95d08c17ad5647c904cb48854ec30ba470 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Sun, 12 Nov 2023 03:14:34 +0200 Subject: [PATCH 088/192] add more network metrics, new load generator Signed-off-by: Andrei Sandu --- Cargo.lock | 1 + polkadot/node/subsystem-bench/Cargo.toml | 2 +- .../src/availability/configuration.rs | 7 +- .../subsystem-bench/src/availability/mod.rs | 335 +++++++++++++----- .../src/availability/network.rs | 149 +++++++- .../src/availability/test_env.rs | 63 ++++ .../subsystem-bench/src/subsystem-bench.rs | 25 +- 7 files changed, 461 insertions(+), 121 deletions(-) create mode 100644 polkadot/node/subsystem-bench/src/availability/test_env.rs diff --git a/Cargo.lock b/Cargo.lock index 05355cad0e2c..ee80ffb6e815 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13012,6 +13012,7 @@ dependencies = [ "env_logger 0.9.3", "futures", "futures-timer", + "itertools 0.11.0", "log", "parity-scale-codec", "polkadot-availability-recovery", diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index 01b992d15fc6..c5d62d3aa74f 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -45,7 +45,7 @@ sp-application-crypto = { path = "../../../substrate/primitives/application-cryp sc-network = { path = "../../../substrate/client/network" } sc-service = { path = "../../../substrate/client/service" } polkadot-node-metrics = { path = "../metrics" } - +itertools = "0.11.0" polkadot-primitives-test-helpers = { path = "../../primitives/test-helpers" } prometheus_endpoint = { package = "substrate-prometheus-endpoint", path = "../../../substrate/utils/prometheus" } prometheus = { version = "0.13.0", default-features = false } diff --git a/polkadot/node/subsystem-bench/src/availability/configuration.rs b/polkadot/node/subsystem-bench/src/availability/configuration.rs index 1355c67edea0..cf142de06634 100644 --- a/polkadot/node/subsystem-bench/src/availability/configuration.rs +++ b/polkadot/node/subsystem-bench/src/availability/configuration.rs @@ -37,6 +37,8 @@ pub struct TestConfiguration { /// The PoV size pub pov_sizes: Vec, /// The amount of bandiwdth remote validators have. + pub peer_bandwidth: usize, + /// The amount of bandiwdth our node has. pub bandwidth: usize, /// Optional peer emulation latency pub latency: Option, @@ -55,6 +57,7 @@ impl Default for TestConfiguration { n_cores: 10, pov_sizes: vec![5 * 1024 * 1024], bandwidth: 60 * 1024 * 1024, + peer_bandwidth: 60 * 1024 * 1024, latency: None, error: 0, num_loops: 1, @@ -76,8 +79,8 @@ impl TestConfiguration { n_cores, n_validators, pov_sizes, - // HW specs node bandwidth bandwidth: 50 * 1024 * 1024, + peer_bandwidth: 50 * 1024 * 1024, // No latency latency: None, error: 0, @@ -98,6 +101,7 @@ impl TestConfiguration { n_validators, pov_sizes, bandwidth: 50 * 1024 * 1024, + peer_bandwidth: 50 * 1024 * 1024, latency: Some(PeerLatency { min_latency: Duration::from_millis(1), max_latency: Duration::from_millis(100), @@ -120,6 +124,7 @@ impl TestConfiguration { n_validators, pov_sizes, bandwidth: 50 * 1024 * 1024, + peer_bandwidth: 50 * 1024 * 1024, latency: Some(PeerLatency { min_latency: Duration::from_millis(10), max_latency: Duration::from_millis(500), diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 7903ba08b616..4f821f819908 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -13,8 +13,11 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use itertools::Itertools; use std::{ collections::HashMap, + iter::Cycle, + ops::{Div, Sub}, sync::Arc, time::{Duration, Instant}, }; @@ -24,6 +27,8 @@ use futures::{ stream::FuturesUnordered, FutureExt, SinkExt, StreamExt, }; +use futures_timer::Delay; + use polkadot_node_metrics::metrics::Metrics; use polkadot_availability_recovery::AvailabilityRecoverySubsystem; @@ -48,6 +53,8 @@ use polkadot_node_subsystem::{ }; use std::net::{Ipv4Addr, SocketAddr}; +mod test_env; + const LOG_TARGET: &str = "subsystem-bench::availability"; use polkadot_node_primitives::{AvailableData, ErasureChunk}; @@ -119,6 +126,8 @@ pub struct TestEnvironment { state: TestState, // A handle to the network emulator. network: NetworkEmulator, + // Configuration/env metrics + metrics: TestEnvironmentMetrics, } impl TestEnvironment { @@ -131,11 +140,13 @@ impl TestEnvironment { task_manager.spawn_handle(), state.config().use_fast_path, ); - + let metrics = + TestEnvironmentMetrics::new(®istry).expect("Metrics need to be registered"); let mut network = NetworkEmulator::new( state.config().n_validators, - state.config().bandwidth, + state.config().peer_bandwidth, task_manager.spawn_handle(), + ®istry, ); // Copy sender for later when we need to inject messages in to the subsystem. @@ -143,13 +154,31 @@ impl TestEnvironment { let task_state = state.clone(); let task_network = network.clone(); + let spawn_handle = task_manager.spawn_handle(); + + // Our node rate limiting + let mut rx_limiter = RateLimit::new(10, state.config.bandwidth); + let (ingress_tx, mut ingress_rx) = tokio::sync::mpsc::unbounded_channel::(); + let our_network_stats = network.peer_stats(0); + + spawn_handle.spawn_blocking("our-node-rx", "test-environment", async move { + while let Some(action) = ingress_rx.recv().await { + let size = action.size(); + + // account for our node receiving the data. + our_network_stats.inc_received(size); + + rx_limiter.reap(size).await; + action.run().await; + } + }); // We need to start a receiver to process messages from the subsystem. // This mocks an overseer and all dependent subsystems task_manager.spawn_handle().spawn_blocking( "test-environment", "test-environment", - async move { Self::env_task(virtual_overseer, task_state, task_network).await }, + async move { Self::env_task(virtual_overseer, task_state, task_network, ingress_tx).await }, ); let registry_clone = registry.clone(); @@ -164,13 +193,17 @@ impl TestEnvironment { .unwrap(); }); - TestEnvironment { task_manager, registry, to_subsystem, instance, state, network } + TestEnvironment { task_manager, registry, to_subsystem, instance, state, network, metrics } } pub fn config(&self) -> &TestConfiguration { self.state.config() } + pub fn network(&self) -> &NetworkEmulator { + &self.network + } + /// Produce a randomized duration between `min` and `max`. fn random_latency(maybe_peer_latency: Option<&PeerLatency>) -> Option { if let Some(peer_latency) = maybe_peer_latency { @@ -183,24 +216,51 @@ impl TestEnvironment { } } + pub fn metrics(&self) -> &TestEnvironmentMetrics { + &self.metrics + } + /// Generate a random error based on `probability`. /// `probability` should be a number between 0 and 100. fn random_error(probability: usize) -> bool { Uniform::from(0..=99).sample(&mut thread_rng()) < probability } - pub fn respond_to_send_request(state: &mut TestState, request: Requests) -> NetworkAction { + pub fn request_size(request: &Requests) -> u64 { + match request { + Requests::ChunkFetchingV1(outgoing_request) => + outgoing_request.payload.encoded_size() as u64, + Requests::AvailableDataFetchingV1(outgoing_request) => + outgoing_request.payload.encoded_size() as u64, + _ => panic!("received an unexpected request"), + } + } + + pub fn respond_to_send_request( + state: &mut TestState, + request: Requests, + ingress_tx: tokio::sync::mpsc::UnboundedSender, + ) -> NetworkAction { match request { Requests::ChunkFetchingV1(outgoing_request) => { let validator_index = outgoing_request.payload.index.0 as usize; - let chunk: ChunkResponse = - state.chunks.get(&outgoing_request.payload.candidate_hash).unwrap() - [validator_index] - .clone() - .into(); - let size = chunk.encoded_size(); + let candidate_hash = outgoing_request.payload.candidate_hash; + + let candidate_index = state + .candidate_hashes + .get(&candidate_hash) + .expect("candidate was generated previously; qed"); + gum::warn!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); + + let chunk: ChunkResponse = state.chunks.get(*candidate_index as usize).unwrap() + [validator_index] + .clone() + .into(); + let mut size = chunk.encoded_size(); let response = if Self::random_error(state.config().error) { + // Error will not account to any bandwidth used. + size = 0; Err(RequestFailure::Network(OutboundFailure::ConnectionClosed)) } else { Ok(req_res::v1::ChunkFetchingResponse::from(Some(chunk)).encode()) @@ -211,21 +271,39 @@ impl TestEnvironment { } .boxed(); + let future_wrapper = async move { + // Forward the response to the ingress channel of our node. + // On receive side we apply our node receiving rate limit. + let action = NetworkAction::new(validator_index, future, size, None); + ingress_tx.send(action).unwrap(); + } + .boxed(); + NetworkAction::new( validator_index, - future, + future_wrapper, size, // Generate a random latency based on configuration. Self::random_latency(state.config().latency.as_ref()), ) }, Requests::AvailableDataFetchingV1(outgoing_request) => { + println!("{:?}", outgoing_request); // TODO: do better, by implementing diff authority ids and mapping network actions // to authority id, let validator_index = Uniform::from(0..state.config().n_validators).sample(&mut thread_rng()); + + let candidate_hash = outgoing_request.payload.candidate_hash; + let candidate_index = state + .candidate_hashes + .get(&candidate_hash) + .expect("candidate was generated previously; qed"); + gum::warn!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); + let available_data = - state.candidates.get(&outgoing_request.payload.candidate_hash).unwrap().clone(); + state.available_data.get(*candidate_index as usize).unwrap().clone(); + let size = available_data.encoded_size(); let response = if Self::random_error(state.config().error) { @@ -240,9 +318,17 @@ impl TestEnvironment { } .boxed(); + let future_wrapper = async move { + // Forward the response to the ingress channel of our node. + // On receive side we apply our node receiving rate limit. + let action = NetworkAction::new(validator_index, future, size, None); + ingress_tx.send(action).unwrap(); + } + .boxed(); + NetworkAction::new( validator_index, - future, + future_wrapper, size, // Generate a random latency based on configuration. Self::random_latency(state.config().latency.as_ref()), @@ -258,11 +344,12 @@ impl TestEnvironment { mut ctx: TestSubsystemContextHandle, mut state: TestState, mut network: NetworkEmulator, + ingress_tx: tokio::sync::mpsc::UnboundedSender, ) { loop { futures::select! { message = ctx.recv().fuse() => { - gum::debug!(target: LOG_TARGET, ?message, "Env task received message"); + gum::trace!(target: LOG_TARGET, ?message, "Env task received message"); match message { AllMessages::NetworkBridgeTx( @@ -272,7 +359,9 @@ impl TestEnvironment { ) ) => { for request in requests { - let action = Self::respond_to_send_request(&mut state, request); + network.inc_sent(Self::request_size(&request)); + let action = Self::respond_to_send_request(&mut state, request, ingress_tx.clone()); + // Account for our node sending the request over the emulated network. network.submit_peer_action(action.index(), action); } }, @@ -287,7 +376,10 @@ impl TestEnvironment { AllMessages::AvailabilityStore( AvailabilityStoreMessage::QueryChunkSize(candidate_hash, tx) ) => { - let chunk_size = state.chunks.get(&candidate_hash).unwrap()[0].encoded_size(); + let candidate_index = state.candidate_hashes.get(&candidate_hash).expect("candidate was generated previously; qed"); + gum::info!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); + + let chunk_size = state.chunks.get(*candidate_index as usize).unwrap()[0].encoded_size(); let _ = tx.send(Some(chunk_size)); } AllMessages::RuntimeApi(RuntimeApiMessage::Request( @@ -345,8 +437,8 @@ impl AvailabilityRecoverySubsystemInstance { ) -> (Self, TestSubsystemContextHandle) { let (context, virtual_overseer) = make_buffered_subsystem_context( spawn_task_handle.clone(), - 4096 * 4, - "availability-recovery", + 128, + "availability-recovery-subsystem", ); let (collation_req_receiver, req_cfg) = IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); @@ -378,8 +470,6 @@ impl AvailabilityRecoverySubsystemInstance { } } -const TIMEOUT: Duration = Duration::from_millis(300); - // We use this to bail out sending messages to the subsystem if it is overloaded such that // the time of flight is breaches 5s. // This should eventually be a test parameter. @@ -387,9 +477,13 @@ const MAX_TIME_OF_FLIGHT: Duration = Duration::from_millis(5000); use sp_keyring::Sr25519Keyring; -use crate::availability::network::NetworkAction; +use crate::availability::network::{ActionFuture, NetworkAction}; -use self::{configuration::PeerLatency, network::NetworkEmulator}; +use self::{ + configuration::PeerLatency, + network::{NetworkEmulator, RateLimit}, + test_env::TestEnvironmentMetrics, +}; #[derive(Clone)] pub struct TestState { @@ -398,15 +492,22 @@ pub struct TestState { validator_authority_id: Vec, // The test node validator index. validator_index: ValidatorIndex, - // Per core candidates receipts. - candidate_receipts: Vec, session_index: SessionIndex, - + pov_sizes: Cycle>, + // Generated candidate receipts to be used in the test + candidates: Cycle>, + candidates_generated: usize, + // Map from pov size to candidate index + pov_size_to_candidate: HashMap, + // Map from generated candidate hashes to candidate index in `available_data` + // and `chunks`. + candidate_hashes: HashMap, persisted_validation_data: PersistedValidationData, - /// A per size pov mapping to available data. - candidates: HashMap, - chunks: HashMap>, + candidate_receipts: Vec, + available_data: Vec, + chunks: Vec>, + /// Next candidate index in config: TestConfiguration, } @@ -415,10 +516,6 @@ impl TestState { &self.config } - fn candidate(&self, candidate_index: usize) -> CandidateReceipt { - self.candidate_receipts.get(candidate_index).unwrap().clone() - } - async fn respond_none_to_available_data_query( &self, tx: oneshot::Sender>, @@ -455,9 +552,17 @@ impl TestState { send_chunk: impl Fn(usize) -> bool, tx: oneshot::Sender>, ) { + gum::info!(target: LOG_TARGET, ?candidate_hash, "respond_to_query_all_request"); + + let candidate_index = self + .candidate_hashes + .get(&candidate_hash) + .expect("candidate was generated previously; qed"); + gum::info!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); + let v = self .chunks - .get(&candidate_hash) + .get(*candidate_index as usize) .unwrap() .iter() .filter(|c| send_chunk(c.index.0 as usize)) @@ -467,6 +572,41 @@ impl TestState { let _ = tx.send(v); } + pub fn next_candidate(&mut self) -> Option { + let candidate = self.candidates.next(); + let candidate_hash = candidate.as_ref().unwrap().hash(); + gum::trace!(target: LOG_TARGET, "Next candidate selected {:?}", candidate_hash); + candidate + } + + /// Generate candidates to be used in the test. + pub fn generate_candidates(&mut self, count: usize) { + gum::info!(target: LOG_TARGET, "Pre-generating {} candidates.", count); + + // Generate all candidates + self.candidates = (0..count) + .map(|index| { + let pov_size = self.pov_sizes.next().expect("This is a cycle; qed"); + let candidate_index = *self + .pov_size_to_candidate + .get(&pov_size) + .expect("pov_size always exists; qed"); + let mut candidate_receipt = self.candidate_receipts[candidate_index].clone(); + + // Make it unique. + candidate_receipt.descriptor.relay_parent = Hash::from_low_u64_be(index as u64); + // Store the new candidate in the state + self.candidate_hashes.insert(candidate_receipt.hash(), candidate_index); + + gum::info!(target: LOG_TARGET, candidate_hash = ?candidate_receipt.hash(), "new candidate"); + + candidate_receipt + }) + .collect::>() + .into_iter() + .cycle(); + } + pub fn new(config: TestConfiguration) -> Self { let validators = (0..config.n_validators as u64) .into_iter() @@ -476,9 +616,10 @@ impl TestState { let validator_public = validator_pubkeys(&validators); let validator_authority_id = validator_authority_id(&validators); let validator_index = ValidatorIndex(0); + let mut chunks = Vec::new(); + let mut available_data = Vec::new(); + let mut candidate_receipts = Vec::new(); let mut pov_size_to_candidate = HashMap::new(); - let mut chunks = HashMap::new(); - let mut candidates = HashMap::new(); let session_index = 10; // we use it for all candidates. @@ -489,59 +630,54 @@ impl TestState { relay_parent_storage_root: Default::default(), }; - // Create initial candidate receipts - let mut candidate_receipts = config - .pov_sizes - .iter() - .map(|_index| dummy_candidate_receipt(dummy_hash())) - .collect::>(); - - for (index, pov_size) in config.pov_sizes.iter().enumerate() { - let candidate = &mut candidate_receipts[index]; - // a hack to make candidate unique. - candidate.descriptor.relay_parent = Hash::from_low_u64_be(index as u64); - - // We reuse candidates of same size, to speed up the test startup. - let (erasure_root, available_data, new_chunks) = - pov_size_to_candidate.entry(pov_size).or_insert_with(|| { - let pov = PoV { block_data: BlockData(vec![index as u8; *pov_size]) }; - - let available_data = AvailableData { - validation_data: persisted_validation_data.clone(), - pov: Arc::new(pov), - }; + // For each unique pov we create a candidate receipt. + for (index, pov_size) in config.pov_sizes.iter().cloned().unique().enumerate() { + gum::info!(target: LOG_TARGET, index, pov_size, "Generating template candidates"); - let (new_chunks, erasure_root) = derive_erasure_chunks_with_proofs_and_root( - validators.len(), - &available_data, - |_, _| {}, - ); + let mut candidate_receipt = dummy_candidate_receipt(dummy_hash()); + let pov = PoV { block_data: BlockData(vec![index as u8; pov_size]) }; - candidate.descriptor.erasure_root = erasure_root; + let new_available_data = AvailableData { + validation_data: persisted_validation_data.clone(), + pov: Arc::new(pov), + }; - chunks.insert(candidate.hash(), new_chunks.clone()); - candidates.insert(candidate.hash(), available_data.clone()); + let (new_chunks, erasure_root) = derive_erasure_chunks_with_proofs_and_root( + validators.len(), + &new_available_data, + |_, _| {}, + ); - (erasure_root, available_data, new_chunks) - }); + candidate_receipt.descriptor.erasure_root = erasure_root; - candidate.descriptor.erasure_root = *erasure_root; - candidates.insert(candidate.hash(), available_data.clone()); - chunks.insert(candidate.hash(), new_chunks.clone()); + chunks.push(new_chunks); + available_data.push(new_available_data); + pov_size_to_candidate.insert(pov_size, index); + candidate_receipts.push(candidate_receipt); } - Self { + let pov_sizes = config.pov_sizes.clone().into_iter().cycle(); + let mut state = Self { validators, validator_public, validator_authority_id, validator_index, - candidate_receipts, session_index, persisted_validation_data, - candidates, + available_data, + candidate_receipts, chunks, config, - } + pov_size_to_candidate, + pov_sizes, + candidates_generated: 0, + candidate_hashes: HashMap::new(), + candidates: Vec::new().into_iter().cycle(), + }; + + gum::info!(target: LOG_TARGET, "Created test environment."); + + state } } @@ -593,12 +729,19 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { let mut batch = FuturesUnordered::new(); let mut availability_bytes = 0u128; + env.metrics().set_n_validators(config.n_validators); + env.metrics().set_n_cores(config.n_cores); + env.metrics().set_pov_size(config.pov_sizes[0]); + let mut completed_count = 0; + for loop_num in 0..env.config().num_loops { gum::info!(target: LOG_TARGET, loop_num, "Starting loop"); + env.metrics().set_current_loop(loop_num); + let loop_start_ts = Instant::now(); for candidate_num in 0..config.n_cores as u64 { - let candidate = env.state.candidate(candidate_num as usize); - + let candidate = + env.state.next_candidate().expect("We always send up to n_cores*num_loops; qed"); let (tx, rx) = oneshot::channel(); batch.push(rx); @@ -609,32 +752,40 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { tx, )) .await; - - // // TODO: select between futures unordered of rx await and timer to send next request. - // if batch.len() >= config.max_parallel_recoveries { - // for rx in std::mem::take(&mut batch) { - // let available_data = rx.await.unwrap().unwrap(); - // availability_bytes += available_data.encoded_size() as u128; - // } - // } } + gum::info!("{} requests pending, {} completed", batch.len(), completed_count); while let Some(completed) = batch.next().await { let available_data = completed.unwrap().unwrap(); availability_bytes += available_data.encoded_size() as u128; } + + let block_time_delta = + Duration::from_secs(6).saturating_sub(Instant::now().sub(loop_start_ts)); + gum::info!(target: LOG_TARGET, "Sleeping till end of block {}ms", block_time_delta.as_millis()); + tokio::time::sleep(block_time_delta).await; } - println!("Waiting for subsystem to complete work... {} requests ", batch.len()); env.send_signal(OverseerSignal::Conclude).await; let duration = start_marker.elapsed().as_millis(); - let tput = ((availability_bytes) / duration) * 1000; - println!("Benchmark completed in {:?}ms", duration); - println!("Throughput: {}KiB/s", tput / 1024); + let availability_bytes = availability_bytes / 1024; + gum::info!("Benchmark completed in {:?}ms", duration); + gum::info!("Throughput: {} KiB/block", availability_bytes / env.config().num_loops as u128); + gum::info!( + "Block time: {} ms", + start_marker.elapsed().as_millis() / env.config().num_loops as u128 + ); + + let stats = env.network.stats(); + gum::info!( + "Total received from network: {} MiB", + stats + .iter() + .enumerate() + .map(|(index, stats)| stats.tx_bytes_total as u128) + .sum::() / + (1024 * 1024) + ); - let stats = env.network.stats().await; - for (index, stat) in stats.iter().enumerate() { - println!("Validator #{} : {:?}", index, stat); - } tokio::time::sleep(Duration::from_secs(1)).await; } diff --git a/polkadot/node/subsystem-bench/src/availability/network.rs b/polkadot/node/subsystem-bench/src/availability/network.rs index 02af817e691f..948fbae445e1 100644 --- a/polkadot/node/subsystem-bench/src/availability/network.rs +++ b/polkadot/node/subsystem-bench/src/availability/network.rs @@ -20,7 +20,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use tokio::sync::mpsc::UnboundedSender; // An emulated node egress traffic rate_limiter. #[derive(Debug)] -struct RateLimit { +pub struct RateLimit { // How often we refill credits in buckets tick_rate: usize, // Total ticks @@ -142,9 +142,8 @@ impl PeerEmulator { spawn_task_handle .clone() .spawn("peer-emulator", "test-environment", async move { - let mut rate_limiter = RateLimit::new(20, bandwidth); - let rx_bytes_total = 0; - let mut tx_bytes_total = 0u128; + // Rate limit peer send. + let mut rate_limiter = RateLimit::new(10, bandwidth); loop { let stats_clone = stats.clone(); let maybe_action: Option = actions_rx.recv().await; @@ -158,14 +157,12 @@ impl PeerEmulator { async move { tokio::time::sleep(latency).await; action.run().await; - stats_clone - .tx_bytes_total - .fetch_add(size as u64, Ordering::Relaxed); + stats_clone.inc_sent(size); }, ) } else { action.run().await; - stats_clone.tx_bytes_total.fetch_add(size as u64, Ordering::Relaxed); + stats_clone.inc_sent(size); } } else { break @@ -195,11 +192,43 @@ pub struct NetworkAction { latency: Option, } +unsafe impl Send for NetworkAction {} + /// Book keeping of sent and received bytes. -#[derive(Debug, Default)] pub struct PeerEmulatorStats { - pub rx_bytes_total: AtomicU64, - pub tx_bytes_total: AtomicU64, + rx_bytes_total: AtomicU64, + tx_bytes_total: AtomicU64, + metrics: Metrics, + peer_index: usize, +} + +impl PeerEmulatorStats { + pub(crate) fn new(peer_index: usize, metrics: Metrics) -> Self { + Self { + metrics, + rx_bytes_total: AtomicU64::from(0), + tx_bytes_total: AtomicU64::from(0), + peer_index, + } + } + + pub fn inc_sent(&self, bytes: usize) { + self.tx_bytes_total.fetch_add(bytes as u64, Ordering::Relaxed); + self.metrics.on_peer_sent(self.peer_index, bytes as u64); + } + + pub fn inc_received(&self, bytes: usize) { + self.rx_bytes_total.fetch_add(bytes as u64, Ordering::Relaxed); + self.metrics.on_peer_received(self.peer_index, bytes as u64); + } + + pub fn sent(&self) -> u64 { + self.tx_bytes_total.load(Ordering::Relaxed) + } + + pub fn received(&self) -> u64 { + self.rx_bytes_total.load(Ordering::Relaxed) + } } #[derive(Debug, Default)] @@ -229,21 +258,31 @@ impl NetworkAction { // Implements network latency, bandwidth and error. #[derive(Clone)] pub struct NetworkEmulator { - // Per peer network emulation + // Per peer network emulation. peers: Vec, + // Per peer stats. stats: Vec>, + // Metrics + metrics: Metrics, } impl NetworkEmulator { - pub fn new(n_peers: usize, bandwidth: usize, spawn_task_handle: SpawnTaskHandle) -> Self { + pub fn new( + n_peers: usize, + bandwidth: usize, + spawn_task_handle: SpawnTaskHandle, + registry: &Registry, + ) -> Self { + let metrics = Metrics::new(®istry).expect("Metrics always register succesfully"); + let (stats, peers) = (0..n_peers) - .map(|_index| { - let stats = Arc::new(PeerEmulatorStats::default()); + .map(|peer_index| { + let stats = Arc::new(PeerEmulatorStats::new(peer_index, metrics.clone())); (stats.clone(), PeerEmulator::new(bandwidth, spawn_task_handle.clone(), stats)) }) .unzip(); - Self { peers, stats } + Self { peers, stats, metrics } } pub fn submit_peer_action(&mut self, index: usize, action: NetworkAction) { @@ -251,15 +290,87 @@ impl NetworkEmulator { } // Returns the sent/received stats for all peers. - pub async fn stats(&mut self) -> Vec { + pub fn peer_stats(&mut self, peer_index: usize) -> Arc { + self.stats[peer_index].clone() + } + + // Returns the sent/received stats for all peers. + pub fn stats(&mut self) -> Vec { let r = self .stats .iter() .map(|stats| PeerStats { - rx_bytes_total: stats.rx_bytes_total.load(Ordering::Relaxed), - tx_bytes_total: stats.tx_bytes_total.load(Ordering::Relaxed), + rx_bytes_total: stats.received(), + tx_bytes_total: stats.sent(), }) .collect::>(); r } + + // Increment bytes sent by our node (the node that contains the subsystem under test) + pub fn inc_sent(&self, bytes: u64) { + // Our node always is peer 0. + self.metrics.on_peer_sent(0, bytes); + } + + // Increment bytes received by our node (the node that contains the subsystem under test) + pub fn inc_received(&self, bytes: u64) { + // Our node always is peer 0. + self.metrics.on_peer_received(0, bytes); + } +} + +use polkadot_node_subsystem_util::metrics::{ + self, + prometheus::{self, Counter, CounterVec, Histogram, Opts, PrometheusError, Registry}, +}; + +/// Emulated network metrics. +#[derive(Clone)] +pub(crate) struct Metrics { + /// Number of bytes sent per peer. + peer_total_sent: CounterVec, + /// Number of received sent per peer. + peer_total_received: CounterVec, +} + +impl Metrics { + pub fn new(registry: &Registry) -> Result { + Ok(Self { + peer_total_sent: prometheus::register( + CounterVec::new( + Opts::new( + "subsystem_benchmark_network_peer_total_bytes_sent", + "Total number of bytes a peer has sent.", + ), + &["peer"], + )?, + registry, + )?, + peer_total_received: prometheus::register( + CounterVec::new( + Opts::new( + "subsystem_benchmark_network_peer_total_bytes_received", + "Total number of bytes a peer has received.", + ), + &["peer"], + )?, + registry, + )?, + }) + } + + /// Increment total sent for a peer. + pub fn on_peer_sent(&self, peer_index: usize, bytes: u64) { + self.peer_total_sent + .with_label_values(vec![format!("node{}", peer_index).as_str()].as_slice()) + .inc_by(bytes); + } + + /// Increment total receioved for a peer. + pub fn on_peer_received(&self, peer_index: usize, bytes: u64) { + self.peer_total_received + .with_label_values(vec![format!("node{}", peer_index).as_str()].as_slice()) + .inc_by(bytes); + } } diff --git a/polkadot/node/subsystem-bench/src/availability/test_env.rs b/polkadot/node/subsystem-bench/src/availability/test_env.rs new file mode 100644 index 000000000000..f67c132f4eb4 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/availability/test_env.rs @@ -0,0 +1,63 @@ +use super::*; +use polkadot_node_subsystem_util::metrics::{ + self, + prometheus::{self, Counter, Gauge, Histogram, Opts, PrometheusError, Registry, U64}, +}; + +/// Test environment/configuration metrics +#[derive(Clone)] +pub struct TestEnvironmentMetrics { + /// Number of bytes sent per peer. + n_validators: Gauge, + /// Number of received sent per peer. + n_cores: Gauge, + /// PoV size + pov_size: Gauge, + /// Current loop + current_loop: Gauge, +} + +impl TestEnvironmentMetrics { + pub fn new(registry: &Registry) -> Result { + Ok(Self { + n_validators: prometheus::register( + Gauge::new( + "subsystem_benchmark_n_validators", + "Total number of validators in the test", + )?, + registry, + )?, + n_cores: prometheus::register( + Gauge::new( + "subsystem_benchmark_n_cores", + "Number of cores we fetch availability for each loop", + )?, + registry, + )?, + pov_size: prometheus::register( + Gauge::new("subsystem_benchmark_pov_size", "The pov size")?, + registry, + )?, + current_loop: prometheus::register( + Gauge::new("subsystem_benchmark_current_loop", "The current test loop")?, + registry, + )?, + }) + } + + pub fn set_n_validators(&self, n_validators: usize) { + self.n_validators.set(n_validators as u64); + } + + pub fn set_n_cores(&self, n_cores: usize) { + self.n_cores.set(n_cores as u64); + } + + pub fn set_current_loop(&self, current_loop: usize) { + self.current_loop.set(current_loop as u64); + } + + pub fn set_pov_size(&self, pov_size: usize) { + self.pov_size.set(pov_size as u64); + } +} diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index ba66d06fe320..bdd8d93313bb 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -90,6 +90,10 @@ struct BenchCli { /// The bandwidth of simulated remote peers in KiB pub peer_bandwidth: Option, + #[clap(short, long)] + /// The bandwidth of our simulated node in KiB + pub bandwidth: Option, + #[clap(long, value_parser=le_100)] /// Simulated connection error rate [0-100]. pub peer_error: Option, @@ -124,10 +128,9 @@ impl BenchCli { let runtime = new_runtime(); let registry = Registry::new(); - let registry_clone = registry.clone(); let mut pov_sizes = Vec::new(); - pov_sizes.append(&mut vec![5 * 1024 * 1024; 200]); + pov_sizes.append(&mut vec![10 * 1024 * 1024; 200]); let mut test_config = match self.target { BenchmarkTarget::DataAvailabilityRead(options) => match self.network { @@ -170,14 +173,20 @@ impl BenchCli { } if let Some(bandwidth) = self.peer_bandwidth { + // CLI expects bw in KiB + test_config.peer_bandwidth = bandwidth * 1024; + } + + if let Some(bandwidth) = self.bandwidth { // CLI expects bw in KiB test_config.bandwidth = bandwidth * 1024; } - let state = TestState::new(test_config); - let mut env = TestEnvironment::new(runtime.handle().clone(), state, registry.clone()); + let candidate_count = test_config.n_cores * test_config.num_loops; - let runtime_handle = runtime.handle().clone(); + let mut state = TestState::new(test_config); + state.generate_candidates(candidate_count); + let mut env = TestEnvironment::new(runtime.handle().clone(), state, registry.clone()); println!("{:?}", env.config()); @@ -230,9 +239,9 @@ impl BenchCli { fn main() -> eyre::Result<()> { color_eyre::install()?; let _ = env_logger::builder() - .is_test(true) - .filter(Some(LOG_TARGET), log::LevelFilter::Info) - .try_init(); + .filter(Some("hyper"), log::LevelFilter::Info) + .try_init() + .unwrap(); let cli: BenchCli = BenchCli::parse(); cli.launch()?; From d1b9fa39aaa98cf7e20b2108399a887780255d3b Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 14 Nov 2023 12:20:24 +0200 Subject: [PATCH 089/192] refactor Signed-off-by: Andrei Sandu --- Cargo.lock | 1 + polkadot/node/subsystem-bench/Cargo.toml | 1 + .../subsystem-bench/src/availability/mod.rs | 105 ++++++++++-------- .../src/availability/test_env.rs | 63 ----------- .../node/subsystem-bench/src/core/display.rs | 15 +++ .../node/subsystem-bench/src/core/keyring.rs | 46 ++++++++ polkadot/node/subsystem-bench/src/core/mod.rs | 80 +++++++++++++ .../src/{availability => core}/network.rs | 48 +++++--- .../node/subsystem-bench/src/core/test_env.rs | 102 +++++++++++++++++ .../subsystem-bench/src/subsystem-bench.rs | 41 +++++-- 10 files changed, 371 insertions(+), 131 deletions(-) delete mode 100644 polkadot/node/subsystem-bench/src/availability/test_env.rs create mode 100644 polkadot/node/subsystem-bench/src/core/display.rs create mode 100644 polkadot/node/subsystem-bench/src/core/keyring.rs create mode 100644 polkadot/node/subsystem-bench/src/core/mod.rs rename polkadot/node/subsystem-bench/src/{availability => core}/network.rs (86%) create mode 100644 polkadot/node/subsystem-bench/src/core/test_env.rs diff --git a/Cargo.lock b/Cargo.lock index ee80ffb6e815..9e93536d4f32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13028,6 +13028,7 @@ dependencies = [ "polkadot-primitives-test-helpers", "prometheus", "rand 0.8.5", + "sc-keystore", "sc-network", "sc-service", "sp-application-crypto", diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index c5d62d3aa74f..72c8c3ac3c4d 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -27,6 +27,7 @@ color-eyre = { version = "0.6.1", default-features = false } assert_matches = "1.5" async-trait = "0.1.57" sp-keystore = { path = "../../../substrate/primitives/keystore" } +sc-keystore = { path = "../../../substrate/client/keystore" } sp-core = { path = "../../../substrate/primitives/core" } clap = { version = "4.4.6", features = ["derive"] } futures = "0.3.21" diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 4f821f819908..5f856ec1780f 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -22,6 +22,10 @@ use std::{ time::{Duration, Instant}, }; +use sc_keystore::LocalKeystore; +use sp_application_crypto::AppCrypto; +use sp_keystore::{Keystore, KeystorePtr}; + use futures::{ channel::{mpsc, oneshot}, stream::FuturesUnordered, @@ -53,7 +57,7 @@ use polkadot_node_subsystem::{ }; use std::net::{Ipv4Addr, SocketAddr}; -mod test_env; +use super::core::{keyring::Keyring, network::*, test_env::TestEnvironmentMetrics}; const LOG_TARGET: &str = "subsystem-bench::availability"; @@ -71,9 +75,8 @@ use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; use sc_service::{SpawnTaskHandle, TaskManager}; mod configuration; -mod network; -pub use configuration::TestConfiguration; +pub use configuration::{PeerLatency, TestConfiguration}; // Deterministic genesis hash for protocol names const GENESIS_HASH: Hash = Hash::repeat_byte(0xff); @@ -140,10 +143,12 @@ impl TestEnvironment { task_manager.spawn_handle(), state.config().use_fast_path, ); + let metrics = TestEnvironmentMetrics::new(®istry).expect("Metrics need to be registered"); let mut network = NetworkEmulator::new( state.config().n_validators, + state.validator_authority_id.clone(), state.config().peer_bandwidth, task_manager.spawn_handle(), ®istry, @@ -243,7 +248,7 @@ impl TestEnvironment { ) -> NetworkAction { match request { Requests::ChunkFetchingV1(outgoing_request) => { - let validator_index = outgoing_request.payload.index.0 as usize; + let validator_index: usize = outgoing_request.payload.index.0 as usize; let candidate_hash = outgoing_request.payload.candidate_hash; let candidate_index = state @@ -266,6 +271,12 @@ impl TestEnvironment { Ok(req_res::v1::ChunkFetchingResponse::from(Some(chunk)).encode()) }; + let authority_discovery_id = match outgoing_request.peer { + req_res::Recipient::Authority(authority_discovery_id) => authority_discovery_id, + _ => panic!("Peer recipient not supported yet"), + }; + let authority_discovery_id_clone = authority_discovery_id.clone(); + let future = async move { let _ = outgoing_request.pending_response.send(response); } @@ -274,13 +285,14 @@ impl TestEnvironment { let future_wrapper = async move { // Forward the response to the ingress channel of our node. // On receive side we apply our node receiving rate limit. - let action = NetworkAction::new(validator_index, future, size, None); + let action = + NetworkAction::new(authority_discovery_id_clone, future, size, None); ingress_tx.send(action).unwrap(); } .boxed(); NetworkAction::new( - validator_index, + authority_discovery_id, future_wrapper, size, // Generate a random latency based on configuration. @@ -288,12 +300,6 @@ impl TestEnvironment { ) }, Requests::AvailableDataFetchingV1(outgoing_request) => { - println!("{:?}", outgoing_request); - // TODO: do better, by implementing diff authority ids and mapping network actions - // to authority id, - let validator_index = - Uniform::from(0..state.config().n_validators).sample(&mut thread_rng()); - let candidate_hash = outgoing_request.payload.candidate_hash; let candidate_index = state .candidate_hashes @@ -318,16 +324,23 @@ impl TestEnvironment { } .boxed(); + let authority_discovery_id = match outgoing_request.peer { + req_res::Recipient::Authority(authority_discovery_id) => authority_discovery_id, + _ => panic!("Peer recipient not supported yet"), + }; + let authority_discovery_id_clone = authority_discovery_id.clone(); + let future_wrapper = async move { // Forward the response to the ingress channel of our node. // On receive side we apply our node receiving rate limit. - let action = NetworkAction::new(validator_index, future, size, None); + let action = + NetworkAction::new(authority_discovery_id_clone, future, size, None); ingress_tx.send(action).unwrap(); } .boxed(); NetworkAction::new( - validator_index, + authority_discovery_id, future_wrapper, size, // Generate a random latency based on configuration. @@ -362,7 +375,7 @@ impl TestEnvironment { network.inc_sent(Self::request_size(&request)); let action = Self::respond_to_send_request(&mut state, request, ingress_tx.clone()); // Account for our node sending the request over the emulated network. - network.submit_peer_action(action.index(), action); + network.submit_peer_action(action.peer(), action); } }, AllMessages::AvailabilityStore(AvailabilityStoreMessage::QueryAvailableData(_candidate_hash, tx)) => { @@ -470,25 +483,24 @@ impl AvailabilityRecoverySubsystemInstance { } } +pub fn random_pov_size(min_pov_size: usize, max_pov_size: usize) -> usize { + random_uniform_sample(min_pov_size, max_pov_size) +} + +fn random_uniform_sample + From>(min_value: T, max_value: T) -> T { + Uniform::from(min_value.into()..=max_value.into()) + .sample(&mut thread_rng()) + .into() +} + // We use this to bail out sending messages to the subsystem if it is overloaded such that // the time of flight is breaches 5s. // This should eventually be a test parameter. const MAX_TIME_OF_FLIGHT: Duration = Duration::from_millis(5000); -use sp_keyring::Sr25519Keyring; - -use crate::availability::network::{ActionFuture, NetworkAction}; - -use self::{ - configuration::PeerLatency, - network::{NetworkEmulator, RateLimit}, - test_env::TestEnvironmentMetrics, -}; - #[derive(Clone)] pub struct TestState { - validators: Vec, - validator_public: IndexedVec, + validator_public: Vec, validator_authority_id: Vec, // The test node validator index. validator_index: ValidatorIndex, @@ -531,7 +543,7 @@ impl TestState { let validator_groups = my_vec.chunks(5).map(|x| Vec::from(x)).collect::>(); SessionInfo { - validators: self.validator_public.clone(), + validators: self.validator_public.clone().into(), discovery_keys: self.validator_authority_id.clone(), validator_groups: IndexedVec::>::from(validator_groups), assignment_keys: vec![], @@ -608,13 +620,24 @@ impl TestState { } pub fn new(config: TestConfiguration) -> Self { - let validators = (0..config.n_validators as u64) - .into_iter() - .map(|_v| Sr25519Keyring::Alice) + let keystore: KeystorePtr = Arc::new(LocalKeystore::in_memory()); + + let keyrings = (0..config.n_validators) + .map(|peer_index| Keyring::new(format!("Node{}", peer_index).into())) .collect::>(); - let validator_public = validator_pubkeys(&validators); - let validator_authority_id = validator_authority_id(&validators); + // Generate `AuthorityDiscoveryId`` for each peer + let validator_public: Vec = keyrings + .iter() + .map(|keyring: &Keyring| keyring.clone().public().into()) + .collect::>(); + + let validator_authority_id: Vec = keyrings + .iter() + .map({ |keyring| keyring.clone().public().into() }) + .collect::>() + .into(); + let validator_index = ValidatorIndex(0); let mut chunks = Vec::new(); let mut available_data = Vec::new(); @@ -643,7 +666,7 @@ impl TestState { }; let (new_chunks, erasure_root) = derive_erasure_chunks_with_proofs_and_root( - validators.len(), + config.n_validators, &new_available_data, |_, _| {}, ); @@ -658,7 +681,6 @@ impl TestState { let pov_sizes = config.pov_sizes.clone().into_iter().cycle(); let mut state = Self { - validators, validator_public, validator_authority_id, validator_index, @@ -681,14 +703,6 @@ impl TestState { } } -fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> IndexedVec { - val_ids.iter().map(|v| v.public().into()).collect() -} - -fn validator_authority_id(val_ids: &[Sr25519Keyring]) -> Vec { - val_ids.iter().map(|v| v.public().into()).collect() -} - fn derive_erasure_chunks_with_proofs_and_root( n_validators: usize, available_data: &AvailableData, @@ -731,8 +745,6 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { env.metrics().set_n_validators(config.n_validators); env.metrics().set_n_cores(config.n_cores); - env.metrics().set_pov_size(config.pov_sizes[0]); - let mut completed_count = 0; for loop_num in 0..env.config().num_loops { gum::info!(target: LOG_TARGET, loop_num, "Starting loop"); @@ -754,9 +766,10 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { .await; } - gum::info!("{} requests pending, {} completed", batch.len(), completed_count); + gum::info!("{} requests pending", batch.len()); while let Some(completed) = batch.next().await { let available_data = completed.unwrap().unwrap(); + env.metrics().on_pov_size(available_data.encoded_size()); availability_bytes += available_data.encoded_size() as u128; } diff --git a/polkadot/node/subsystem-bench/src/availability/test_env.rs b/polkadot/node/subsystem-bench/src/availability/test_env.rs deleted file mode 100644 index f67c132f4eb4..000000000000 --- a/polkadot/node/subsystem-bench/src/availability/test_env.rs +++ /dev/null @@ -1,63 +0,0 @@ -use super::*; -use polkadot_node_subsystem_util::metrics::{ - self, - prometheus::{self, Counter, Gauge, Histogram, Opts, PrometheusError, Registry, U64}, -}; - -/// Test environment/configuration metrics -#[derive(Clone)] -pub struct TestEnvironmentMetrics { - /// Number of bytes sent per peer. - n_validators: Gauge, - /// Number of received sent per peer. - n_cores: Gauge, - /// PoV size - pov_size: Gauge, - /// Current loop - current_loop: Gauge, -} - -impl TestEnvironmentMetrics { - pub fn new(registry: &Registry) -> Result { - Ok(Self { - n_validators: prometheus::register( - Gauge::new( - "subsystem_benchmark_n_validators", - "Total number of validators in the test", - )?, - registry, - )?, - n_cores: prometheus::register( - Gauge::new( - "subsystem_benchmark_n_cores", - "Number of cores we fetch availability for each loop", - )?, - registry, - )?, - pov_size: prometheus::register( - Gauge::new("subsystem_benchmark_pov_size", "The pov size")?, - registry, - )?, - current_loop: prometheus::register( - Gauge::new("subsystem_benchmark_current_loop", "The current test loop")?, - registry, - )?, - }) - } - - pub fn set_n_validators(&self, n_validators: usize) { - self.n_validators.set(n_validators as u64); - } - - pub fn set_n_cores(&self, n_cores: usize) { - self.n_cores.set(n_cores as u64); - } - - pub fn set_current_loop(&self, current_loop: usize) { - self.current_loop.set(current_loop as u64); - } - - pub fn set_pov_size(&self, pov_size: usize) { - self.pov_size.set(pov_size as u64); - } -} diff --git a/polkadot/node/subsystem-bench/src/core/display.rs b/polkadot/node/subsystem-bench/src/core/display.rs new file mode 100644 index 000000000000..47483d33a42a --- /dev/null +++ b/polkadot/node/subsystem-bench/src/core/display.rs @@ -0,0 +1,15 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . diff --git a/polkadot/node/subsystem-bench/src/core/keyring.rs b/polkadot/node/subsystem-bench/src/core/keyring.rs new file mode 100644 index 000000000000..40e8d60d0cd1 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/core/keyring.rs @@ -0,0 +1,46 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +pub use sp_core::sr25519; +use sp_core::{ + sr25519::{Pair, Public, Signature}, + ByteArray, Pair as PairT, H256, +}; +use std::{collections::HashMap, ops::Deref}; + +/// Set of test accounts. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Keyring { + name: String, +} + +impl Keyring { + pub fn new(name: String) -> Keyring { + Self { name } + } + + pub fn pair(self) -> Pair { + Pair::from_string(&format!("//{}", self.name), None).expect("input is always good; qed") + } + + pub fn public(self) -> Public { + self.pair().public() + } + + pub fn to_seed(self) -> String { + format!("//{}", self.name) + } +} diff --git a/polkadot/node/subsystem-bench/src/core/mod.rs b/polkadot/node/subsystem-bench/src/core/mod.rs new file mode 100644 index 000000000000..4b9db3144f54 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/core/mod.rs @@ -0,0 +1,80 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use itertools::Itertools; +use std::{ + collections::HashMap, + iter::Cycle, + ops::{Div, Sub}, + sync::Arc, + time::{Duration, Instant}, +}; + +use sc_keystore::LocalKeystore; +use sp_application_crypto::AppCrypto; +use sp_keystore::{Keystore, KeystorePtr}; + +use futures::{ + channel::{mpsc, oneshot}, + stream::FuturesUnordered, + FutureExt, SinkExt, StreamExt, +}; +use futures_timer::Delay; + +use polkadot_node_metrics::metrics::Metrics; + +use polkadot_availability_recovery::AvailabilityRecoverySubsystem; + +use parity_scale_codec::Encode; +use polkadot_node_network_protocol::request_response::{ + self as req_res, v1::ChunkResponse, IncomingRequest, ReqProtocolNames, Requests, +}; +use rand::{distributions::Uniform, prelude::Distribution, seq::IteratorRandom, thread_rng}; + +use prometheus::Registry; +use sc_network::{config::RequestResponseConfig, OutboundFailure, RequestFailure}; + +use polkadot_erasure_coding::{branches, obtain_chunks_v1 as obtain_chunks}; +use polkadot_node_primitives::{BlockData, PoV, Proof}; +use polkadot_node_subsystem::{ + messages::{ + AllMessages, AvailabilityRecoveryMessage, AvailabilityStoreMessage, NetworkBridgeTxMessage, + RuntimeApiMessage, RuntimeApiRequest, + }, + ActiveLeavesUpdate, FromOrchestra, OverseerSignal, Subsystem, +}; +use std::net::{Ipv4Addr, SocketAddr}; + +use super::core::{keyring::Keyring, network::*, test_env::TestEnvironmentMetrics}; + +const LOG_TARGET: &str = "subsystem-bench::core"; + +use polkadot_node_primitives::{AvailableData, ErasureChunk}; + +use polkadot_node_subsystem_test_helpers::{ + make_buffered_subsystem_context, mock::new_leaf, TestSubsystemContextHandle, +}; +use polkadot_node_subsystem_util::TimeoutExt; +use polkadot_primitives::{ + AuthorityDiscoveryId, CandidateHash, CandidateReceipt, GroupIndex, Hash, HeadData, IndexedVec, + PersistedValidationData, SessionIndex, SessionInfo, ValidatorId, ValidatorIndex, +}; +use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; +use sc_service::{SpawnTaskHandle, TaskManager}; + +pub mod keyring; +pub mod network; +pub mod test_env; diff --git a/polkadot/node/subsystem-bench/src/availability/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs similarity index 86% rename from polkadot/node/subsystem-bench/src/availability/network.rs rename to polkadot/node/subsystem-bench/src/core/network.rs index 948fbae445e1..170ab45e35a3 100644 --- a/polkadot/node/subsystem-bench/src/availability/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -124,7 +124,9 @@ mod tests { } } -// A network peer emulator +// A network peer emulator. It spawns a task that accepts `NetworkActions` and +// executes them with a configurable delay and bandwidth constraints. Tipically +// these actions wrap a future that performs a channel send to the subsystem(s) under test. #[derive(Clone)] struct PeerEmulator { // The queue of requests waiting to be served by the emulator @@ -186,8 +188,8 @@ pub struct NetworkAction { run: ActionFuture, // The payload size that we simulate sending from a peer size: usize, - // Peer index - index: usize, + // Peer which should run the action. + peer: AuthorityDiscoveryId, // The amount of time to delay the polling `run` latency: Option, } @@ -237,8 +239,13 @@ pub struct PeerStats { pub tx_bytes_total: u64, } impl NetworkAction { - pub fn new(index: usize, run: ActionFuture, size: usize, latency: Option) -> Self { - Self { run, size, index, latency } + pub fn new( + peer: AuthorityDiscoveryId, + run: ActionFuture, + size: usize, + latency: Option, + ) -> Self { + Self { run, size, peer, latency } } pub fn size(&self) -> usize { @@ -249,44 +256,55 @@ impl NetworkAction { self.run.await; } - pub fn index(&self) -> usize { - self.index + pub fn peer(&self) -> AuthorityDiscoveryId { + self.peer.clone() } } -// Mocks the network bridge and an arbitrary number of connected peer nodes. -// Implements network latency, bandwidth and error. +/// Mocks the network bridge and an arbitrary number of connected peer nodes. +/// Implements network latency, bandwidth and connection errors. #[derive(Clone)] pub struct NetworkEmulator { // Per peer network emulation. peers: Vec, - // Per peer stats. + /// Per peer stats. stats: Vec>, - // Metrics + /// Network throughput metrics metrics: Metrics, + /// Each emulated peer is a validator. + validator_authority_ids: HashMap, } impl NetworkEmulator { pub fn new( n_peers: usize, + validator_authority_ids: Vec, bandwidth: usize, spawn_task_handle: SpawnTaskHandle, registry: &Registry, ) -> Self { let metrics = Metrics::new(®istry).expect("Metrics always register succesfully"); + let mut validator_authority_id_mapping = HashMap::new(); + // Create a `PeerEmulator` for each peer. let (stats, peers) = (0..n_peers) - .map(|peer_index| { + .zip(validator_authority_ids.into_iter()) + .map(|(peer_index, authority_id)| { + validator_authority_id_mapping.insert(authority_id, peer_index); let stats = Arc::new(PeerEmulatorStats::new(peer_index, metrics.clone())); (stats.clone(), PeerEmulator::new(bandwidth, spawn_task_handle.clone(), stats)) }) .unzip(); - Self { peers, stats, metrics } + Self { peers, stats, metrics, validator_authority_ids: validator_authority_id_mapping } } - pub fn submit_peer_action(&mut self, index: usize, action: NetworkAction) { - let _ = self.peers[index].send(action); + pub fn submit_peer_action(&mut self, peer: AuthorityDiscoveryId, action: NetworkAction) { + let index = self + .validator_authority_ids + .get(&peer) + .expect("all test authorities are valid; qed"); + self.peers[*index].send(action); } // Returns the sent/received stats for all peers. diff --git a/polkadot/node/subsystem-bench/src/core/test_env.rs b/polkadot/node/subsystem-bench/src/core/test_env.rs new file mode 100644 index 000000000000..c20b96d642af --- /dev/null +++ b/polkadot/node/subsystem-bench/src/core/test_env.rs @@ -0,0 +1,102 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use super::*; +use polkadot_node_subsystem_util::metrics::{ + self, + prometheus::{ + self, Counter, Gauge, Histogram, HistogramVec, Opts, PrometheusError, Registry, U64, + }, +}; + +const MIB: f64 = 1024.0*1024.0; + +/// Test environment/configuration metrics +#[derive(Clone)] +pub struct TestEnvironmentMetrics { + /// Number of bytes sent per peer. + n_validators: Gauge, + /// Number of received sent per peer. + n_cores: Gauge, + /// PoV size + pov_size: Histogram, + /// Current loop + current_loop: Gauge, +} + +impl TestEnvironmentMetrics { + pub fn new(registry: &Registry) -> Result { + let mut buckets = prometheus::exponential_buckets(16384.0, 2.0, 9) + .expect("arguments are always valid; qed"); + buckets.extend(vec![ + 5.0 * MIB, + 6.0 * MIB, + 7.0 * MIB, + 8.0 * MIB, + 9.0 * MIB, + 10.0 * MIB, + ]); + + Ok(Self { + n_validators: prometheus::register( + Gauge::new( + "subsystem_benchmark_n_validators", + "Total number of validators in the test", + )?, + registry, + )?, + n_cores: prometheus::register( + Gauge::new( + "subsystem_benchmark_n_cores", + "Number of cores we fetch availability for each loop", + )?, + registry, + )?, + current_loop: prometheus::register( + Gauge::new("subsystem_benchmark_current_loop", "The current test loop")?, + registry, + )?, + pov_size: prometheus::register( + Histogram::with_opts( + prometheus::HistogramOpts::new( + "subsystem_benchmark_pov_size", + "The compressed size of the proof of validity of a candidate", + ) + .buckets( + buckets + ), + )?, + registry, + )?, + }) + } + + pub fn set_n_validators(&self, n_validators: usize) { + self.n_validators.set(n_validators as u64); + } + + pub fn set_n_cores(&self, n_cores: usize) { + self.n_cores.set(n_cores as u64); + } + + pub fn set_current_loop(&self, current_loop: usize) { + self.current_loop.set(current_loop as u64); + } + + pub fn on_pov_size(&self, pov_size: usize) { + self.pov_size.observe(pov_size as f64); + } +} diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index bdd8d93313bb..9e581555d761 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -22,8 +22,9 @@ use prometheus::proto::LabelPair; use std::time::Duration; pub(crate) mod availability; +pub(crate) mod core; -use availability::{TestConfiguration, TestEnvironment, TestState}; +use availability::{random_pov_size, TestConfiguration, TestEnvironment, TestState}; const LOG_TARGET: &str = "subsystem-bench"; use clap_num::number_range; @@ -62,6 +63,14 @@ pub struct DataAvailabilityReadOptions { /// Number of validators to fetch chunks from. pub n_validators: usize, + #[clap(long, ignore_case = true, default_value_t = 5120)] + /// The minimum pov size in KiB + pub min_pov_size: usize, + + #[clap(long, ignore_case = true, default_value_t = 5120)] + /// The maximum pov size bytes + pub max_pov_size: usize, + #[clap(short, long, default_value_t = false)] /// Turbo boost AD Read by fetching from backers first. Tipically this is only faster if nodes /// have enough bandwidth. @@ -129,9 +138,6 @@ impl BenchCli { let runtime = new_runtime(); let registry = Registry::new(); - let mut pov_sizes = Vec::new(); - pov_sizes.append(&mut vec![10 * 1024 * 1024; 200]); - let mut test_config = match self.target { BenchmarkTarget::DataAvailabilityRead(options) => match self.network { NetworkEmulation::Healthy => TestConfiguration::healthy_network( @@ -139,21 +145,42 @@ impl BenchCli { options.fetch_from_backers, options.n_validators, options.n_cores, - pov_sizes, + (0..options.n_cores) + .map(|_| { + random_pov_size( + options.min_pov_size * 1024, + options.max_pov_size * 1024, + ) + }) + .collect(), ), NetworkEmulation::Degraded => TestConfiguration::degraded_network( options.num_loops, options.fetch_from_backers, options.n_validators, options.n_cores, - pov_sizes, + (0..options.n_cores) + .map(|_| { + random_pov_size( + options.min_pov_size * 1024, + options.max_pov_size * 1024, + ) + }) + .collect(), ), NetworkEmulation::Ideal => TestConfiguration::ideal_network( options.num_loops, options.fetch_from_backers, options.n_validators, options.n_cores, - pov_sizes, + (0..options.n_cores) + .map(|_| { + random_pov_size( + options.min_pov_size * 1024, + options.max_pov_size * 1024, + ) + }) + .collect(), ), }, }; From c5937ab840c56a812f840332fdbb295b23c10823 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 15 Nov 2023 08:42:41 +0200 Subject: [PATCH 090/192] pretty cli + minor refactor + remove unused Signed-off-by: Andrei Sandu --- Cargo.lock | 61 +++- cumulus/pallets/xcmp-queue/src/bridging.rs | 4 +- polkadot/node/subsystem-bench/Cargo.toml | 3 + .../src/availability/configuration.rs | 83 +++++- .../subsystem-bench/src/availability/mod.rs | 120 ++++---- .../node/subsystem-bench/src/core/display.rs | 276 ++++++++++++++++++ .../node/subsystem-bench/src/core/keyring.rs | 10 +- polkadot/node/subsystem-bench/src/core/mod.rs | 56 +--- .../node/subsystem-bench/src/core/network.rs | 4 +- .../node/subsystem-bench/src/core/test_env.rs | 39 +-- .../subsystem-bench/src/subsystem-bench.rs | 99 +++---- .../node/subsystem-test-helpers/src/lib.rs | 7 + 12 files changed, 537 insertions(+), 225 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e93536d4f32..73fc3cbdeccc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2764,6 +2764,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "colored" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2674ec482fbc38012cf31e6c42ba0177b431a0cb6f15fe40efa5aab1bda516f6" +dependencies = [ + "is-terminal", + "lazy_static", + "windows-sys 0.48.0", +] + [[package]] name = "comfy-table" version = "7.0.1" @@ -8568,7 +8579,7 @@ dependencies = [ "itertools 0.10.5", "tar", "tempfile", - "toml_edit", + "toml_edit 0.19.14", ] [[package]] @@ -13009,6 +13020,7 @@ dependencies = [ "clap 4.4.6", "clap-num", "color-eyre", + "colored", "env_logger 0.9.3", "futures", "futures-timer", @@ -13031,12 +13043,14 @@ dependencies = [ "sc-keystore", "sc-network", "sc-service", + "serde", "sp-application-crypto", "sp-core", "sp-keyring", "sp-keystore", "substrate-prometheus-endpoint", "tokio", + "toml 0.8.8", "tracing-gum", ] @@ -13441,7 +13455,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" dependencies = [ "once_cell", - "toml_edit", + "toml_edit 0.19.14", ] [[package]] @@ -16276,18 +16290,18 @@ checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" dependencies = [ "proc-macro2", "quote", @@ -16316,9 +16330,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" dependencies = [ "serde", ] @@ -18819,14 +18833,26 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_edit 0.19.14", +] + +[[package]] +name = "toml" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit 0.21.0", ] [[package]] name = "toml_datetime" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" dependencies = [ "serde", ] @@ -18844,6 +18870,19 @@ dependencies = [ "winnow", ] +[[package]] +name = "toml_edit" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +dependencies = [ + "indexmap 2.0.0", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" diff --git a/cumulus/pallets/xcmp-queue/src/bridging.rs b/cumulus/pallets/xcmp-queue/src/bridging.rs index 0fc3f1f39ea3..53238fe2bf7a 100644 --- a/cumulus/pallets/xcmp-queue/src/bridging.rs +++ b/cumulus/pallets/xcmp-queue/src/bridging.rs @@ -55,7 +55,9 @@ impl, Runtime: crate::Config> let sibling_bridge_hub_id: ParaId = SiblingBridgeHubParaId::get(); // let's find the channel's state with the sibling parachain, - let Some((outbound_state, queued_pages)) = pallet::Pallet::::outbound_channel_state(sibling_bridge_hub_id) else { + let Some((outbound_state, queued_pages)) = + pallet::Pallet::::outbound_channel_state(sibling_bridge_hub_id) + else { return false }; // suspended channel => it is congested diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index 72c8c3ac3c4d..3308b6fe1052 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -24,6 +24,7 @@ polkadot-primitives = { path = "../../primitives" } polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-availability-recovery = { path = "../network/availability-recovery", features=["subsystem-benchmarks"]} color-eyre = { version = "0.6.1", default-features = false } +colored = "2.0.4" assert_matches = "1.5" async-trait = "0.1.57" sp-keystore = { path = "../../../substrate/primitives/keystore" } @@ -50,6 +51,8 @@ itertools = "0.11.0" polkadot-primitives-test-helpers = { path = "../../primitives/test-helpers" } prometheus_endpoint = { package = "substrate-prometheus-endpoint", path = "../../../substrate/utils/prometheus" } prometheus = { version = "0.13.0", default-features = false } +toml = "0.8.8" +serde = "1.0.192" [features] default = [] diff --git a/polkadot/node/subsystem-bench/src/availability/configuration.rs b/polkadot/node/subsystem-bench/src/availability/configuration.rs index cf142de06634..2d29d23811da 100644 --- a/polkadot/node/subsystem-bench/src/availability/configuration.rs +++ b/polkadot/node/subsystem-bench/src/availability/configuration.rs @@ -14,10 +14,12 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use super::*; +use std::path::Path; +use super::*; +use serde::{Deserialize,Serialize}; /// Peer response latency configuration. -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct PeerLatency { /// Min latency for `NetworkAction` completion. pub min_latency: Duration, @@ -26,7 +28,7 @@ pub struct PeerLatency { } /// The test input parameters -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct TestConfiguration { /// Configuration for the `availability-recovery` subsystem. pub use_fast_path: bool, @@ -34,8 +36,13 @@ pub struct TestConfiguration { pub n_validators: usize, /// Number of cores pub n_cores: usize, - /// The PoV size - pub pov_sizes: Vec, + /// The min PoV size + pub min_pov_size: usize, + /// The max PoV size, + pub max_pov_size: usize, + /// Randomly sampled pov_sizes + #[serde(skip)] + pov_sizes: Vec, /// The amount of bandiwdth remote validators have. pub peer_bandwidth: usize, /// The amount of bandiwdth our node has. @@ -44,31 +51,72 @@ pub struct TestConfiguration { pub latency: Option, /// Error probability pub error: usize, - /// Number of loops - /// In one loop `n_cores` candidates are recovered - pub num_loops: usize, + /// Number of blocks + /// In one block `n_cores` candidates are recovered + pub num_blocks: usize, } + impl Default for TestConfiguration { fn default() -> Self { Self { use_fast_path: false, - n_validators: 10, + n_validators: 100, n_cores: 10, pov_sizes: vec![5 * 1024 * 1024], bandwidth: 60 * 1024 * 1024, peer_bandwidth: 60 * 1024 * 1024, latency: None, error: 0, - num_loops: 1, + num_blocks: 1, + min_pov_size: 5*1024*1024, + max_pov_size: 5*1024*1024, } } } +fn generate_pov_sizes(count: usize, min: usize, max: usize) -> Vec { + (0..count).map(|_| random_pov_size(min, max)).collect() +} + +#[derive(Serialize,Deserialize)] +pub struct TestSequence { + #[serde(rename(serialize = "TestConfiguration", deserialize = "TestConfiguration"))] + test_configurations: Vec +} + +impl TestSequence { + pub fn to_vec(mut self) -> Vec { + // Generate Pov sizes + + for config in self.test_configurations.iter_mut() { + config.pov_sizes = generate_pov_sizes(config.n_cores, config.min_pov_size, config.max_pov_size); + } + + self.test_configurations + } +} + +impl TestSequence { + pub fn new_from_file(path: &Path) -> std::io::Result { + let string = String::from_utf8(std::fs::read(&path)?).expect("File is valid UTF8"); + Ok(toml::from_str(&string).expect("File is valid test sequence TOML")) + } +} + impl TestConfiguration { + pub fn write_to_disk(&self) { + // Serialize a slice of configurations + let toml = toml::to_string(&TestSequence{ test_configurations: vec![self.clone()] }).unwrap(); + std::fs::write("last_test.toml", toml).unwrap(); + } + + pub fn pov_sizes(&self) -> &[usize] { + &self.pov_sizes + } /// An unconstrained standard configuration matching Polkadot/Kusama pub fn ideal_network( - num_loops: usize, + num_blocks: usize, use_fast_path: bool, n_validators: usize, n_cores: usize, @@ -84,12 +132,13 @@ impl TestConfiguration { // No latency latency: None, error: 0, - num_loops, + num_blocks, + ..Default::default() } } pub fn healthy_network( - num_loops: usize, + num_blocks: usize, use_fast_path: bool, n_validators: usize, n_cores: usize, @@ -107,12 +156,13 @@ impl TestConfiguration { max_latency: Duration::from_millis(100), }), error: 3, - num_loops, + num_blocks, + ..Default::default() } } pub fn degraded_network( - num_loops: usize, + num_blocks: usize, use_fast_path: bool, n_validators: usize, n_cores: usize, @@ -130,7 +180,8 @@ impl TestConfiguration { max_latency: Duration::from_millis(500), }), error: 33, - num_loops, + num_blocks, + ..Default::default() } } } diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 5f856ec1780f..2c9f3e735afb 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -17,22 +17,18 @@ use itertools::Itertools; use std::{ collections::HashMap, iter::Cycle, - ops::{Div, Sub}, + ops::Sub, sync::Arc, time::{Duration, Instant}, }; -use sc_keystore::LocalKeystore; -use sp_application_crypto::AppCrypto; -use sp_keystore::{Keystore, KeystorePtr}; +use colored::Colorize; use futures::{ channel::{mpsc, oneshot}, stream::FuturesUnordered, FutureExt, SinkExt, StreamExt, }; -use futures_timer::Delay; - use polkadot_node_metrics::metrics::Metrics; use polkadot_availability_recovery::AvailabilityRecoverySubsystem; @@ -41,7 +37,7 @@ use parity_scale_codec::Encode; use polkadot_node_network_protocol::request_response::{ self as req_res, v1::ChunkResponse, IncomingRequest, ReqProtocolNames, Requests, }; -use rand::{distributions::Uniform, prelude::Distribution, seq::IteratorRandom, thread_rng}; +use rand::{distributions::Uniform, prelude::Distribution, thread_rng}; use prometheus::Registry; use sc_network::{config::RequestResponseConfig, OutboundFailure, RequestFailure}; @@ -74,9 +70,9 @@ use polkadot_primitives::{ use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; use sc_service::{SpawnTaskHandle, TaskManager}; -mod configuration; +pub mod configuration; -pub use configuration::{PeerLatency, TestConfiguration}; +pub use configuration::{PeerLatency, TestConfiguration, TestSequence}; // Deterministic genesis hash for protocol names const GENESIS_HASH: Hash = Hash::repeat_byte(0xff); @@ -205,8 +201,12 @@ impl TestEnvironment { self.state.config() } - pub fn network(&self) -> &NetworkEmulator { - &self.network + pub fn network(&mut self) -> &mut NetworkEmulator { + &mut self.network + } + + pub fn registry(&self) -> &Registry { + &self.registry } /// Produce a randomized duration between `min` and `max`. @@ -361,7 +361,14 @@ impl TestEnvironment { ) { loop { futures::select! { - message = ctx.recv().fuse() => { + maybe_message = ctx.maybe_recv().fuse() => { + let message = if let Some(message) = maybe_message{ + message + } else { + gum::info!("{}", "Test completed".bright_blue()); + return + }; + gum::trace!(target: LOG_TARGET, ?message, "Env task received message"); match message { @@ -390,7 +397,7 @@ impl TestEnvironment { AvailabilityStoreMessage::QueryChunkSize(candidate_hash, tx) ) => { let candidate_index = state.candidate_hashes.get(&candidate_hash).expect("candidate was generated previously; qed"); - gum::info!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); + gum::debug!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); let chunk_size = state.chunks.get(*candidate_index as usize).unwrap()[0].encoded_size(); let _ = tx.send(Some(chunk_size)); @@ -564,13 +571,11 @@ impl TestState { send_chunk: impl Fn(usize) -> bool, tx: oneshot::Sender>, ) { - gum::info!(target: LOG_TARGET, ?candidate_hash, "respond_to_query_all_request"); - let candidate_index = self .candidate_hashes .get(&candidate_hash) .expect("candidate was generated previously; qed"); - gum::info!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); + gum::debug!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); let v = self .chunks @@ -593,7 +598,7 @@ impl TestState { /// Generate candidates to be used in the test. pub fn generate_candidates(&mut self, count: usize) { - gum::info!(target: LOG_TARGET, "Pre-generating {} candidates.", count); + gum::info!(target: LOG_TARGET,"{}", format!("Pre-generating {} candidates.", count).bright_blue()); // Generate all candidates self.candidates = (0..count) @@ -610,7 +615,7 @@ impl TestState { // Store the new candidate in the state self.candidate_hashes.insert(candidate_receipt.hash(), candidate_index); - gum::info!(target: LOG_TARGET, candidate_hash = ?candidate_receipt.hash(), "new candidate"); + gum::debug!(target: LOG_TARGET, candidate_hash = ?candidate_receipt.hash(), "new candidate"); candidate_receipt }) @@ -620,8 +625,6 @@ impl TestState { } pub fn new(config: TestConfiguration) -> Self { - let keystore: KeystorePtr = Arc::new(LocalKeystore::in_memory()); - let keyrings = (0..config.n_validators) .map(|peer_index| Keyring::new(format!("Node{}", peer_index).into())) .collect::>(); @@ -634,7 +637,7 @@ impl TestState { let validator_authority_id: Vec = keyrings .iter() - .map({ |keyring| keyring.clone().public().into() }) + .map(|keyring| keyring.clone().public().into()) .collect::>() .into(); @@ -654,8 +657,8 @@ impl TestState { }; // For each unique pov we create a candidate receipt. - for (index, pov_size) in config.pov_sizes.iter().cloned().unique().enumerate() { - gum::info!(target: LOG_TARGET, index, pov_size, "Generating template candidates"); + for (index, pov_size) in config.pov_sizes().iter().cloned().unique().enumerate() { + gum::info!(target: LOG_TARGET, index, pov_size, "{}", "Generating template candidate".bright_blue()); let mut candidate_receipt = dummy_candidate_receipt(dummy_hash()); let pov = PoV { block_data: BlockData(vec![index as u8; pov_size]) }; @@ -679,8 +682,10 @@ impl TestState { candidate_receipts.push(candidate_receipt); } - let pov_sizes = config.pov_sizes.clone().into_iter().cycle(); - let mut state = Self { + let pov_sizes = config.pov_sizes().to_vec().into_iter().cycle(); + gum::info!(target: LOG_TARGET, "{}","Created test environment.".bright_blue()); + + Self { validator_public, validator_authority_id, validator_index, @@ -695,11 +700,7 @@ impl TestState { candidates_generated: 0, candidate_hashes: HashMap::new(), candidates: Vec::new().into_iter().cycle(), - }; - - gum::info!(target: LOG_TARGET, "Created test environment."); - - state + } } } @@ -746,27 +747,29 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { env.metrics().set_n_validators(config.n_validators); env.metrics().set_n_cores(config.n_cores); - for loop_num in 0..env.config().num_loops { - gum::info!(target: LOG_TARGET, loop_num, "Starting loop"); - env.metrics().set_current_loop(loop_num); + for block_num in 0..env.config().num_blocks { + gum::info!(target: LOG_TARGET, "Current block {}/{}", block_num, env.config().num_blocks); + env.metrics().set_current_block(block_num); - let loop_start_ts = Instant::now(); + let block_start_ts = Instant::now(); for candidate_num in 0..config.n_cores as u64 { let candidate = - env.state.next_candidate().expect("We always send up to n_cores*num_loops; qed"); + env.state.next_candidate().expect("We always send up to n_cores*num_blocks; qed"); let (tx, rx) = oneshot::channel(); batch.push(rx); env.send_message(AvailabilityRecoveryMessage::RecoverAvailableData( candidate.clone(), 1, - Some(GroupIndex(candidate_num as u32 % (config.n_cores / 5) as u32)), + Some(GroupIndex( + candidate_num as u32 % (std::cmp::max(5, config.n_cores) / 5) as u32, + )), tx, )) .await; } - gum::info!("{} requests pending", batch.len()); + gum::info!("{}", format!("{} requests pending", batch.len()).bright_black()); while let Some(completed) = batch.next().await { let available_data = completed.unwrap().unwrap(); env.metrics().on_pov_size(available_data.encoded_size()); @@ -774,31 +777,44 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { } let block_time_delta = - Duration::from_secs(6).saturating_sub(Instant::now().sub(loop_start_ts)); - gum::info!(target: LOG_TARGET, "Sleeping till end of block {}ms", block_time_delta.as_millis()); + Duration::from_secs(6).saturating_sub(Instant::now().sub(block_start_ts)); + gum::info!(target: LOG_TARGET,"{}", format!("Sleeping till end of block ({}ms)", block_time_delta.as_millis()).bright_black()); tokio::time::sleep(block_time_delta).await; } env.send_signal(OverseerSignal::Conclude).await; let duration = start_marker.elapsed().as_millis(); let availability_bytes = availability_bytes / 1024; - gum::info!("Benchmark completed in {:?}ms", duration); - gum::info!("Throughput: {} KiB/block", availability_bytes / env.config().num_loops as u128); + gum::info!("Benchmark completed in {}", format!("{:?}ms", duration).cyan()); gum::info!( - "Block time: {} ms", - start_marker.elapsed().as_millis() / env.config().num_loops as u128 + "Throughput: {}", + format!("{} KiB/block", availability_bytes / env.config().num_blocks as u128).bright_red() + ); + gum::info!( + "Block time: {}", + format!("{} ms", start_marker.elapsed().as_millis() / env.config().num_blocks as u128).red() ); - let stats = env.network.stats(); + let stats = env.network().stats(); gum::info!( - "Total received from network: {} MiB", - stats - .iter() - .enumerate() - .map(|(index, stats)| stats.tx_bytes_total as u128) - .sum::() / - (1024 * 1024) + "Total received from network: {}", + format!( + "{} MiB", + stats + .iter() + .enumerate() + .map(|(_index, stats)| stats.tx_bytes_total as u128) + .sum::() / (1024 * 1024) + ) + .cyan() ); - tokio::time::sleep(Duration::from_secs(1)).await; + let test_metrics = super::core::display::parse_metrics(&env.registry()); + let subsystem_cpu_metrics = + test_metrics.subset_with_label_value("task_group", "availability-recovery-subsystem"); + gum::info!(target: LOG_TARGET, "Total subsystem CPU usage {}", format!("{:.2}s", subsystem_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum")).bright_purple()); + + let test_env_cpu_metrics = + test_metrics.subset_with_label_value("task_group", "test-environment"); + gum::info!(target: LOG_TARGET, "Total test environment CPU usage {}", format!("{:.2}s", test_env_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum")).bright_purple()); } diff --git a/polkadot/node/subsystem-bench/src/core/display.rs b/polkadot/node/subsystem-bench/src/core/display.rs index 47483d33a42a..4b63f45c5f8a 100644 --- a/polkadot/node/subsystem-bench/src/core/display.rs +++ b/polkadot/node/subsystem-bench/src/core/display.rs @@ -13,3 +13,279 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +//! Some helper methods for parsing prometheus metrics to a format that can be +//! displayed in the CLI. +//! +//! Currently histogram buckets are skipped. +use super::LOG_TARGET; +use colored::Colorize; +use prometheus::{ + proto::{MetricFamily, MetricType}, + Registry, +}; +use std::fmt::Display; + +#[derive(Default)] +pub struct MetricCollection(Vec); + +impl From> for MetricCollection { + fn from(metrics: Vec) -> Self { + MetricCollection(metrics) + } +} + +impl MetricCollection { + pub fn get(&self, name: &str) -> Vec<&TestMetric> { + self.all().into_iter().filter(|metric| &metric.name == name).collect() + } + + pub fn all(&self) -> &Vec { + &self.0 + } + + /// Sums up all metrics with the given name in the collection + pub fn sum_by(&self, name: &str) -> f64 { + self.all() + .into_iter() + .filter(|metric| &metric.name == name) + .map(|metric| metric.value) + .sum() + } + + pub fn subset_with_label_value(&self, label_name: &str, label_value: &str) -> MetricCollection { + self.0 + .iter() + .filter_map(|metric| { + if let Some(index) = metric.label_names.iter().position(|label| label == label_name) + { + if Some(&String::from(label_value)) == metric.label_values.get(index) { + Some(metric.clone()) + } else { + None + } + } else { + None + } + }) + .collect::>() + .into() + } +} + +impl Display for MetricCollection { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "")?; + let metrics = self.all(); + for metric in metrics { + writeln!(f, "{}", metric)?; + } + Ok(()) + } +} +#[derive(Debug, Clone)] +pub struct TestMetric { + name: String, + label_names: Vec, + label_values: Vec, + value: f64, +} + +impl Display for TestMetric { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "({} = {}) [{:?}, {:?}]", + self.name.cyan(), + format!("{}", self.value).white(), + self.label_names, + self.label_values + ) + } +} + +// fn encode_impl( +// &self, +// metric_families: &[MetricFamily], +// writer: &mut dyn WriteUtf8, +// ) -> Result<()> { for mf in metric_families { // Fail-fast checks. check_metric_family(mf)?; + +// // Write `# HELP` header. +// let name = mf.get_name(); +// let help = mf.get_help(); +// if !help.is_empty() { +// writer.write_all("# HELP ")?; +// writer.write_all(name)?; +// writer.write_all(" ")?; +// writer.write_all(&escape_string(help, false))?; +// writer.write_all("\n")?; +// } + +// // Write `# TYPE` header. +// let metric_type = mf.get_field_type(); +// let lowercase_type = format!("{:?}", metric_type).to_lowercase(); +// writer.write_all("# TYPE ")?; +// writer.write_all(name)?; +// writer.write_all(" ")?; +// writer.write_all(&lowercase_type)?; +// writer.write_all("\n")?; + +// for m in mf.get_metric() { +// match metric_type { +// MetricType::COUNTER => { +// write_sample(writer, name, None, m, None, m.get_counter().get_value())?; +// } +// MetricType::GAUGE => { +// write_sample(writer, name, None, m, None, m.get_gauge().get_value())?; +// } +// MetricType::HISTOGRAM => { +// let h = m.get_histogram(); + +// let mut inf_seen = false; +// for b in h.get_bucket() { +// let upper_bound = b.get_upper_bound(); +// write_sample( +// writer, +// name, +// Some("_bucket"), +// m, +// Some((BUCKET_LABEL, &upper_bound.to_string())), +// b.get_cumulative_count() as f64, +// )?; +// if upper_bound.is_sign_positive() && upper_bound.is_infinite() { +// inf_seen = true; +// } +// } +// if !inf_seen { +// write_sample( +// writer, +// name, +// Some("_bucket"), +// m, +// Some((BUCKET_LABEL, POSITIVE_INF)), +// h.get_sample_count() as f64, +// )?; +// } + +// write_sample(writer, name, Some("_sum"), m, None, h.get_sample_sum())?; + +// write_sample( +// writer, +// name, +// Some("_count"), +// m, +// None, +// h.get_sample_count() as f64, +// )?; +// } +// MetricType::SUMMARY => { +// let s = m.get_summary(); + +// for q in s.get_quantile() { +// write_sample( +// writer, +// name, +// None, +// m, +// Some((QUANTILE, &q.get_quantile().to_string())), +// q.get_value(), +// )?; +// } + +// write_sample(writer, name, Some("_sum"), m, None, s.get_sample_sum())?; + +// write_sample( +// writer, +// name, +// Some("_count"), +// m, +// None, +// s.get_sample_count() as f64, +// )?; +// } +// MetricType::UNTYPED => { +// unimplemented!(); +// } +// } +// } +// } + +// Ok(()) +// } + +// Returns `false` if metric should be skipped. +fn check_metric_family(mf: &MetricFamily) -> bool { + if mf.get_metric().is_empty() { + gum::error!(target: LOG_TARGET, "MetricFamily has no metrics: {:?}", mf); + return false + } + if mf.get_name().is_empty() { + gum::error!(target: LOG_TARGET, "MetricFamily has no name: {:?}", mf); + return false + } + + true +} + +pub fn parse_metrics(registry: &Registry) -> MetricCollection { + let metric_families = registry.gather(); + let mut test_metrics = Vec::new(); + for mf in metric_families { + if !check_metric_family(&mf) { + continue + } + + let name: String = mf.get_name().into(); + let metric_type = mf.get_field_type(); + for m in mf.get_metric() { + let (label_names, label_values): (Vec, Vec) = m + .get_label() + .iter() + .map(|pair| (String::from(pair.get_name()), String::from(pair.get_value()))) + .unzip(); + + match metric_type { + MetricType::COUNTER => { + test_metrics.push(TestMetric { + name: name.clone(), + label_names, + label_values, + value: m.get_counter().get_value(), + }); + }, + MetricType::GAUGE => { + test_metrics.push(TestMetric { + name: name.clone(), + label_names, + label_values, + value: m.get_gauge().get_value(), + }); + }, + MetricType::HISTOGRAM => { + let h = m.get_histogram(); + let h_name = name.clone() + "_sum".into(); + test_metrics.push(TestMetric { + name: h_name, + label_names: label_names.clone(), + label_values: label_values.clone(), + value: h.get_sample_sum(), + }); + + let h_name = name.clone() + "_count".into(); + test_metrics.push(TestMetric { + name: h_name, + label_names, + label_values, + value: h.get_sample_sum(), + }); + }, + MetricType::SUMMARY => { + unimplemented!(); + }, + MetricType::UNTYPED => { + unimplemented!(); + }, + } + } + } + test_metrics.into() +} diff --git a/polkadot/node/subsystem-bench/src/core/keyring.rs b/polkadot/node/subsystem-bench/src/core/keyring.rs index 40e8d60d0cd1..2d9aa348a922 100644 --- a/polkadot/node/subsystem-bench/src/core/keyring.rs +++ b/polkadot/node/subsystem-bench/src/core/keyring.rs @@ -16,11 +16,9 @@ pub use sp_core::sr25519; use sp_core::{ - sr25519::{Pair, Public, Signature}, - ByteArray, Pair as PairT, H256, + sr25519::{Pair, Public}, + Pair as PairT, }; -use std::{collections::HashMap, ops::Deref}; - /// Set of test accounts. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Keyring { @@ -39,8 +37,4 @@ impl Keyring { pub fn public(self) -> Public { self.pair().public() } - - pub fn to_seed(self) -> String { - format!("//{}", self.name) - } } diff --git a/polkadot/node/subsystem-bench/src/core/mod.rs b/polkadot/node/subsystem-bench/src/core/mod.rs index 4b9db3144f54..0d7b5c3c4015 100644 --- a/polkadot/node/subsystem-bench/src/core/mod.rs +++ b/polkadot/node/subsystem-bench/src/core/mod.rs @@ -14,67 +14,17 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use itertools::Itertools; use std::{ collections::HashMap, - iter::Cycle, - ops::{Div, Sub}, sync::Arc, time::{Duration, Instant}, }; - -use sc_keystore::LocalKeystore; -use sp_application_crypto::AppCrypto; -use sp_keystore::{Keystore, KeystorePtr}; - -use futures::{ - channel::{mpsc, oneshot}, - stream::FuturesUnordered, - FutureExt, SinkExt, StreamExt, -}; -use futures_timer::Delay; - -use polkadot_node_metrics::metrics::Metrics; - -use polkadot_availability_recovery::AvailabilityRecoverySubsystem; - -use parity_scale_codec::Encode; -use polkadot_node_network_protocol::request_response::{ - self as req_res, v1::ChunkResponse, IncomingRequest, ReqProtocolNames, Requests, -}; -use rand::{distributions::Uniform, prelude::Distribution, seq::IteratorRandom, thread_rng}; - -use prometheus::Registry; -use sc_network::{config::RequestResponseConfig, OutboundFailure, RequestFailure}; - -use polkadot_erasure_coding::{branches, obtain_chunks_v1 as obtain_chunks}; -use polkadot_node_primitives::{BlockData, PoV, Proof}; -use polkadot_node_subsystem::{ - messages::{ - AllMessages, AvailabilityRecoveryMessage, AvailabilityStoreMessage, NetworkBridgeTxMessage, - RuntimeApiMessage, RuntimeApiRequest, - }, - ActiveLeavesUpdate, FromOrchestra, OverseerSignal, Subsystem, -}; -use std::net::{Ipv4Addr, SocketAddr}; - -use super::core::{keyring::Keyring, network::*, test_env::TestEnvironmentMetrics}; - const LOG_TARGET: &str = "subsystem-bench::core"; -use polkadot_node_primitives::{AvailableData, ErasureChunk}; - -use polkadot_node_subsystem_test_helpers::{ - make_buffered_subsystem_context, mock::new_leaf, TestSubsystemContextHandle, -}; -use polkadot_node_subsystem_util::TimeoutExt; -use polkadot_primitives::{ - AuthorityDiscoveryId, CandidateHash, CandidateReceipt, GroupIndex, Hash, HeadData, IndexedVec, - PersistedValidationData, SessionIndex, SessionInfo, ValidatorId, ValidatorIndex, -}; -use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; -use sc_service::{SpawnTaskHandle, TaskManager}; +use polkadot_primitives::AuthorityDiscoveryId; +use sc_service::SpawnTaskHandle; pub mod keyring; pub mod network; pub mod test_env; +pub mod display; \ No newline at end of file diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 170ab45e35a3..9250762f9987 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -15,7 +15,6 @@ // along with Polkadot. If not, see . use super::*; use prometheus_endpoint::U64; -use sc_network::network_state::Peer; use std::sync::atomic::{AtomicU64, Ordering}; use tokio::sync::mpsc::UnboundedSender; // An emulated node egress traffic rate_limiter. @@ -339,8 +338,7 @@ impl NetworkEmulator { } use polkadot_node_subsystem_util::metrics::{ - self, - prometheus::{self, Counter, CounterVec, Histogram, Opts, PrometheusError, Registry}, + prometheus::{CounterVec, Opts, PrometheusError, Registry}, }; /// Emulated network metrics. diff --git a/polkadot/node/subsystem-bench/src/core/test_env.rs b/polkadot/node/subsystem-bench/src/core/test_env.rs index c20b96d642af..153d5bdf95c7 100644 --- a/polkadot/node/subsystem-bench/src/core/test_env.rs +++ b/polkadot/node/subsystem-bench/src/core/test_env.rs @@ -14,15 +14,11 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use super::*; -use polkadot_node_subsystem_util::metrics::{ - self, - prometheus::{ - self, Counter, Gauge, Histogram, HistogramVec, Opts, PrometheusError, Registry, U64, - }, +use polkadot_node_subsystem_util::metrics::prometheus::{ + self, Gauge, Histogram, PrometheusError, Registry, U64, }; -const MIB: f64 = 1024.0*1024.0; +const MIB: f64 = 1024.0 * 1024.0; /// Test environment/configuration metrics #[derive(Clone)] @@ -33,22 +29,15 @@ pub struct TestEnvironmentMetrics { n_cores: Gauge, /// PoV size pov_size: Histogram, - /// Current loop - current_loop: Gauge, + /// Current block + current_block: Gauge, } impl TestEnvironmentMetrics { pub fn new(registry: &Registry) -> Result { let mut buckets = prometheus::exponential_buckets(16384.0, 2.0, 9) - .expect("arguments are always valid; qed"); - buckets.extend(vec![ - 5.0 * MIB, - 6.0 * MIB, - 7.0 * MIB, - 8.0 * MIB, - 9.0 * MIB, - 10.0 * MIB, - ]); + .expect("arguments are always valid; qed"); + buckets.extend(vec![5.0 * MIB, 6.0 * MIB, 7.0 * MIB, 8.0 * MIB, 9.0 * MIB, 10.0 * MIB]); Ok(Self { n_validators: prometheus::register( @@ -61,12 +50,12 @@ impl TestEnvironmentMetrics { n_cores: prometheus::register( Gauge::new( "subsystem_benchmark_n_cores", - "Number of cores we fetch availability for each loop", + "Number of cores we fetch availability for each block", )?, registry, )?, - current_loop: prometheus::register( - Gauge::new("subsystem_benchmark_current_loop", "The current test loop")?, + current_block: prometheus::register( + Gauge::new("subsystem_benchmark_current_block", "The current test block")?, registry, )?, pov_size: prometheus::register( @@ -75,9 +64,7 @@ impl TestEnvironmentMetrics { "subsystem_benchmark_pov_size", "The compressed size of the proof of validity of a candidate", ) - .buckets( - buckets - ), + .buckets(buckets), )?, registry, )?, @@ -92,8 +79,8 @@ impl TestEnvironmentMetrics { self.n_cores.set(n_cores as u64); } - pub fn set_current_loop(&self, current_loop: usize) { - self.current_loop.set(current_loop as u64); + pub fn set_current_block(&self, current_block: usize) { + self.current_block.set(current_block as u64); } pub fn on_pov_size(&self, pov_size: usize) { diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 9e581555d761..3cffd2ec427e 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -18,8 +18,9 @@ //! CI regression testing. use clap::Parser; use color_eyre::eyre; -use prometheus::proto::LabelPair; -use std::time::Duration; + +use colored::Colorize; +use std::{time::Duration, path::Path}; pub(crate) mod availability; pub(crate) mod core; @@ -77,15 +78,29 @@ pub struct DataAvailabilityReadOptions { pub fetch_from_backers: bool, #[clap(short, long, ignore_case = true, default_value_t = 1)] - /// Number of times to loop fetching for each core. - pub num_loops: usize, + /// Number of times to block fetching for each core. + pub num_blocks: usize, } + + +#[derive(Debug, clap::Parser)] +#[clap(rename_all = "kebab-case")] +#[allow(missing_docs)] +pub struct TestSequenceOptions { + #[clap(short, long, ignore_case = true)] + pub path: String, +} + + + /// Define the supported benchmarks targets #[derive(Debug, Parser)] #[command(about = "Target subsystems", version, rename_all = "kebab-case")] enum BenchmarkTarget { /// Benchmark availability recovery strategies. DataAvailabilityRead(DataAvailabilityReadOptions), + /// Run a test sequence specified in a file + TestSequence(TestSequenceOptions), } #[derive(Debug, Parser)] @@ -131,17 +146,31 @@ fn new_runtime() -> tokio::runtime::Runtime { impl BenchCli { /// Launch a malus node. fn launch(self) -> eyre::Result<()> { - use prometheus::{proto::MetricType, Registry, TextEncoder}; - - println!("Preparing {:?} benchmarks", self.target); + use prometheus::Registry; let runtime = new_runtime(); - let registry = Registry::new(); let mut test_config = match self.target { + BenchmarkTarget::TestSequence(options) => { + let test_sequence = availability::TestSequence::new_from_file(Path::new(&options.path)).expect("File exists").to_vec(); + let num_steps = test_sequence.len(); + gum::info!("{}", format!("Sequence contains {} step(s)",num_steps).bright_purple()); + for (index, test_config) in test_sequence.into_iter().enumerate(){ + gum::info!("{}", format!("Current step {}/{}", index + 1, num_steps).bright_purple()); + + let candidate_count = test_config.n_cores * test_config.num_blocks; + + let mut state = TestState::new(test_config); + state.generate_candidates(candidate_count); + let mut env = TestEnvironment::new(runtime.handle().clone(), state, Registry::new()); + + runtime.block_on(availability::bench_chunk_recovery(&mut env)); + } + return Ok(()) + } BenchmarkTarget::DataAvailabilityRead(options) => match self.network { NetworkEmulation::Healthy => TestConfiguration::healthy_network( - options.num_loops, + options.num_blocks, options.fetch_from_backers, options.n_validators, options.n_cores, @@ -155,7 +184,7 @@ impl BenchCli { .collect(), ), NetworkEmulation::Degraded => TestConfiguration::degraded_network( - options.num_loops, + options.num_blocks, options.fetch_from_backers, options.n_validators, options.n_cores, @@ -169,7 +198,7 @@ impl BenchCli { .collect(), ), NetworkEmulation::Ideal => TestConfiguration::ideal_network( - options.num_loops, + options.num_blocks, options.fetch_from_backers, options.n_validators, options.n_cores, @@ -209,56 +238,15 @@ impl BenchCli { test_config.bandwidth = bandwidth * 1024; } - let candidate_count = test_config.n_cores * test_config.num_loops; + let candidate_count = test_config.n_cores * test_config.num_blocks; + test_config.write_to_disk(); let mut state = TestState::new(test_config); state.generate_candidates(candidate_count); - let mut env = TestEnvironment::new(runtime.handle().clone(), state, registry.clone()); - - println!("{:?}", env.config()); + let mut env = TestEnvironment::new(runtime.handle().clone(), state, Registry::new()); runtime.block_on(availability::bench_chunk_recovery(&mut env)); - let metric_families = registry.gather(); - - for familiy in metric_families { - let metric_type = familiy.get_field_type(); - - for metric in familiy.get_metric() { - match metric_type { - MetricType::HISTOGRAM => { - let h = metric.get_histogram(); - - let labels = metric.get_label(); - // Skip test env usage. - let mut env_label = LabelPair::default(); - env_label.set_name("task_group".into()); - env_label.set_value("test-environment".into()); - - let mut is_env_metric = false; - for label_pair in labels { - if &env_label == label_pair { - is_env_metric = true; - break - } - } - - if !is_env_metric { - println!( - "{:?} CPU seconds used: {:?}", - familiy.get_name(), - h.get_sample_sum() - ); - } - }, - _ => {}, - } - } - } - // encoder.encode(&metric_families, &mut buffer).unwrap(); - - // Output to the standard output. - // println!("Metrics: {}", String::from_utf8(buffer).unwrap()); Ok(()) } } @@ -267,6 +255,7 @@ fn main() -> eyre::Result<()> { color_eyre::install()?; let _ = env_logger::builder() .filter(Some("hyper"), log::LevelFilter::Info) + .filter(None, log::LevelFilter::Info) .try_init() .unwrap(); diff --git a/polkadot/node/subsystem-test-helpers/src/lib.rs b/polkadot/node/subsystem-test-helpers/src/lib.rs index 5393ccafa6f3..1c3c47150ac6 100644 --- a/polkadot/node/subsystem-test-helpers/src/lib.rs +++ b/polkadot/node/subsystem-test-helpers/src/lib.rs @@ -279,6 +279,13 @@ impl TestSubsystemContextHandle { .expect("Test subsystem no longer live") } + /// Receive the next message from the subsystem. + pub async fn maybe_recv(&mut self) -> Option { + self.try_recv() + .timeout(Self::TIMEOUT) + .await + .expect("`fn recv` does not timeout") + } /// Receive the next message from the subsystem, or `None` if the channel has been closed. pub async fn try_recv(&mut self) -> Option { self.rx From d6c259df9ff7eaaa5f7207364b87a7f3a76b165e Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 15 Nov 2023 18:39:29 +0200 Subject: [PATCH 091/192] update Signed-off-by: Andrei Sandu --- .../network/availability-recovery/src/lib.rs | 3 +- .../subsystem-bench/src/availability/cli.rs | 17 ++++ .../src/availability/configuration.rs | 19 ++--- .../subsystem-bench/src/availability/mod.rs | 27 +++++-- polkadot/node/subsystem-bench/src/core/mod.rs | 2 +- .../node/subsystem-bench/src/core/network.rs | 8 +- .../node/subsystem-bench/src/core/test_env.rs | 10 +++ .../subsystem-bench/src/subsystem-bench.rs | 41 ++++++---- .../node/subsystem-bench/test_sequence.toml | 77 +++++++++++++++++++ 9 files changed, 165 insertions(+), 39 deletions(-) create mode 100644 polkadot/node/subsystem-bench/src/availability/cli.rs create mode 100644 polkadot/node/subsystem-bench/test_sequence.toml diff --git a/polkadot/node/network/availability-recovery/src/lib.rs b/polkadot/node/network/availability-recovery/src/lib.rs index ffb634ad76e2..6dafcf4ccfc8 100644 --- a/polkadot/node/network/availability-recovery/src/lib.rs +++ b/polkadot/node/network/availability-recovery/src/lib.rs @@ -582,6 +582,7 @@ impl AvailabilityRecoverySubsystem { } } + /// Starts the inner subsystem loop. pub async fn run(self, mut ctx: Context) -> SubsystemResult<()> { let mut state = State::default(); let Self { mut req_receiver, metrics, recovery_strategy_kind, bypass_availability_store } = @@ -726,8 +727,6 @@ impl AvailabilityRecoverySubsystem { } } output = state.ongoing_recoveries.select_next_some() => { - // No caching for benchmark. - #[cfg(not(feature = "subsystem-benchmarks"))] if let Some((candidate_hash, result)) = output { if let Ok(recovery) = CachedRecovery::try_from(result) { state.availability_lru.insert(candidate_hash, recovery); diff --git a/polkadot/node/subsystem-bench/src/availability/cli.rs b/polkadot/node/subsystem-bench/src/availability/cli.rs new file mode 100644 index 000000000000..43a938f2abea --- /dev/null +++ b/polkadot/node/subsystem-bench/src/availability/cli.rs @@ -0,0 +1,17 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use super::*; diff --git a/polkadot/node/subsystem-bench/src/availability/configuration.rs b/polkadot/node/subsystem-bench/src/availability/configuration.rs index 2d29d23811da..cbad4a2dc1b8 100644 --- a/polkadot/node/subsystem-bench/src/availability/configuration.rs +++ b/polkadot/node/subsystem-bench/src/availability/configuration.rs @@ -17,7 +17,7 @@ use std::path::Path; use super::*; -use serde::{Deserialize,Serialize}; +use serde::{Deserialize, Serialize}; /// Peer response latency configuration. #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct PeerLatency { @@ -56,7 +56,6 @@ pub struct TestConfiguration { pub num_blocks: usize, } - impl Default for TestConfiguration { fn default() -> Self { Self { @@ -69,8 +68,8 @@ impl Default for TestConfiguration { latency: None, error: 0, num_blocks: 1, - min_pov_size: 5*1024*1024, - max_pov_size: 5*1024*1024, + min_pov_size: 5 * 1024 * 1024, + max_pov_size: 5 * 1024 * 1024, } } } @@ -79,10 +78,10 @@ fn generate_pov_sizes(count: usize, min: usize, max: usize) -> Vec { (0..count).map(|_| random_pov_size(min, max)).collect() } -#[derive(Serialize,Deserialize)] +#[derive(Serialize, Deserialize)] pub struct TestSequence { #[serde(rename(serialize = "TestConfiguration", deserialize = "TestConfiguration"))] - test_configurations: Vec + test_configurations: Vec, } impl TestSequence { @@ -90,14 +89,15 @@ impl TestSequence { // Generate Pov sizes for config in self.test_configurations.iter_mut() { - config.pov_sizes = generate_pov_sizes(config.n_cores, config.min_pov_size, config.max_pov_size); + config.pov_sizes = + generate_pov_sizes(config.n_cores, config.min_pov_size, config.max_pov_size); } self.test_configurations } } -impl TestSequence { +impl TestSequence { pub fn new_from_file(path: &Path) -> std::io::Result { let string = String::from_utf8(std::fs::read(&path)?).expect("File is valid UTF8"); Ok(toml::from_str(&string).expect("File is valid test sequence TOML")) @@ -107,7 +107,8 @@ impl TestSequence { impl TestConfiguration { pub fn write_to_disk(&self) { // Serialize a slice of configurations - let toml = toml::to_string(&TestSequence{ test_configurations: vec![self.clone()] }).unwrap(); + let toml = + toml::to_string(&TestSequence { test_configurations: vec![self.clone()] }).unwrap(); std::fs::write("last_test.toml", toml).unwrap(); } diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 2c9f3e735afb..0a0830ff9975 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -748,13 +748,15 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { env.metrics().set_n_cores(config.n_cores); for block_num in 0..env.config().num_blocks { - gum::info!(target: LOG_TARGET, "Current block {}/{}", block_num, env.config().num_blocks); + gum::info!(target: LOG_TARGET, "Current block {}/{}", block_num + 1, env.config().num_blocks); env.metrics().set_current_block(block_num); let block_start_ts = Instant::now(); for candidate_num in 0..config.n_cores as u64 { - let candidate = - env.state.next_candidate().expect("We always send up to n_cores*num_blocks; qed"); + let candidate = env + .state + .next_candidate() + .expect("We always send up to n_cores*num_blocks; qed"); let (tx, rx) = oneshot::channel(); batch.push(rx); @@ -769,7 +771,7 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { .await; } - gum::info!("{}", format!("{} requests pending", batch.len()).bright_black()); + gum::info!("{}", format!("{} recoveries pending", batch.len()).bright_black()); while let Some(completed) = batch.next().await { let available_data = completed.unwrap().unwrap(); env.metrics().on_pov_size(available_data.encoded_size()); @@ -778,6 +780,10 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { let block_time_delta = Duration::from_secs(6).saturating_sub(Instant::now().sub(block_start_ts)); + + let block_time = Instant::now().sub(block_start_ts).as_millis() as u64; + env.metrics().set_block_time(block_time); + gum::info!("Block time {}", format!("{:?}ms", block_time).cyan()); gum::info!(target: LOG_TARGET,"{}", format!("Sleeping till end of block ({}ms)", block_time_delta.as_millis()).bright_black()); tokio::time::sleep(block_time_delta).await; } @@ -785,14 +791,15 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { env.send_signal(OverseerSignal::Conclude).await; let duration = start_marker.elapsed().as_millis(); let availability_bytes = availability_bytes / 1024; - gum::info!("Benchmark completed in {}", format!("{:?}ms", duration).cyan()); + gum::info!("All blocks processed in {}", format!("{:?}ms", duration).cyan()); gum::info!( "Throughput: {}", format!("{} KiB/block", availability_bytes / env.config().num_blocks as u128).bright_red() ); gum::info!( "Block time: {}", - format!("{} ms", start_marker.elapsed().as_millis() / env.config().num_blocks as u128).red() + format!("{} ms", start_marker.elapsed().as_millis() / env.config().num_blocks as u128) + .red() ); let stats = env.network().stats(); @@ -812,9 +819,13 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { let test_metrics = super::core::display::parse_metrics(&env.registry()); let subsystem_cpu_metrics = test_metrics.subset_with_label_value("task_group", "availability-recovery-subsystem"); - gum::info!(target: LOG_TARGET, "Total subsystem CPU usage {}", format!("{:.2}s", subsystem_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum")).bright_purple()); + let total_cpu = subsystem_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); + gum::info!(target: LOG_TARGET, "Total subsystem CPU usage {}", format!("{:.2}s", total_cpu).bright_purple()); + gum::info!(target: LOG_TARGET, "CPU usage per block {}", format!("{:.2}s", total_cpu/env.config().num_blocks as f64).bright_purple()); let test_env_cpu_metrics = test_metrics.subset_with_label_value("task_group", "test-environment"); - gum::info!(target: LOG_TARGET, "Total test environment CPU usage {}", format!("{:.2}s", test_env_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum")).bright_purple()); + let total_cpu = test_env_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); + gum::info!(target: LOG_TARGET, "Total test environment CPU usage {}", format!("{:.2}s", total_cpu).bright_purple()); + gum::info!(target: LOG_TARGET, "CPU usage per block {}", format!("{:.2}s", total_cpu/env.config().num_blocks as f64).bright_purple()); } diff --git a/polkadot/node/subsystem-bench/src/core/mod.rs b/polkadot/node/subsystem-bench/src/core/mod.rs index 0d7b5c3c4015..2e9e0364273e 100644 --- a/polkadot/node/subsystem-bench/src/core/mod.rs +++ b/polkadot/node/subsystem-bench/src/core/mod.rs @@ -24,7 +24,7 @@ const LOG_TARGET: &str = "subsystem-bench::core"; use polkadot_primitives::AuthorityDiscoveryId; use sc_service::SpawnTaskHandle; +pub mod display; pub mod keyring; pub mod network; pub mod test_env; -pub mod display; \ No newline at end of file diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 9250762f9987..629d09df694c 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -14,9 +14,11 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . use super::*; +use colored::Colorize; use prometheus_endpoint::U64; use std::sync::atomic::{AtomicU64, Ordering}; use tokio::sync::mpsc::UnboundedSender; + // An emulated node egress traffic rate_limiter. #[derive(Debug)] pub struct RateLimit { @@ -282,6 +284,8 @@ impl NetworkEmulator { spawn_task_handle: SpawnTaskHandle, registry: &Registry, ) -> Self { + gum::info!(target: LOG_TARGET, "{}",format!("Initializing network emulation for {} peers.", n_peers).bright_blue()); + let metrics = Metrics::new(®istry).expect("Metrics always register succesfully"); let mut validator_authority_id_mapping = HashMap::new(); @@ -337,8 +341,8 @@ impl NetworkEmulator { } } -use polkadot_node_subsystem_util::metrics::{ - prometheus::{CounterVec, Opts, PrometheusError, Registry}, +use polkadot_node_subsystem_util::metrics::prometheus::{ + self, CounterVec, Opts, PrometheusError, Registry, }; /// Emulated network metrics. diff --git a/polkadot/node/subsystem-bench/src/core/test_env.rs b/polkadot/node/subsystem-bench/src/core/test_env.rs index 153d5bdf95c7..e6b09a1c13e6 100644 --- a/polkadot/node/subsystem-bench/src/core/test_env.rs +++ b/polkadot/node/subsystem-bench/src/core/test_env.rs @@ -31,6 +31,8 @@ pub struct TestEnvironmentMetrics { pov_size: Histogram, /// Current block current_block: Gauge, + /// Current block + block_time: Gauge, } impl TestEnvironmentMetrics { @@ -58,6 +60,10 @@ impl TestEnvironmentMetrics { Gauge::new("subsystem_benchmark_current_block", "The current test block")?, registry, )?, + block_time: prometheus::register( + Gauge::new("subsystem_benchmark_block_time", "The time it takes for the target subsystems(s) to complete all the requests in a block")?, + registry, + )?, pov_size: prometheus::register( Histogram::with_opts( prometheus::HistogramOpts::new( @@ -83,6 +89,10 @@ impl TestEnvironmentMetrics { self.current_block.set(current_block as u64); } + pub fn set_block_time(&self, block_time_ms: u64) { + self.block_time.set(block_time_ms); + } + pub fn on_pov_size(&self, pov_size: usize) { self.pov_size.observe(pov_size as f64); } diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 3cffd2ec427e..280172662453 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -20,7 +20,7 @@ use clap::Parser; use color_eyre::eyre; use colored::Colorize; -use std::{time::Duration, path::Path}; +use std::{path::Path, time::Duration}; pub(crate) mod availability; pub(crate) mod core; @@ -82,7 +82,6 @@ pub struct DataAvailabilityReadOptions { pub num_blocks: usize, } - #[derive(Debug, clap::Parser)] #[clap(rename_all = "kebab-case")] #[allow(missing_docs)] @@ -91,12 +90,10 @@ pub struct TestSequenceOptions { pub path: String, } - - /// Define the supported benchmarks targets #[derive(Debug, Parser)] -#[command(about = "Target subsystems", version, rename_all = "kebab-case")] -enum BenchmarkTarget { +#[command(about = "Test objectives", version, rename_all = "kebab-case")] +enum TestObjective { /// Benchmark availability recovery strategies. DataAvailabilityRead(DataAvailabilityReadOptions), /// Run a test sequence specified in a file @@ -131,7 +128,7 @@ struct BenchCli { pub peer_max_latency: Option, #[command(subcommand)] - pub target: BenchmarkTarget, + pub objective: TestObjective, } fn new_runtime() -> tokio::runtime::Runtime { @@ -150,25 +147,35 @@ impl BenchCli { let runtime = new_runtime(); - let mut test_config = match self.target { - BenchmarkTarget::TestSequence(options) => { - let test_sequence = availability::TestSequence::new_from_file(Path::new(&options.path)).expect("File exists").to_vec(); + let mut test_config = match self.objective { + TestObjective::TestSequence(options) => { + let test_sequence = + availability::TestSequence::new_from_file(Path::new(&options.path)) + .expect("File exists") + .to_vec(); let num_steps = test_sequence.len(); - gum::info!("{}", format!("Sequence contains {} step(s)",num_steps).bright_purple()); - for (index, test_config) in test_sequence.into_iter().enumerate(){ - gum::info!("{}", format!("Current step {}/{}", index + 1, num_steps).bright_purple()); + gum::info!( + "{}", + format!("Sequence contains {} step(s)", num_steps).bright_purple() + ); + for (index, test_config) in test_sequence.into_iter().enumerate() { + gum::info!( + "{}", + format!("Current step {}/{}", index + 1, num_steps).bright_purple() + ); let candidate_count = test_config.n_cores * test_config.num_blocks; let mut state = TestState::new(test_config); state.generate_candidates(candidate_count); - let mut env = TestEnvironment::new(runtime.handle().clone(), state, Registry::new()); - + let mut env = + TestEnvironment::new(runtime.handle().clone(), state, Registry::new()); + runtime.block_on(availability::bench_chunk_recovery(&mut env)); } return Ok(()) - } - BenchmarkTarget::DataAvailabilityRead(options) => match self.network { + }, + TestObjective::DataAvailabilityRead(options) => match self.network { NetworkEmulation::Healthy => TestConfiguration::healthy_network( options.num_blocks, options.fetch_from_backers, diff --git a/polkadot/node/subsystem-bench/test_sequence.toml b/polkadot/node/subsystem-bench/test_sequence.toml new file mode 100644 index 000000000000..d32477b9efe9 --- /dev/null +++ b/polkadot/node/subsystem-bench/test_sequence.toml @@ -0,0 +1,77 @@ +[[TestConfiguration]] +use_fast_path = false +n_validators = 300 +n_cores = 20 +min_pov_size = 5242880 +max_pov_size = 5242880 +peer_bandwidth = 128000 +bandwidth = 52428800 +error = 33 +num_blocks = 5 + +[TestConfiguration.latency.min_latency] +secs = 0 +nanos = 1000000 + +[TestConfiguration.latency.max_latency] +secs = 0 +nanos = 100000000 + +[[TestConfiguration]] +use_fast_path = false +n_validators = 500 +n_cores = 20 +min_pov_size = 5242880 +max_pov_size = 5242880 +peer_bandwidth = 128000 +bandwidth = 52428800 +error = 33 +num_blocks = 5 + +[TestConfiguration.latency.min_latency] +secs = 0 +nanos = 1000000 + +[TestConfiguration.latency.max_latency] +secs = 0 +nanos = 1000000000 + + +[[TestConfiguration]] +use_fast_path = false +n_validators = 1000 +n_cores = 20 +min_pov_size = 5242880 +max_pov_size = 5242880 +peer_bandwidth = 128000 +bandwidth = 52428800 +error = 33 +num_blocks = 5 + +[TestConfiguration.latency.min_latency] +secs = 0 +nanos = 1000000 + +[TestConfiguration.latency.max_latency] +secs = 0 +nanos = 1000000000 + + +[[TestConfiguration]] +use_fast_path = false +n_validators = 2000 +n_cores = 20 +min_pov_size = 5242880 +max_pov_size = 5242880 +peer_bandwidth = 128000 +bandwidth = 52428800 +error = 33 +num_blocks = 5 + +[TestConfiguration.latency.min_latency] +secs = 0 +nanos = 1000000 + +[TestConfiguration.latency.max_latency] +secs = 0 +nanos = 1000000000 From 050529b68ca2402e79e593968571606580d5ca7a Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 15 Nov 2023 18:47:05 +0200 Subject: [PATCH 092/192] remove comment Signed-off-by: Andrei Sandu --- polkadot/node/subsystem-bench/src/subsystem-bench.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 280172662453..7dcc8a15074a 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -141,7 +141,6 @@ fn new_runtime() -> tokio::runtime::Runtime { } impl BenchCli { - /// Launch a malus node. fn launch(self) -> eyre::Result<()> { use prometheus::Registry; From cb38be5c505863df72567efa3a0e3489b9bc42eb Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 17 Nov 2023 11:38:23 +0200 Subject: [PATCH 093/192] separate cli options for availability Signed-off-by: Andrei Sandu --- .../subsystem-bench/src/availability/cli.rs | 44 +++++++++++++++- .../subsystem-bench/src/availability/mod.rs | 7 +-- .../src/core/{test_env.rs => environment.rs} | 2 + polkadot/node/subsystem-bench/src/core/mod.rs | 2 +- .../subsystem-bench/src/subsystem-bench.rs | 52 ++----------------- 5 files changed, 55 insertions(+), 52 deletions(-) rename polkadot/node/subsystem-bench/src/core/{test_env.rs => environment.rs} (98%) diff --git a/polkadot/node/subsystem-bench/src/availability/cli.rs b/polkadot/node/subsystem-bench/src/availability/cli.rs index 43a938f2abea..ef4d7e6f631a 100644 --- a/polkadot/node/subsystem-bench/src/availability/cli.rs +++ b/polkadot/node/subsystem-bench/src/availability/cli.rs @@ -14,4 +14,46 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use super::*; +#[derive(Debug, clap::Parser, Clone)] +#[clap(rename_all = "kebab-case")] +#[allow(missing_docs)] +pub struct NetworkOptions {} + +#[derive(clap::ValueEnum, Clone, Copy, Debug, PartialEq)] +#[value(rename_all = "kebab-case")] +#[non_exhaustive] +pub enum NetworkEmulation { + Ideal, + Healthy, + Degraded, +} + +#[derive(Debug, clap::Parser)] +#[clap(rename_all = "kebab-case")] +#[allow(missing_docs)] +pub struct DataAvailabilityReadOptions { + #[clap(long, ignore_case = true, default_value_t = 100)] + /// Number of cores to fetch availability for. + pub n_cores: usize, + + #[clap(long, ignore_case = true, default_value_t = 500)] + /// Number of validators to fetch chunks from. + pub n_validators: usize, + + #[clap(long, ignore_case = true, default_value_t = 5120)] + /// The minimum pov size in KiB + pub min_pov_size: usize, + + #[clap(long, ignore_case = true, default_value_t = 5120)] + /// The maximum pov size bytes + pub max_pov_size: usize, + + #[clap(short, long, default_value_t = false)] + /// Turbo boost AD Read by fetching from backers first. Tipically this is only faster if nodes + /// have enough bandwidth. + pub fetch_from_backers: bool, + + #[clap(short, long, ignore_case = true, default_value_t = 1)] + /// Number of times to block fetching for each core. + pub num_blocks: usize, +} diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 0a0830ff9975..8866348ea22b 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -53,7 +53,7 @@ use polkadot_node_subsystem::{ }; use std::net::{Ipv4Addr, SocketAddr}; -use super::core::{keyring::Keyring, network::*, test_env::TestEnvironmentMetrics}; +use super::core::{environment::TestEnvironmentMetrics, keyring::Keyring, network::*}; const LOG_TARGET: &str = "subsystem-bench::availability"; @@ -70,8 +70,9 @@ use polkadot_primitives::{ use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; use sc_service::{SpawnTaskHandle, TaskManager}; +mod cli; pub mod configuration; - +pub use cli::{DataAvailabilityReadOptions, NetworkEmulation, NetworkOptions}; pub use configuration::{PeerLatency, TestConfiguration, TestSequence}; // Deterministic genesis hash for protocol names @@ -162,7 +163,7 @@ impl TestEnvironment { let (ingress_tx, mut ingress_rx) = tokio::sync::mpsc::unbounded_channel::(); let our_network_stats = network.peer_stats(0); - spawn_handle.spawn_blocking("our-node-rx", "test-environment", async move { + spawn_handle.spawn_blocking("node0-rx", "test-environment", async move { while let Some(action) = ingress_rx.recv().await { let size = action.size(); diff --git a/polkadot/node/subsystem-bench/src/core/test_env.rs b/polkadot/node/subsystem-bench/src/core/environment.rs similarity index 98% rename from polkadot/node/subsystem-bench/src/core/test_env.rs rename to polkadot/node/subsystem-bench/src/core/environment.rs index e6b09a1c13e6..6a680799972d 100644 --- a/polkadot/node/subsystem-bench/src/core/test_env.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use super::*; +use network::NetworkEmulator; use polkadot_node_subsystem_util::metrics::prometheus::{ self, Gauge, Histogram, PrometheusError, Registry, U64, }; diff --git a/polkadot/node/subsystem-bench/src/core/mod.rs b/polkadot/node/subsystem-bench/src/core/mod.rs index 2e9e0364273e..564fb7148fa0 100644 --- a/polkadot/node/subsystem-bench/src/core/mod.rs +++ b/polkadot/node/subsystem-bench/src/core/mod.rs @@ -25,6 +25,6 @@ use polkadot_primitives::AuthorityDiscoveryId; use sc_service::SpawnTaskHandle; pub mod display; +pub mod environment; pub mod keyring; pub mod network; -pub mod test_env; diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 7dcc8a15074a..42efb7fd63c8 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -25,10 +25,13 @@ use std::{path::Path, time::Duration}; pub(crate) mod availability; pub(crate) mod core; -use availability::{random_pov_size, TestConfiguration, TestEnvironment, TestState}; -const LOG_TARGET: &str = "subsystem-bench"; +use availability::{ + random_pov_size, DataAvailabilityReadOptions, NetworkEmulation, TestConfiguration, + TestEnvironment, TestState, +}; use clap_num::number_range; +const LOG_TARGET: &str = "subsystem-bench"; fn le_100(s: &str) -> Result { number_range(s, 0, 100) @@ -37,51 +40,6 @@ fn le_100(s: &str) -> Result { fn le_5000(s: &str) -> Result { number_range(s, 0, 5000) } - -#[derive(Debug, clap::Parser, Clone)] -#[clap(rename_all = "kebab-case")] -#[allow(missing_docs)] -pub struct NetworkOptions {} - -#[derive(clap::ValueEnum, Clone, Copy, Debug, PartialEq)] -#[value(rename_all = "kebab-case")] -#[non_exhaustive] -pub enum NetworkEmulation { - Ideal, - Healthy, - Degraded, -} - -#[derive(Debug, clap::Parser)] -#[clap(rename_all = "kebab-case")] -#[allow(missing_docs)] -pub struct DataAvailabilityReadOptions { - #[clap(long, ignore_case = true, default_value_t = 100)] - /// Number of cores to fetch availability for. - pub n_cores: usize, - - #[clap(long, ignore_case = true, default_value_t = 500)] - /// Number of validators to fetch chunks from. - pub n_validators: usize, - - #[clap(long, ignore_case = true, default_value_t = 5120)] - /// The minimum pov size in KiB - pub min_pov_size: usize, - - #[clap(long, ignore_case = true, default_value_t = 5120)] - /// The maximum pov size bytes - pub max_pov_size: usize, - - #[clap(short, long, default_value_t = false)] - /// Turbo boost AD Read by fetching from backers first. Tipically this is only faster if nodes - /// have enough bandwidth. - pub fetch_from_backers: bool, - - #[clap(short, long, ignore_case = true, default_value_t = 1)] - /// Number of times to block fetching for each core. - pub num_blocks: usize, -} - #[derive(Debug, clap::Parser)] #[clap(rename_all = "kebab-case")] #[allow(missing_docs)] From 24a736afb7727f2cc4780748edcc873692928503 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 17 Nov 2023 13:07:40 +0200 Subject: [PATCH 094/192] implement unified and extensible configuration Signed-off-by: Andrei Sandu --- Cargo.lock | 52 +++-- polkadot/node/subsystem-bench/Cargo.toml | 2 +- .../subsystem-bench/src/availability/cli.rs | 23 +-- .../src/availability/configuration.rs | 169 +--------------- .../subsystem-bench/src/availability/mod.rs | 32 +-- polkadot/node/subsystem-bench/src/cli.rs | 65 ++++++ .../subsystem-bench/src/core/configuration.rs | 190 ++++++++++++++++++ polkadot/node/subsystem-bench/src/core/mod.rs | 1 + .../node/subsystem-bench/src/core/network.rs | 1 - .../subsystem-bench/src/subsystem-bench.rs | 100 ++++----- .../node/subsystem-bench/test_sequence.toml | 77 ------- .../node/subsystem-bench/test_sequence.yaml | 56 ++++++ 12 files changed, 398 insertions(+), 370 deletions(-) create mode 100644 polkadot/node/subsystem-bench/src/cli.rs create mode 100644 polkadot/node/subsystem-bench/src/core/configuration.rs delete mode 100644 polkadot/node/subsystem-bench/test_sequence.toml create mode 100644 polkadot/node/subsystem-bench/test_sequence.yaml diff --git a/Cargo.lock b/Cargo.lock index 73fc3cbdeccc..b40a40db47b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8579,7 +8579,7 @@ dependencies = [ "itertools 0.10.5", "tar", "tempfile", - "toml_edit 0.19.14", + "toml_edit", ] [[package]] @@ -13044,13 +13044,13 @@ dependencies = [ "sc-network", "sc-service", "serde", + "serde_yaml", "sp-application-crypto", "sp-core", "sp-keyring", "sp-keystore", "substrate-prometheus-endpoint", "tokio", - "toml 0.8.8", "tracing-gum", ] @@ -13455,7 +13455,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" dependencies = [ "once_cell", - "toml_edit 0.19.14", + "toml_edit", ] [[package]] @@ -16349,6 +16349,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cc7a1570e38322cfe4154732e5110f887ea57e22b76f4bfd32b5bdd3368666c" +dependencies = [ + "indexmap 2.0.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "serial_test" version = "2.0.0" @@ -18833,19 +18846,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.19.14", -] - -[[package]] -name = "toml" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit 0.21.0", + "toml_edit", ] [[package]] @@ -18870,19 +18871,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "toml_edit" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" -dependencies = [ - "indexmap 2.0.0", - "serde", - "serde_spanned", - "toml_datetime", - "winnow", -] - [[package]] name = "tower" version = "0.4.13" @@ -19325,6 +19313,12 @@ dependencies = [ "subtle 2.4.1", ] +[[package]] +name = "unsafe-libyaml" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" + [[package]] name = "unsigned-varint" version = "0.7.1" diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index 3308b6fe1052..9dab8dce8455 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -51,8 +51,8 @@ itertools = "0.11.0" polkadot-primitives-test-helpers = { path = "../../primitives/test-helpers" } prometheus_endpoint = { package = "substrate-prometheus-endpoint", path = "../../../substrate/utils/prometheus" } prometheus = { version = "0.13.0", default-features = false } -toml = "0.8.8" serde = "1.0.192" +serde_yaml = "0.9" [features] default = [] diff --git a/polkadot/node/subsystem-bench/src/availability/cli.rs b/polkadot/node/subsystem-bench/src/availability/cli.rs index ef4d7e6f631a..06fb2966d878 100644 --- a/polkadot/node/subsystem-bench/src/availability/cli.rs +++ b/polkadot/node/subsystem-bench/src/availability/cli.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use serde::{Deserialize, Serialize}; #[derive(Debug, clap::Parser, Clone)] #[clap(rename_all = "kebab-case")] #[allow(missing_docs)] @@ -28,32 +29,12 @@ pub enum NetworkEmulation { Degraded, } -#[derive(Debug, clap::Parser)] +#[derive(Debug, Clone, Serialize, Deserialize, clap::Parser)] #[clap(rename_all = "kebab-case")] #[allow(missing_docs)] pub struct DataAvailabilityReadOptions { - #[clap(long, ignore_case = true, default_value_t = 100)] - /// Number of cores to fetch availability for. - pub n_cores: usize, - - #[clap(long, ignore_case = true, default_value_t = 500)] - /// Number of validators to fetch chunks from. - pub n_validators: usize, - - #[clap(long, ignore_case = true, default_value_t = 5120)] - /// The minimum pov size in KiB - pub min_pov_size: usize, - - #[clap(long, ignore_case = true, default_value_t = 5120)] - /// The maximum pov size bytes - pub max_pov_size: usize, - #[clap(short, long, default_value_t = false)] /// Turbo boost AD Read by fetching from backers first. Tipically this is only faster if nodes /// have enough bandwidth. pub fetch_from_backers: bool, - - #[clap(short, long, ignore_case = true, default_value_t = 1)] - /// Number of times to block fetching for each core. - pub num_blocks: usize, } diff --git a/polkadot/node/subsystem-bench/src/availability/configuration.rs b/polkadot/node/subsystem-bench/src/availability/configuration.rs index cbad4a2dc1b8..f96b8e2cb7ce 100644 --- a/polkadot/node/subsystem-bench/src/availability/configuration.rs +++ b/polkadot/node/subsystem-bench/src/availability/configuration.rs @@ -14,175 +14,12 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use std::path::Path; - use super::*; use serde::{Deserialize, Serialize}; -/// Peer response latency configuration. -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct PeerLatency { - /// Min latency for `NetworkAction` completion. - pub min_latency: Duration, - /// Max latency or `NetworkAction` completion. - pub max_latency: Duration, -} /// The test input parameters -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct TestConfiguration { - /// Configuration for the `availability-recovery` subsystem. +#[derive(Clone, Default, Debug, Serialize, Deserialize)] +pub struct AvailabilityRecoveryConfiguration { + /// Prefer the fast path (try fetch from backers first) pub use_fast_path: bool, - /// Number of validators - pub n_validators: usize, - /// Number of cores - pub n_cores: usize, - /// The min PoV size - pub min_pov_size: usize, - /// The max PoV size, - pub max_pov_size: usize, - /// Randomly sampled pov_sizes - #[serde(skip)] - pov_sizes: Vec, - /// The amount of bandiwdth remote validators have. - pub peer_bandwidth: usize, - /// The amount of bandiwdth our node has. - pub bandwidth: usize, - /// Optional peer emulation latency - pub latency: Option, - /// Error probability - pub error: usize, - /// Number of blocks - /// In one block `n_cores` candidates are recovered - pub num_blocks: usize, -} - -impl Default for TestConfiguration { - fn default() -> Self { - Self { - use_fast_path: false, - n_validators: 100, - n_cores: 10, - pov_sizes: vec![5 * 1024 * 1024], - bandwidth: 60 * 1024 * 1024, - peer_bandwidth: 60 * 1024 * 1024, - latency: None, - error: 0, - num_blocks: 1, - min_pov_size: 5 * 1024 * 1024, - max_pov_size: 5 * 1024 * 1024, - } - } -} - -fn generate_pov_sizes(count: usize, min: usize, max: usize) -> Vec { - (0..count).map(|_| random_pov_size(min, max)).collect() -} - -#[derive(Serialize, Deserialize)] -pub struct TestSequence { - #[serde(rename(serialize = "TestConfiguration", deserialize = "TestConfiguration"))] - test_configurations: Vec, -} - -impl TestSequence { - pub fn to_vec(mut self) -> Vec { - // Generate Pov sizes - - for config in self.test_configurations.iter_mut() { - config.pov_sizes = - generate_pov_sizes(config.n_cores, config.min_pov_size, config.max_pov_size); - } - - self.test_configurations - } -} - -impl TestSequence { - pub fn new_from_file(path: &Path) -> std::io::Result { - let string = String::from_utf8(std::fs::read(&path)?).expect("File is valid UTF8"); - Ok(toml::from_str(&string).expect("File is valid test sequence TOML")) - } -} - -impl TestConfiguration { - pub fn write_to_disk(&self) { - // Serialize a slice of configurations - let toml = - toml::to_string(&TestSequence { test_configurations: vec![self.clone()] }).unwrap(); - std::fs::write("last_test.toml", toml).unwrap(); - } - - pub fn pov_sizes(&self) -> &[usize] { - &self.pov_sizes - } - /// An unconstrained standard configuration matching Polkadot/Kusama - pub fn ideal_network( - num_blocks: usize, - use_fast_path: bool, - n_validators: usize, - n_cores: usize, - pov_sizes: Vec, - ) -> TestConfiguration { - Self { - use_fast_path, - n_cores, - n_validators, - pov_sizes, - bandwidth: 50 * 1024 * 1024, - peer_bandwidth: 50 * 1024 * 1024, - // No latency - latency: None, - error: 0, - num_blocks, - ..Default::default() - } - } - - pub fn healthy_network( - num_blocks: usize, - use_fast_path: bool, - n_validators: usize, - n_cores: usize, - pov_sizes: Vec, - ) -> TestConfiguration { - Self { - use_fast_path, - n_cores, - n_validators, - pov_sizes, - bandwidth: 50 * 1024 * 1024, - peer_bandwidth: 50 * 1024 * 1024, - latency: Some(PeerLatency { - min_latency: Duration::from_millis(1), - max_latency: Duration::from_millis(100), - }), - error: 3, - num_blocks, - ..Default::default() - } - } - - pub fn degraded_network( - num_blocks: usize, - use_fast_path: bool, - n_validators: usize, - n_cores: usize, - pov_sizes: Vec, - ) -> TestConfiguration { - Self { - use_fast_path, - n_cores, - n_validators, - pov_sizes, - bandwidth: 50 * 1024 * 1024, - peer_bandwidth: 50 * 1024 * 1024, - latency: Some(PeerLatency { - min_latency: Duration::from_millis(10), - max_latency: Duration::from_millis(500), - }), - error: 33, - num_blocks, - ..Default::default() - } - } } diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 8866348ea22b..9be15c576e3a 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -53,12 +53,18 @@ use polkadot_node_subsystem::{ }; use std::net::{Ipv4Addr, SocketAddr}; -use super::core::{environment::TestEnvironmentMetrics, keyring::Keyring, network::*}; +use super::core::{ + configuration::{PeerLatency, TestConfiguration}, + environment::TestEnvironmentMetrics, + keyring::Keyring, + network::*, +}; const LOG_TARGET: &str = "subsystem-bench::availability"; use polkadot_node_primitives::{AvailableData, ErasureChunk}; +use super::cli::TestObjective; use polkadot_node_subsystem_test_helpers::{ make_buffered_subsystem_context, mock::new_leaf, TestSubsystemContextHandle, }; @@ -73,7 +79,7 @@ use sc_service::{SpawnTaskHandle, TaskManager}; mod cli; pub mod configuration; pub use cli::{DataAvailabilityReadOptions, NetworkEmulation, NetworkOptions}; -pub use configuration::{PeerLatency, TestConfiguration, TestSequence}; +pub use configuration::AvailabilityRecoveryConfiguration; // Deterministic genesis hash for protocol names const GENESIS_HASH: Hash = Hash::repeat_byte(0xff); @@ -138,7 +144,10 @@ impl TestEnvironment { let (instance, virtual_overseer) = AvailabilityRecoverySubsystemInstance::new( ®istry, task_manager.spawn_handle(), - state.config().use_fast_path, + match &state.config().objective { + TestObjective::DataAvailabilityRead(options) => options.fetch_from_backers, + _ => panic!("Unexpected objective"), + }, ); let metrics = @@ -491,16 +500,6 @@ impl AvailabilityRecoverySubsystemInstance { } } -pub fn random_pov_size(min_pov_size: usize, max_pov_size: usize) -> usize { - random_uniform_sample(min_pov_size, max_pov_size) -} - -fn random_uniform_sample + From>(min_value: T, max_value: T) -> T { - Uniform::from(min_value.into()..=max_value.into()) - .sample(&mut thread_rng()) - .into() -} - // We use this to bail out sending messages to the subsystem if it is overloaded such that // the time of flight is breaches 5s. // This should eventually be a test parameter. @@ -508,6 +507,9 @@ const MAX_TIME_OF_FLIGHT: Duration = Duration::from_millis(5000); #[derive(Clone)] pub struct TestState { + // Full test configuration + config: TestConfiguration, + // State starts here. validator_public: Vec, validator_authority_id: Vec, // The test node validator index. @@ -527,8 +529,6 @@ pub struct TestState { candidate_receipts: Vec, available_data: Vec, chunks: Vec>, - /// Next candidate index in - config: TestConfiguration, } impl TestState { @@ -687,6 +687,7 @@ impl TestState { gum::info!(target: LOG_TARGET, "{}","Created test environment.".bright_blue()); Self { + config, validator_public, validator_authority_id, validator_index, @@ -695,7 +696,6 @@ impl TestState { available_data, candidate_receipts, chunks, - config, pov_size_to_candidate, pov_sizes, candidates_generated: 0, diff --git a/polkadot/node/subsystem-bench/src/cli.rs b/polkadot/node/subsystem-bench/src/cli.rs new file mode 100644 index 000000000000..2f00ad2f3585 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/cli.rs @@ -0,0 +1,65 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . +use super::availability::{ + AvailabilityRecoveryConfiguration, DataAvailabilityReadOptions, NetworkEmulation, + TestEnvironment, TestState, +}; +use serde::{Deserialize, Serialize}; + +use super::core::configuration::{PeerLatency, TestConfiguration, TestSequence}; + +#[derive(Debug, Clone, Serialize, Deserialize, clap::Parser)] +#[clap(rename_all = "kebab-case")] +#[allow(missing_docs)] +pub struct TestSequenceOptions { + #[clap(short, long, ignore_case = true)] + pub path: String, +} + +/// Define the supported benchmarks targets +#[derive(Debug, Clone, clap::Parser, Serialize, Deserialize)] +#[command(about = "Test objectives", version, rename_all = "kebab-case")] +pub enum TestObjective { + /// Benchmark availability recovery strategies. + DataAvailabilityRead(DataAvailabilityReadOptions), + /// Run a test sequence specified in a file + TestSequence(TestSequenceOptions), +} + +#[derive(Debug, clap::Parser)] +#[clap(rename_all = "kebab-case")] +#[allow(missing_docs)] +pub struct StandardTestOptions { + #[clap(long, ignore_case = true, default_value_t = 100)] + /// Number of cores to fetch availability for. + pub n_cores: usize, + + #[clap(long, ignore_case = true, default_value_t = 500)] + /// Number of validators to fetch chunks from. + pub n_validators: usize, + + #[clap(long, ignore_case = true, default_value_t = 5120)] + /// The minimum pov size in KiB + pub min_pov_size: usize, + + #[clap(long, ignore_case = true, default_value_t = 5120)] + /// The maximum pov size bytes + pub max_pov_size: usize, + + #[clap(short, long, ignore_case = true, default_value_t = 1)] + /// The number of blocks the test is going to run. + pub num_blocks: usize, +} diff --git a/polkadot/node/subsystem-bench/src/core/configuration.rs b/polkadot/node/subsystem-bench/src/core/configuration.rs new file mode 100644 index 000000000000..017d4023ef65 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/core/configuration.rs @@ -0,0 +1,190 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . +use std::path::Path; + +use crate::availability::AvailabilityRecoveryConfiguration; + +use super::*; +pub use crate::cli::TestObjective; +use rand::{distributions::Uniform, prelude::Distribution, thread_rng}; +use serde::{Deserialize, Serialize}; + +pub fn random_pov_size(min_pov_size: usize, max_pov_size: usize) -> usize { + random_uniform_sample(min_pov_size, max_pov_size) +} + +fn random_uniform_sample + From>(min_value: T, max_value: T) -> T { + Uniform::from(min_value.into()..=max_value.into()) + .sample(&mut thread_rng()) + .into() +} + +/// Peer response latency configuration. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct PeerLatency { + /// Min latency for `NetworkAction` completion. + pub min_latency: Duration, + /// Max latency or `NetworkAction` completion. + pub max_latency: Duration, +} + +/// The test input parameters +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TestConfiguration { + /// The test objective + pub objective: TestObjective, + /// Number of validators + pub n_validators: usize, + /// Number of cores + pub n_cores: usize, + /// The min PoV size + pub min_pov_size: usize, + /// The max PoV size, + pub max_pov_size: usize, + /// Randomly sampled pov_sizes + #[serde(skip)] + pov_sizes: Vec, + /// The amount of bandiwdth remote validators have. + pub peer_bandwidth: usize, + /// The amount of bandiwdth our node has. + pub bandwidth: usize, + /// Optional peer emulation latency + pub latency: Option, + /// Error probability + pub error: usize, + /// Number of blocks + /// In one block `n_cores` candidates are recovered + pub num_blocks: usize, +} + +fn generate_pov_sizes(count: usize, min_kib: usize, max_kib: usize) -> Vec { + (0..count).map(|_| random_pov_size(min_kib * 1024, max_kib * 1024)).collect() +} + +#[derive(Serialize, Deserialize)] +pub struct TestSequence { + #[serde(rename(serialize = "TestConfiguration", deserialize = "TestConfiguration"))] + test_configurations: Vec, +} + +impl TestSequence { + pub fn to_vec(mut self) -> Vec { + self.test_configurations + .into_iter() + .map(|mut config| { + config.pov_sizes = + generate_pov_sizes(config.n_cores, config.min_pov_size, config.max_pov_size); + config + }) + .collect() + } +} + +impl TestSequence { + pub fn new_from_file(path: &Path) -> std::io::Result { + let string = String::from_utf8(std::fs::read(&path)?).expect("File is valid UTF8"); + Ok(serde_yaml::from_str(&string).expect("File is valid test sequence YA")) + } +} + +impl TestConfiguration { + pub fn write_to_disk(&self) { + // Serialize a slice of configurations + let yaml = serde_yaml::to_string(&TestSequence { test_configurations: vec![self.clone()] }) + .unwrap(); + std::fs::write("last_test.yaml", yaml).unwrap(); + } + + pub fn pov_sizes(&self) -> &[usize] { + &self.pov_sizes + } + /// An unconstrained standard configuration matching Polkadot/Kusama + pub fn ideal_network( + objective: TestObjective, + num_blocks: usize, + n_validators: usize, + n_cores: usize, + min_pov_size: usize, + max_pov_size: usize, + ) -> TestConfiguration { + Self { + objective, + n_cores, + n_validators, + pov_sizes: generate_pov_sizes(n_cores, min_pov_size, max_pov_size), + bandwidth: 50 * 1024 * 1024, + peer_bandwidth: 50 * 1024 * 1024, + // No latency + latency: None, + error: 0, + num_blocks, + min_pov_size, + max_pov_size, + } + } + + pub fn healthy_network( + objective: TestObjective, + num_blocks: usize, + n_validators: usize, + n_cores: usize, + min_pov_size: usize, + max_pov_size: usize, + ) -> TestConfiguration { + Self { + objective, + n_cores, + n_validators, + pov_sizes: generate_pov_sizes(n_cores, min_pov_size, max_pov_size), + bandwidth: 50 * 1024 * 1024, + peer_bandwidth: 50 * 1024 * 1024, + latency: Some(PeerLatency { + min_latency: Duration::from_millis(1), + max_latency: Duration::from_millis(100), + }), + error: 3, + num_blocks, + min_pov_size, + max_pov_size, + } + } + + pub fn degraded_network( + objective: TestObjective, + num_blocks: usize, + n_validators: usize, + n_cores: usize, + min_pov_size: usize, + max_pov_size: usize, + ) -> TestConfiguration { + Self { + objective, + n_cores, + n_validators, + pov_sizes: generate_pov_sizes(n_cores, min_pov_size, max_pov_size), + bandwidth: 50 * 1024 * 1024, + peer_bandwidth: 50 * 1024 * 1024, + latency: Some(PeerLatency { + min_latency: Duration::from_millis(10), + max_latency: Duration::from_millis(500), + }), + error: 33, + num_blocks, + min_pov_size, + max_pov_size, + } + } +} diff --git a/polkadot/node/subsystem-bench/src/core/mod.rs b/polkadot/node/subsystem-bench/src/core/mod.rs index 564fb7148fa0..06aa58f7256b 100644 --- a/polkadot/node/subsystem-bench/src/core/mod.rs +++ b/polkadot/node/subsystem-bench/src/core/mod.rs @@ -24,6 +24,7 @@ const LOG_TARGET: &str = "subsystem-bench::core"; use polkadot_primitives::AuthorityDiscoveryId; use sc_service::SpawnTaskHandle; +pub mod configuration; pub mod display; pub mod environment; pub mod keyring; diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 629d09df694c..f20bb919dedb 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -113,7 +113,6 @@ mod tests { let end = Instant::now(); - // assert_eq!(end - start, Duration::from_secs(1)); println!("duration: {}", (end - start).as_millis()); // Allow up to `budget/max_refill` error tolerance diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 42efb7fd63c8..b94e594e5945 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -23,12 +23,16 @@ use colored::Colorize; use std::{path::Path, time::Duration}; pub(crate) mod availability; +pub(crate) mod cli; pub(crate) mod core; use availability::{ - random_pov_size, DataAvailabilityReadOptions, NetworkEmulation, TestConfiguration, + AvailabilityRecoveryConfiguration, DataAvailabilityReadOptions, NetworkEmulation, TestEnvironment, TestState, }; +use cli::TestObjective; + +use core::configuration::{PeerLatency, TestConfiguration, TestSequence}; use clap_num::number_range; const LOG_TARGET: &str = "subsystem-bench"; @@ -40,23 +44,6 @@ fn le_100(s: &str) -> Result { fn le_5000(s: &str) -> Result { number_range(s, 0, 5000) } -#[derive(Debug, clap::Parser)] -#[clap(rename_all = "kebab-case")] -#[allow(missing_docs)] -pub struct TestSequenceOptions { - #[clap(short, long, ignore_case = true)] - pub path: String, -} - -/// Define the supported benchmarks targets -#[derive(Debug, Parser)] -#[command(about = "Test objectives", version, rename_all = "kebab-case")] -enum TestObjective { - /// Benchmark availability recovery strategies. - DataAvailabilityRead(DataAvailabilityReadOptions), - /// Run a test sequence specified in a file - TestSequence(TestSequenceOptions), -} #[derive(Debug, Parser)] #[allow(missing_docs)] @@ -65,6 +52,9 @@ struct BenchCli { /// The type of network to be emulated pub network: NetworkEmulation, + #[clap(flatten)] + pub standard_configuration: cli::StandardTestOptions, + #[clap(short, long)] /// The bandwidth of simulated remote peers in KiB pub peer_bandwidth: Option, @@ -86,7 +76,7 @@ struct BenchCli { pub peer_max_latency: Option, #[command(subcommand)] - pub objective: TestObjective, + pub objective: cli::TestObjective, } fn new_runtime() -> tokio::runtime::Runtime { @@ -104,10 +94,11 @@ impl BenchCli { let runtime = new_runtime(); + let configuration = self.standard_configuration; let mut test_config = match self.objective { TestObjective::TestSequence(options) => { let test_sequence = - availability::TestSequence::new_from_file(Path::new(&options.path)) + core::configuration::TestSequence::new_from_file(Path::new(&options.path)) .expect("File exists") .to_vec(); let num_steps = test_sequence.len(); @@ -117,8 +108,17 @@ impl BenchCli { ); for (index, test_config) in test_sequence.into_iter().enumerate() { gum::info!( - "{}", - format!("Current step {}/{}", index + 1, num_steps).bright_purple() + "{}, {}, {}, {}, {}, {}", + format!("Step {}/{}", index + 1, num_steps).bright_purple(), + format!("n_validators = {}", test_config.n_validators).blue(), + format!("n_cores = {}", test_config.n_cores).blue(), + format!( + "pov_size = {} - {}", + test_config.min_pov_size, test_config.max_pov_size + ) + .bright_black(), + format!("error = {}", test_config.error).bright_black(), + format!("latency = {:?}", test_config.latency).bright_black(), ); let candidate_count = test_config.n_cores * test_config.num_blocks; @@ -132,48 +132,30 @@ impl BenchCli { } return Ok(()) }, - TestObjective::DataAvailabilityRead(options) => match self.network { + TestObjective::DataAvailabilityRead(ref options) => match self.network { NetworkEmulation::Healthy => TestConfiguration::healthy_network( - options.num_blocks, - options.fetch_from_backers, - options.n_validators, - options.n_cores, - (0..options.n_cores) - .map(|_| { - random_pov_size( - options.min_pov_size * 1024, - options.max_pov_size * 1024, - ) - }) - .collect(), + self.objective, + configuration.num_blocks, + configuration.n_validators, + configuration.n_cores, + configuration.min_pov_size, + configuration.max_pov_size, ), NetworkEmulation::Degraded => TestConfiguration::degraded_network( - options.num_blocks, - options.fetch_from_backers, - options.n_validators, - options.n_cores, - (0..options.n_cores) - .map(|_| { - random_pov_size( - options.min_pov_size * 1024, - options.max_pov_size * 1024, - ) - }) - .collect(), + self.objective, + configuration.num_blocks, + configuration.n_validators, + configuration.n_cores, + configuration.min_pov_size, + configuration.max_pov_size, ), NetworkEmulation::Ideal => TestConfiguration::ideal_network( - options.num_blocks, - options.fetch_from_backers, - options.n_validators, - options.n_cores, - (0..options.n_cores) - .map(|_| { - random_pov_size( - options.min_pov_size * 1024, - options.max_pov_size * 1024, - ) - }) - .collect(), + self.objective, + configuration.num_blocks, + configuration.n_validators, + configuration.n_cores, + configuration.min_pov_size, + configuration.max_pov_size, ), }, }; diff --git a/polkadot/node/subsystem-bench/test_sequence.toml b/polkadot/node/subsystem-bench/test_sequence.toml deleted file mode 100644 index d32477b9efe9..000000000000 --- a/polkadot/node/subsystem-bench/test_sequence.toml +++ /dev/null @@ -1,77 +0,0 @@ -[[TestConfiguration]] -use_fast_path = false -n_validators = 300 -n_cores = 20 -min_pov_size = 5242880 -max_pov_size = 5242880 -peer_bandwidth = 128000 -bandwidth = 52428800 -error = 33 -num_blocks = 5 - -[TestConfiguration.latency.min_latency] -secs = 0 -nanos = 1000000 - -[TestConfiguration.latency.max_latency] -secs = 0 -nanos = 100000000 - -[[TestConfiguration]] -use_fast_path = false -n_validators = 500 -n_cores = 20 -min_pov_size = 5242880 -max_pov_size = 5242880 -peer_bandwidth = 128000 -bandwidth = 52428800 -error = 33 -num_blocks = 5 - -[TestConfiguration.latency.min_latency] -secs = 0 -nanos = 1000000 - -[TestConfiguration.latency.max_latency] -secs = 0 -nanos = 1000000000 - - -[[TestConfiguration]] -use_fast_path = false -n_validators = 1000 -n_cores = 20 -min_pov_size = 5242880 -max_pov_size = 5242880 -peer_bandwidth = 128000 -bandwidth = 52428800 -error = 33 -num_blocks = 5 - -[TestConfiguration.latency.min_latency] -secs = 0 -nanos = 1000000 - -[TestConfiguration.latency.max_latency] -secs = 0 -nanos = 1000000000 - - -[[TestConfiguration]] -use_fast_path = false -n_validators = 2000 -n_cores = 20 -min_pov_size = 5242880 -max_pov_size = 5242880 -peer_bandwidth = 128000 -bandwidth = 52428800 -error = 33 -num_blocks = 5 - -[TestConfiguration.latency.min_latency] -secs = 0 -nanos = 1000000 - -[TestConfiguration.latency.max_latency] -secs = 0 -nanos = 1000000000 diff --git a/polkadot/node/subsystem-bench/test_sequence.yaml b/polkadot/node/subsystem-bench/test_sequence.yaml new file mode 100644 index 000000000000..088a7e15729b --- /dev/null +++ b/polkadot/node/subsystem-bench/test_sequence.yaml @@ -0,0 +1,56 @@ +TestConfiguration: +# Test 1 +- objective: !DataAvailabilityRead + fetch_from_backers: false + n_validators: 300 + n_cores: 10 + min_pov_size: 1120 + max_pov_size: 5120 + peer_bandwidth: 52428800 + bandwidth: 52428800 + latency: + min_latency: + secs: 0 + nanos: 1000000 + max_latency: + secs: 0 + nanos: 100000000 + error: 3 + num_blocks: 10 +# Test 2 +- objective: !DataAvailabilityRead + fetch_from_backers: false + n_validators: 500 + n_cores: 10 + min_pov_size: 1120 + max_pov_size: 5120 + peer_bandwidth: 52428800 + bandwidth: 52428800 + latency: + min_latency: + secs: 0 + nanos: 1000000 + max_latency: + secs: 0 + nanos: 100000000 + error: 3 + num_blocks: 10 + +# Test 2 +- objective: !DataAvailabilityRead + fetch_from_backers: false + n_validators: 1000 + n_cores: 10 + min_pov_size: 1120 + max_pov_size: 5120 + peer_bandwidth: 52428800 + bandwidth: 52428800 + latency: + min_latency: + secs: 0 + nanos: 1000000 + max_latency: + secs: 0 + nanos: 100000000 + error: 3 + num_blocks: 10 From 28438650ef36a17528cf45883712786ca9dc034d Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 24 Nov 2023 16:05:51 +0200 Subject: [PATCH 095/192] Prepare to swtich to overseer Signed-off-by: Andrei Sandu --- Cargo.lock | 1 + polkadot/node/subsystem-bench/Cargo.toml | 1 + .../subsystem-bench/src/availability/mod.rs | 20 ++--- .../subsystem-bench/src/core/mock/dummy.rs | 89 +++++++++++++++++++ .../node/subsystem-bench/src/core/mock/mod.rs | 76 ++++++++++++++++ polkadot/node/subsystem-bench/src/core/mod.rs | 1 + .../subsystem-bench/src/core/subsystem.rs | 16 ++++ .../subsystem-bench/src/subsystem-bench.rs | 8 +- .../procedural/src/pallet/expand/warnings.rs | 8 +- 9 files changed, 199 insertions(+), 21 deletions(-) create mode 100644 polkadot/node/subsystem-bench/src/core/mock/dummy.rs create mode 100644 polkadot/node/subsystem-bench/src/core/mock/mod.rs create mode 100644 polkadot/node/subsystem-bench/src/core/subsystem.rs diff --git a/Cargo.lock b/Cargo.lock index b40a40db47b5..44a9093710ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13027,6 +13027,7 @@ dependencies = [ "itertools 0.11.0", "log", "parity-scale-codec", + "paste", "polkadot-availability-recovery", "polkadot-erasure-coding", "polkadot-node-metrics", diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index 9dab8dce8455..d1c68c6e5f54 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -53,6 +53,7 @@ prometheus_endpoint = { package = "substrate-prometheus-endpoint", path = "../.. prometheus = { version = "0.13.0", default-features = false } serde = "1.0.192" serde_yaml = "0.9" +paste = "1.0.14" [features] default = [] diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 9be15c576e3a..8bd28b02bd73 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -129,7 +129,7 @@ pub struct TestEnvironment { // for the whole duration of the test. instance: AvailabilityRecoverySubsystemInstance, // The test intial state. The current state is owned by `env_task`. - state: TestState, + config: TestConfiguration, // A handle to the network emulator. network: NetworkEmulator, // Configuration/env metrics @@ -140,6 +140,7 @@ impl TestEnvironment { // Create a new test environment with specified initial state and prometheus registry. // We use prometheus metrics to collect per job task poll time and subsystem metrics. pub fn new(runtime: tokio::runtime::Handle, state: TestState, registry: Registry) -> Self { + let config = state.config().clone(); let task_manager: TaskManager = TaskManager::new(runtime.clone(), Some(®istry)).unwrap(); let (instance, virtual_overseer) = AvailabilityRecoverySubsystemInstance::new( ®istry, @@ -153,9 +154,9 @@ impl TestEnvironment { let metrics = TestEnvironmentMetrics::new(®istry).expect("Metrics need to be registered"); let mut network = NetworkEmulator::new( - state.config().n_validators, + config.n_validators, state.validator_authority_id.clone(), - state.config().peer_bandwidth, + config.peer_bandwidth, task_manager.spawn_handle(), ®istry, ); @@ -168,7 +169,7 @@ impl TestEnvironment { let spawn_handle = task_manager.spawn_handle(); // Our node rate limiting - let mut rx_limiter = RateLimit::new(10, state.config.bandwidth); + let mut rx_limiter = RateLimit::new(10, config.bandwidth); let (ingress_tx, mut ingress_rx) = tokio::sync::mpsc::unbounded_channel::(); let our_network_stats = network.peer_stats(0); @@ -204,11 +205,11 @@ impl TestEnvironment { .unwrap(); }); - TestEnvironment { task_manager, registry, to_subsystem, instance, state, network, metrics } + TestEnvironment { task_manager, registry, to_subsystem, instance, config, network, metrics } } pub fn config(&self) -> &TestConfiguration { - self.state.config() + &self.config } pub fn network(&mut self) -> &mut NetworkEmulator { @@ -457,8 +458,6 @@ impl TestEnvironment { } } -/// Implementation for chunks only -/// TODO: all recovery methods. impl AvailabilityRecoverySubsystemInstance { pub fn new( registry: &Registry, @@ -732,7 +731,7 @@ fn derive_erasure_chunks_with_proofs_and_root( (erasure_chunks, root) } -pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { +pub async fn bench_chunk_recovery(env: &mut TestEnvironment, mut state: TestState) { let config = env.config().clone(); env.send_signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( @@ -754,8 +753,7 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment) { let block_start_ts = Instant::now(); for candidate_num in 0..config.n_cores as u64 { - let candidate = env - .state + let candidate = state .next_candidate() .expect("We always send up to n_cores*num_blocks; qed"); let (tx, rx) = oneshot::channel(); diff --git a/polkadot/node/subsystem-bench/src/core/mock/dummy.rs b/polkadot/node/subsystem-bench/src/core/mock/dummy.rs new file mode 100644 index 000000000000..122fc23ac52f --- /dev/null +++ b/polkadot/node/subsystem-bench/src/core/mock/dummy.rs @@ -0,0 +1,89 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . +//! Dummy subsystem mocks. +//! +use paste::paste; + +use futures::{channel::oneshot, select, Future, FutureExt}; +use polkadot_node_subsystem::{ + overseer, AllMessages, FromOrchestra, HeadSupportsParachains, Overseer, OverseerConnector, + OverseerHandle, SpawnGlue, SpawnedSubsystem, Subsystem, SubsystemError, +}; +use std::time::Duration; +use tokio::time::sleep; + +macro_rules! mock { + // Just query by relay parent + ($subsystem_name:ident) => { + paste! { + pub struct [] {} + #[overseer::subsystem($subsystem_name, error=SubsystemError, prefix=self::overseer)] + impl [] { + fn start(self, ctx: Context) -> SpawnedSubsystem { + let future = self.run(ctx).map(|_| Ok(())).boxed(); + + SpawnedSubsystem { name: stringify!($subsystem_name), future } + } + } + + #[overseer::contextbounds($subsystem_name, prefix = self::overseer)] + impl [] { + async fn run(self, mut ctx: Context) { + let mut count_total_msg = 0; + loop { + futures::select!{ + _msg = ctx.recv().fuse() => { + count_total_msg +=1; + } + _ = sleep(Duration::from_secs(6)).fuse() => { + if count_total_msg > 0 { + gum::info!(target: "mock-subsystems", "Subsystem {} processed {} messages since last time", stringify!($subsystem_name), count_total_msg); + } + count_total_msg = 0; + } + } + } + } + } + } + }; +} + +mock!(AvailabilityStore); +mock!(StatementDistribution); +mock!(BitfieldSigning); +mock!(BitfieldDistribution); +mock!(Provisioner); +mock!(NetworkBridgeRx); +mock!(CollationGeneration); +mock!(CollatorProtocol); +mock!(GossipSupport); +mock!(DisputeDistribution); +mock!(DisputeCoordinator); +mock!(ProspectiveParachains); +mock!(PvfChecker); +mock!(CandidateBacking); +mock!(AvailabilityDistribution); +mock!(CandidateValidation); +mock!(AvailabilityRecovery); +mock!(NetworkBridgeTx); +mock!(ChainApi); +mock!(ChainSelection); +mock!(ApprovalVoting); +mock!(ApprovalDistribution); +mock!(RuntimeApi); + + diff --git a/polkadot/node/subsystem-bench/src/core/mock/mod.rs b/polkadot/node/subsystem-bench/src/core/mock/mod.rs new file mode 100644 index 000000000000..f13e87c8683b --- /dev/null +++ b/polkadot/node/subsystem-bench/src/core/mock/mod.rs @@ -0,0 +1,76 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use polkadot_node_subsystem::{ + overseer, AllMessages, FromOrchestra, HeadSupportsParachains, Overseer, OverseerConnector, + OverseerHandle, SpawnGlue, SpawnedSubsystem, Subsystem, +}; +use polkadot_node_subsystem_types::Hash; + +pub mod dummy; +mod temp; + +use dummy::*; +use sc_service::SpawnTaskHandle; + +struct AlwaysSupportsParachains {} +#[async_trait::async_trait] +impl HeadSupportsParachains for AlwaysSupportsParachains { + async fn head_supports_parachains(&self, _head: &Hash) -> bool { + true + } +} + +pub fn new_overseer_with_dummy_subsystems(spawn_task_handle: SpawnTaskHandle) { + // Initialize a mock overseer. + // All subsystem except approval_voting and approval_distribution are mock subsystems. + let spawner_glue = SpawnGlue(spawn_task_handle); + let overseer_connector = OverseerConnector::with_event_capacity(64000); + let builder = Overseer::builder() + .approval_voting(MockApprovalVoting {}) + .approval_distribution(MockApprovalDistribution {}) + .availability_recovery(MockAvailabilityRecovery {}) + .candidate_validation(MockCandidateValidation {}) + .chain_api(MockChainApi { }) + .chain_selection(MockChainSelection {}) + .dispute_coordinator(MockDisputeCoordinator {}) + .runtime_api(MockRuntimeApi { }) + .network_bridge_tx(MockNetworkBridgeTx {}) + .availability_distribution(MockAvailabilityDistribution {}) + .availability_store(MockAvailabilityStore {}) + .pvf_checker(MockPvfChecker {}) + .candidate_backing(MockCandidateBacking {}) + .statement_distribution(MockStatementDistribution {}) + .bitfield_signing(MockBitfieldSigning {}) + .bitfield_distribution(MockBitfieldDistribution {}) + .provisioner(MockProvisioner {}) + .network_bridge_rx(MockNetworkBridgeRx {}) + .collation_generation(MockCollationGeneration {}) + .collator_protocol(MockCollatorProtocol {}) + .gossip_support(MockGossipSupport {}) + .dispute_distribution(MockDisputeDistribution {}) + .prospective_parachains(MockProspectiveParachains {}) + .activation_external_listeners(Default::default()) + .span_per_active_leaf(Default::default()) + .active_leaves(Default::default()) + .metrics(Default::default()) + .supports_parachains(AlwaysSupportsParachains {}) + .spawner(spawner_glue); + + let (mock_overseer, mock_overseer_handle) = + builder.build_with_connector(overseer_connector).expect("Should not fail"); + +} \ No newline at end of file diff --git a/polkadot/node/subsystem-bench/src/core/mod.rs b/polkadot/node/subsystem-bench/src/core/mod.rs index 06aa58f7256b..af2abf0860cd 100644 --- a/polkadot/node/subsystem-bench/src/core/mod.rs +++ b/polkadot/node/subsystem-bench/src/core/mod.rs @@ -29,3 +29,4 @@ pub mod display; pub mod environment; pub mod keyring; pub mod network; +pub mod mock; diff --git a/polkadot/node/subsystem-bench/src/core/subsystem.rs b/polkadot/node/subsystem-bench/src/core/subsystem.rs new file mode 100644 index 000000000000..c61e641d255d --- /dev/null +++ b/polkadot/node/subsystem-bench/src/core/subsystem.rs @@ -0,0 +1,16 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index b94e594e5945..ca561e5c4955 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -126,9 +126,9 @@ impl BenchCli { let mut state = TestState::new(test_config); state.generate_candidates(candidate_count); let mut env = - TestEnvironment::new(runtime.handle().clone(), state, Registry::new()); + TestEnvironment::new(runtime.handle().clone(), state.clone(), Registry::new()); - runtime.block_on(availability::bench_chunk_recovery(&mut env)); + runtime.block_on(availability::bench_chunk_recovery(&mut env, state)); } return Ok(()) }, @@ -189,9 +189,9 @@ impl BenchCli { let mut state = TestState::new(test_config); state.generate_candidates(candidate_count); - let mut env = TestEnvironment::new(runtime.handle().clone(), state, Registry::new()); + let mut env = TestEnvironment::new(runtime.handle().clone(), state.clone(), Registry::new()); - runtime.block_on(availability::bench_chunk_recovery(&mut env)); + runtime.block_on(availability::bench_chunk_recovery(&mut env, state)); Ok(()) } diff --git a/substrate/frame/support/procedural/src/pallet/expand/warnings.rs b/substrate/frame/support/procedural/src/pallet/expand/warnings.rs index 6ce2097c2684..030e3ddaf323 100644 --- a/substrate/frame/support/procedural/src/pallet/expand/warnings.rs +++ b/substrate/frame/support/procedural/src/pallet/expand/warnings.rs @@ -33,9 +33,7 @@ pub(crate) fn weight_witness_warning( if dev_mode { return } - let CallWeightDef::Immediate(w) = &method.weight else { - return - }; + let CallWeightDef::Immediate(w) = &method.weight else { return }; let partial_warning = Warning::new_deprecated("UncheckedWeightWitness") .old("not check weight witness data") @@ -66,9 +64,7 @@ pub(crate) fn weight_constant_warning( if dev_mode { return } - let syn::Expr::Lit(lit) = weight else { - return - }; + let syn::Expr::Lit(lit) = weight else { return }; let warning = Warning::new_deprecated("ConstantWeight") .index(warnings.len()) From b17a1477ede5840d882a69d44f8e0a40eb986c56 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 27 Nov 2023 11:28:53 +0200 Subject: [PATCH 096/192] add mocked subsystems Signed-off-by: Andrei Sandu --- Cargo.lock | 1 + polkadot/node/subsystem-bench/Cargo.toml | 1 + .../src/availability/configuration.rs | 1 - .../subsystem-bench/src/availability/mod.rs | 5 +- polkadot/node/subsystem-bench/src/cli.rs | 7 +- .../subsystem-bench/src/core/configuration.rs | 22 +- .../subsystem-bench/src/core/environment.rs | 2 - .../subsystem-bench/src/core/mock/av_store.rs | 127 +++++++++ .../subsystem-bench/src/core/mock/dummy.rs | 10 +- .../node/subsystem-bench/src/core/mock/mod.rs | 97 ++++--- .../src/core/mock/network_bridge.rs | 262 ++++++++++++++++++ .../src/core/mock/runtime_api.rs | 107 +++++++ polkadot/node/subsystem-bench/src/core/mod.rs | 2 +- .../subsystem-bench/src/subsystem-bench.rs | 21 +- 14 files changed, 587 insertions(+), 78 deletions(-) create mode 100644 polkadot/node/subsystem-bench/src/core/mock/av_store.rs create mode 100644 polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs create mode 100644 polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs diff --git a/Cargo.lock b/Cargo.lock index 15cda46316f9..b349886761ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13433,6 +13433,7 @@ dependencies = [ "futures-timer", "itertools 0.11.0", "log", + "orchestra", "parity-scale-codec", "paste", "polkadot-availability-recovery", diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index d1c68c6e5f54..8296874c0dab 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -54,6 +54,7 @@ prometheus = { version = "0.13.0", default-features = false } serde = "1.0.192" serde_yaml = "0.9" paste = "1.0.14" +orchestra = { version = "0.3.3", default-features = false, features=["futures_channel"] } [features] default = [] diff --git a/polkadot/node/subsystem-bench/src/availability/configuration.rs b/polkadot/node/subsystem-bench/src/availability/configuration.rs index f96b8e2cb7ce..1274862a8e4a 100644 --- a/polkadot/node/subsystem-bench/src/availability/configuration.rs +++ b/polkadot/node/subsystem-bench/src/availability/configuration.rs @@ -14,7 +14,6 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use super::*; use serde::{Deserialize, Serialize}; /// The test input parameters diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 8bd28b02bd73..3f9598505074 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -753,9 +753,8 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment, mut state: TestStat let block_start_ts = Instant::now(); for candidate_num in 0..config.n_cores as u64 { - let candidate = state - .next_candidate() - .expect("We always send up to n_cores*num_blocks; qed"); + let candidate = + state.next_candidate().expect("We always send up to n_cores*num_blocks; qed"); let (tx, rx) = oneshot::channel(); batch.push(rx); diff --git a/polkadot/node/subsystem-bench/src/cli.rs b/polkadot/node/subsystem-bench/src/cli.rs index 2f00ad2f3585..ee67a01d449e 100644 --- a/polkadot/node/subsystem-bench/src/cli.rs +++ b/polkadot/node/subsystem-bench/src/cli.rs @@ -13,14 +13,9 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use super::availability::{ - AvailabilityRecoveryConfiguration, DataAvailabilityReadOptions, NetworkEmulation, - TestEnvironment, TestState, -}; +use super::availability::DataAvailabilityReadOptions; use serde::{Deserialize, Serialize}; -use super::core::configuration::{PeerLatency, TestConfiguration, TestSequence}; - #[derive(Debug, Clone, Serialize, Deserialize, clap::Parser)] #[clap(rename_all = "kebab-case")] #[allow(missing_docs)] diff --git a/polkadot/node/subsystem-bench/src/core/configuration.rs b/polkadot/node/subsystem-bench/src/core/configuration.rs index 017d4023ef65..4526505c3a64 100644 --- a/polkadot/node/subsystem-bench/src/core/configuration.rs +++ b/polkadot/node/subsystem-bench/src/core/configuration.rs @@ -15,8 +15,6 @@ // along with Polkadot. If not, see . use std::path::Path; -use crate::availability::AvailabilityRecoveryConfiguration; - use super::*; pub use crate::cli::TestObjective; use rand::{distributions::Uniform, prelude::Distribution, thread_rng}; @@ -81,7 +79,7 @@ pub struct TestSequence { } impl TestSequence { - pub fn to_vec(mut self) -> Vec { + pub fn to_vec(self) -> Vec { self.test_configurations .into_iter() .map(|mut config| { @@ -188,3 +186,21 @@ impl TestConfiguration { } } } + +/// Produce a randomized duration between `min` and `max`. +pub fn random_latency(maybe_peer_latency: Option<&PeerLatency>) -> Option { + if let Some(peer_latency) = maybe_peer_latency { + Some( + Uniform::from(peer_latency.min_latency..=peer_latency.max_latency) + .sample(&mut thread_rng()), + ) + } else { + None + } +} + +/// Generate a random error based on `probability`. +/// `probability` should be a number between 0 and 100. +pub fn random_error(probability: usize) -> bool { + Uniform::from(0..=99).sample(&mut thread_rng()) < probability +} diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 6a680799972d..e6b09a1c13e6 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -14,8 +14,6 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use super::*; -use network::NetworkEmulator; use polkadot_node_subsystem_util::metrics::prometheus::{ self, Gauge, Histogram, PrometheusError, Registry, U64, }; diff --git a/polkadot/node/subsystem-bench/src/core/mock/av_store.rs b/polkadot/node/subsystem-bench/src/core/mock/av_store.rs new file mode 100644 index 000000000000..e84aeba5b6b7 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/core/mock/av_store.rs @@ -0,0 +1,127 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . +//! +//! A generic av store subsystem mockup suitable to be used in benchmarks. + +use parity_scale_codec::Encode; +use polkadot_primitives::CandidateHash; + +use std::collections::HashMap; + +use futures::{channel::oneshot, FutureExt}; + +use polkadot_node_primitives::ErasureChunk; + +use polkadot_node_subsystem::{ + messages::AvailabilityStoreMessage, overseer, SpawnedSubsystem, SubsystemError, +}; + +pub struct AvailabilityStoreState { + candidate_hashes: HashMap, + chunks: Vec>, +} + +const LOG_TARGET: &str = "subsystem-bench::av-store-mock"; + +/// A mock of the availability store subsystem. This one also generates all the +/// candidates that a +pub struct MockAvailabilityStore { + state: AvailabilityStoreState, +} + +impl MockAvailabilityStore { + pub fn new( + chunks: Vec>, + candidate_hashes: HashMap, + ) -> MockAvailabilityStore { + Self { state: AvailabilityStoreState { chunks, candidate_hashes } } + } + + async fn respond_to_query_all_request( + &self, + candidate_hash: CandidateHash, + send_chunk: impl Fn(usize) -> bool, + tx: oneshot::Sender>, + ) { + let candidate_index = self + .state + .candidate_hashes + .get(&candidate_hash) + .expect("candidate was generated previously; qed"); + gum::debug!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); + + let v = self + .state + .chunks + .get(*candidate_index as usize) + .unwrap() + .iter() + .filter(|c| send_chunk(c.index.0 as usize)) + .cloned() + .collect(); + + let _ = tx.send(v); + } +} + +#[overseer::subsystem(AvailabilityStore, error=SubsystemError, prefix=self::overseer)] +impl MockAvailabilityStore { + fn start(self, ctx: Context) -> SpawnedSubsystem { + let future = self.run(ctx).map(|_| Ok(())).boxed(); + + SpawnedSubsystem { name: "av-store-mock-subsystem", future } + } +} + +#[overseer::contextbounds(AvailabilityStore, prefix = self::overseer)] +impl MockAvailabilityStore { + async fn run(self, mut ctx: Context) { + loop { + let msg = ctx.recv().await.expect("Overseer never fails us"); + + match msg { + orchestra::FromOrchestra::Signal(_) => {}, + orchestra::FromOrchestra::Communication { msg } => match msg { + AvailabilityStoreMessage::QueryAvailableData(_candidate_hash, tx) => { + // We never have the full available data. + let _ = tx.send(None); + }, + AvailabilityStoreMessage::QueryAllChunks(candidate_hash, tx) => { + // We always have our own chunk. + self.respond_to_query_all_request(candidate_hash, |index| index == 0, tx) + .await; + }, + AvailabilityStoreMessage::QueryChunkSize(candidate_hash, tx) => { + let candidate_index = self + .state + .candidate_hashes + .get(&candidate_hash) + .expect("candidate was generated previously; qed"); + gum::debug!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); + + let chunk_size = self.state.chunks.get(*candidate_index as usize).unwrap() + [0] + .encoded_size(); + let _ = tx.send(Some(chunk_size)); + }, + _ => { + unimplemented!("Unexpected runtime-api message") + }, + }, + } + } + } +} diff --git a/polkadot/node/subsystem-bench/src/core/mock/dummy.rs b/polkadot/node/subsystem-bench/src/core/mock/dummy.rs index 122fc23ac52f..196cc81f1e82 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/dummy.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/dummy.rs @@ -14,14 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . //! Dummy subsystem mocks. -//! use paste::paste; -use futures::{channel::oneshot, select, Future, FutureExt}; -use polkadot_node_subsystem::{ - overseer, AllMessages, FromOrchestra, HeadSupportsParachains, Overseer, OverseerConnector, - OverseerHandle, SpawnGlue, SpawnedSubsystem, Subsystem, SubsystemError, -}; +use futures::FutureExt; +use polkadot_node_subsystem::{overseer, SpawnedSubsystem, SubsystemError}; use std::time::Duration; use tokio::time::sleep; @@ -85,5 +81,3 @@ mock!(ChainSelection); mock!(ApprovalVoting); mock!(ApprovalDistribution); mock!(RuntimeApi); - - diff --git a/polkadot/node/subsystem-bench/src/core/mock/mod.rs b/polkadot/node/subsystem-bench/src/core/mock/mod.rs index f13e87c8683b..df874de31a7c 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/mod.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/mod.rs @@ -15,18 +15,20 @@ // along with Polkadot. If not, see . use polkadot_node_subsystem::{ - overseer, AllMessages, FromOrchestra, HeadSupportsParachains, Overseer, OverseerConnector, - OverseerHandle, SpawnGlue, SpawnedSubsystem, Subsystem, + HeadSupportsParachains, Overseer, OverseerConnector, OverseerHandle, SpawnGlue, }; use polkadot_node_subsystem_types::Hash; +pub mod av_store; pub mod dummy; -mod temp; +pub mod network_bridge; +pub mod runtime_api; + +pub(crate) use dummy::*; -use dummy::*; use sc_service::SpawnTaskHandle; -struct AlwaysSupportsParachains {} +pub struct AlwaysSupportsParachains {} #[async_trait::async_trait] impl HeadSupportsParachains for AlwaysSupportsParachains { async fn head_supports_parachains(&self, _head: &Hash) -> bool { @@ -34,43 +36,50 @@ impl HeadSupportsParachains for AlwaysSupportsParachains { } } -pub fn new_overseer_with_dummy_subsystems(spawn_task_handle: SpawnTaskHandle) { - // Initialize a mock overseer. - // All subsystem except approval_voting and approval_distribution are mock subsystems. - let spawner_glue = SpawnGlue(spawn_task_handle); - let overseer_connector = OverseerConnector::with_event_capacity(64000); - let builder = Overseer::builder() - .approval_voting(MockApprovalVoting {}) - .approval_distribution(MockApprovalDistribution {}) - .availability_recovery(MockAvailabilityRecovery {}) - .candidate_validation(MockCandidateValidation {}) - .chain_api(MockChainApi { }) - .chain_selection(MockChainSelection {}) - .dispute_coordinator(MockDisputeCoordinator {}) - .runtime_api(MockRuntimeApi { }) - .network_bridge_tx(MockNetworkBridgeTx {}) - .availability_distribution(MockAvailabilityDistribution {}) - .availability_store(MockAvailabilityStore {}) - .pvf_checker(MockPvfChecker {}) - .candidate_backing(MockCandidateBacking {}) - .statement_distribution(MockStatementDistribution {}) - .bitfield_signing(MockBitfieldSigning {}) - .bitfield_distribution(MockBitfieldDistribution {}) - .provisioner(MockProvisioner {}) - .network_bridge_rx(MockNetworkBridgeRx {}) - .collation_generation(MockCollationGeneration {}) - .collator_protocol(MockCollatorProtocol {}) - .gossip_support(MockGossipSupport {}) - .dispute_distribution(MockDisputeDistribution {}) - .prospective_parachains(MockProspectiveParachains {}) - .activation_external_listeners(Default::default()) - .span_per_active_leaf(Default::default()) - .active_leaves(Default::default()) - .metrics(Default::default()) - .supports_parachains(AlwaysSupportsParachains {}) - .spawner(spawner_glue); - - let (mock_overseer, mock_overseer_handle) = - builder.build_with_connector(overseer_connector).expect("Should not fail"); +// An orchestra with dummy subsystems +macro_rules! dummy_builder { + ($spawn_task_handle: ident) => { + // Initialize a mock overseer. + // All subsystem except approval_voting and approval_distribution are mock subsystems. + Overseer::builder() + .approval_voting(MockApprovalVoting {}) + .approval_distribution(MockApprovalDistribution {}) + .availability_recovery(MockAvailabilityRecovery {}) + .candidate_validation(MockCandidateValidation {}) + .chain_api(MockChainApi {}) + .chain_selection(MockChainSelection {}) + .dispute_coordinator(MockDisputeCoordinator {}) + .runtime_api(MockRuntimeApi {}) + .network_bridge_tx(MockNetworkBridgeTx {}) + .availability_distribution(MockAvailabilityDistribution {}) + .availability_store(MockAvailabilityStore {}) + .pvf_checker(MockPvfChecker {}) + .candidate_backing(MockCandidateBacking {}) + .statement_distribution(MockStatementDistribution {}) + .bitfield_signing(MockBitfieldSigning {}) + .bitfield_distribution(MockBitfieldDistribution {}) + .provisioner(MockProvisioner {}) + .network_bridge_rx(MockNetworkBridgeRx {}) + .collation_generation(MockCollationGeneration {}) + .collator_protocol(MockCollatorProtocol {}) + .gossip_support(MockGossipSupport {}) + .dispute_distribution(MockDisputeDistribution {}) + .prospective_parachains(MockProspectiveParachains {}) + .activation_external_listeners(Default::default()) + .span_per_active_leaf(Default::default()) + .active_leaves(Default::default()) + .metrics(Default::default()) + .supports_parachains(AlwaysSupportsParachains {}) + .spawner(SpawnGlue($spawn_task_handle)) + }; +} -} \ No newline at end of file +pub fn new_overseer_with_dummy_subsystems( + spawn_task_handle: SpawnTaskHandle, +) -> (Overseer, AlwaysSupportsParachains>, OverseerHandle) { + let overseer_connector = OverseerConnector::with_event_capacity(64000); + let dummy = dummy_builder!(spawn_task_handle); + let builder = dummy.replace_chain_api(|_| MockChainApi {}); + // let (mock_overseer, mock_overseer_handle) = + builder.build_with_connector(overseer_connector).expect("Should not fail") +} diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs new file mode 100644 index 000000000000..cd374f8c18db --- /dev/null +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -0,0 +1,262 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . +//! +//! A generic av store subsystem mockup suitable to be used in benchmarks. + +use parity_scale_codec::Encode; + +use std::collections::HashMap; + +use futures::FutureExt; + +use polkadot_node_primitives::{AvailableData, ErasureChunk}; + +use polkadot_primitives::CandidateHash; +use sc_network::{OutboundFailure, RequestFailure}; + +use polkadot_node_subsystem::{ + messages::NetworkBridgeTxMessage, overseer, SpawnedSubsystem, SubsystemError, +}; + +use polkadot_node_network_protocol::request_response::{ + self as req_res, v1::ChunkResponse, Requests, +}; + +use crate::core::{ + configuration::{random_error, random_latency, TestConfiguration}, + network::{NetworkAction, NetworkEmulator, RateLimit}, +}; + +/// The availability store state of all emulated peers. +/// The network bridge tx mock will respond to requests as if the request is being serviced +/// by a remote peer on the network +pub struct NetworkAvailabilityState { + candidate_hashes: HashMap, + available_data: Vec, + chunks: Vec>, +} + +const LOG_TARGET: &str = "subsystem-bench::network-bridge-tx-mock"; + +/// A mock of the network bridge tx subsystem. +pub struct MockNetworkBridgeTx { + /// The test configurationg + config: TestConfiguration, + /// The network availability state + availabilty: NetworkAvailabilityState, + /// A network emulator instance + network: NetworkEmulator, +} + +impl MockNetworkBridgeTx { + pub fn new( + config: TestConfiguration, + availabilty: NetworkAvailabilityState, + network: NetworkEmulator, + ) -> MockNetworkBridgeTx { + Self { config, availabilty, network } + } + + pub fn respond_to_send_request( + &mut self, + request: Requests, + ingress_tx: &mut tokio::sync::mpsc::UnboundedSender, + ) -> NetworkAction { + let ingress_tx = ingress_tx.clone(); + + match request { + Requests::ChunkFetchingV1(outgoing_request) => { + let validator_index: usize = outgoing_request.payload.index.0 as usize; + let candidate_hash = outgoing_request.payload.candidate_hash; + + let candidate_index = self + .availabilty + .candidate_hashes + .get(&candidate_hash) + .expect("candidate was generated previously; qed"); + gum::warn!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); + + let chunk: ChunkResponse = + self.availabilty.chunks.get(*candidate_index as usize).unwrap() + [validator_index] + .clone() + .into(); + let mut size = chunk.encoded_size(); + + let response = if random_error(self.config.error) { + // Error will not account to any bandwidth used. + size = 0; + Err(RequestFailure::Network(OutboundFailure::ConnectionClosed)) + } else { + Ok(req_res::v1::ChunkFetchingResponse::from(Some(chunk)).encode()) + }; + + let authority_discovery_id = match outgoing_request.peer { + req_res::Recipient::Authority(authority_discovery_id) => authority_discovery_id, + _ => unimplemented!("Peer recipient not supported yet"), + }; + let authority_discovery_id_clone = authority_discovery_id.clone(); + + let future = async move { + let _ = outgoing_request.pending_response.send(response); + } + .boxed(); + + let future_wrapper = async move { + // Forward the response to the ingress channel of our node. + // On receive side we apply our node receiving rate limit. + let action = + NetworkAction::new(authority_discovery_id_clone, future, size, None); + ingress_tx.send(action).unwrap(); + } + .boxed(); + + NetworkAction::new( + authority_discovery_id, + future_wrapper, + size, + // Generate a random latency based on configuration. + random_latency(self.config.latency.as_ref()), + ) + }, + Requests::AvailableDataFetchingV1(outgoing_request) => { + let candidate_hash = outgoing_request.payload.candidate_hash; + let candidate_index = self + .availabilty + .candidate_hashes + .get(&candidate_hash) + .expect("candidate was generated previously; qed"); + gum::warn!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); + + let available_data = + self.availabilty.available_data.get(*candidate_index as usize).unwrap().clone(); + + let size = available_data.encoded_size(); + + let response = if random_error(self.config.error) { + Err(RequestFailure::Network(OutboundFailure::ConnectionClosed)) + } else { + Ok(req_res::v1::AvailableDataFetchingResponse::from(Some(available_data)) + .encode()) + }; + + let future = async move { + let _ = outgoing_request.pending_response.send(response); + } + .boxed(); + + let authority_discovery_id = match outgoing_request.peer { + req_res::Recipient::Authority(authority_discovery_id) => authority_discovery_id, + _ => unimplemented!("Peer recipient not supported yet"), + }; + let authority_discovery_id_clone = authority_discovery_id.clone(); + + let future_wrapper = async move { + // Forward the response to the ingress channel of our node. + // On receive side we apply our node receiving rate limit. + let action = + NetworkAction::new(authority_discovery_id_clone, future, size, None); + ingress_tx.send(action).unwrap(); + } + .boxed(); + + NetworkAction::new( + authority_discovery_id, + future_wrapper, + size, + // Generate a random latency based on configuration. + random_latency(self.config.latency.as_ref()), + ) + }, + _ => panic!("received an unexpected request"), + } + } +} + +#[overseer::subsystem(NetworkBridgeTx, error=SubsystemError, prefix=self::overseer)] +impl MockNetworkBridgeTx { + fn start(self, ctx: Context) -> SpawnedSubsystem { + let future = self.run(ctx).map(|_| Ok(())).boxed(); + + SpawnedSubsystem { name: "network-bridge-tx-mock-subsystem", future } + } +} + +#[overseer::contextbounds(NetworkBridgeTx, prefix = self::overseer)] +impl MockNetworkBridgeTx { + async fn run(mut self, mut ctx: Context) { + let (mut ingress_tx, mut ingress_rx) = + tokio::sync::mpsc::unbounded_channel::(); + + // Initialize our node bandwidth limits. + let mut rx_limiter = RateLimit::new(10, self.config.bandwidth); + + // Get a handle to our node network emulation stats. + let our_network_stats = self.network.peer_stats(0); + // This task will handle receipt of messages on our simulated network of the node. + let _ = ctx + .spawn_blocking( + "node0-rx", + async move { + while let Some(action) = ingress_rx.recv().await { + let size = action.size(); + + // account for our node receiving the data. + our_network_stats.inc_received(size); + + rx_limiter.reap(size).await; + action.run().await; + } + } + .boxed(), + ) + .expect("We never fail to spawn tasks"); + + // Main subsystem loop. + loop { + let msg = ctx.recv().await.expect("Overseer never fails us"); + + match msg { + orchestra::FromOrchestra::Signal(_) => {}, + orchestra::FromOrchestra::Communication { msg } => match msg { + NetworkBridgeTxMessage::SendRequests(requests, _if_disconnected) => { + for request in requests { + self.network.inc_sent(request_size(&request)); + let action = self.respond_to_send_request(request, &mut ingress_tx); + // Will account for our node sending the request over the emulated + // network. + self.network.submit_peer_action(action.peer(), action); + } + }, + _ => { + unimplemented!("Unexpected runtime-api message") + }, + }, + } + } + } +} + +// A helper to determine the request payload size. +fn request_size(request: &Requests) -> u64 { + match request { + Requests::ChunkFetchingV1(outgoing_request) => + outgoing_request.payload.encoded_size() as u64, + Requests::AvailableDataFetchingV1(outgoing_request) => + outgoing_request.payload.encoded_size() as u64, + _ => panic!("received an unexpected request"), + } +} diff --git a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs new file mode 100644 index 000000000000..e8c1098b97f0 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs @@ -0,0 +1,107 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . +//! +//! A generic runtime api subsystem mockup suitable to be used in benchmarks. + +use polkadot_primitives::{ + AuthorityDiscoveryId, GroupIndex, IndexedVec, SessionInfo, ValidatorId, ValidatorIndex, +}; + +use polkadot_node_subsystem::{ + messages::{RuntimeApiMessage, RuntimeApiRequest}, + overseer, SpawnedSubsystem, SubsystemError, +}; + +use crate::core::configuration::TestConfiguration; +use futures::FutureExt; + +pub struct RuntimeApiState { + validator_public: Vec, + validator_authority_id: Vec, +} + +pub struct MockRuntimeApi { + state: RuntimeApiState, + config: TestConfiguration, +} + +impl MockRuntimeApi { + pub fn new( + config: TestConfiguration, + validator_public: Vec, + validator_authority_id: Vec, + ) -> MockRuntimeApi { + Self { state: RuntimeApiState { validator_public, validator_authority_id }, config } + } + + fn session_info(&self) -> SessionInfo { + let all_validators = (0..self.config.n_validators) + .map(|i| ValidatorIndex(i as _)) + .collect::>(); + + let validator_groups = all_validators.chunks(5).map(|x| Vec::from(x)).collect::>(); + + SessionInfo { + validators: self.state.validator_public.clone().into(), + discovery_keys: self.state.validator_authority_id.clone(), + validator_groups: IndexedVec::>::from(validator_groups), + assignment_keys: vec![], + n_cores: self.config.n_cores as u32, + zeroth_delay_tranche_width: 0, + relay_vrf_modulo_samples: 0, + n_delay_tranches: 0, + no_show_slots: 0, + needed_approvals: 0, + active_validator_indices: vec![], + dispute_period: 6, + random_seed: [0u8; 32], + } + } +} + +#[overseer::subsystem(RuntimeApi, error=SubsystemError, prefix=self::overseer)] +impl MockRuntimeApi { + fn start(self, ctx: Context) -> SpawnedSubsystem { + let future = self.run(ctx).map(|_| Ok(())).boxed(); + + SpawnedSubsystem { name: "runtime-api-mock-subsystem", future } + } +} + +#[overseer::contextbounds(RuntimeApi, prefix = self::overseer)] +impl MockRuntimeApi { + async fn run(self, mut ctx: Context) { + loop { + let msg = ctx.recv().await.expect("Overseer never fails us"); + + match msg { + orchestra::FromOrchestra::Signal(_) => {}, + orchestra::FromOrchestra::Communication { msg } => match msg { + RuntimeApiMessage::Request( + _request, + RuntimeApiRequest::SessionInfo(_session_index, sender), + ) => { + let _ = sender.send(Ok(Some(self.session_info()))); + }, + // Long term TODO: implement more as needed. + _ => { + unimplemented!("Unexpected runtime-api message") + }, + }, + } + } + } +} diff --git a/polkadot/node/subsystem-bench/src/core/mod.rs b/polkadot/node/subsystem-bench/src/core/mod.rs index af2abf0860cd..11ca03dbda4c 100644 --- a/polkadot/node/subsystem-bench/src/core/mod.rs +++ b/polkadot/node/subsystem-bench/src/core/mod.rs @@ -28,5 +28,5 @@ pub mod configuration; pub mod display; pub mod environment; pub mod keyring; -pub mod network; pub mod mock; +pub mod network; diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index ca561e5c4955..ce9e8aa3a8be 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -26,16 +26,13 @@ pub(crate) mod availability; pub(crate) mod cli; pub(crate) mod core; -use availability::{ - AvailabilityRecoveryConfiguration, DataAvailabilityReadOptions, NetworkEmulation, - TestEnvironment, TestState, -}; +use availability::{NetworkEmulation, TestEnvironment, TestState}; use cli::TestObjective; -use core::configuration::{PeerLatency, TestConfiguration, TestSequence}; +use core::configuration::TestConfiguration; use clap_num::number_range; -const LOG_TARGET: &str = "subsystem-bench"; +// const LOG_TARGET: &str = "subsystem-bench"; fn le_100(s: &str) -> Result { number_range(s, 0, 100) @@ -125,14 +122,17 @@ impl BenchCli { let mut state = TestState::new(test_config); state.generate_candidates(candidate_count); - let mut env = - TestEnvironment::new(runtime.handle().clone(), state.clone(), Registry::new()); + let mut env = TestEnvironment::new( + runtime.handle().clone(), + state.clone(), + Registry::new(), + ); runtime.block_on(availability::bench_chunk_recovery(&mut env, state)); } return Ok(()) }, - TestObjective::DataAvailabilityRead(ref options) => match self.network { + TestObjective::DataAvailabilityRead(ref _options) => match self.network { NetworkEmulation::Healthy => TestConfiguration::healthy_network( self.objective, configuration.num_blocks, @@ -189,7 +189,8 @@ impl BenchCli { let mut state = TestState::new(test_config); state.generate_candidates(candidate_count); - let mut env = TestEnvironment::new(runtime.handle().clone(), state.clone(), Registry::new()); + let mut env = + TestEnvironment::new(runtime.handle().clone(), state.clone(), Registry::new()); runtime.block_on(availability::bench_chunk_recovery(&mut env, state)); From 4724d8c98d6b47643cbccb9e00dc7e550dad3a78 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 27 Nov 2023 19:43:15 +0200 Subject: [PATCH 097/192] full overseer based implementation complete Signed-off-by: Andrei Sandu --- polkadot/node/overseer/src/lib.rs | 2 + .../subsystem-bench/src/availability/mod.rs | 637 ++++++------------ .../subsystem-bench/src/core/configuration.rs | 33 + .../subsystem-bench/src/core/environment.rs | 28 + .../subsystem-bench/src/core/mock/av_store.rs | 8 +- .../subsystem-bench/src/core/mock/dummy.rs | 25 +- .../node/subsystem-bench/src/core/mock/mod.rs | 26 +- .../src/core/mock/network_bridge.rs | 7 +- .../src/core/mock/runtime_api.rs | 28 +- .../node/subsystem-bench/src/core/network.rs | 2 +- .../subsystem-bench/src/subsystem-bench.rs | 40 +- .../node/subsystem-test-helpers/src/mock.rs | 12 +- 12 files changed, 347 insertions(+), 501 deletions(-) diff --git a/polkadot/node/overseer/src/lib.rs b/polkadot/node/overseer/src/lib.rs index da99546a44f7..f4eddf1f41ce 100644 --- a/polkadot/node/overseer/src/lib.rs +++ b/polkadot/node/overseer/src/lib.rs @@ -276,6 +276,7 @@ impl From> for BlockInfo { /// An event from outside the overseer scope, such /// as the substrate framework or user interaction. +#[derive(Debug)] pub enum Event { /// A new block was imported. /// @@ -300,6 +301,7 @@ pub enum Event { } /// Some request from outer world. +#[derive(Debug)] pub enum ExternalRequest { /// Wait for the activation of a particular hash /// and be notified by means of the return channel. diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 3f9598505074..54a3cd961319 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -21,14 +21,16 @@ use std::{ sync::Arc, time::{Duration, Instant}, }; +use tokio::runtime::{Handle, Runtime}; + +use polkadot_node_subsystem::{ + BlockInfo, Event, Overseer, OverseerConnector, OverseerHandle, SpawnGlue, +}; +use sc_network::request_responses::ProtocolConfig; use colored::Colorize; -use futures::{ - channel::{mpsc, oneshot}, - stream::FuturesUnordered, - FutureExt, SinkExt, StreamExt, -}; +use futures::{channel::oneshot, stream::FuturesUnordered, FutureExt, SinkExt, StreamExt}; use polkadot_node_metrics::metrics::Metrics; use polkadot_availability_recovery::AvailabilityRecoverySubsystem; @@ -40,23 +42,30 @@ use polkadot_node_network_protocol::request_response::{ use rand::{distributions::Uniform, prelude::Distribution, thread_rng}; use prometheus::Registry; -use sc_network::{config::RequestResponseConfig, OutboundFailure, RequestFailure}; +use sc_network::{OutboundFailure, RequestFailure}; use polkadot_erasure_coding::{branches, obtain_chunks_v1 as obtain_chunks}; use polkadot_node_primitives::{BlockData, PoV, Proof}; use polkadot_node_subsystem::{ - messages::{ - AllMessages, AvailabilityRecoveryMessage, AvailabilityStoreMessage, NetworkBridgeTxMessage, - RuntimeApiMessage, RuntimeApiRequest, - }, - ActiveLeavesUpdate, FromOrchestra, OverseerSignal, Subsystem, + messages::{AllMessages, AvailabilityRecoveryMessage}, + ActiveLeavesUpdate, OverseerSignal, }; use std::net::{Ipv4Addr, SocketAddr}; +use crate::core::{ + configuration::TestAuthorities, + environment::TestEnvironmentDependencies, + mock::{ + av_store, + network_bridge::{self, MockNetworkBridgeTx, NetworkAvailabilityState}, + runtime_api, MockAvailabilityStore, MockRuntimeApi, + }, +}; + use super::core::{ configuration::{PeerLatency, TestConfiguration}, environment::TestEnvironmentMetrics, - keyring::Keyring, + mock::dummy_builder, network::*, }; @@ -64,14 +73,12 @@ const LOG_TARGET: &str = "subsystem-bench::availability"; use polkadot_node_primitives::{AvailableData, ErasureChunk}; -use super::cli::TestObjective; -use polkadot_node_subsystem_test_helpers::{ - make_buffered_subsystem_context, mock::new_leaf, TestSubsystemContextHandle, -}; +use super::{cli::TestObjective, core::mock::AlwaysSupportsParachains}; +use polkadot_node_subsystem_test_helpers::mock::new_block_import_event; use polkadot_node_subsystem_util::TimeoutExt; use polkadot_primitives::{ - AuthorityDiscoveryId, CandidateHash, CandidateReceipt, GroupIndex, Hash, HeadData, IndexedVec, - PersistedValidationData, SessionIndex, SessionInfo, ValidatorId, ValidatorIndex, + CandidateHash, CandidateReceipt, GroupIndex, Hash, HeadData, PersistedValidationData, + SessionIndex, ValidatorIndex, }; use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; use sc_service::{SpawnTaskHandle, TaskManager}; @@ -81,28 +88,22 @@ pub mod configuration; pub use cli::{DataAvailabilityReadOptions, NetworkEmulation, NetworkOptions}; pub use configuration::AvailabilityRecoveryConfiguration; -// Deterministic genesis hash for protocol names +// A dummy genesis hash const GENESIS_HASH: Hash = Hash::repeat_byte(0xff); -struct AvailabilityRecoverySubsystemInstance { - _protocol_config: RequestResponseConfig, -} - -/// The test environment is responsible for creating an instance of the availability recovery -/// subsystem and connecting it to an emulated overseer. +/// The test environment is the high level wrapper of all things required to test +/// a certain subsystem. /// /// ## Mockups -/// We emulate the following subsystems: -/// - runtime api -/// - network bridge -/// - availability store +/// The overseer is passed in during construction and it can host an arbitrary number of +/// real subsystems instances and the corresponding mocked instances such that the real +/// subsystems can get their messages answered. /// /// As the subsystem's performance depends on network connectivity, the test environment /// emulates validator nodes on the network, see `NetworkEmulator`. The network emulation /// is configurable in terms of peer bandwidth, latency and connection error rate using /// uniform distribution sampling. /// -/// The mockup logic is implemented in `env_task` which owns and advances the `TestState`. /// /// ## Usage /// `TestEnvironment` is used in tests to send `Overseer` messages or signals to the subsystem @@ -121,13 +122,14 @@ pub struct TestEnvironment { // A task manager that tracks task poll durations allows us to measure // per task CPU usage as we do in the Polkadot node. task_manager: TaskManager, + // Our runtime + runtime: tokio::runtime::Runtime, + // A runtime handle + runtime_handle: tokio::runtime::Handle, // The Prometheus metrics registry registry: Registry, - // A channel to the availability recovery subsystem - to_subsystem: mpsc::Sender>, - // Subsystem instance, currently keeps req/response protocol channel senders - // for the whole duration of the test. - instance: AvailabilityRecoverySubsystemInstance, + // A handle to the lovely overseer + overseer_handle: OverseerHandle, // The test intial state. The current state is owned by `env_task`. config: TestConfiguration, // A handle to the network emulator. @@ -136,62 +138,142 @@ pub struct TestEnvironment { metrics: TestEnvironmentMetrics, } -impl TestEnvironment { - // Create a new test environment with specified initial state and prometheus registry. - // We use prometheus metrics to collect per job task poll time and subsystem metrics. - pub fn new(runtime: tokio::runtime::Handle, state: TestState, registry: Registry) -> Self { - let config = state.config().clone(); - let task_manager: TaskManager = TaskManager::new(runtime.clone(), Some(®istry)).unwrap(); - let (instance, virtual_overseer) = AvailabilityRecoverySubsystemInstance::new( - ®istry, - task_manager.spawn_handle(), - match &state.config().objective { - TestObjective::DataAvailabilityRead(options) => options.fetch_from_backers, - _ => panic!("Unexpected objective"), - }, - ); +fn build_overseer( + spawn_task_handle: SpawnTaskHandle, + runtime_api: MockRuntimeApi, + av_store: MockAvailabilityStore, + network_bridge: MockNetworkBridgeTx, + availability_recovery: AvailabilityRecoverySubsystem, +) -> (Overseer, AlwaysSupportsParachains>, OverseerHandle) { + let overseer_connector = OverseerConnector::with_event_capacity(64000); + let dummy = dummy_builder!(spawn_task_handle); + let builder = dummy + .replace_runtime_api(|_| runtime_api) + .replace_availability_store(|_| av_store) + .replace_network_bridge_tx(|_| network_bridge) + .replace_availability_recovery(|_| availability_recovery); + + builder.build_with_connector(overseer_connector).expect("Should not fail") +} - let metrics = - TestEnvironmentMetrics::new(®istry).expect("Metrics need to be registered"); - let mut network = NetworkEmulator::new( - config.n_validators, - state.validator_authority_id.clone(), - config.peer_bandwidth, - task_manager.spawn_handle(), - ®istry, - ); - - // Copy sender for later when we need to inject messages in to the subsystem. - let to_subsystem = virtual_overseer.tx.clone(); - - let task_state = state.clone(); - let task_network = network.clone(); - let spawn_handle = task_manager.spawn_handle(); +/// Takes a test configuration and uses it to creates the `TestEnvironment`. +pub fn prepare_test( + config: TestConfiguration, + state: &mut TestState, +) -> (TestEnvironment, ProtocolConfig) { + prepare_test_inner(config, state, TestEnvironmentDependencies::default()) +} + +/// Takes a test configuration and uses it to creates the `TestEnvironment`. +pub fn prepare_test_with_dependencies( + config: TestConfiguration, + state: &mut TestState, + dependencies: TestEnvironmentDependencies, +) -> (TestEnvironment, ProtocolConfig) { + prepare_test_inner(config, state, dependencies) +} + +fn prepare_test_inner( + config: TestConfiguration, + state: &mut TestState, + dependencies: TestEnvironmentDependencies, +) -> (TestEnvironment, ProtocolConfig) { + // We need to first create the high level test state object. + // This will then be decomposed into per subsystem states. + let candidate_count = config.n_cores * config.num_blocks; + state.generate_candidates(candidate_count); + + // Generate test authorities. + let test_authorities = config.generate_authorities(); + + let runtime_api = runtime_api::MockRuntimeApi::new( + config.clone(), + test_authorities.validator_public.clone(), + test_authorities.validator_authority_id.clone(), + ); - // Our node rate limiting - let mut rx_limiter = RateLimit::new(10, config.bandwidth); - let (ingress_tx, mut ingress_rx) = tokio::sync::mpsc::unbounded_channel::(); - let our_network_stats = network.peer_stats(0); + let av_store = + av_store::MockAvailabilityStore::new(state.chunks.clone(), state.candidate_hashes.clone()); + + let availability_state = NetworkAvailabilityState { + candidate_hashes: state.candidate_hashes.clone(), + available_data: state.available_data.clone(), + chunks: state.chunks.clone(), + }; + + let network = NetworkEmulator::new( + config.n_validators.clone(), + test_authorities.validator_authority_id.clone(), + config.peer_bandwidth, + dependencies.task_manager.spawn_handle(), + &dependencies.registry, + ); - spawn_handle.spawn_blocking("node0-rx", "test-environment", async move { - while let Some(action) = ingress_rx.recv().await { - let size = action.size(); + let network_bridge_tx = network_bridge::MockNetworkBridgeTx::new( + config.clone(), + availability_state, + network.clone(), + ); + + let use_fast_path = match &state.config().objective { + TestObjective::DataAvailabilityRead(options) => options.fetch_from_backers, + _ => panic!("Unexpected objective"), + }; + + let (collation_req_receiver, req_cfg) = + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); + + let subsystem = if use_fast_path { + AvailabilityRecoverySubsystem::with_fast_path( + collation_req_receiver, + Metrics::try_register(&dependencies.registry).unwrap(), + ) + } else { + AvailabilityRecoverySubsystem::with_chunks_only( + collation_req_receiver, + Metrics::try_register(&dependencies.registry).unwrap(), + ) + }; + + let (overseer, overseer_handle) = build_overseer( + dependencies.task_manager.spawn_handle(), + runtime_api, + av_store, + network_bridge_tx, + subsystem, + ); - // account for our node receiving the data. - our_network_stats.inc_received(size); + ( + TestEnvironment::new( + dependencies.task_manager, + config, + dependencies.registry, + dependencies.runtime, + network, + overseer, + overseer_handle, + ), + req_cfg, + ) +} - rx_limiter.reap(size).await; - action.run().await; - } - }); +impl TestEnvironment { + // Create a new test environment with specified initial state and prometheus registry. + // We use prometheus metrics to collect per job task poll time and subsystem metrics. + pub fn new( + task_manager: TaskManager, + config: TestConfiguration, + registry: Registry, + runtime: Runtime, + network: NetworkEmulator, + overseer: Overseer, AlwaysSupportsParachains>, + overseer_handle: OverseerHandle, + ) -> Self { + let metrics = + TestEnvironmentMetrics::new(®istry).expect("Metrics need to be registered"); - // We need to start a receiver to process messages from the subsystem. - // This mocks an overseer and all dependent subsystems - task_manager.spawn_handle().spawn_blocking( - "test-environment", - "test-environment", - async move { Self::env_task(virtual_overseer, task_state, task_network, ingress_tx).await }, - ); + let spawn_handle = task_manager.spawn_handle(); + spawn_handle.spawn_blocking("overseer", "overseer", overseer.run()); let registry_clone = registry.clone(); task_manager @@ -205,7 +287,16 @@ impl TestEnvironment { .unwrap(); }); - TestEnvironment { task_manager, registry, to_subsystem, instance, config, network, metrics } + TestEnvironment { + task_manager, + runtime_handle: runtime.handle().clone(), + runtime, + registry, + overseer_handle, + config, + network, + metrics, + } } pub fn config(&self) -> &TestConfiguration { @@ -236,266 +327,20 @@ impl TestEnvironment { &self.metrics } - /// Generate a random error based on `probability`. - /// `probability` should be a number between 0 and 100. - fn random_error(probability: usize) -> bool { - Uniform::from(0..=99).sample(&mut thread_rng()) < probability - } - - pub fn request_size(request: &Requests) -> u64 { - match request { - Requests::ChunkFetchingV1(outgoing_request) => - outgoing_request.payload.encoded_size() as u64, - Requests::AvailableDataFetchingV1(outgoing_request) => - outgoing_request.payload.encoded_size() as u64, - _ => panic!("received an unexpected request"), - } - } - - pub fn respond_to_send_request( - state: &mut TestState, - request: Requests, - ingress_tx: tokio::sync::mpsc::UnboundedSender, - ) -> NetworkAction { - match request { - Requests::ChunkFetchingV1(outgoing_request) => { - let validator_index: usize = outgoing_request.payload.index.0 as usize; - let candidate_hash = outgoing_request.payload.candidate_hash; - - let candidate_index = state - .candidate_hashes - .get(&candidate_hash) - .expect("candidate was generated previously; qed"); - gum::warn!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); - - let chunk: ChunkResponse = state.chunks.get(*candidate_index as usize).unwrap() - [validator_index] - .clone() - .into(); - let mut size = chunk.encoded_size(); - - let response = if Self::random_error(state.config().error) { - // Error will not account to any bandwidth used. - size = 0; - Err(RequestFailure::Network(OutboundFailure::ConnectionClosed)) - } else { - Ok(req_res::v1::ChunkFetchingResponse::from(Some(chunk)).encode()) - }; - - let authority_discovery_id = match outgoing_request.peer { - req_res::Recipient::Authority(authority_discovery_id) => authority_discovery_id, - _ => panic!("Peer recipient not supported yet"), - }; - let authority_discovery_id_clone = authority_discovery_id.clone(); - - let future = async move { - let _ = outgoing_request.pending_response.send(response); - } - .boxed(); - - let future_wrapper = async move { - // Forward the response to the ingress channel of our node. - // On receive side we apply our node receiving rate limit. - let action = - NetworkAction::new(authority_discovery_id_clone, future, size, None); - ingress_tx.send(action).unwrap(); - } - .boxed(); - - NetworkAction::new( - authority_discovery_id, - future_wrapper, - size, - // Generate a random latency based on configuration. - Self::random_latency(state.config().latency.as_ref()), - ) - }, - Requests::AvailableDataFetchingV1(outgoing_request) => { - let candidate_hash = outgoing_request.payload.candidate_hash; - let candidate_index = state - .candidate_hashes - .get(&candidate_hash) - .expect("candidate was generated previously; qed"); - gum::warn!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); - - let available_data = - state.available_data.get(*candidate_index as usize).unwrap().clone(); - - let size = available_data.encoded_size(); - - let response = if Self::random_error(state.config().error) { - Err(RequestFailure::Network(OutboundFailure::ConnectionClosed)) - } else { - Ok(req_res::v1::AvailableDataFetchingResponse::from(Some(available_data)) - .encode()) - }; - - let future = async move { - let _ = outgoing_request.pending_response.send(response); - } - .boxed(); - - let authority_discovery_id = match outgoing_request.peer { - req_res::Recipient::Authority(authority_discovery_id) => authority_discovery_id, - _ => panic!("Peer recipient not supported yet"), - }; - let authority_discovery_id_clone = authority_discovery_id.clone(); - - let future_wrapper = async move { - // Forward the response to the ingress channel of our node. - // On receive side we apply our node receiving rate limit. - let action = - NetworkAction::new(authority_discovery_id_clone, future, size, None); - ingress_tx.send(action).unwrap(); - } - .boxed(); - - NetworkAction::new( - authority_discovery_id, - future_wrapper, - size, - // Generate a random latency based on configuration. - Self::random_latency(state.config().latency.as_ref()), - ) - }, - _ => panic!("received an unexpected request"), - } - } - - // A task that mocks dependent subsystems based on environment configuration. - // TODO: Spawn real subsystems, user overseer builder. - async fn env_task( - mut ctx: TestSubsystemContextHandle, - mut state: TestState, - mut network: NetworkEmulator, - ingress_tx: tokio::sync::mpsc::UnboundedSender, - ) { - loop { - futures::select! { - maybe_message = ctx.maybe_recv().fuse() => { - let message = if let Some(message) = maybe_message{ - message - } else { - gum::info!("{}", "Test completed".bright_blue()); - return - }; - - gum::trace!(target: LOG_TARGET, ?message, "Env task received message"); - - match message { - AllMessages::NetworkBridgeTx( - NetworkBridgeTxMessage::SendRequests( - requests, - _if_disconnected, - ) - ) => { - for request in requests { - network.inc_sent(Self::request_size(&request)); - let action = Self::respond_to_send_request(&mut state, request, ingress_tx.clone()); - // Account for our node sending the request over the emulated network. - network.submit_peer_action(action.peer(), action); - } - }, - AllMessages::AvailabilityStore(AvailabilityStoreMessage::QueryAvailableData(_candidate_hash, tx)) => { - // TODO: Simulate av store load by delaying the response. - state.respond_none_to_available_data_query(tx).await; - }, - AllMessages::AvailabilityStore(AvailabilityStoreMessage::QueryAllChunks(candidate_hash, tx)) => { - // Test env: We always have our own chunk. - state.respond_to_query_all_request(candidate_hash, |index| index == state.validator_index.0 as usize, tx).await; - }, - AllMessages::AvailabilityStore( - AvailabilityStoreMessage::QueryChunkSize(candidate_hash, tx) - ) => { - let candidate_index = state.candidate_hashes.get(&candidate_hash).expect("candidate was generated previously; qed"); - gum::debug!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); - - let chunk_size = state.chunks.get(*candidate_index as usize).unwrap()[0].encoded_size(); - let _ = tx.send(Some(chunk_size)); - } - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - _relay_parent, - RuntimeApiRequest::SessionInfo( - _session_index, - tx, - ) - )) => { - tx.send(Ok(Some(state.session_info()))).unwrap(); - } - _ => panic!("Unexpected input") - } - } - } - } + pub fn runtime(&self) -> Handle { + self.runtime_handle.clone() } // Send a message to the subsystem under test environment. - pub async fn send_message(&mut self, msg: AvailabilityRecoveryMessage) { - gum::trace!(msg = ?msg, "sending message"); - self.to_subsystem - .send(FromOrchestra::Communication { msg }) + pub async fn send_message(&mut self, msg: Event) { + self.overseer_handle + .send(msg) .timeout(MAX_TIME_OF_FLIGHT) .await .unwrap_or_else(|| { panic!("{}ms maximum time of flight breached", MAX_TIME_OF_FLIGHT.as_millis()) }) - .unwrap(); - } - - // Send a signal to the subsystem under test environment. - pub async fn send_signal(&mut self, signal: OverseerSignal) { - self.to_subsystem - .send(FromOrchestra::Signal(signal)) - .timeout(MAX_TIME_OF_FLIGHT) - .await - .unwrap_or_else(|| { - panic!( - "{}ms is more than enough for sending signals.", - MAX_TIME_OF_FLIGHT.as_millis() - ) - }) - .unwrap(); - } -} - -impl AvailabilityRecoverySubsystemInstance { - pub fn new( - registry: &Registry, - spawn_task_handle: SpawnTaskHandle, - use_fast_path: bool, - ) -> (Self, TestSubsystemContextHandle) { - let (context, virtual_overseer) = make_buffered_subsystem_context( - spawn_task_handle.clone(), - 128, - "availability-recovery-subsystem", - ); - let (collation_req_receiver, req_cfg) = - IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); - - let subsystem = if use_fast_path { - AvailabilityRecoverySubsystem::with_fast_path( - collation_req_receiver, - Metrics::try_register(®istry).unwrap(), - ) - } else { - AvailabilityRecoverySubsystem::with_chunks_only( - collation_req_receiver, - Metrics::try_register(®istry).unwrap(), - ) - }; - - let spawned_subsystem = subsystem.start(context); - let subsystem_future = async move { - spawned_subsystem.future.await.unwrap(); - }; - - spawn_task_handle.spawn_blocking( - spawned_subsystem.name, - spawned_subsystem.name, - subsystem_future, - ); - - (Self { _protocol_config: req_cfg }, virtual_overseer) + .expect("send never fails"); } } @@ -509,8 +354,7 @@ pub struct TestState { // Full test configuration config: TestConfiguration, // State starts here. - validator_public: Vec, - validator_authority_id: Vec, + test_authorities: TestAuthorities, // The test node validator index. validator_index: ValidatorIndex, session_index: SessionIndex, @@ -535,60 +379,6 @@ impl TestState { &self.config } - async fn respond_none_to_available_data_query( - &self, - tx: oneshot::Sender>, - ) { - let _ = tx.send(None); - } - - fn session_info(&self) -> SessionInfo { - let my_vec = (0..self.config().n_validators) - .map(|i| ValidatorIndex(i as _)) - .collect::>(); - - let validator_groups = my_vec.chunks(5).map(|x| Vec::from(x)).collect::>(); - - SessionInfo { - validators: self.validator_public.clone().into(), - discovery_keys: self.validator_authority_id.clone(), - validator_groups: IndexedVec::>::from(validator_groups), - assignment_keys: vec![], - n_cores: self.config().n_cores as u32, - zeroth_delay_tranche_width: 0, - relay_vrf_modulo_samples: 0, - n_delay_tranches: 0, - no_show_slots: 0, - needed_approvals: 0, - active_validator_indices: vec![], - dispute_period: 6, - random_seed: [0u8; 32], - } - } - async fn respond_to_query_all_request( - &self, - candidate_hash: CandidateHash, - send_chunk: impl Fn(usize) -> bool, - tx: oneshot::Sender>, - ) { - let candidate_index = self - .candidate_hashes - .get(&candidate_hash) - .expect("candidate was generated previously; qed"); - gum::debug!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); - - let v = self - .chunks - .get(*candidate_index as usize) - .unwrap() - .iter() - .filter(|c| send_chunk(c.index.0 as usize)) - .cloned() - .collect(); - - let _ = tx.send(v); - } - pub fn next_candidate(&mut self) -> Option { let candidate = self.candidates.next(); let candidate_hash = candidate.as_ref().unwrap().hash(); @@ -596,6 +386,10 @@ impl TestState { candidate } + pub fn authorities(&self) -> &TestAuthorities { + &self.test_authorities + } + /// Generate candidates to be used in the test. pub fn generate_candidates(&mut self, count: usize) { gum::info!(target: LOG_TARGET,"{}", format!("Pre-generating {} candidates.", count).bright_blue()); @@ -624,22 +418,9 @@ impl TestState { .cycle(); } - pub fn new(config: TestConfiguration) -> Self { - let keyrings = (0..config.n_validators) - .map(|peer_index| Keyring::new(format!("Node{}", peer_index).into())) - .collect::>(); - - // Generate `AuthorityDiscoveryId`` for each peer - let validator_public: Vec = keyrings - .iter() - .map(|keyring: &Keyring| keyring.clone().public().into()) - .collect::>(); - - let validator_authority_id: Vec = keyrings - .iter() - .map(|keyring| keyring.clone().public().into()) - .collect::>() - .into(); + pub fn new(config: &TestConfiguration) -> Self { + let config = config.clone(); + let test_authorities = config.generate_authorities(); let validator_index = ValidatorIndex(0); let mut chunks = Vec::new(); @@ -687,8 +468,7 @@ impl TestState { Self { config, - validator_public, - validator_authority_id, + test_authorities, validator_index, session_index, persisted_validation_data, @@ -734,11 +514,7 @@ fn derive_erasure_chunks_with_proofs_and_root( pub async fn bench_chunk_recovery(env: &mut TestEnvironment, mut state: TestState) { let config = env.config().clone(); - env.send_signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( - Hash::repeat_byte(1), - 1, - )))) - .await; + env.send_message(new_block_import_event(Hash::repeat_byte(1), 1)).await; let start_marker = Instant::now(); let mut batch = FuturesUnordered::new(); @@ -758,15 +534,20 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment, mut state: TestStat let (tx, rx) = oneshot::channel(); batch.push(rx); - env.send_message(AvailabilityRecoveryMessage::RecoverAvailableData( - candidate.clone(), - 1, - Some(GroupIndex( - candidate_num as u32 % (std::cmp::max(5, config.n_cores) / 5) as u32, - )), - tx, - )) - .await; + let message = Event::MsgToSubsystem { + msg: AllMessages::AvailabilityRecovery( + AvailabilityRecoveryMessage::RecoverAvailableData( + candidate.clone(), + 1, + Some(GroupIndex( + candidate_num as u32 % (std::cmp::max(5, config.n_cores) / 5) as u32, + )), + tx, + ), + ), + origin: LOG_TARGET, + }; + env.send_message(message).await; } gum::info!("{}", format!("{} recoveries pending", batch.len()).bright_black()); @@ -786,8 +567,8 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment, mut state: TestStat tokio::time::sleep(block_time_delta).await; } - env.send_signal(OverseerSignal::Conclude).await; - let duration = start_marker.elapsed().as_millis(); + env.send_message(Event::Stop).await; + let duration: u128 = start_marker.elapsed().as_millis(); let availability_bytes = availability_bytes / 1024; gum::info!("All blocks processed in {}", format!("{:?}ms", duration).cyan()); gum::info!( diff --git a/polkadot/node/subsystem-bench/src/core/configuration.rs b/polkadot/node/subsystem-bench/src/core/configuration.rs index 4526505c3a64..f8fdcf2973eb 100644 --- a/polkadot/node/subsystem-bench/src/core/configuration.rs +++ b/polkadot/node/subsystem-bench/src/core/configuration.rs @@ -16,7 +16,10 @@ use std::path::Path; use super::*; +use keyring::Keyring; + pub use crate::cli::TestObjective; +use polkadot_primitives::ValidatorId; use rand::{distributions::Uniform, prelude::Distribution, thread_rng}; use serde::{Deserialize, Serialize}; @@ -98,6 +101,14 @@ impl TestSequence { } } +/// Helper struct for authority related state. +#[derive(Clone)] +pub struct TestAuthorities { + pub keyrings: Vec, + pub validator_public: Vec, + pub validator_authority_id: Vec, +} + impl TestConfiguration { pub fn write_to_disk(&self) { // Serialize a slice of configurations @@ -109,6 +120,28 @@ impl TestConfiguration { pub fn pov_sizes(&self) -> &[usize] { &self.pov_sizes } + + /// Generates the authority keys we need for the network emulation. + pub fn generate_authorities(&self) -> TestAuthorities { + let keyrings = (0..self.n_validators) + .map(|peer_index| Keyring::new(format!("Node{}", peer_index).into())) + .collect::>(); + + // Generate `AuthorityDiscoveryId`` for each peer + let validator_public: Vec = keyrings + .iter() + .map(|keyring: &Keyring| keyring.clone().public().into()) + .collect::>(); + + let validator_authority_id: Vec = keyrings + .iter() + .map(|keyring| keyring.clone().public().into()) + .collect::>() + .into(); + + TestAuthorities { keyrings, validator_public, validator_authority_id } + } + /// An unconstrained standard configuration matching Polkadot/Kusama pub fn ideal_network( objective: TestObjective, diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index e6b09a1c13e6..c9cc6ae40410 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -17,6 +17,7 @@ use polkadot_node_subsystem_util::metrics::prometheus::{ self, Gauge, Histogram, PrometheusError, Registry, U64, }; +use sc_service::TaskManager; const MIB: f64 = 1024.0 * 1024.0; @@ -97,3 +98,30 @@ impl TestEnvironmentMetrics { self.pov_size.observe(pov_size as f64); } } + +fn new_runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_multi_thread() + .thread_name("subsystem-bench") + .enable_all() + .thread_stack_size(3 * 1024 * 1024) + .build() + .unwrap() +} + +/// Wrapper for dependencies +pub struct TestEnvironmentDependencies { + pub registry: Registry, + pub task_manager: TaskManager, + pub runtime: tokio::runtime::Runtime, +} + +impl Default for TestEnvironmentDependencies { + fn default() -> Self { + let runtime = new_runtime(); + let registry = Registry::new(); + let task_manager: TaskManager = + TaskManager::new(runtime.handle().clone(), Some(®istry)).unwrap(); + + Self { runtime, registry, task_manager } + } +} diff --git a/polkadot/node/subsystem-bench/src/core/mock/av_store.rs b/polkadot/node/subsystem-bench/src/core/mock/av_store.rs index e84aeba5b6b7..7f6ff2abfe9e 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/av_store.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/av_store.rs @@ -89,22 +89,28 @@ impl MockAvailabilityStore { #[overseer::contextbounds(AvailabilityStore, prefix = self::overseer)] impl MockAvailabilityStore { async fn run(self, mut ctx: Context) { + gum::debug!(target: LOG_TARGET, "Subsystem running"); loop { let msg = ctx.recv().await.expect("Overseer never fails us"); match msg { orchestra::FromOrchestra::Signal(_) => {}, orchestra::FromOrchestra::Communication { msg } => match msg { - AvailabilityStoreMessage::QueryAvailableData(_candidate_hash, tx) => { + AvailabilityStoreMessage::QueryAvailableData(candidate_hash, tx) => { + gum::debug!(target: LOG_TARGET, candidate_hash = ?candidate_hash, "Responding to QueryAvailableData"); + // We never have the full available data. let _ = tx.send(None); }, AvailabilityStoreMessage::QueryAllChunks(candidate_hash, tx) => { // We always have our own chunk. + gum::debug!(target: LOG_TARGET, candidate_hash = ?candidate_hash, "Responding to QueryAllChunks"); self.respond_to_query_all_request(candidate_hash, |index| index == 0, tx) .await; }, AvailabilityStoreMessage::QueryChunkSize(candidate_hash, tx) => { + gum::debug!(target: LOG_TARGET, candidate_hash = ?candidate_hash, "Responding to QueryChunkSize"); + let candidate_index = self .state .candidate_hashes diff --git a/polkadot/node/subsystem-bench/src/core/mock/dummy.rs b/polkadot/node/subsystem-bench/src/core/mock/dummy.rs index 196cc81f1e82..998153875ede 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/dummy.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/dummy.rs @@ -21,6 +21,8 @@ use polkadot_node_subsystem::{overseer, SpawnedSubsystem, SubsystemError}; use std::time::Duration; use tokio::time::sleep; +const LOG_TARGET: &str = "subsystem-bench::mockery"; + macro_rules! mock { // Just query by relay parent ($subsystem_name:ident) => { @@ -41,15 +43,22 @@ macro_rules! mock { let mut count_total_msg = 0; loop { futures::select!{ - _msg = ctx.recv().fuse() => { - count_total_msg +=1; - } - _ = sleep(Duration::from_secs(6)).fuse() => { - if count_total_msg > 0 { - gum::info!(target: "mock-subsystems", "Subsystem {} processed {} messages since last time", stringify!($subsystem_name), count_total_msg); + msg = ctx.recv().fuse() => { + match msg.unwrap() { + orchestra::FromOrchestra::Signal(_) => {}, + orchestra::FromOrchestra::Communication { msg } => { + gum::debug!(target: LOG_TARGET, msg = ?msg, "mocked subsystem received message"); + } + } + + count_total_msg +=1; + } + _ = sleep(Duration::from_secs(6)).fuse() => { + if count_total_msg > 0 { + gum::trace!(target: LOG_TARGET, "Subsystem {} processed {} messages since last time", stringify!($subsystem_name), count_total_msg); + } + count_total_msg = 0; } - count_total_msg = 0; - } } } } diff --git a/polkadot/node/subsystem-bench/src/core/mock/mod.rs b/polkadot/node/subsystem-bench/src/core/mock/mod.rs index df874de31a7c..d59642e96058 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/mod.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/mod.rs @@ -14,9 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use polkadot_node_subsystem::{ - HeadSupportsParachains, Overseer, OverseerConnector, OverseerHandle, SpawnGlue, -}; +use polkadot_node_subsystem::HeadSupportsParachains; use polkadot_node_subsystem_types::Hash; pub mod av_store; @@ -24,9 +22,9 @@ pub mod dummy; pub mod network_bridge; pub mod runtime_api; -pub(crate) use dummy::*; - -use sc_service::SpawnTaskHandle; +pub use av_store::*; +pub use network_bridge::*; +pub use runtime_api::*; pub struct AlwaysSupportsParachains {} #[async_trait::async_trait] @@ -38,7 +36,9 @@ impl HeadSupportsParachains for AlwaysSupportsParachains { // An orchestra with dummy subsystems macro_rules! dummy_builder { - ($spawn_task_handle: ident) => { + ($spawn_task_handle: ident) => {{ + use super::core::mock::dummy::*; + // Initialize a mock overseer. // All subsystem except approval_voting and approval_distribution are mock subsystems. Overseer::builder() @@ -71,15 +71,7 @@ macro_rules! dummy_builder { .metrics(Default::default()) .supports_parachains(AlwaysSupportsParachains {}) .spawner(SpawnGlue($spawn_task_handle)) - }; + }}; } -pub fn new_overseer_with_dummy_subsystems( - spawn_task_handle: SpawnTaskHandle, -) -> (Overseer, AlwaysSupportsParachains>, OverseerHandle) { - let overseer_connector = OverseerConnector::with_event_capacity(64000); - let dummy = dummy_builder!(spawn_task_handle); - let builder = dummy.replace_chain_api(|_| MockChainApi {}); - // let (mock_overseer, mock_overseer_handle) = - builder.build_with_connector(overseer_connector).expect("Should not fail") -} +pub(crate) use dummy_builder; diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index cd374f8c18db..a6d07c3d4a20 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -44,9 +44,9 @@ use crate::core::{ /// The network bridge tx mock will respond to requests as if the request is being serviced /// by a remote peer on the network pub struct NetworkAvailabilityState { - candidate_hashes: HashMap, - available_data: Vec, - chunks: Vec>, + pub candidate_hashes: HashMap, + pub available_data: Vec, + pub chunks: Vec>, } const LOG_TARGET: &str = "subsystem-bench::network-bridge-tx-mock"; @@ -234,6 +234,7 @@ impl MockNetworkBridgeTx { orchestra::FromOrchestra::Communication { msg } => match msg { NetworkBridgeTxMessage::SendRequests(requests, _if_disconnected) => { for request in requests { + gum::debug!(target: LOG_TARGET, request = ?request, "Processing request"); self.network.inc_sent(request_size(&request)); let action = self.respond_to_send_request(request, &mut ingress_tx); // Will account for our node sending the request over the emulated diff --git a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs index e8c1098b97f0..a106eb130991 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs @@ -28,6 +28,8 @@ use polkadot_node_subsystem::{ use crate::core::configuration::TestConfiguration; use futures::FutureExt; +const LOG_TARGET: &str = "subsystem-bench::runtime-api-mock"; + pub struct RuntimeApiState { validator_public: Vec, validator_authority_id: Vec, @@ -89,17 +91,21 @@ impl MockRuntimeApi { match msg { orchestra::FromOrchestra::Signal(_) => {}, - orchestra::FromOrchestra::Communication { msg } => match msg { - RuntimeApiMessage::Request( - _request, - RuntimeApiRequest::SessionInfo(_session_index, sender), - ) => { - let _ = sender.send(Ok(Some(self.session_info()))); - }, - // Long term TODO: implement more as needed. - _ => { - unimplemented!("Unexpected runtime-api message") - }, + orchestra::FromOrchestra::Communication { msg } => { + gum::debug!(target: LOG_TARGET, msg=?msg, "recv message"); + + match msg { + RuntimeApiMessage::Request( + _request, + RuntimeApiRequest::SessionInfo(_session_index, sender), + ) => { + let _ = sender.send(Ok(Some(self.session_info()))); + }, + // Long term TODO: implement more as needed. + _ => { + unimplemented!("Unexpected runtime-api message") + }, + } }, } } diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index f20bb919dedb..80d961babe03 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -186,7 +186,7 @@ pub type ActionFuture = std::pin::Pin + std pub struct NetworkAction { // The function that performs the action run: ActionFuture, - // The payload size that we simulate sending from a peer + // The payload size that we simulate sending/receiving from a peer size: usize, // Peer which should run the action. peer: AuthorityDiscoveryId, diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index ce9e8aa3a8be..4460315c35c5 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -26,7 +26,7 @@ pub(crate) mod availability; pub(crate) mod cli; pub(crate) mod core; -use availability::{NetworkEmulation, TestEnvironment, TestState}; +use availability::{prepare_test, NetworkEmulation, TestEnvironment, TestState}; use cli::TestObjective; use core::configuration::TestConfiguration; @@ -76,21 +76,8 @@ struct BenchCli { pub objective: cli::TestObjective, } -fn new_runtime() -> tokio::runtime::Runtime { - tokio::runtime::Builder::new_multi_thread() - .thread_name("subsystem-bench") - .enable_all() - .thread_stack_size(3 * 1024 * 1024) - .build() - .unwrap() -} - impl BenchCli { fn launch(self) -> eyre::Result<()> { - use prometheus::Registry; - - let runtime = new_runtime(); - let configuration = self.standard_configuration; let mut test_config = match self.objective { TestObjective::TestSequence(options) => { @@ -120,15 +107,9 @@ impl BenchCli { let candidate_count = test_config.n_cores * test_config.num_blocks; - let mut state = TestState::new(test_config); - state.generate_candidates(candidate_count); - let mut env = TestEnvironment::new( - runtime.handle().clone(), - state.clone(), - Registry::new(), - ); - - runtime.block_on(availability::bench_chunk_recovery(&mut env, state)); + let mut state = TestState::new(&test_config); + let (mut env, _protocol_config) = prepare_test(test_config, &mut state); + env.runtime().block_on(availability::bench_chunk_recovery(&mut env, state)); } return Ok(()) }, @@ -185,14 +166,11 @@ impl BenchCli { } let candidate_count = test_config.n_cores * test_config.num_blocks; - test_config.write_to_disk(); - - let mut state = TestState::new(test_config); - state.generate_candidates(candidate_count); - let mut env = - TestEnvironment::new(runtime.handle().clone(), state.clone(), Registry::new()); + // test_config.write_to_disk(); - runtime.block_on(availability::bench_chunk_recovery(&mut env, state)); + let mut state = TestState::new(&test_config); + let (mut env, _protocol_config) = prepare_test(test_config, &mut state); + env.runtime().block_on(availability::bench_chunk_recovery(&mut env, state)); Ok(()) } @@ -202,7 +180,7 @@ fn main() -> eyre::Result<()> { color_eyre::install()?; let _ = env_logger::builder() .filter(Some("hyper"), log::LevelFilter::Info) - .filter(None, log::LevelFilter::Info) + // .filter(None, log::LevelFilter::Trace) .try_init() .unwrap(); diff --git a/polkadot/node/subsystem-test-helpers/src/mock.rs b/polkadot/node/subsystem-test-helpers/src/mock.rs index 522bc3c2cc4f..11e77b6e8968 100644 --- a/polkadot/node/subsystem-test-helpers/src/mock.rs +++ b/polkadot/node/subsystem-test-helpers/src/mock.rs @@ -16,7 +16,7 @@ use std::sync::Arc; -use polkadot_node_subsystem::{jaeger, ActivatedLeaf}; +use polkadot_node_subsystem::{jaeger, ActivatedLeaf, Event, BlockInfo}; use sc_client_api::UnpinHandle; use sc_keystore::LocalKeystore; use sc_utils::mpsc::tracing_unbounded; @@ -59,3 +59,13 @@ pub fn new_leaf(hash: Hash, number: BlockNumber) -> ActivatedLeaf { span: Arc::new(jaeger::Span::Disabled), } } + +/// Create a new leaf with the given hash and number. +pub fn new_block_import_event(hash: Hash, number: BlockNumber) -> Event { + Event::BlockImported(BlockInfo { + hash, + parent_hash: Hash::default(), + number, + unpin_handle: dummy_unpin_handle(hash), + }) +} From 7aed30f13be1c8cf6de43e49945dd419613037f2 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 27 Nov 2023 19:55:18 +0200 Subject: [PATCH 098/192] make clean Signed-off-by: Andrei Sandu --- .../subsystem-bench/src/availability/mod.rs | 185 +----------------- .../subsystem-bench/src/core/environment.rs | 157 ++++++++++++++- .../subsystem-bench/src/core/subsystem.rs | 16 -- .../subsystem-bench/src/subsystem-bench.rs | 13 +- 4 files changed, 171 insertions(+), 200 deletions(-) delete mode 100644 polkadot/node/subsystem-bench/src/core/subsystem.rs diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 54a3cd961319..e1974794cb8d 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -21,36 +21,24 @@ use std::{ sync::Arc, time::{Duration, Instant}, }; -use tokio::runtime::{Handle, Runtime}; -use polkadot_node_subsystem::{ - BlockInfo, Event, Overseer, OverseerConnector, OverseerHandle, SpawnGlue, -}; +use crate::TestEnvironment; +use polkadot_node_subsystem::{Event, Overseer, OverseerConnector, OverseerHandle, SpawnGlue}; use sc_network::request_responses::ProtocolConfig; use colored::Colorize; -use futures::{channel::oneshot, stream::FuturesUnordered, FutureExt, SinkExt, StreamExt}; +use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; use polkadot_node_metrics::metrics::Metrics; use polkadot_availability_recovery::AvailabilityRecoverySubsystem; +use crate::GENESIS_HASH; use parity_scale_codec::Encode; -use polkadot_node_network_protocol::request_response::{ - self as req_res, v1::ChunkResponse, IncomingRequest, ReqProtocolNames, Requests, -}; -use rand::{distributions::Uniform, prelude::Distribution, thread_rng}; - -use prometheus::Registry; -use sc_network::{OutboundFailure, RequestFailure}; - use polkadot_erasure_coding::{branches, obtain_chunks_v1 as obtain_chunks}; +use polkadot_node_network_protocol::request_response::{IncomingRequest, ReqProtocolNames}; use polkadot_node_primitives::{BlockData, PoV, Proof}; -use polkadot_node_subsystem::{ - messages::{AllMessages, AvailabilityRecoveryMessage}, - ActiveLeavesUpdate, OverseerSignal, -}; -use std::net::{Ipv4Addr, SocketAddr}; +use polkadot_node_subsystem::messages::{AllMessages, AvailabilityRecoveryMessage}; use crate::core::{ configuration::TestAuthorities, @@ -62,12 +50,7 @@ use crate::core::{ }, }; -use super::core::{ - configuration::{PeerLatency, TestConfiguration}, - environment::TestEnvironmentMetrics, - mock::dummy_builder, - network::*, -}; +use super::core::{configuration::TestConfiguration, mock::dummy_builder, network::*}; const LOG_TARGET: &str = "subsystem-bench::availability"; @@ -75,69 +58,18 @@ use polkadot_node_primitives::{AvailableData, ErasureChunk}; use super::{cli::TestObjective, core::mock::AlwaysSupportsParachains}; use polkadot_node_subsystem_test_helpers::mock::new_block_import_event; -use polkadot_node_subsystem_util::TimeoutExt; use polkadot_primitives::{ CandidateHash, CandidateReceipt, GroupIndex, Hash, HeadData, PersistedValidationData, - SessionIndex, ValidatorIndex, + ValidatorIndex, }; use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; -use sc_service::{SpawnTaskHandle, TaskManager}; +use sc_service::SpawnTaskHandle; mod cli; pub mod configuration; pub use cli::{DataAvailabilityReadOptions, NetworkEmulation, NetworkOptions}; pub use configuration::AvailabilityRecoveryConfiguration; -// A dummy genesis hash -const GENESIS_HASH: Hash = Hash::repeat_byte(0xff); - -/// The test environment is the high level wrapper of all things required to test -/// a certain subsystem. -/// -/// ## Mockups -/// The overseer is passed in during construction and it can host an arbitrary number of -/// real subsystems instances and the corresponding mocked instances such that the real -/// subsystems can get their messages answered. -/// -/// As the subsystem's performance depends on network connectivity, the test environment -/// emulates validator nodes on the network, see `NetworkEmulator`. The network emulation -/// is configurable in terms of peer bandwidth, latency and connection error rate using -/// uniform distribution sampling. -/// -/// -/// ## Usage -/// `TestEnvironment` is used in tests to send `Overseer` messages or signals to the subsystem -/// under test. -/// -/// ## Collecting test metrics -/// -/// ### Prometheus -/// A prometheus endpoint is exposed while the test is running. A local Prometheus instance -/// can scrape it every 1s and a Grafana dashboard is the preferred way of visualizing -/// the performance characteristics of the subsystem. -/// -/// ### CLI -/// A subset of the Prometheus metrics are printed at the end of the test. -pub struct TestEnvironment { - // A task manager that tracks task poll durations allows us to measure - // per task CPU usage as we do in the Polkadot node. - task_manager: TaskManager, - // Our runtime - runtime: tokio::runtime::Runtime, - // A runtime handle - runtime_handle: tokio::runtime::Handle, - // The Prometheus metrics registry - registry: Registry, - // A handle to the lovely overseer - overseer_handle: OverseerHandle, - // The test intial state. The current state is owned by `env_task`. - config: TestConfiguration, - // A handle to the network emulator. - network: NetworkEmulator, - // Configuration/env metrics - metrics: TestEnvironmentMetrics, -} - fn build_overseer( spawn_task_handle: SpawnTaskHandle, runtime_api: MockRuntimeApi, @@ -257,107 +189,12 @@ fn prepare_test_inner( ) } -impl TestEnvironment { - // Create a new test environment with specified initial state and prometheus registry. - // We use prometheus metrics to collect per job task poll time and subsystem metrics. - pub fn new( - task_manager: TaskManager, - config: TestConfiguration, - registry: Registry, - runtime: Runtime, - network: NetworkEmulator, - overseer: Overseer, AlwaysSupportsParachains>, - overseer_handle: OverseerHandle, - ) -> Self { - let metrics = - TestEnvironmentMetrics::new(®istry).expect("Metrics need to be registered"); - - let spawn_handle = task_manager.spawn_handle(); - spawn_handle.spawn_blocking("overseer", "overseer", overseer.run()); - - let registry_clone = registry.clone(); - task_manager - .spawn_handle() - .spawn_blocking("prometheus", "test-environment", async move { - prometheus_endpoint::init_prometheus( - SocketAddr::new(std::net::IpAddr::V4(Ipv4Addr::LOCALHOST), 9999), - registry_clone, - ) - .await - .unwrap(); - }); - - TestEnvironment { - task_manager, - runtime_handle: runtime.handle().clone(), - runtime, - registry, - overseer_handle, - config, - network, - metrics, - } - } - - pub fn config(&self) -> &TestConfiguration { - &self.config - } - - pub fn network(&mut self) -> &mut NetworkEmulator { - &mut self.network - } - - pub fn registry(&self) -> &Registry { - &self.registry - } - - /// Produce a randomized duration between `min` and `max`. - fn random_latency(maybe_peer_latency: Option<&PeerLatency>) -> Option { - if let Some(peer_latency) = maybe_peer_latency { - Some( - Uniform::from(peer_latency.min_latency..=peer_latency.max_latency) - .sample(&mut thread_rng()), - ) - } else { - None - } - } - - pub fn metrics(&self) -> &TestEnvironmentMetrics { - &self.metrics - } - - pub fn runtime(&self) -> Handle { - self.runtime_handle.clone() - } - - // Send a message to the subsystem under test environment. - pub async fn send_message(&mut self, msg: Event) { - self.overseer_handle - .send(msg) - .timeout(MAX_TIME_OF_FLIGHT) - .await - .unwrap_or_else(|| { - panic!("{}ms maximum time of flight breached", MAX_TIME_OF_FLIGHT.as_millis()) - }) - .expect("send never fails"); - } -} - -// We use this to bail out sending messages to the subsystem if it is overloaded such that -// the time of flight is breaches 5s. -// This should eventually be a test parameter. -const MAX_TIME_OF_FLIGHT: Duration = Duration::from_millis(5000); - #[derive(Clone)] pub struct TestState { // Full test configuration config: TestConfiguration, // State starts here. test_authorities: TestAuthorities, - // The test node validator index. - validator_index: ValidatorIndex, - session_index: SessionIndex, pov_sizes: Cycle>, // Generated candidate receipts to be used in the test candidates: Cycle>, @@ -422,12 +259,10 @@ impl TestState { let config = config.clone(); let test_authorities = config.generate_authorities(); - let validator_index = ValidatorIndex(0); let mut chunks = Vec::new(); let mut available_data = Vec::new(); let mut candidate_receipts = Vec::new(); let mut pov_size_to_candidate = HashMap::new(); - let session_index = 10; // we use it for all candidates. let persisted_validation_data = PersistedValidationData { @@ -469,8 +304,6 @@ impl TestState { Self { config, test_authorities, - validator_index, - session_index, persisted_validation_data, available_data, candidate_receipts, diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index c9cc6ae40410..4fd752675074 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -14,10 +14,23 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use crate::{ + core::{configuration::PeerLatency, mock::AlwaysSupportsParachains, network::NetworkEmulator}, + TestConfiguration, +}; +use core::time::Duration; +use polkadot_node_subsystem::{Event, Overseer, OverseerHandle, SpawnGlue, TimeoutExt}; +use polkadot_node_subsystem_types::Hash; use polkadot_node_subsystem_util::metrics::prometheus::{ self, Gauge, Histogram, PrometheusError, Registry, U64, }; -use sc_service::TaskManager; +use rand::{ + distributions::{Distribution, Uniform}, + thread_rng, +}; +use sc_service::{SpawnTaskHandle, TaskManager}; +use std::net::{Ipv4Addr, SocketAddr}; +use tokio::runtime::{Handle, Runtime}; const MIB: f64 = 1024.0 * 1024.0; @@ -125,3 +138,145 @@ impl Default for TestEnvironmentDependencies { Self { runtime, registry, task_manager } } } + +// A dummy genesis hash +pub const GENESIS_HASH: Hash = Hash::repeat_byte(0xff); + +// We use this to bail out sending messages to the subsystem if it is overloaded such that +// the time of flight is breaches 5s. +// This should eventually be a test parameter. +const MAX_TIME_OF_FLIGHT: Duration = Duration::from_millis(5000); + +/// The test environment is the high level wrapper of all things required to test +/// a certain subsystem. +/// +/// ## Mockups +/// The overseer is passed in during construction and it can host an arbitrary number of +/// real subsystems instances and the corresponding mocked instances such that the real +/// subsystems can get their messages answered. +/// +/// As the subsystem's performance depends on network connectivity, the test environment +/// emulates validator nodes on the network, see `NetworkEmulator`. The network emulation +/// is configurable in terms of peer bandwidth, latency and connection error rate using +/// uniform distribution sampling. +/// +/// +/// ## Usage +/// `TestEnvironment` is used in tests to send `Overseer` messages or signals to the subsystem +/// under test. +/// +/// ## Collecting test metrics +/// +/// ### Prometheus +/// A prometheus endpoint is exposed while the test is running. A local Prometheus instance +/// can scrape it every 1s and a Grafana dashboard is the preferred way of visualizing +/// the performance characteristics of the subsystem. +/// +/// ### CLI +/// A subset of the Prometheus metrics are printed at the end of the test. +pub struct TestEnvironment { + // A task manager that tracks task poll durations allows us to measure + // per task CPU usage as we do in the Polkadot node. + task_manager: TaskManager, + // Our runtime + runtime: tokio::runtime::Runtime, + // A runtime handle + runtime_handle: tokio::runtime::Handle, + // The Prometheus metrics registry + registry: Registry, + // A handle to the lovely overseer + overseer_handle: OverseerHandle, + // The test intial state. The current state is owned by `env_task`. + config: TestConfiguration, + // A handle to the network emulator. + network: NetworkEmulator, + // Configuration/env metrics + metrics: TestEnvironmentMetrics, +} + +impl TestEnvironment { + // Create a new test environment with specified initial state and prometheus registry. + // We use prometheus metrics to collect per job task poll time and subsystem metrics. + pub fn new( + task_manager: TaskManager, + config: TestConfiguration, + registry: Registry, + runtime: Runtime, + network: NetworkEmulator, + overseer: Overseer, AlwaysSupportsParachains>, + overseer_handle: OverseerHandle, + ) -> Self { + let metrics = + TestEnvironmentMetrics::new(®istry).expect("Metrics need to be registered"); + + let spawn_handle = task_manager.spawn_handle(); + spawn_handle.spawn_blocking("overseer", "overseer", overseer.run()); + + let registry_clone = registry.clone(); + task_manager + .spawn_handle() + .spawn_blocking("prometheus", "test-environment", async move { + prometheus_endpoint::init_prometheus( + SocketAddr::new(std::net::IpAddr::V4(Ipv4Addr::LOCALHOST), 9999), + registry_clone, + ) + .await + .unwrap(); + }); + + TestEnvironment { + task_manager, + runtime_handle: runtime.handle().clone(), + runtime, + registry, + overseer_handle, + config, + network, + metrics, + } + } + + pub fn config(&self) -> &TestConfiguration { + &self.config + } + + pub fn network(&mut self) -> &mut NetworkEmulator { + &mut self.network + } + + pub fn registry(&self) -> &Registry { + &self.registry + } + + /// Produce a randomized duration between `min` and `max`. + fn random_latency(maybe_peer_latency: Option<&PeerLatency>) -> Option { + if let Some(peer_latency) = maybe_peer_latency { + Some( + Uniform::from(peer_latency.min_latency..=peer_latency.max_latency) + .sample(&mut thread_rng()), + ) + } else { + None + } + } + + pub fn metrics(&self) -> &TestEnvironmentMetrics { + &self.metrics + } + + pub fn runtime(&self) -> Handle { + self.runtime_handle.clone() + } + + // Send a message to the subsystem under test environment. + pub async fn send_message(&mut self, msg: Event) { + self.overseer_handle + .send(msg) + .timeout(MAX_TIME_OF_FLIGHT) + .await + .unwrap_or_else(|| { + panic!("{}ms maximum time of flight breached", MAX_TIME_OF_FLIGHT.as_millis()) + }) + .expect("send never fails"); + } +} diff --git a/polkadot/node/subsystem-bench/src/core/subsystem.rs b/polkadot/node/subsystem-bench/src/core/subsystem.rs deleted file mode 100644 index c61e641d255d..000000000000 --- a/polkadot/node/subsystem-bench/src/core/subsystem.rs +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 4460315c35c5..51ce8fc1d5ea 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -26,10 +26,13 @@ pub(crate) mod availability; pub(crate) mod cli; pub(crate) mod core; -use availability::{prepare_test, NetworkEmulation, TestEnvironment, TestState}; +use availability::{prepare_test, NetworkEmulation, TestState}; use cli::TestObjective; -use core::configuration::TestConfiguration; +use core::{ + configuration::TestConfiguration, + environment::{TestEnvironment, GENESIS_HASH}, +}; use clap_num::number_range; // const LOG_TARGET: &str = "subsystem-bench"; @@ -105,8 +108,6 @@ impl BenchCli { format!("latency = {:?}", test_config.latency).bright_black(), ); - let candidate_count = test_config.n_cores * test_config.num_blocks; - let mut state = TestState::new(&test_config); let (mut env, _protocol_config) = prepare_test(test_config, &mut state); env.runtime().block_on(availability::bench_chunk_recovery(&mut env, state)); @@ -165,11 +166,9 @@ impl BenchCli { test_config.bandwidth = bandwidth * 1024; } - let candidate_count = test_config.n_cores * test_config.num_blocks; - // test_config.write_to_disk(); - let mut state = TestState::new(&test_config); let (mut env, _protocol_config) = prepare_test(test_config, &mut state); + // test_config.write_to_disk(); env.runtime().block_on(availability::bench_chunk_recovery(&mut env, state)); Ok(()) From b51485bfe995c8891107b8c8618dc4b972c1d0c0 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 27 Nov 2023 20:00:10 +0200 Subject: [PATCH 099/192] more cleaning Signed-off-by: Andrei Sandu --- .../subsystem-bench/src/availability/mod.rs | 13 +----- .../subsystem-bench/src/core/environment.rs | 41 ++++++++----------- 2 files changed, 18 insertions(+), 36 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index e1974794cb8d..02ec794dc745 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -175,18 +175,7 @@ fn prepare_test_inner( subsystem, ); - ( - TestEnvironment::new( - dependencies.task_manager, - config, - dependencies.registry, - dependencies.runtime, - network, - overseer, - overseer_handle, - ), - req_cfg, - ) + (TestEnvironment::new(dependencies, config, network, overseer, overseer_handle), req_cfg) } #[derive(Clone)] diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 4fd752675074..d213d24c9af7 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -30,7 +30,7 @@ use rand::{ }; use sc_service::{SpawnTaskHandle, TaskManager}; use std::net::{Ipv4Addr, SocketAddr}; -use tokio::runtime::{Handle, Runtime}; +use tokio::runtime::Handle; const MIB: f64 = 1024.0 * 1024.0; @@ -175,15 +175,10 @@ const MAX_TIME_OF_FLIGHT: Duration = Duration::from_millis(5000); /// ### CLI /// A subset of the Prometheus metrics are printed at the end of the test. pub struct TestEnvironment { - // A task manager that tracks task poll durations allows us to measure - // per task CPU usage as we do in the Polkadot node. - task_manager: TaskManager, - // Our runtime - runtime: tokio::runtime::Runtime, + // Test dependencies + dependencies: TestEnvironmentDependencies, // A runtime handle runtime_handle: tokio::runtime::Handle, - // The Prometheus metrics registry - registry: Registry, // A handle to the lovely overseer overseer_handle: OverseerHandle, // The test intial state. The current state is owned by `env_task`. @@ -198,37 +193,35 @@ impl TestEnvironment { // Create a new test environment with specified initial state and prometheus registry. // We use prometheus metrics to collect per job task poll time and subsystem metrics. pub fn new( - task_manager: TaskManager, + dependencies: TestEnvironmentDependencies, config: TestConfiguration, - registry: Registry, - runtime: Runtime, network: NetworkEmulator, overseer: Overseer, AlwaysSupportsParachains>, overseer_handle: OverseerHandle, ) -> Self { - let metrics = - TestEnvironmentMetrics::new(®istry).expect("Metrics need to be registered"); + let metrics = TestEnvironmentMetrics::new(&dependencies.registry) + .expect("Metrics need to be registered"); - let spawn_handle = task_manager.spawn_handle(); + let spawn_handle = dependencies.task_manager.spawn_handle(); spawn_handle.spawn_blocking("overseer", "overseer", overseer.run()); - let registry_clone = registry.clone(); - task_manager - .spawn_handle() - .spawn_blocking("prometheus", "test-environment", async move { + let registry_clone = dependencies.registry.clone(); + dependencies.task_manager.spawn_handle().spawn_blocking( + "prometheus", + "test-environment", + async move { prometheus_endpoint::init_prometheus( SocketAddr::new(std::net::IpAddr::V4(Ipv4Addr::LOCALHOST), 9999), registry_clone, ) .await .unwrap(); - }); + }, + ); TestEnvironment { - task_manager, - runtime_handle: runtime.handle().clone(), - runtime, - registry, + runtime_handle: dependencies.runtime.handle().clone(), + dependencies, overseer_handle, config, network, @@ -245,7 +238,7 @@ impl TestEnvironment { } pub fn registry(&self) -> &Registry { - &self.registry + &self.dependencies.registry } /// Produce a randomized duration between `min` and `max`. From 7e464447d7db427223089a2d01aa38048f7c8927 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 27 Nov 2023 20:11:06 +0200 Subject: [PATCH 100/192] more cleaning Signed-off-by: Andrei Sandu --- .../subsystem-bench/src/availability/mod.rs | 5 +- .../subsystem-bench/src/core/configuration.rs | 1 + .../node/subsystem-bench/src/core/display.rs | 109 ------------------ .../subsystem-bench/src/core/environment.rs | 20 +--- .../node/subsystem-bench/src/core/network.rs | 1 + 5 files changed, 5 insertions(+), 131 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 02ec794dc745..5546d9cc357f 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -97,6 +97,7 @@ pub fn prepare_test( } /// Takes a test configuration and uses it to creates the `TestEnvironment`. +#[allow(unused)] pub fn prepare_test_with_dependencies( config: TestConfiguration, state: &mut TestState, @@ -212,10 +213,6 @@ impl TestState { candidate } - pub fn authorities(&self) -> &TestAuthorities { - &self.test_authorities - } - /// Generate candidates to be used in the test. pub fn generate_candidates(&mut self, count: usize) { gum::info!(target: LOG_TARGET,"{}", format!("Pre-generating {} candidates.", count).bright_blue()); diff --git a/polkadot/node/subsystem-bench/src/core/configuration.rs b/polkadot/node/subsystem-bench/src/core/configuration.rs index f8fdcf2973eb..35fa51790c91 100644 --- a/polkadot/node/subsystem-bench/src/core/configuration.rs +++ b/polkadot/node/subsystem-bench/src/core/configuration.rs @@ -110,6 +110,7 @@ pub struct TestAuthorities { } impl TestConfiguration { + #[allow(unused)] pub fn write_to_disk(&self) { // Serialize a slice of configurations let yaml = serde_yaml::to_string(&TestSequence { test_configurations: vec![self.clone()] }) diff --git a/polkadot/node/subsystem-bench/src/core/display.rs b/polkadot/node/subsystem-bench/src/core/display.rs index 4b63f45c5f8a..921c22b2059e 100644 --- a/polkadot/node/subsystem-bench/src/core/display.rs +++ b/polkadot/node/subsystem-bench/src/core/display.rs @@ -103,115 +103,6 @@ impl Display for TestMetric { } } -// fn encode_impl( -// &self, -// metric_families: &[MetricFamily], -// writer: &mut dyn WriteUtf8, -// ) -> Result<()> { for mf in metric_families { // Fail-fast checks. check_metric_family(mf)?; - -// // Write `# HELP` header. -// let name = mf.get_name(); -// let help = mf.get_help(); -// if !help.is_empty() { -// writer.write_all("# HELP ")?; -// writer.write_all(name)?; -// writer.write_all(" ")?; -// writer.write_all(&escape_string(help, false))?; -// writer.write_all("\n")?; -// } - -// // Write `# TYPE` header. -// let metric_type = mf.get_field_type(); -// let lowercase_type = format!("{:?}", metric_type).to_lowercase(); -// writer.write_all("# TYPE ")?; -// writer.write_all(name)?; -// writer.write_all(" ")?; -// writer.write_all(&lowercase_type)?; -// writer.write_all("\n")?; - -// for m in mf.get_metric() { -// match metric_type { -// MetricType::COUNTER => { -// write_sample(writer, name, None, m, None, m.get_counter().get_value())?; -// } -// MetricType::GAUGE => { -// write_sample(writer, name, None, m, None, m.get_gauge().get_value())?; -// } -// MetricType::HISTOGRAM => { -// let h = m.get_histogram(); - -// let mut inf_seen = false; -// for b in h.get_bucket() { -// let upper_bound = b.get_upper_bound(); -// write_sample( -// writer, -// name, -// Some("_bucket"), -// m, -// Some((BUCKET_LABEL, &upper_bound.to_string())), -// b.get_cumulative_count() as f64, -// )?; -// if upper_bound.is_sign_positive() && upper_bound.is_infinite() { -// inf_seen = true; -// } -// } -// if !inf_seen { -// write_sample( -// writer, -// name, -// Some("_bucket"), -// m, -// Some((BUCKET_LABEL, POSITIVE_INF)), -// h.get_sample_count() as f64, -// )?; -// } - -// write_sample(writer, name, Some("_sum"), m, None, h.get_sample_sum())?; - -// write_sample( -// writer, -// name, -// Some("_count"), -// m, -// None, -// h.get_sample_count() as f64, -// )?; -// } -// MetricType::SUMMARY => { -// let s = m.get_summary(); - -// for q in s.get_quantile() { -// write_sample( -// writer, -// name, -// None, -// m, -// Some((QUANTILE, &q.get_quantile().to_string())), -// q.get_value(), -// )?; -// } - -// write_sample(writer, name, Some("_sum"), m, None, s.get_sample_sum())?; - -// write_sample( -// writer, -// name, -// Some("_count"), -// m, -// None, -// s.get_sample_count() as f64, -// )?; -// } -// MetricType::UNTYPED => { -// unimplemented!(); -// } -// } -// } -// } - -// Ok(()) -// } - // Returns `false` if metric should be skipped. fn check_metric_family(mf: &MetricFamily) -> bool { if mf.get_metric().is_empty() { diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index d213d24c9af7..28e98c6b42d0 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -15,7 +15,7 @@ // along with Polkadot. If not, see . use crate::{ - core::{configuration::PeerLatency, mock::AlwaysSupportsParachains, network::NetworkEmulator}, + core::{mock::AlwaysSupportsParachains, network::NetworkEmulator}, TestConfiguration, }; use core::time::Duration; @@ -24,10 +24,6 @@ use polkadot_node_subsystem_types::Hash; use polkadot_node_subsystem_util::metrics::prometheus::{ self, Gauge, Histogram, PrometheusError, Registry, U64, }; -use rand::{ - distributions::{Distribution, Uniform}, - thread_rng, -}; use sc_service::{SpawnTaskHandle, TaskManager}; use std::net::{Ipv4Addr, SocketAddr}; use tokio::runtime::Handle; @@ -181,7 +177,7 @@ pub struct TestEnvironment { runtime_handle: tokio::runtime::Handle, // A handle to the lovely overseer overseer_handle: OverseerHandle, - // The test intial state. The current state is owned by `env_task`. + // The test configuration. config: TestConfiguration, // A handle to the network emulator. network: NetworkEmulator, @@ -241,18 +237,6 @@ impl TestEnvironment { &self.dependencies.registry } - /// Produce a randomized duration between `min` and `max`. - fn random_latency(maybe_peer_latency: Option<&PeerLatency>) -> Option { - if let Some(peer_latency) = maybe_peer_latency { - Some( - Uniform::from(peer_latency.min_latency..=peer_latency.max_latency) - .sample(&mut thread_rng()), - ) - } else { - None - } - } - pub fn metrics(&self) -> &TestEnvironmentMetrics { &self.metrics } diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 80d961babe03..f5532087e35c 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -334,6 +334,7 @@ impl NetworkEmulator { } // Increment bytes received by our node (the node that contains the subsystem under test) + #[allow(unused)] pub fn inc_received(&self, bytes: u64) { // Our node always is peer 0. self.metrics.on_peer_received(0, bytes); From d3df9279adbe6bfb78856e16ff4502d09245c25f Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 27 Nov 2023 22:37:48 +0200 Subject: [PATCH 101/192] proper overseer control Signed-off-by: Andrei Sandu --- Cargo.lock | 1 + polkadot/node/subsystem-bench/Cargo.toml | 1 + .../subsystem-bench/src/availability/mod.rs | 56 ++++++++----------- .../node/subsystem-bench/src/core/display.rs | 1 + .../subsystem-bench/src/core/environment.rs | 31 ++++++++-- .../subsystem-bench/src/core/mock/av_store.rs | 9 ++- .../subsystem-bench/src/core/mock/dummy.rs | 10 +++- .../src/core/mock/network_bridge.rs | 8 ++- .../src/core/mock/runtime_api.rs | 29 +++++----- .../subsystem-bench/src/subsystem-bench.rs | 2 + .../node/subsystem-test-helpers/src/mock.rs | 8 +-- 11 files changed, 91 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b349886761ad..197807b7fa8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13445,6 +13445,7 @@ dependencies = [ "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-types", "polkadot-node-subsystem-util", + "polkadot-overseer", "polkadot-primitives", "polkadot-primitives-test-helpers", "prometheus", diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index 8296874c0dab..f775a1ff9efe 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -24,6 +24,7 @@ polkadot-primitives = { path = "../../primitives" } polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-availability-recovery = { path = "../network/availability-recovery", features=["subsystem-benchmarks"]} color-eyre = { version = "0.6.1", default-features = false } +polkadot-overseer = { path = "../overseer" } colored = "2.0.4" assert_matches = "1.5" async-trait = "0.1.57" diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 5546d9cc357f..6282a6b63f01 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -23,7 +23,9 @@ use std::{ }; use crate::TestEnvironment; -use polkadot_node_subsystem::{Event, Overseer, OverseerConnector, OverseerHandle, SpawnGlue}; +use polkadot_node_subsystem::{Overseer, OverseerConnector, SpawnGlue}; +use polkadot_overseer::Handle as OverseerHandle; + use sc_network::request_responses::ProtocolConfig; use colored::Colorize; @@ -41,7 +43,6 @@ use polkadot_node_primitives::{BlockData, PoV, Proof}; use polkadot_node_subsystem::messages::{AllMessages, AvailabilityRecoveryMessage}; use crate::core::{ - configuration::TestAuthorities, environment::TestEnvironmentDependencies, mock::{ av_store, @@ -57,7 +58,7 @@ const LOG_TARGET: &str = "subsystem-bench::availability"; use polkadot_node_primitives::{AvailableData, ErasureChunk}; use super::{cli::TestObjective, core::mock::AlwaysSupportsParachains}; -use polkadot_node_subsystem_test_helpers::mock::new_block_import_event; +use polkadot_node_subsystem_test_helpers::mock::new_block_import_info; use polkadot_primitives::{ CandidateHash, CandidateReceipt, GroupIndex, Hash, HeadData, PersistedValidationData, ValidatorIndex, @@ -85,7 +86,10 @@ fn build_overseer( .replace_network_bridge_tx(|_| network_bridge) .replace_availability_recovery(|_| availability_recovery); - builder.build_with_connector(overseer_connector).expect("Should not fail") + let (overseer, raw_handle) = + builder.build_with_connector(overseer_connector).expect("Should not fail"); + + (overseer, OverseerHandle::new(raw_handle)) } /// Takes a test configuration and uses it to creates the `TestEnvironment`. @@ -119,11 +123,7 @@ fn prepare_test_inner( // Generate test authorities. let test_authorities = config.generate_authorities(); - let runtime_api = runtime_api::MockRuntimeApi::new( - config.clone(), - test_authorities.validator_public.clone(), - test_authorities.validator_authority_id.clone(), - ); + let runtime_api = runtime_api::MockRuntimeApi::new(config.clone(), test_authorities.clone()); let av_store = av_store::MockAvailabilityStore::new(state.chunks.clone(), state.candidate_hashes.clone()); @@ -136,7 +136,7 @@ fn prepare_test_inner( let network = NetworkEmulator::new( config.n_validators.clone(), - test_authorities.validator_authority_id.clone(), + test_authorities.validator_authority_id, config.peer_bandwidth, dependencies.task_manager.spawn_handle(), &dependencies.registry, @@ -183,18 +183,14 @@ fn prepare_test_inner( pub struct TestState { // Full test configuration config: TestConfiguration, - // State starts here. - test_authorities: TestAuthorities, pov_sizes: Cycle>, // Generated candidate receipts to be used in the test candidates: Cycle>, - candidates_generated: usize, // Map from pov size to candidate index pov_size_to_candidate: HashMap, // Map from generated candidate hashes to candidate index in `available_data` // and `chunks`. candidate_hashes: HashMap, - persisted_validation_data: PersistedValidationData, candidate_receipts: Vec, available_data: Vec, @@ -243,7 +239,6 @@ impl TestState { pub fn new(config: &TestConfiguration) -> Self { let config = config.clone(); - let test_authorities = config.generate_authorities(); let mut chunks = Vec::new(); let mut available_data = Vec::new(); @@ -289,14 +284,11 @@ impl TestState { Self { config, - test_authorities, - persisted_validation_data, available_data, candidate_receipts, chunks, pov_size_to_candidate, pov_sizes, - candidates_generated: 0, candidate_hashes: HashMap::new(), candidates: Vec::new().into_iter().cycle(), } @@ -333,7 +325,7 @@ fn derive_erasure_chunks_with_proofs_and_root( pub async fn bench_chunk_recovery(env: &mut TestEnvironment, mut state: TestState) { let config = env.config().clone(); - env.send_message(new_block_import_event(Hash::repeat_byte(1), 1)).await; + env.import_block(new_block_import_info(Hash::repeat_byte(1), 1)).await; let start_marker = Instant::now(); let mut batch = FuturesUnordered::new(); @@ -353,19 +345,16 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment, mut state: TestStat let (tx, rx) = oneshot::channel(); batch.push(rx); - let message = Event::MsgToSubsystem { - msg: AllMessages::AvailabilityRecovery( - AvailabilityRecoveryMessage::RecoverAvailableData( - candidate.clone(), - 1, - Some(GroupIndex( - candidate_num as u32 % (std::cmp::max(5, config.n_cores) / 5) as u32, - )), - tx, - ), + let message = AllMessages::AvailabilityRecovery( + AvailabilityRecoveryMessage::RecoverAvailableData( + candidate.clone(), + 1, + Some(GroupIndex( + candidate_num as u32 % (std::cmp::max(5, config.n_cores) / 5) as u32, + )), + tx, ), - origin: LOG_TARGET, - }; + ); env.send_message(message).await; } @@ -386,7 +375,8 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment, mut state: TestStat tokio::time::sleep(block_time_delta).await; } - env.send_message(Event::Stop).await; + env.stop().await; + let duration: u128 = start_marker.elapsed().as_millis(); let availability_bytes = availability_bytes / 1024; gum::info!("All blocks processed in {}", format!("{:?}ms", duration).cyan()); @@ -416,7 +406,7 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment, mut state: TestStat let test_metrics = super::core::display::parse_metrics(&env.registry()); let subsystem_cpu_metrics = - test_metrics.subset_with_label_value("task_group", "availability-recovery-subsystem"); + test_metrics.subset_with_label_value("task_group", "availability-recovery"); let total_cpu = subsystem_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); gum::info!(target: LOG_TARGET, "Total subsystem CPU usage {}", format!("{:.2}s", total_cpu).bright_purple()); gum::info!(target: LOG_TARGET, "CPU usage per block {}", format!("{:.2}s", total_cpu/env.config().num_blocks as f64).bright_purple()); diff --git a/polkadot/node/subsystem-bench/src/core/display.rs b/polkadot/node/subsystem-bench/src/core/display.rs index 921c22b2059e..13ea7d375e95 100644 --- a/polkadot/node/subsystem-bench/src/core/display.rs +++ b/polkadot/node/subsystem-bench/src/core/display.rs @@ -13,6 +13,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +// //! Some helper methods for parsing prometheus metrics to a format that can be //! displayed in the CLI. //! diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 28e98c6b42d0..fd09de9169a4 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -19,11 +19,15 @@ use crate::{ TestConfiguration, }; use core::time::Duration; -use polkadot_node_subsystem::{Event, Overseer, OverseerHandle, SpawnGlue, TimeoutExt}; +use polkadot_overseer::{BlockInfo, Handle as OverseerHandle}; + +use polkadot_node_subsystem::{messages::AllMessages, Overseer, SpawnGlue, TimeoutExt}; use polkadot_node_subsystem_types::Hash; use polkadot_node_subsystem_util::metrics::prometheus::{ self, Gauge, Histogram, PrometheusError, Registry, U64, }; + +use sc_network::peer_store::LOG_TARGET; use sc_service::{SpawnTaskHandle, TaskManager}; use std::net::{Ipv4Addr, SocketAddr}; use tokio::runtime::Handle; @@ -199,8 +203,8 @@ impl TestEnvironment { .expect("Metrics need to be registered"); let spawn_handle = dependencies.task_manager.spawn_handle(); - spawn_handle.spawn_blocking("overseer", "overseer", overseer.run()); + spawn_handle.spawn_blocking("overseer", "overseer", overseer.run()); let registry_clone = dependencies.registry.clone(); dependencies.task_manager.spawn_handle().spawn_blocking( "prometheus", @@ -246,14 +250,29 @@ impl TestEnvironment { } // Send a message to the subsystem under test environment. - pub async fn send_message(&mut self, msg: Event) { + pub async fn send_message(&mut self, msg: AllMessages) { self.overseer_handle - .send(msg) + .send_msg(msg, LOG_TARGET) .timeout(MAX_TIME_OF_FLIGHT) .await .unwrap_or_else(|| { panic!("{}ms maximum time of flight breached", MAX_TIME_OF_FLIGHT.as_millis()) - }) - .expect("send never fails"); + }); + } + + // Send a signal to the subsystem under test environment. + pub async fn import_block(&mut self, block: BlockInfo) { + self.overseer_handle + .block_imported(block) + .timeout(MAX_TIME_OF_FLIGHT) + .await + .unwrap_or_else(|| { + panic!("{}ms maximum time of flight breached", MAX_TIME_OF_FLIGHT.as_millis()) + }); + } + + // Stop overseer and subsystems. + pub async fn stop(&mut self) { + self.overseer_handle.stop().await; } } diff --git a/polkadot/node/subsystem-bench/src/core/mock/av_store.rs b/polkadot/node/subsystem-bench/src/core/mock/av_store.rs index 7f6ff2abfe9e..1ff7d1728af9 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/av_store.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/av_store.rs @@ -29,6 +29,8 @@ use polkadot_node_subsystem::{ messages::AvailabilityStoreMessage, overseer, SpawnedSubsystem, SubsystemError, }; +use polkadot_node_subsystem_types::OverseerSignal; + pub struct AvailabilityStoreState { candidate_hashes: HashMap, chunks: Vec>, @@ -82,7 +84,7 @@ impl MockAvailabilityStore { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx).map(|_| Ok(())).boxed(); - SpawnedSubsystem { name: "av-store-mock-subsystem", future } + SpawnedSubsystem { name: "test-environment", future } } } @@ -94,7 +96,10 @@ impl MockAvailabilityStore { let msg = ctx.recv().await.expect("Overseer never fails us"); match msg { - orchestra::FromOrchestra::Signal(_) => {}, + orchestra::FromOrchestra::Signal(signal) => match signal { + OverseerSignal::Conclude => return, + _ => {}, + }, orchestra::FromOrchestra::Communication { msg } => match msg { AvailabilityStoreMessage::QueryAvailableData(candidate_hash, tx) => { gum::debug!(target: LOG_TARGET, candidate_hash = ?candidate_hash, "Responding to QueryAvailableData"); diff --git a/polkadot/node/subsystem-bench/src/core/mock/dummy.rs b/polkadot/node/subsystem-bench/src/core/mock/dummy.rs index 998153875ede..0628368a49c0 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/dummy.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/dummy.rs @@ -33,7 +33,8 @@ macro_rules! mock { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx).map(|_| Ok(())).boxed(); - SpawnedSubsystem { name: stringify!($subsystem_name), future } + // The name will appear in substrate CPU task metrics as `task_group`.` + SpawnedSubsystem { name: "test-environment", future } } } @@ -45,7 +46,12 @@ macro_rules! mock { futures::select!{ msg = ctx.recv().fuse() => { match msg.unwrap() { - orchestra::FromOrchestra::Signal(_) => {}, + orchestra::FromOrchestra::Signal(signal) => { + match signal { + polkadot_node_subsystem_types::OverseerSignal::Conclude => {return}, + _ => {} + } + }, orchestra::FromOrchestra::Communication { msg } => { gum::debug!(target: LOG_TARGET, msg = ?msg, "mocked subsystem received message"); } diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index a6d07c3d4a20..144a16b9f14b 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -17,6 +17,7 @@ //! A generic av store subsystem mockup suitable to be used in benchmarks. use parity_scale_codec::Encode; +use polkadot_node_subsystem_types::OverseerSignal; use std::collections::HashMap; @@ -191,7 +192,7 @@ impl MockNetworkBridgeTx { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx).map(|_| Ok(())).boxed(); - SpawnedSubsystem { name: "network-bridge-tx-mock-subsystem", future } + SpawnedSubsystem { name: "test-environment", future } } } @@ -230,7 +231,10 @@ impl MockNetworkBridgeTx { let msg = ctx.recv().await.expect("Overseer never fails us"); match msg { - orchestra::FromOrchestra::Signal(_) => {}, + orchestra::FromOrchestra::Signal(signal) => match signal { + OverseerSignal::Conclude => return, + _ => {}, + }, orchestra::FromOrchestra::Communication { msg } => match msg { NetworkBridgeTxMessage::SendRequests(requests, _if_disconnected) => { for request in requests { diff --git a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs index a106eb130991..9cbe025ae806 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs @@ -16,23 +16,21 @@ //! //! A generic runtime api subsystem mockup suitable to be used in benchmarks. -use polkadot_primitives::{ - AuthorityDiscoveryId, GroupIndex, IndexedVec, SessionInfo, ValidatorId, ValidatorIndex, -}; +use polkadot_primitives::{GroupIndex, IndexedVec, SessionInfo, ValidatorIndex}; use polkadot_node_subsystem::{ messages::{RuntimeApiMessage, RuntimeApiRequest}, overseer, SpawnedSubsystem, SubsystemError, }; +use polkadot_node_subsystem_types::OverseerSignal; -use crate::core::configuration::TestConfiguration; +use crate::core::configuration::{TestAuthorities, TestConfiguration}; use futures::FutureExt; const LOG_TARGET: &str = "subsystem-bench::runtime-api-mock"; pub struct RuntimeApiState { - validator_public: Vec, - validator_authority_id: Vec, + authorities: TestAuthorities, } pub struct MockRuntimeApi { @@ -41,12 +39,8 @@ pub struct MockRuntimeApi { } impl MockRuntimeApi { - pub fn new( - config: TestConfiguration, - validator_public: Vec, - validator_authority_id: Vec, - ) -> MockRuntimeApi { - Self { state: RuntimeApiState { validator_public, validator_authority_id }, config } + pub fn new(config: TestConfiguration, authorities: TestAuthorities) -> MockRuntimeApi { + Self { state: RuntimeApiState { authorities }, config } } fn session_info(&self) -> SessionInfo { @@ -57,8 +51,8 @@ impl MockRuntimeApi { let validator_groups = all_validators.chunks(5).map(|x| Vec::from(x)).collect::>(); SessionInfo { - validators: self.state.validator_public.clone().into(), - discovery_keys: self.state.validator_authority_id.clone(), + validators: self.state.authorities.validator_public.clone().into(), + discovery_keys: self.state.authorities.validator_authority_id.clone(), validator_groups: IndexedVec::>::from(validator_groups), assignment_keys: vec![], n_cores: self.config.n_cores as u32, @@ -79,7 +73,7 @@ impl MockRuntimeApi { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx).map(|_| Ok(())).boxed(); - SpawnedSubsystem { name: "runtime-api-mock-subsystem", future } + SpawnedSubsystem { name: "test-environment", future } } } @@ -90,7 +84,10 @@ impl MockRuntimeApi { let msg = ctx.recv().await.expect("Overseer never fails us"); match msg { - orchestra::FromOrchestra::Signal(_) => {}, + orchestra::FromOrchestra::Signal(signal) => match signal { + OverseerSignal::Conclude => return, + _ => {}, + }, orchestra::FromOrchestra::Communication { msg } => { gum::debug!(target: LOG_TARGET, msg=?msg, "recv message"); diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 51ce8fc1d5ea..f9261d848778 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -179,6 +179,8 @@ fn main() -> eyre::Result<()> { color_eyre::install()?; let _ = env_logger::builder() .filter(Some("hyper"), log::LevelFilter::Info) + // Avoid `Terminating due to subsystem exit subsystem` warnings + .filter(Some("polkadot_overseer"), log::LevelFilter::Error) // .filter(None, log::LevelFilter::Trace) .try_init() .unwrap(); diff --git a/polkadot/node/subsystem-test-helpers/src/mock.rs b/polkadot/node/subsystem-test-helpers/src/mock.rs index 11e77b6e8968..fc2dd6a4e34e 100644 --- a/polkadot/node/subsystem-test-helpers/src/mock.rs +++ b/polkadot/node/subsystem-test-helpers/src/mock.rs @@ -16,7 +16,7 @@ use std::sync::Arc; -use polkadot_node_subsystem::{jaeger, ActivatedLeaf, Event, BlockInfo}; +use polkadot_node_subsystem::{jaeger, ActivatedLeaf,BlockInfo}; use sc_client_api::UnpinHandle; use sc_keystore::LocalKeystore; use sc_utils::mpsc::tracing_unbounded; @@ -61,11 +61,11 @@ pub fn new_leaf(hash: Hash, number: BlockNumber) -> ActivatedLeaf { } /// Create a new leaf with the given hash and number. -pub fn new_block_import_event(hash: Hash, number: BlockNumber) -> Event { - Event::BlockImported(BlockInfo { +pub fn new_block_import_info(hash: Hash, number: BlockNumber) -> BlockInfo { + BlockInfo { hash, parent_hash: Hash::default(), number, unpin_handle: dummy_unpin_handle(hash), - }) + } } From 7557768d740a87336cb0479d78ebe2c7b816e6b2 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 27 Nov 2023 23:06:17 +0200 Subject: [PATCH 102/192] refactor CLI display of env stats Signed-off-by: Andrei Sandu --- .../subsystem-bench/src/availability/mod.rs | 27 +------- .../subsystem-bench/src/core/environment.rs | 63 +++++++++++++++++-- .../src/core/mock/network_bridge.rs | 2 + .../node/subsystem-bench/src/core/network.rs | 2 +- 4 files changed, 63 insertions(+), 31 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 6282a6b63f01..ae4e743205e3 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -390,30 +390,5 @@ pub async fn bench_chunk_recovery(env: &mut TestEnvironment, mut state: TestStat .red() ); - let stats = env.network().stats(); - gum::info!( - "Total received from network: {}", - format!( - "{} MiB", - stats - .iter() - .enumerate() - .map(|(_index, stats)| stats.tx_bytes_total as u128) - .sum::() / (1024 * 1024) - ) - .cyan() - ); - - let test_metrics = super::core::display::parse_metrics(&env.registry()); - let subsystem_cpu_metrics = - test_metrics.subset_with_label_value("task_group", "availability-recovery"); - let total_cpu = subsystem_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); - gum::info!(target: LOG_TARGET, "Total subsystem CPU usage {}", format!("{:.2}s", total_cpu).bright_purple()); - gum::info!(target: LOG_TARGET, "CPU usage per block {}", format!("{:.2}s", total_cpu/env.config().num_blocks as f64).bright_purple()); - - let test_env_cpu_metrics = - test_metrics.subset_with_label_value("task_group", "test-environment"); - let total_cpu = test_env_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); - gum::info!(target: LOG_TARGET, "Total test environment CPU usage {}", format!("{:.2}s", total_cpu).bright_purple()); - gum::info!(target: LOG_TARGET, "CPU usage per block {}", format!("{:.2}s", total_cpu/env.config().num_blocks as f64).bright_purple()); + gum::info!("{}", &env); } diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index fd09de9169a4..24d10ecb1fa1 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -18,6 +18,7 @@ use crate::{ core::{mock::AlwaysSupportsParachains, network::NetworkEmulator}, TestConfiguration, }; +use colored::Colorize; use core::time::Duration; use polkadot_overseer::{BlockInfo, Handle as OverseerHandle}; @@ -29,7 +30,10 @@ use polkadot_node_subsystem_util::metrics::prometheus::{ use sc_network::peer_store::LOG_TARGET; use sc_service::{SpawnTaskHandle, TaskManager}; -use std::net::{Ipv4Addr, SocketAddr}; +use std::{ + fmt::Display, + net::{Ipv4Addr, SocketAddr}, +}; use tokio::runtime::Handle; const MIB: f64 = 1024.0 * 1024.0; @@ -233,8 +237,8 @@ impl TestEnvironment { &self.config } - pub fn network(&mut self) -> &mut NetworkEmulator { - &mut self.network + pub fn network(&self) -> &NetworkEmulator { + &self.network } pub fn registry(&self) -> &Registry { @@ -260,7 +264,7 @@ impl TestEnvironment { }); } - // Send a signal to the subsystem under test environment. + // Send an `ActiveLeavesUpdate` signal to all subsystems under test. pub async fn import_block(&mut self, block: BlockInfo) { self.overseer_handle .block_imported(block) @@ -276,3 +280,54 @@ impl TestEnvironment { self.overseer_handle.stop().await; } } + +impl Display for TestEnvironment { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let stats = self.network().stats(); + + writeln!(f, "\n")?; + writeln!( + f, + "Total received from network: {}", + format!( + "{} MiB", + stats + .iter() + .enumerate() + .map(|(_index, stats)| stats.tx_bytes_total as u128) + .sum::() / (1024 * 1024) + ) + .cyan() + )?; + writeln!( + f, + "Total sent to network: {}", + format!("{} KiB", stats[0].tx_bytes_total / (1024)).cyan() + )?; + + let test_metrics = super::display::parse_metrics(self.registry()); + let subsystem_cpu_metrics = + test_metrics.subset_with_label_value("task_group", "availability-recovery"); + let total_cpu = subsystem_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); + writeln!(f, "Total subsystem CPU usage {}", format!("{:.2}s", total_cpu).bright_purple())?; + writeln!( + f, + "CPU usage per block {}", + format!("{:.2}s", total_cpu / self.config().num_blocks as f64).bright_purple() + )?; + + let test_env_cpu_metrics = + test_metrics.subset_with_label_value("task_group", "test-environment"); + let total_cpu = test_env_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); + writeln!( + f, + "Total test environment CPU usage {}", + format!("{:.2}s", total_cpu).bright_purple() + )?; + writeln!( + f, + "CPU usage per block {}", + format!("{:.2}s", total_cpu / self.config().num_blocks as f64).bright_purple() + ) + } +} diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index 144a16b9f14b..a45cacd0241a 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -80,6 +80,8 @@ impl MockNetworkBridgeTx { match request { Requests::ChunkFetchingV1(outgoing_request) => { + self.network.peer_stats(0).inc_sent(outgoing_request.payload.encoded_size()); + let validator_index: usize = outgoing_request.payload.index.0 as usize; let candidate_hash = outgoing_request.payload.candidate_hash; diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index f5532087e35c..f36c0967466b 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -315,7 +315,7 @@ impl NetworkEmulator { } // Returns the sent/received stats for all peers. - pub fn stats(&mut self) -> Vec { + pub fn stats(&self) -> Vec { let r = self .stats .iter() From 787dc00bc7c411becbc17e04eea29fa91d1f8e00 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 28 Nov 2023 10:13:11 +0200 Subject: [PATCH 103/192] Add grafana dashboards for DA read Signed-off-by: Andrei Sandu --- .../src/grafana/availability-read.json | 1872 +++++++++++++++++ .../src/grafana/task-cpu-usage.json | 755 +++++++ 2 files changed, 2627 insertions(+) create mode 100644 polkadot/node/subsystem-bench/src/grafana/availability-read.json create mode 100644 polkadot/node/subsystem-bench/src/grafana/task-cpu-usage.json diff --git a/polkadot/node/subsystem-bench/src/grafana/availability-read.json b/polkadot/node/subsystem-bench/src/grafana/availability-read.json new file mode 100644 index 000000000000..4fbbe1f58731 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/grafana/availability-read.json @@ -0,0 +1,1872 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Subsystem and test environment metrics", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 90, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "editorMode": "code", + "expr": "subsystem_benchmark_n_validators{}", + "instant": false, + "legendFormat": "n_vaidators", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "editorMode": "code", + "expr": "subsystem_benchmark_n_cores{}", + "hide": false, + "instant": false, + "legendFormat": "n_cores", + "range": true, + "refId": "B" + } + ], + "title": "Test configuration", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 31, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$data_source" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 57, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "mean", + "min", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.2", + "repeat": "nodename", + "targets": [ + { + "datasource": { + "uid": "$data_source" + }, + "editorMode": "code", + "expr": "sum(rate(substrate_tasks_polling_duration_sum{}[2s])) by ($cpu_group_by)", + "interval": "", + "legendFormat": "{{task_group}}", + "range": true, + "refId": "A" + } + ], + "title": "All tasks CPU usage breakdown", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$data_source" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 6 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 93, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "mean", + "min", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$data_source" + }, + "editorMode": "code", + "expr": "increase(substrate_tasks_polling_duration_sum{task_group=\"availability-recovery-subsystem\"}[6s])", + "interval": "", + "legendFormat": "{{task_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Availability subsystem CPU usage per block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$data_source" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 94, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$data_source" + }, + "editorMode": "code", + "expr": "sum(substrate_tasks_polling_duration_sum{}) by ($cpu_group_by)", + "interval": "", + "legendFormat": "{{task_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Total CPU burn", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$data_source" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "dark-red", + "value": 6000 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 95, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$data_source" + }, + "editorMode": "code", + "expr": "subsystem_benchmark_block_time", + "interval": "", + "legendFormat": "Instant block time", + "range": true, + "refId": "A" + } + ], + "title": "Block time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 89, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "editorMode": "code", + "expr": "sum(rate(subsystem_benchmark_network_peer_total_bytes_sent{}[5s]))", + "instant": false, + "legendFormat": "Received", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "editorMode": "code", + "expr": "sum(rate(subsystem_benchmark_network_peer_total_bytes_received{}[5s]))", + "hide": false, + "instant": false, + "legendFormat": "Sent", + "range": true, + "refId": "B" + } + ], + "title": "Emulated network throughput ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 52 + }, + "id": 88, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "editorMode": "code", + "expr": "rate(subsystem_benchmark_network_peer_total_bytes_received{}[10s])", + "instant": false, + "legendFormat": "Received by {{peer}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "editorMode": "code", + "expr": "rate(subsystem_benchmark_network_peer_total_bytes_sent{}[10s])", + "hide": false, + "instant": false, + "legendFormat": "Sent by {{peer}}", + "range": true, + "refId": "B" + } + ], + "title": "Emulated peer throughput", + "type": "timeseries" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateInferno", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 12, + "y": 52 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 92, + "interval": "1s", + "legend": { + "show": true + }, + "maxDataPoints": 1340, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": { + "decimals": 0 + }, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Inferno", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "bytes" + } + }, + "pluginVersion": "10.1.1", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(subsystem_benchmark_pov_size_bucket{}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "B" + } + ], + "title": "Recovered PoV sizes", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 0, + "transformations": [], + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "Number of erasure-encoded chunks of data belonging to candidate blocks. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic", + "seriesBy": "max" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "chunks/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 43, + "interval": "1s", + "maxDataPoints": 1340, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(polkadot_parachain_availability_recovery_chunk_requests_issued{}[10s]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Chunks requested", + "queryType": "randomWalk", + "refId": "B" + } + ], + "title": "Availability", + "transformations": [], + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 77 + }, + "id": 35, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Availability subystem metrics", + "type": "row" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateInferno", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 78 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 68, + "interval": "1s", + "legend": { + "show": true + }, + "maxDataPoints": 1340, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": { + "decimals": 0 + }, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Inferno", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.1.1", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(polkadot_parachain_availability_recovery_time_total_bucket{}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "B" + } + ], + "title": "Time to recover a PoV", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 0, + "transformations": [], + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateInferno", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 78 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 67, + "interval": "1s", + "legend": { + "show": true + }, + "maxDataPoints": 1340, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": { + "decimals": 0 + }, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Inferno", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.1.1", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(polkadot_parachain_availability_recovery_time_chunk_request_bucket{}[$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Chunk request duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 0, + "transformations": [], + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "bitfields", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic", + "seriesBy": "max" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 88 + }, + "id": 85, + "interval": "1s", + "maxDataPoints": 1340, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(polkadot_parachain_availability_recovery_bytes_total{}[30s])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Bytes recovered", + "queryType": "randomWalk", + "refId": "B" + } + ], + "title": "Recovery throughtput", + "transformations": [], + "type": "timeseries" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateInferno", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 88 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 84, + "interval": "1s", + "legend": { + "show": true + }, + "maxDataPoints": 1340, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": { + "decimals": 0 + }, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Inferno", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.1.1", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(polkadot_parachain_availability_reencode_chunks_bucket{}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "B" + } + ], + "title": "Re-encoding chunks timing", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 0, + "transformations": [], + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateInferno", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 98 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 83, + "interval": "1s", + "legend": { + "show": true + }, + "maxDataPoints": 1340, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": { + "decimals": 0 + }, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Inferno", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.1.1", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(polkadot_parachain_availability_recovery_time_erasure_recovery_bucket{}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "B" + } + ], + "title": "Erasure recovery (no I/O)", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 0, + "transformations": [], + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "Number of erasure-encoded chunks of data belonging to candidate blocks. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic", + "seriesBy": "max" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 108 + }, + "id": 86, + "interval": "1s", + "maxDataPoints": 1340, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(polkadot_parachain_availability_recovery_recoveries_finished{}[1s]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Finished", + "queryType": "randomWalk", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(polkadot_parachain_availability_recovery_recovieries_started{}[1s]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Started", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Recoveries", + "transformations": [], + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 118 + }, + "id": 2, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Approval voting", + "type": "row" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "subsystem", + "benchmark" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "hide": 0, + "includeAll": false, + "label": "Source of data", + "multi": false, + "name": "data_source", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "task_name", + "value": "task_name" + }, + "description": "Sum CPU usage by task name or task group.", + "hide": 0, + "includeAll": false, + "label": "Group CPU usage", + "multi": false, + "name": "cpu_group_by", + "options": [ + { + "selected": true, + "text": "task_name", + "value": "task_name" + }, + { + "selected": false, + "text": "task_group", + "value": "task_group" + } + ], + "query": "task_name, task_group", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s" + ] + }, + "timezone": "utc", + "title": "Data Availability Read", + "uid": "asdadasd1", + "version": 56, + "weekStart": "" + } \ No newline at end of file diff --git a/polkadot/node/subsystem-bench/src/grafana/task-cpu-usage.json b/polkadot/node/subsystem-bench/src/grafana/task-cpu-usage.json new file mode 100644 index 000000000000..90763444abf1 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/grafana/task-cpu-usage.json @@ -0,0 +1,755 @@ +{ + "annotations": { + "list": [ + { + "$$hashKey": "object:326", + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "limit": 100, + "name": "Annotations & Alerts", + "showIn": 0, + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + }, + { + "$$hashKey": "object:327", + "datasource": { + "uid": "$data_source" + }, + "enable": true, + "expr": "increase(${metric_namespace}_tasks_ended_total{reason=\"panic\", node=~\"${nodename}\"}[10m])", + "hide": true, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Task panics", + "rawQuery": "SELECT\n extract(epoch from time_column) AS time,\n text_column as text,\n tags_column as tags\nFROM\n metric_table\nWHERE\n $__timeFilter(time_column)\n", + "showIn": 0, + "step": "10m", + "tags": [], + "textFormat": "{{node}} - {{task_name}}", + "titleFormat": "Panic!", + "type": "tags" + }, + { + "$$hashKey": "object:621", + "datasource": { + "uid": "$data_source" + }, + "enable": true, + "expr": "changes(${metric_namespace}_process_start_time_seconds{node=~\"${nodename}\"}[10m])", + "hide": false, + "iconColor": "#8AB8FF", + "name": "Node reboots", + "showIn": 0, + "step": "10m", + "textFormat": "{{node}}", + "titleFormat": "Reboots" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 29, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Tasks", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 11, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "nodename", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$data_source" + }, + "editorMode": "code", + "expr": "sum(rate(substrate_tasks_polling_duration_sum{}[$__rate_interval])) by (task_name)", + "interval": "", + "legendFormat": "{{task_name}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU time spent on each task", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2721", + "format": "percentunit", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2722", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 30, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "nodename", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$data_source" + }, + "editorMode": "code", + "expr": "rate(substrate_tasks_polling_duration_count{}[$__rate_interval])", + "interval": "", + "legendFormat": "{{task_name}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Task polling rate per second", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2571", + "format": "cps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2572", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 43, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "nodename", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$data_source" + }, + "editorMode": "code", + "expr": "increase(substrate_tasks_polling_duration_sum{}[$__rate_interval]) / increase(substrate_tasks_polling_duration_count{}[$__rate_interval])", + "interval": "", + "legendFormat": "{{task_name}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Average time it takes to call Future::poll()", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2571", + "format": "s", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:2572", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 22 + }, + "hiddenSeries": false, + "id": 15, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "nodename", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": true, + "targets": [ + { + "datasource": { + "uid": "$data_source" + }, + "editorMode": "code", + "expr": "increase(substrate_tasks_spawned_total{}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{task_name}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of tasks started", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:771", + "format": "short", + "logBase": 10, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:772", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 2, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "nodename", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$data_source" + }, + "editorMode": "code", + "expr": "substrate_tasks_spawned_total{} - sum(substrate_tasks_ended_total{}) without(reason)\n\n# Fallback if tasks_ended_total is null for that task\nor on(task_name) substrate_tasks_spawned_total{}", + "interval": "", + "legendFormat": "{{task_name}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of tasks running", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:919", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:920", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 34 + }, + "hiddenSeries": false, + "id": 7, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "nodename", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": true, + "targets": [ + { + "datasource": { + "uid": "$data_source" + }, + "editorMode": "code", + "expr": "irate(substrate_tasks_polling_duration_bucket{le=\"+Inf\"}[$__rate_interval])\n - ignoring(le)\n irate(substrate_tasks_polling_duration_bucket{le=\"1.024\"}[$__rate_interval]) > 0", + "interval": "", + "legendFormat": "{{task_name}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of calls to `Future::poll` that took more than one second", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3040", + "format": "cps", + "label": "Calls to `Future::poll`/second", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3041", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 27, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Unbounded Channels", + "type": "row" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "utc", + "title": "Substrate Service Tasks with substrate prefix", + "uid": "S7sc-M_Gk", + "version": 17, + "weekStart": "" + } \ No newline at end of file From cd18f8de2d4963c4fafe05423290c25a667be190 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 28 Nov 2023 10:14:09 +0200 Subject: [PATCH 104/192] network stats fixes Signed-off-by: Andrei Sandu --- .../subsystem-bench/src/availability/mod.rs | 10 ------ .../subsystem-bench/src/core/configuration.rs | 7 +++-- .../node/subsystem-bench/src/core/display.rs | 4 +-- .../subsystem-bench/src/core/environment.rs | 2 +- .../src/core/mock/network_bridge.rs | 31 +++++++++++++------ polkadot/node/subsystem-bench/src/core/mod.rs | 8 ----- .../node/subsystem-bench/src/core/network.rs | 23 ++++++++++++-- 7 files changed, 49 insertions(+), 36 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index ae4e743205e3..a5f1a0866a5b 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -100,16 +100,6 @@ pub fn prepare_test( prepare_test_inner(config, state, TestEnvironmentDependencies::default()) } -/// Takes a test configuration and uses it to creates the `TestEnvironment`. -#[allow(unused)] -pub fn prepare_test_with_dependencies( - config: TestConfiguration, - state: &mut TestState, - dependencies: TestEnvironmentDependencies, -) -> (TestEnvironment, ProtocolConfig) { - prepare_test_inner(config, state, dependencies) -} - fn prepare_test_inner( config: TestConfiguration, state: &mut TestState, diff --git a/polkadot/node/subsystem-bench/src/core/configuration.rs b/polkadot/node/subsystem-bench/src/core/configuration.rs index 35fa51790c91..340b5c03ab84 100644 --- a/polkadot/node/subsystem-bench/src/core/configuration.rs +++ b/polkadot/node/subsystem-bench/src/core/configuration.rs @@ -13,13 +13,14 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use std::path::Path; - +// +//! Test configuration definition and helpers. use super::*; use keyring::Keyring; +use std::{path::Path, time::Duration}; pub use crate::cli::TestObjective; -use polkadot_primitives::ValidatorId; +use polkadot_primitives::{AuthorityDiscoveryId, ValidatorId}; use rand::{distributions::Uniform, prelude::Distribution, thread_rng}; use serde::{Deserialize, Serialize}; diff --git a/polkadot/node/subsystem-bench/src/core/display.rs b/polkadot/node/subsystem-bench/src/core/display.rs index 13ea7d375e95..f21a8b907d11 100644 --- a/polkadot/node/subsystem-bench/src/core/display.rs +++ b/polkadot/node/subsystem-bench/src/core/display.rs @@ -14,8 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . // -//! Some helper methods for parsing prometheus metrics to a format that can be -//! displayed in the CLI. +//! Display implementations and helper methods for parsing prometheus metrics +//! to a format that can be displayed in the CLI. //! //! Currently histogram buckets are skipped. use super::LOG_TARGET; diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 24d10ecb1fa1..5c04071c442f 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -13,7 +13,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . - +//! Test environment implementation use crate::{ core::{mock::AlwaysSupportsParachains, network::NetworkEmulator}, TestConfiguration, diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index a45cacd0241a..c14a3895e238 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -80,7 +80,16 @@ impl MockNetworkBridgeTx { match request { Requests::ChunkFetchingV1(outgoing_request) => { + let authority_discovery_id = match outgoing_request.peer { + req_res::Recipient::Authority(authority_discovery_id) => authority_discovery_id, + _ => unimplemented!("Peer recipient not supported yet"), + }; + // Account our sent request bytes. self.network.peer_stats(0).inc_sent(outgoing_request.payload.encoded_size()); + // Account for remote received request bytes. + self.network + .peer_stats_by_id(authority_discovery_id.clone()) + .inc_received(outgoing_request.payload.encoded_size()); let validator_index: usize = outgoing_request.payload.index.0 as usize; let candidate_hash = outgoing_request.payload.candidate_hash; @@ -107,10 +116,6 @@ impl MockNetworkBridgeTx { Ok(req_res::v1::ChunkFetchingResponse::from(Some(chunk)).encode()) }; - let authority_discovery_id = match outgoing_request.peer { - req_res::Recipient::Authority(authority_discovery_id) => authority_discovery_id, - _ => unimplemented!("Peer recipient not supported yet"), - }; let authority_discovery_id_clone = authority_discovery_id.clone(); let future = async move { @@ -142,7 +147,18 @@ impl MockNetworkBridgeTx { .candidate_hashes .get(&candidate_hash) .expect("candidate was generated previously; qed"); - gum::warn!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); + gum::debug!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); + + let authority_discovery_id = match outgoing_request.peer { + req_res::Recipient::Authority(authority_discovery_id) => authority_discovery_id, + _ => unimplemented!("Peer recipient not supported yet"), + }; + // Account our sent request bytes. + self.network.peer_stats(0).inc_sent(outgoing_request.payload.encoded_size()); + // Account for remote received request bytes. + self.network + .peer_stats_by_id(authority_discovery_id.clone()) + .inc_received(outgoing_request.payload.encoded_size()); let available_data = self.availabilty.available_data.get(*candidate_index as usize).unwrap().clone(); @@ -161,10 +177,6 @@ impl MockNetworkBridgeTx { } .boxed(); - let authority_discovery_id = match outgoing_request.peer { - req_res::Recipient::Authority(authority_discovery_id) => authority_discovery_id, - _ => unimplemented!("Peer recipient not supported yet"), - }; let authority_discovery_id_clone = authority_discovery_id.clone(); let future_wrapper = async move { @@ -243,6 +255,7 @@ impl MockNetworkBridgeTx { gum::debug!(target: LOG_TARGET, request = ?request, "Processing request"); self.network.inc_sent(request_size(&request)); let action = self.respond_to_send_request(request, &mut ingress_tx); + // Will account for our node sending the request over the emulated // network. self.network.submit_peer_action(action.peer(), action); diff --git a/polkadot/node/subsystem-bench/src/core/mod.rs b/polkadot/node/subsystem-bench/src/core/mod.rs index 11ca03dbda4c..282788d143b4 100644 --- a/polkadot/node/subsystem-bench/src/core/mod.rs +++ b/polkadot/node/subsystem-bench/src/core/mod.rs @@ -14,16 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use std::{ - collections::HashMap, - sync::Arc, - time::{Duration, Instant}, -}; const LOG_TARGET: &str = "subsystem-bench::core"; -use polkadot_primitives::AuthorityDiscoveryId; -use sc_service::SpawnTaskHandle; - pub mod configuration; pub mod display; pub mod environment; diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index f36c0967466b..40809ce36e8d 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -15,8 +15,17 @@ // along with Polkadot. If not, see . use super::*; use colored::Colorize; +use polkadot_primitives::AuthorityDiscoveryId; use prometheus_endpoint::U64; -use std::sync::atomic::{AtomicU64, Ordering}; +use sc_service::SpawnTaskHandle; +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; use tokio::sync::mpsc::UnboundedSender; // An emulated node egress traffic rate_limiter. @@ -309,11 +318,20 @@ impl NetworkEmulator { self.peers[*index].send(action); } - // Returns the sent/received stats for all peers. + // Returns the sent/received stats for `peer_index`. pub fn peer_stats(&mut self, peer_index: usize) -> Arc { self.stats[peer_index].clone() } + // Returns the sent/received stats for `peer`. + pub fn peer_stats_by_id(&mut self, peer: AuthorityDiscoveryId) -> Arc { + let peer_index = self + .validator_authority_ids + .get(&peer) + .expect("all test authorities are valid; qed"); + self.stats[*peer_index].clone() + } + // Returns the sent/received stats for all peers. pub fn stats(&self) -> Vec { let r = self @@ -334,7 +352,6 @@ impl NetworkEmulator { } // Increment bytes received by our node (the node that contains the subsystem under test) - #[allow(unused)] pub fn inc_received(&self, bytes: u64) { // Our node always is peer 0. self.metrics.on_peer_received(0, bytes); From e8506b3d663a408b67cbff21749c3d273aa0c031 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 28 Nov 2023 10:16:41 +0200 Subject: [PATCH 105/192] move examples and grafana Signed-off-by: Andrei Sandu --- .../{src => }/grafana/availability-read.json | 0 .../{src => }/grafana/task-cpu-usage.json | 0 .../examples/availability_read.yaml} | 27 ++++++++++--------- 3 files changed, 14 insertions(+), 13 deletions(-) rename polkadot/node/subsystem-bench/{src => }/grafana/availability-read.json (100%) rename polkadot/node/subsystem-bench/{src => }/grafana/task-cpu-usage.json (100%) rename polkadot/node/subsystem-bench/{test_sequence.yaml => src/examples/availability_read.yaml} (75%) diff --git a/polkadot/node/subsystem-bench/src/grafana/availability-read.json b/polkadot/node/subsystem-bench/grafana/availability-read.json similarity index 100% rename from polkadot/node/subsystem-bench/src/grafana/availability-read.json rename to polkadot/node/subsystem-bench/grafana/availability-read.json diff --git a/polkadot/node/subsystem-bench/src/grafana/task-cpu-usage.json b/polkadot/node/subsystem-bench/grafana/task-cpu-usage.json similarity index 100% rename from polkadot/node/subsystem-bench/src/grafana/task-cpu-usage.json rename to polkadot/node/subsystem-bench/grafana/task-cpu-usage.json diff --git a/polkadot/node/subsystem-bench/test_sequence.yaml b/polkadot/node/subsystem-bench/src/examples/availability_read.yaml similarity index 75% rename from polkadot/node/subsystem-bench/test_sequence.yaml rename to polkadot/node/subsystem-bench/src/examples/availability_read.yaml index 088a7e15729b..889309e64a2b 100644 --- a/polkadot/node/subsystem-bench/test_sequence.yaml +++ b/polkadot/node/subsystem-bench/src/examples/availability_read.yaml @@ -1,10 +1,10 @@ TestConfiguration: # Test 1 - objective: !DataAvailabilityRead - fetch_from_backers: false + fetch_from_backers: true n_validators: 300 - n_cores: 10 - min_pov_size: 1120 + n_cores: 20 + min_pov_size: 5120 max_pov_size: 5120 peer_bandwidth: 52428800 bandwidth: 52428800 @@ -16,13 +16,14 @@ TestConfiguration: secs: 0 nanos: 100000000 error: 3 - num_blocks: 10 + num_blocks: 3 + # Test 2 - objective: !DataAvailabilityRead - fetch_from_backers: false + fetch_from_backers: true n_validators: 500 - n_cores: 10 - min_pov_size: 1120 + n_cores: 20 + min_pov_size: 5120 max_pov_size: 5120 peer_bandwidth: 52428800 bandwidth: 52428800 @@ -34,14 +35,14 @@ TestConfiguration: secs: 0 nanos: 100000000 error: 3 - num_blocks: 10 + num_blocks: 3 -# Test 2 +# Test 3 - objective: !DataAvailabilityRead - fetch_from_backers: false + fetch_from_backers: true n_validators: 1000 - n_cores: 10 - min_pov_size: 1120 + n_cores: 20 + min_pov_size: 5120 max_pov_size: 5120 peer_bandwidth: 52428800 bandwidth: 52428800 @@ -53,4 +54,4 @@ TestConfiguration: secs: 0 nanos: 100000000 error: 3 - num_blocks: 10 + num_blocks: 3 From cbb677202c14fe04c013100c7910dfeecdf26b5b Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 28 Nov 2023 14:52:42 +0200 Subject: [PATCH 106/192] Add readme Signed-off-by: Andrei Sandu --- polkadot/node/subsystem-bench/README.md | 182 +++++++++++++++++- .../{src => }/examples/availability_read.yaml | 0 .../subsystem-bench/src/availability/cli.rs | 5 +- .../subsystem-bench/src/availability/mod.rs | 2 +- polkadot/node/subsystem-bench/src/cli.rs | 2 +- .../node/subsystem-bench/src/core/display.rs | 18 +- .../subsystem-bench/src/subsystem-bench.rs | 24 +-- 7 files changed, 211 insertions(+), 22 deletions(-) rename polkadot/node/subsystem-bench/{src => }/examples/availability_read.yaml (100%) diff --git a/polkadot/node/subsystem-bench/README.md b/polkadot/node/subsystem-bench/README.md index 8843f9883116..4ed25ff9078c 100644 --- a/polkadot/node/subsystem-bench/README.md +++ b/polkadot/node/subsystem-bench/README.md @@ -1,6 +1,182 @@ # Subsystem benchmark client -Run subsystem performance tests in isolation. +Run parachain consensus stress and performance tests on your development machine. + +## Motivation +The parachain consensus node implementation spans across many modules which we call subsystems. Each subsystem is responsible for a small part of logic of the parachain consensus pipeline, but in general the most load and performance issues are localized in just a few core subsystems like `availability-recovery`, `approval-voting` or `dispute-coordinator`. In the absence of this client, we would run large test nets in order to load/stress test these parts of the system. Setting up and making sense of the amount of data produced by such a large test is very expensive, hard to orchestrate and is a huge development time sink. + +This tool aims to solve this problem by making it easy to: +- set up and run core subsystem load tests locally on your development machine +- iterate and conclude faster when benchmarking new optimizations or comparing implementations +- automate and keep track of performance regressions in CI runs +- simulate various networking topologies, bandwidth and connectivity issues + +## Test environment setup + +`cargo build --profile=testnet --bin subsystem-bench -p polkadot-subsystem-bench` + +The output binary will be placed in `target/testnet/subsystem-bench`. + +### Test metrics +Subsystem, CPU usage and network metrics are exposed via a prometheus endpoint during the test execution. +A small subset of these collected metrics are displayed in the CLI, but for an in depth analysys of the test results, a local Grafana/Prometheus stack is needed. + +### Install Prometheus +Please follow the [official installation guide](https://prometheus.io/docs/prometheus/latest/installation/) for your platform/OS. + +After succesfully installing and starting up Prometheus, we need to alter it's configuration such that it +will scrape the benchmark prometheus endpoint `127.0.0.1:9999`. Please check the prometheus official documentation +regarding the location of `prometheus.yml`. On MacOS for example the full path `/opt/homebrew/etc/prometheus.yml` + +prometheus.yml: +``` +global: + scrape_interval: 5s + +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] + - job_name: "subsystem-bench" + scrape_interval: 0s500ms + static_configs: + - targets: ['localhost:9999'] +``` + +To complete this step restart Prometheus server such that it picks up the new configuration. +### Install and setup Grafana + +Follow the [installation guide](https://grafana.com/docs/grafana/latest/setup-grafana/installation/) relevant +to your operating system. + +Once you have the installation up and running, configure the local Prometheus as a data source by following +[this guide](https://grafana.com/docs/grafana/latest/datasources/prometheus/configure-prometheus-data-source/) + +#### Import dashboards + +Follow [this guide](https://grafana.com/docs/grafana/latest/dashboards/manage-dashboards/#export-and-import-dashboards) to import the dashboards from the repository `grafana` folder. + +## Running existing tests + +To run a test, you need to first choose a test objective. Currently, we support the following: + +``` +target/testnet/subsystem-bench --help +The almighty Subsystem Benchmark Tool™️ + +Usage: subsystem-bench [OPTIONS] + +Commands: + data-availability-read Benchmark availability recovery strategies + test-sequence Run a test sequence specified in a file + help Print this message or the help of the given subcommand(s) + +``` + +The `test-sequence` is a special test objective that wraps up an arbitrary number of test objectives. It is tipically used to run a suite of tests defined in a `yaml` file like in this [example](examples/availability_read.yaml). + +### Standard test options + +``` +Options: + --network The type of network to be emulated [default: ideal] [possible values: ideal, + healthy, degraded] + --n-cores Number of cores to fetch availability for [default: 100] + --n-validators Number of validators to fetch chunks from [default: 500] + --min-pov-size The minimum pov size in KiB [default: 5120] + --max-pov-size The maximum pov size bytes [default: 5120] + -n, --num-blocks The number of blocks the test is going to run [default: 1] + -p, --peer-bandwidth The bandwidth of simulated remote peers in KiB + -b, --bandwidth The bandwidth of our simulated node in KiB + --peer-error Simulated conection error ratio [0-100] + --peer-min-latency Minimum remote peer latency in milliseconds [0-5000] + --peer-max-latency Maximum remote peer latency in milliseconds [0-5000] + -h, --help Print help + -V, --version Print version +``` + +These apply to all test objectives, except `test-sequence` which relies on the values being specified in a file. + +### Test objectives +Each test objective can have it's specific configuration options, in contrast with the standard test options. + +For `data-availability-read` the recovery strategy to be used is configurable. +``` +target/testnet/subsystem-bench data-availability-read --help +Benchmark availability recovery strategies + +Usage: subsystem-bench data-availability-read [OPTIONS] + +Options: + -f, --fetch-from-backers Turbo boost AD Read by fetching the full availability datafrom backers first. Saves CPU as we + don't need to re-construct from chunks. Tipically this is only faster if nodes have enough + bandwidth + -h, --help Print help +``` +### Understanding the test configuration +A single test configuration `TestConfiguration` struct applies to a single run of a certain test objective. + +The configuration describes the following important parameters that influence the test duration and resource +usage: +- how many validators are on the emulated network (`n_validators`) +- how many cores per block the subsystem will have to do work on (`n_cores`) +- for how many blocks the test should run (`num_blocks`) + +From the perspective of the subsystem under test, this means that it will receive an `ActiveLeavesUpdate` signal +followed by an arbitrary amount of messages. The process repeat itself for `num_blocks`. These messages are generally test payloads pre-generated before the test run, or constructed on pre-genereated paylods. For example the `AvailabilityRecoveryMessage::RecoverAvailableData` message includes a `CandidateReceipt` which is generated before the test is started. + +### Example run + +Let's run an availabilty read test which will recover availability for 10 cores with max PoV size on a 500 +node validator network. + +``` + target/testnet/subsystem-bench --n-cores 10 data-availability-read +[2023-11-28T09:01:59Z INFO subsystem_bench::core::display] n_validators = 500, n_cores = 10, pov_size = 5120 - 5120, error = 0, latency = None +[2023-11-28T09:01:59Z INFO subsystem-bench::availability] Generating template candidate index=0 pov_size=5242880 +[2023-11-28T09:01:59Z INFO subsystem-bench::availability] Created test environment. +[2023-11-28T09:01:59Z INFO subsystem-bench::availability] Pre-generating 10 candidates. +[2023-11-28T09:02:01Z INFO subsystem-bench::core] Initializing network emulation for 500 peers. +[2023-11-28T09:02:01Z INFO substrate_prometheus_endpoint] 〽️ Prometheus exporter started at 127.0.0.1:9999 +[2023-11-28T09:02:01Z INFO subsystem-bench::availability] Current block 1/1 +[2023-11-28T09:02:01Z INFO subsystem_bench::availability] 10 recoveries pending +[2023-11-28T09:02:04Z INFO subsystem_bench::availability] Block time 3231ms +[2023-11-28T09:02:04Z INFO subsystem-bench::availability] Sleeping till end of block (2768ms) +[2023-11-28T09:02:07Z INFO subsystem_bench::availability] All blocks processed in 6001ms +[2023-11-28T09:02:07Z INFO subsystem_bench::availability] Throughput: 51200 KiB/block +[2023-11-28T09:02:07Z INFO subsystem_bench::availability] Block time: 6001 ms +[2023-11-28T09:02:07Z INFO subsystem_bench::availability] + + Total received from network: 66 MiB + Total sent to network: 58 KiB + Total subsystem CPU usage 4.16s + CPU usage per block 4.16s + Total test environment CPU usage 0.00s + CPU usage per block 0.00s +``` +### Test logs + +You can select node cateogries and verbosity as with the Polkadot clien, simply setting `RUST_LOOG="parachain=debug"` turns on debug logs for all parachain consensus subsystems in the test. + +### View test metrics + +Assuming the Grafana/Prometheus stack installation steps completed succesfully, you should be able to +view the test progress in real time by accessing [this link](http://localhost:3000/goto/i1vzLpNSR?orgId=1). + +Now run `target/testnet/subsystem-bench test-sequence --path polkadot/node/subsystem-bench/examples/availability_read.yaml` and view the metrics in real time and spot differences between different `n_valiator` values. + +## Create new test objectives +This tool is intended to make it easy to write new test objectives that focus individual subsystems, +or even multiple subsystems (for example `approval-distribution` and `approval-voting`). + +A special kind of test objectives are performance regression tests for the CI pipeline. These should be sequences +of tests that check the performance characteristics (such as CPU usage, speed) of the subsystem under test in both happy and negative scenarios (low bandwidth, network errors and low connectivity). + +### Reuaseble test components +To faster write a new test objective you need to use some higher level wrappers and logic: `TestEnvironment` `TestConfiguration`, `TestAuthorities`, `NetworkEmulator`. To create the `TestEnvironment` you will +need to also build an `Overseer`, but that should be easy using the mockups for subsystems in`core::mock`. + +### Mocking +Ideally we want to have a single mock implementation for subsystems that can be minimally configured to +be used in different tests. A good example is `runtime-api` which currently only responds to session information requests based on static data. It can be easily extended to service other requests. -Currently implemented benchmarks: -* `availability-recovery` diff --git a/polkadot/node/subsystem-bench/src/examples/availability_read.yaml b/polkadot/node/subsystem-bench/examples/availability_read.yaml similarity index 100% rename from polkadot/node/subsystem-bench/src/examples/availability_read.yaml rename to polkadot/node/subsystem-bench/examples/availability_read.yaml diff --git a/polkadot/node/subsystem-bench/src/availability/cli.rs b/polkadot/node/subsystem-bench/src/availability/cli.rs index 06fb2966d878..f86f1bfb700d 100644 --- a/polkadot/node/subsystem-bench/src/availability/cli.rs +++ b/polkadot/node/subsystem-bench/src/availability/cli.rs @@ -34,7 +34,8 @@ pub enum NetworkEmulation { #[allow(missing_docs)] pub struct DataAvailabilityReadOptions { #[clap(short, long, default_value_t = false)] - /// Turbo boost AD Read by fetching from backers first. Tipically this is only faster if nodes - /// have enough bandwidth. + /// Turbo boost AD Read by fetching the full availability datafrom backers first. Saves CPU as + /// we don't need to re-construct from chunks. Tipically this is only faster if nodes have + /// enough bandwidth. pub fetch_from_backers: bool, } diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index a5f1a0866a5b..e5543e5d3904 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -312,7 +312,7 @@ fn derive_erasure_chunks_with_proofs_and_root( (erasure_chunks, root) } -pub async fn bench_chunk_recovery(env: &mut TestEnvironment, mut state: TestState) { +pub async fn benchmark_availability_read(env: &mut TestEnvironment, mut state: TestState) { let config = env.config().clone(); env.import_block(new_block_import_info(Hash::repeat_byte(1), 1)).await; diff --git a/polkadot/node/subsystem-bench/src/cli.rs b/polkadot/node/subsystem-bench/src/cli.rs index ee67a01d449e..3352f33a3503 100644 --- a/polkadot/node/subsystem-bench/src/cli.rs +++ b/polkadot/node/subsystem-bench/src/cli.rs @@ -26,7 +26,7 @@ pub struct TestSequenceOptions { /// Define the supported benchmarks targets #[derive(Debug, Clone, clap::Parser, Serialize, Deserialize)] -#[command(about = "Test objectives", version, rename_all = "kebab-case")] +#[command(rename_all = "kebab-case")] pub enum TestObjective { /// Benchmark availability recovery strategies. DataAvailabilityRead(DataAvailabilityReadOptions), diff --git a/polkadot/node/subsystem-bench/src/core/display.rs b/polkadot/node/subsystem-bench/src/core/display.rs index f21a8b907d11..629fb2edc414 100644 --- a/polkadot/node/subsystem-bench/src/core/display.rs +++ b/polkadot/node/subsystem-bench/src/core/display.rs @@ -18,7 +18,7 @@ //! to a format that can be displayed in the CLI. //! //! Currently histogram buckets are skipped. -use super::LOG_TARGET; +use super::{LOG_TARGET, configuration::TestConfiguration}; use colored::Colorize; use prometheus::{ proto::{MetricFamily, MetricType}, @@ -181,3 +181,19 @@ pub fn parse_metrics(registry: &Registry) -> MetricCollection { } test_metrics.into() } + + +pub fn display_configuration(test_config: &TestConfiguration) { + gum::info!( + "{}, {}, {}, {}, {}", + format!("n_validators = {}", test_config.n_validators).blue(), + format!("n_cores = {}", test_config.n_cores).blue(), + format!( + "pov_size = {} - {}", + test_config.min_pov_size, test_config.max_pov_size + ) + .bright_black(), + format!("error = {}", test_config.error).bright_black(), + format!("latency = {:?}", test_config.latency).bright_black(), + ); +} diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index f9261d848778..a666ee06ad55 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -35,7 +35,8 @@ use core::{ }; use clap_num::number_range; -// const LOG_TARGET: &str = "subsystem-bench"; + +use crate::core::display::display_configuration; fn le_100(s: &str) -> Result { number_range(s, 0, 100) @@ -64,7 +65,7 @@ struct BenchCli { pub bandwidth: Option, #[clap(long, value_parser=le_100)] - /// Simulated connection error rate [0-100]. + /// Simulated conection error ratio [0-100]. pub peer_error: Option, #[clap(long, value_parser=le_5000)] @@ -95,22 +96,14 @@ impl BenchCli { ); for (index, test_config) in test_sequence.into_iter().enumerate() { gum::info!( - "{}, {}, {}, {}, {}, {}", + "{}", format!("Step {}/{}", index + 1, num_steps).bright_purple(), - format!("n_validators = {}", test_config.n_validators).blue(), - format!("n_cores = {}", test_config.n_cores).blue(), - format!( - "pov_size = {} - {}", - test_config.min_pov_size, test_config.max_pov_size - ) - .bright_black(), - format!("error = {}", test_config.error).bright_black(), - format!("latency = {:?}", test_config.latency).bright_black(), ); + display_configuration(&test_config); let mut state = TestState::new(&test_config); let (mut env, _protocol_config) = prepare_test(test_config, &mut state); - env.runtime().block_on(availability::bench_chunk_recovery(&mut env, state)); + env.runtime().block_on(availability::benchmark_availability_read(&mut env, state)); } return Ok(()) }, @@ -166,10 +159,12 @@ impl BenchCli { test_config.bandwidth = bandwidth * 1024; } + display_configuration(&test_config); + let mut state = TestState::new(&test_config); let (mut env, _protocol_config) = prepare_test(test_config, &mut state); // test_config.write_to_disk(); - env.runtime().block_on(availability::bench_chunk_recovery(&mut env, state)); + env.runtime().block_on(availability::benchmark_availability_read(&mut env, state)); Ok(()) } @@ -181,6 +176,7 @@ fn main() -> eyre::Result<()> { .filter(Some("hyper"), log::LevelFilter::Info) // Avoid `Terminating due to subsystem exit subsystem` warnings .filter(Some("polkadot_overseer"), log::LevelFilter::Error) + .filter(None, log::LevelFilter::Info) // .filter(None, log::LevelFilter::Trace) .try_init() .unwrap(); From 1a8087010d36a8712482ba9f03d383e17e547623 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 28 Nov 2023 15:04:32 +0200 Subject: [PATCH 107/192] fmt + readme updates Signed-off-by: Andrei Sandu --- polkadot/node/subsystem-bench/README.md | 22 ++++++++++--------- .../subsystem-bench/src/availability/cli.rs | 4 ++-- .../node/subsystem-bench/src/core/display.rs | 12 ++++------ .../subsystem-bench/src/subsystem-bench.rs | 11 +++++----- 4 files changed, 23 insertions(+), 26 deletions(-) diff --git a/polkadot/node/subsystem-bench/README.md b/polkadot/node/subsystem-bench/README.md index 4ed25ff9078c..5b58dc3a5be4 100644 --- a/polkadot/node/subsystem-bench/README.md +++ b/polkadot/node/subsystem-bench/README.md @@ -3,9 +3,9 @@ Run parachain consensus stress and performance tests on your development machine. ## Motivation -The parachain consensus node implementation spans across many modules which we call subsystems. Each subsystem is responsible for a small part of logic of the parachain consensus pipeline, but in general the most load and performance issues are localized in just a few core subsystems like `availability-recovery`, `approval-voting` or `dispute-coordinator`. In the absence of this client, we would run large test nets in order to load/stress test these parts of the system. Setting up and making sense of the amount of data produced by such a large test is very expensive, hard to orchestrate and is a huge development time sink. +The parachain consensus node implementation spans across many modules which we call subsystems. Each subsystem is responsible for a small part of logic of the parachain consensus pipeline, but in general the most load and performance issues are localized in just a few core subsystems like `availability-recovery`, `approval-voting` or `dispute-coordinator`. In the absence such a tool, we would run large test nets to load/stress test these parts of the system. Setting up and making sense of the amount of data produced by such a large test is very expensive, hard to orchestrate and is a huge development time sink. -This tool aims to solve this problem by making it easy to: +This tool aims to solve the problem by making it easy to: - set up and run core subsystem load tests locally on your development machine - iterate and conclude faster when benchmarking new optimizations or comparing implementations - automate and keep track of performance regressions in CI runs @@ -56,7 +56,7 @@ Once you have the installation up and running, configure the local Prometheus as Follow [this guide](https://grafana.com/docs/grafana/latest/dashboards/manage-dashboards/#export-and-import-dashboards) to import the dashboards from the repository `grafana` folder. -## Running existing tests +## How to run a test To run a test, you need to first choose a test objective. Currently, we support the following: @@ -68,12 +68,10 @@ Usage: subsystem-bench [OPTIONS] Commands: data-availability-read Benchmark availability recovery strategies - test-sequence Run a test sequence specified in a file - help Print this message or the help of the given subcommand(s) ``` -The `test-sequence` is a special test objective that wraps up an arbitrary number of test objectives. It is tipically used to run a suite of tests defined in a `yaml` file like in this [example](examples/availability_read.yaml). +Note: `test-sequence` is a special test objective that wraps up an arbitrary number of test objectives. It is tipically used to run a suite of tests defined in a `yaml` file like in this [example](examples/availability_read.yaml). ### Standard test options @@ -123,7 +121,7 @@ usage: - for how many blocks the test should run (`num_blocks`) From the perspective of the subsystem under test, this means that it will receive an `ActiveLeavesUpdate` signal -followed by an arbitrary amount of messages. The process repeat itself for `num_blocks`. These messages are generally test payloads pre-generated before the test run, or constructed on pre-genereated paylods. For example the `AvailabilityRecoveryMessage::RecoverAvailableData` message includes a `CandidateReceipt` which is generated before the test is started. +followed by an arbitrary amount of messages. This process repeats itself for `num_blocks`. The messages are generally test payloads pre-generated before the test run, or constructed on pre-genereated payloads. For example the `AvailabilityRecoveryMessage::RecoverAvailableData` message includes a `CandidateReceipt` which is generated before the test is started. ### Example run @@ -154,14 +152,18 @@ node validator network. Total test environment CPU usage 0.00s CPU usage per block 0.00s ``` + +`Block time` in the context of `data-availability-read` has a different meaning. It measures the amount of time it took the subsystem to finish processing all of the messages sent in the context of the current test block. + + ### Test logs -You can select node cateogries and verbosity as with the Polkadot clien, simply setting `RUST_LOOG="parachain=debug"` turns on debug logs for all parachain consensus subsystems in the test. +You can select log target, subtarget and verbosity just like with Polkadot node CLI, simply setting `RUST_LOOG="parachain=debug"` turns on debug logs for all parachain consensus subsystems in the test. ### View test metrics Assuming the Grafana/Prometheus stack installation steps completed succesfully, you should be able to -view the test progress in real time by accessing [this link](http://localhost:3000/goto/i1vzLpNSR?orgId=1). +view the test progress in real time by accessing [this link](http://localhost:3000/goto/SM5B8pNSR?orgId=1). Now run `target/testnet/subsystem-bench test-sequence --path polkadot/node/subsystem-bench/examples/availability_read.yaml` and view the metrics in real time and spot differences between different `n_valiator` values. @@ -172,7 +174,7 @@ or even multiple subsystems (for example `approval-distribution` and `approval-v A special kind of test objectives are performance regression tests for the CI pipeline. These should be sequences of tests that check the performance characteristics (such as CPU usage, speed) of the subsystem under test in both happy and negative scenarios (low bandwidth, network errors and low connectivity). -### Reuaseble test components +### Reusable test components To faster write a new test objective you need to use some higher level wrappers and logic: `TestEnvironment` `TestConfiguration`, `TestAuthorities`, `NetworkEmulator`. To create the `TestEnvironment` you will need to also build an `Overseer`, but that should be easy using the mockups for subsystems in`core::mock`. diff --git a/polkadot/node/subsystem-bench/src/availability/cli.rs b/polkadot/node/subsystem-bench/src/availability/cli.rs index f86f1bfb700d..8da4a59253c6 100644 --- a/polkadot/node/subsystem-bench/src/availability/cli.rs +++ b/polkadot/node/subsystem-bench/src/availability/cli.rs @@ -34,8 +34,8 @@ pub enum NetworkEmulation { #[allow(missing_docs)] pub struct DataAvailabilityReadOptions { #[clap(short, long, default_value_t = false)] - /// Turbo boost AD Read by fetching the full availability datafrom backers first. Saves CPU as - /// we don't need to re-construct from chunks. Tipically this is only faster if nodes have + /// Turbo boost AD Read by fetching the full availability datafrom backers first. Saves CPU as + /// we don't need to re-construct from chunks. Tipically this is only faster if nodes have /// enough bandwidth. pub fetch_from_backers: bool, } diff --git a/polkadot/node/subsystem-bench/src/core/display.rs b/polkadot/node/subsystem-bench/src/core/display.rs index 629fb2edc414..03a5c13aeb47 100644 --- a/polkadot/node/subsystem-bench/src/core/display.rs +++ b/polkadot/node/subsystem-bench/src/core/display.rs @@ -14,11 +14,11 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . // -//! Display implementations and helper methods for parsing prometheus metrics +//! Display implementations and helper methods for parsing prometheus metrics //! to a format that can be displayed in the CLI. //! //! Currently histogram buckets are skipped. -use super::{LOG_TARGET, configuration::TestConfiguration}; +use super::{configuration::TestConfiguration, LOG_TARGET}; use colored::Colorize; use prometheus::{ proto::{MetricFamily, MetricType}, @@ -182,17 +182,13 @@ pub fn parse_metrics(registry: &Registry) -> MetricCollection { test_metrics.into() } - pub fn display_configuration(test_config: &TestConfiguration) { gum::info!( "{}, {}, {}, {}, {}", format!("n_validators = {}", test_config.n_validators).blue(), format!("n_cores = {}", test_config.n_cores).blue(), - format!( - "pov_size = {} - {}", - test_config.min_pov_size, test_config.max_pov_size - ) - .bright_black(), + format!("pov_size = {} - {}", test_config.min_pov_size, test_config.max_pov_size) + .bright_black(), format!("error = {}", test_config.error).bright_black(), format!("latency = {:?}", test_config.latency).bright_black(), ); diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index a666ee06ad55..5337a13e9729 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -95,15 +95,13 @@ impl BenchCli { format!("Sequence contains {} step(s)", num_steps).bright_purple() ); for (index, test_config) in test_sequence.into_iter().enumerate() { - gum::info!( - "{}", - format!("Step {}/{}", index + 1, num_steps).bright_purple(), - ); + gum::info!("{}", format!("Step {}/{}", index + 1, num_steps).bright_purple(),); display_configuration(&test_config); let mut state = TestState::new(&test_config); let (mut env, _protocol_config) = prepare_test(test_config, &mut state); - env.runtime().block_on(availability::benchmark_availability_read(&mut env, state)); + env.runtime() + .block_on(availability::benchmark_availability_read(&mut env, state)); } return Ok(()) }, @@ -164,7 +162,8 @@ impl BenchCli { let mut state = TestState::new(&test_config); let (mut env, _protocol_config) = prepare_test(test_config, &mut state); // test_config.write_to_disk(); - env.runtime().block_on(availability::benchmark_availability_read(&mut env, state)); + env.runtime() + .block_on(availability::benchmark_availability_read(&mut env, state)); Ok(()) } From eb49ea0277b89ecbd1cc14613dc0f6eb6e0bd9b7 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 28 Nov 2023 15:14:26 +0200 Subject: [PATCH 108/192] update dashboard and sample Signed-off-by: Andrei Sandu --- .../examples/availability_read.yaml | 6 +- .../grafana/availability-read.json | 3418 +++++++++-------- 2 files changed, 1713 insertions(+), 1711 deletions(-) diff --git a/polkadot/node/subsystem-bench/examples/availability_read.yaml b/polkadot/node/subsystem-bench/examples/availability_read.yaml index 889309e64a2b..311ea972141f 100644 --- a/polkadot/node/subsystem-bench/examples/availability_read.yaml +++ b/polkadot/node/subsystem-bench/examples/availability_read.yaml @@ -1,7 +1,7 @@ TestConfiguration: # Test 1 - objective: !DataAvailabilityRead - fetch_from_backers: true + fetch_from_backers: false n_validators: 300 n_cores: 20 min_pov_size: 5120 @@ -20,7 +20,7 @@ TestConfiguration: # Test 2 - objective: !DataAvailabilityRead - fetch_from_backers: true + fetch_from_backers: false n_validators: 500 n_cores: 20 min_pov_size: 5120 @@ -39,7 +39,7 @@ TestConfiguration: # Test 3 - objective: !DataAvailabilityRead - fetch_from_backers: true + fetch_from_backers: false n_validators: 1000 n_cores: 20 min_pov_size: 5120 diff --git a/polkadot/node/subsystem-bench/grafana/availability-read.json b/polkadot/node/subsystem-bench/grafana/availability-read.json index 4fbbe1f58731..31c4ad3c7952 100644 --- a/polkadot/node/subsystem-bench/grafana/availability-read.json +++ b/polkadot/node/subsystem-bench/grafana/availability-read.json @@ -1,1872 +1,1874 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "Subsystem and test environment metrics", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 2, - "links": [], - "liveNow": false, - "panels": [ + "annotations": { + "list": [ { + "builtIn": 1, "datasource": { - "type": "prometheus", - "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "type": "dashboard" + } + ] + }, + "description": "Subsystem and test environment metrics", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": 60000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 90, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" - }, - "editorMode": "code", - "expr": "subsystem_benchmark_n_validators{}", - "instant": false, - "legendFormat": "n_vaidators", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" - }, - "editorMode": "code", - "expr": "subsystem_benchmark_n_cores{}", - "hide": false, - "instant": false, - "legendFormat": "n_cores", - "range": true, - "refId": "B" - } - ], - "title": "Test configuration", - "type": "timeseries" + "overrides": [] }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 90, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "id": 31, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Overview", - "type": "row" + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - { - "datasource": { - "type": "prometheus", - "uid": "$data_source" + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "editorMode": "code", + "expr": "subsystem_benchmark_n_validators{}", + "instant": false, + "legendFormat": "n_vaidators", + "range": true, + "refId": "A" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + { + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "editorMode": "code", + "expr": "subsystem_benchmark_n_cores{}", + "hide": false, + "instant": false, + "legendFormat": "n_cores", + "range": true, + "refId": "B" + } + ], + "title": "Test configuration", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 31, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$data_source" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" }, - "unit": "percentunit" + "thresholdsStyle": { + "mode": "off" + } }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 57, - "interval": "1s", - "options": { - "legend": { - "calcs": [ - "mean", - "min", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "unit": "percentunit" }, - "pluginVersion": "10.0.2", - "repeat": "nodename", - "targets": [ - { - "datasource": { - "uid": "$data_source" - }, - "editorMode": "code", - "expr": "sum(rate(substrate_tasks_polling_duration_sum{}[2s])) by ($cpu_group_by)", - "interval": "", - "legendFormat": "{{task_group}}", - "range": true, - "refId": "A" - } - ], - "title": "All tasks CPU usage breakdown", - "type": "timeseries" + "overrides": [] }, - { - "datasource": { - "type": "prometheus", - "uid": "$data_source" + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 57, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "mean", + "min", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.2", + "repeat": "nodename", + "targets": [ + { + "datasource": { + "uid": "$data_source" + }, + "editorMode": "code", + "expr": "sum(rate(substrate_tasks_polling_duration_sum{}[2s])) by ($cpu_group_by)", + "interval": "", + "legendFormat": "{{task_group}}", + "range": true, + "refId": "A" + } + ], + "title": "All tasks CPU usage breakdown", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$data_source" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "area" - } + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 6 - } - ] + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" }, - "unit": "s" + "thresholdsStyle": { + "mode": "area" + } }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 93, - "interval": "1s", - "options": { - "legend": { - "calcs": [ - "mean", - "min", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 6 + } + ] }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "unit": "s" }, - "pluginVersion": "10.0.2", - "targets": [ - { - "datasource": { - "uid": "$data_source" - }, - "editorMode": "code", - "expr": "increase(substrate_tasks_polling_duration_sum{task_group=\"availability-recovery-subsystem\"}[6s])", - "interval": "", - "legendFormat": "{{task_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Availability subsystem CPU usage per block", - "type": "timeseries" + "overrides": [] }, - { - "datasource": { - "type": "prometheus", - "uid": "$data_source" + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 93, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "mean", + "min", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$data_source" + }, + "editorMode": "code", + "expr": "increase(substrate_tasks_polling_duration_sum{task_group=\"availability-recovery\"}[6s])", + "interval": "", + "legendFormat": "{{task_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Availability subsystem CPU usage per block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$data_source" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 10, - "type": "log" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "unit": "s" + "thresholdsStyle": { + "mode": "off" + } }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 30 - }, - "id": 94, - "interval": "1s", - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "unit": "s" }, - "pluginVersion": "10.0.2", - "targets": [ - { - "datasource": { - "uid": "$data_source" - }, - "editorMode": "code", - "expr": "sum(substrate_tasks_polling_duration_sum{}) by ($cpu_group_by)", - "interval": "", - "legendFormat": "{{task_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Total CPU burn", - "type": "timeseries" + "overrides": [] }, - { - "datasource": { - "type": "prometheus", - "uid": "$data_source" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 94, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$data_source" + }, + "editorMode": "code", + "expr": "sum(substrate_tasks_polling_duration_sum{}) by ($cpu_group_by)", + "interval": "", + "legendFormat": "{{task_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Total CPU burn", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$data_source" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 10, - "type": "log" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "area" - } + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "dark-red", - "value": 6000 - } - ] + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "unit": "ms" + "thresholdsStyle": { + "mode": "area" + } }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 95, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last", - "sortDesc": true + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "dark-red", + "value": 6000 + } + ] }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "unit": "ms" }, - "pluginVersion": "10.0.2", - "targets": [ - { - "datasource": { - "uid": "$data_source" - }, - "editorMode": "code", - "expr": "subsystem_benchmark_block_time", - "interval": "", - "legendFormat": "Instant block time", - "range": true, - "refId": "A" - } - ], - "title": "Block time", - "type": "timeseries" + "overrides": [] }, - { - "datasource": { - "type": "prometheus", - "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 95, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$data_source" + }, + "editorMode": "code", + "expr": "subsystem_benchmark_block_time", + "interval": "", + "legendFormat": "Instant block time", + "range": true, + "refId": "A" + } + ], + "title": "All candidates in block recovery time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "hue", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 2, - "scaleDistribution": { - "log": 2, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "log": 2, + "type": "log" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" }, - "unit": "binBps" + "thresholdsStyle": { + "mode": "off" + } }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 89, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "unit": "binBps" }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" - }, - "editorMode": "code", - "expr": "sum(rate(subsystem_benchmark_network_peer_total_bytes_sent{}[5s]))", - "instant": false, - "legendFormat": "Received", - "range": true, - "refId": "A" + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 89, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" }, - { - "datasource": { - "type": "prometheus", - "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" - }, - "editorMode": "code", - "expr": "sum(rate(subsystem_benchmark_network_peer_total_bytes_received{}[5s]))", - "hide": false, - "instant": false, - "legendFormat": "Sent", - "range": true, - "refId": "B" - } - ], - "title": "Emulated network throughput ", - "type": "timeseries" + "editorMode": "code", + "expr": "sum(rate(subsystem_benchmark_network_peer_total_bytes_received{}[5s]))", + "instant": false, + "legendFormat": "Received", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "editorMode": "code", + "expr": "sum(rate(subsystem_benchmark_network_peer_total_bytes_sent{}[5s]))", + "hide": false, + "instant": false, + "legendFormat": "Sent", + "range": true, + "refId": "B" + } + ], + "title": "Emulated network throughput ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" }, - { - "datasource": { - "type": "prometheus", - "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "unit": "bytes" + "thresholdsStyle": { + "mode": "off" + } }, - "overrides": [] - }, - "gridPos": { - "h": 15, - "w": 12, - "x": 0, - "y": 52 - }, - "id": 88, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "unit": "bytes" }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" - }, - "editorMode": "code", - "expr": "rate(subsystem_benchmark_network_peer_total_bytes_received{}[10s])", - "instant": false, - "legendFormat": "Received by {{peer}}", - "range": true, - "refId": "A" + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 52 + }, + "id": 88, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" }, - { - "datasource": { - "type": "prometheus", - "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + "editorMode": "code", + "expr": "rate(subsystem_benchmark_network_peer_total_bytes_received{}[10s])", + "instant": false, + "legendFormat": "Received by {{peer}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "editorMode": "code", + "expr": "rate(subsystem_benchmark_network_peer_total_bytes_sent{}[10s])", + "hide": false, + "instant": false, + "legendFormat": "Sent by {{peer}}", + "range": true, + "refId": "B" + } + ], + "title": "Emulated peer throughput", + "type": "timeseries" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateInferno", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "editorMode": "code", - "expr": "rate(subsystem_benchmark_network_peer_total_bytes_sent{}[10s])", - "hide": false, - "instant": false, - "legendFormat": "Sent by {{peer}}", - "range": true, - "refId": "B" + "scaleDistribution": { + "type": "linear" + } } - ], - "title": "Emulated peer throughput", - "type": "timeseries" + }, + "overrides": [] }, - { - "cards": {}, + "gridPos": { + "h": 15, + "w": 12, + "x": 12, + "y": 52 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 92, + "interval": "1s", + "legend": { + "show": true + }, + "maxDataPoints": 1340, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": { + "decimals": 0 + }, "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateInferno", "exponent": 0.5, - "mode": "spectrum" + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Inferno", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 }, - "dataFormat": "tsbuckets", - "datasource": { - "type": "prometheus", - "uid": "${data_source}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 15, - "w": 12, - "x": 12, - "y": 52 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 92, - "interval": "1s", "legend": { "show": true }, - "maxDataPoints": 1340, - "options": { - "calculate": false, - "calculation": {}, - "cellGap": 2, - "cellValues": { - "decimals": 0 - }, - "color": { - "exponent": 0.5, - "fill": "#b4ff00", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Inferno", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "showValue": "never", - "tooltip": { - "show": true, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "decimals": 0, - "reverse": false, - "unit": "bytes" - } + "rowsFrame": { + "layout": "auto" }, - "pluginVersion": "10.1.1", - "reverseYBuckets": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${data_source}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(subsystem_benchmark_pov_size_bucket{}[$__rate_interval])) by (le)", - "format": "heatmap", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "B" - } - ], - "title": "Recovered PoV sizes", + "showValue": "never", "tooltip": { "show": true, - "showHistogram": true - }, - "tooltipDecimals": 0, - "transformations": [], - "type": "heatmap", - "xAxis": { - "show": true + "yHistogram": true }, "yAxis": { + "axisPlacement": "left", "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" + "reverse": false, + "unit": "bytes" + } }, - { - "datasource": { - "type": "prometheus", - "uid": "${data_source}" - }, - "description": "Number of erasure-encoded chunks of data belonging to candidate blocks. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic", - "seriesBy": "max" + "pluginVersion": "10.1.1", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(subsystem_benchmark_pov_size_bucket{}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "B" + } + ], + "title": "Recovered PoV sizes", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 0, + "transformations": [], + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "Number of erasure-encoded chunks of data belonging to candidate blocks. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic", + "seriesBy": "max" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" }, - "unit": "chunks/s" + "thresholdsStyle": { + "mode": "off" + } }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 43, - "interval": "1s", - "maxDataPoints": 1340, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "unit": "chunks/s" }, - "pluginVersion": "8.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${data_source}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(polkadot_parachain_availability_recovery_chunk_requests_issued{}[10s]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "Chunks requested", - "queryType": "randomWalk", - "refId": "B" - } - ], - "title": "Availability", - "transformations": [], - "type": "timeseries" + "overrides": [] }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 77 + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 43, + "interval": "1s", + "maxDataPoints": 1340, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "id": 35, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(polkadot_parachain_availability_recovery_chunk_requests_issued{}[10s]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Chunks requested", + "queryType": "randomWalk", + "refId": "B" + } + ], + "title": "Availability", + "transformations": [], + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 77 + }, + "id": 35, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Availability subystem metrics", + "type": "row" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateInferno", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "refId": "A" + "scaleDistribution": { + "type": "linear" + } } - ], - "title": "Availability subystem metrics", - "type": "row" + }, + "overrides": [] }, - { - "cards": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 78 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 68, + "interval": "1s", + "legend": { + "show": true + }, + "maxDataPoints": 1340, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": { + "decimals": 0 + }, "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateInferno", "exponent": 0.5, - "mode": "spectrum" + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Inferno", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 }, - "dataFormat": "tsbuckets", - "datasource": { - "type": "prometheus", - "uid": "${data_source}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 78 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 68, - "interval": "1s", "legend": { "show": true }, - "maxDataPoints": 1340, - "options": { - "calculate": false, - "calculation": {}, - "cellGap": 2, - "cellValues": { - "decimals": 0 - }, - "color": { - "exponent": 0.5, - "fill": "#b4ff00", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Inferno", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "showValue": "never", - "tooltip": { - "show": true, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "decimals": 0, - "reverse": false, - "unit": "s" - } + "rowsFrame": { + "layout": "auto" }, - "pluginVersion": "10.1.1", - "reverseYBuckets": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${data_source}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(polkadot_parachain_availability_recovery_time_total_bucket{}[$__rate_interval])) by (le)", - "format": "heatmap", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "B" - } - ], - "title": "Time to recover a PoV", + "showValue": "never", "tooltip": { "show": true, - "showHistogram": true - }, - "tooltipDecimals": 0, - "transformations": [], - "type": "heatmap", - "xAxis": { - "show": true + "yHistogram": true }, "yAxis": { + "axisPlacement": "left", "decimals": 0, - "format": "s", - "logBase": 1, - "show": true + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.1.1", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(polkadot_parachain_availability_recovery_time_total_bucket{}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "B" + } + ], + "title": "Time to recover a PoV", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 0, + "transformations": [], + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateInferno", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } }, - "yBucketBound": "auto" + "overrides": [] }, - { - "cards": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 78 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 67, + "interval": "1s", + "legend": { + "show": true + }, + "maxDataPoints": 1340, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": { + "decimals": 0 + }, "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateInferno", "exponent": 0.5, - "mode": "spectrum" + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Inferno", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 }, - "dataFormat": "tsbuckets", - "datasource": { - "type": "prometheus", - "uid": "${data_source}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 78 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 67, - "interval": "1s", "legend": { "show": true }, - "maxDataPoints": 1340, - "options": { - "calculate": false, - "calculation": {}, - "cellGap": 2, - "cellValues": { - "decimals": 0 - }, - "color": { - "exponent": 0.5, - "fill": "#b4ff00", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Inferno", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "showValue": "never", - "tooltip": { - "show": true, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "decimals": 0, - "reverse": false, - "unit": "s" - } + "rowsFrame": { + "layout": "auto" }, - "pluginVersion": "10.1.1", - "reverseYBuckets": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${data_source}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(polkadot_parachain_availability_recovery_time_chunk_request_bucket{}[$__rate_interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "Chunk request duration", + "showValue": "never", "tooltip": { "show": true, - "showHistogram": true - }, - "tooltipDecimals": 0, - "transformations": [], - "type": "heatmap", - "xAxis": { - "show": true + "yHistogram": true }, "yAxis": { + "axisPlacement": "left", "decimals": 0, - "format": "bitfields", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" + "reverse": false, + "unit": "s" + } }, - { - "datasource": { - "type": "prometheus", - "uid": "${data_source}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic", - "seriesBy": "max" + "pluginVersion": "10.1.1", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(polkadot_parachain_availability_recovery_time_chunk_request_bucket{}[$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Chunk request duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 0, + "transformations": [], + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "bitfields", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic", + "seriesBy": "max" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" }, - "unit": "Bps" + "thresholdsStyle": { + "mode": "off" + } }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 88 - }, - "id": 85, - "interval": "1s", - "maxDataPoints": 1340, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "unit": "Bps" }, - "pluginVersion": "8.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${data_source}" + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 88 + }, + "id": 85, + "interval": "1s", + "maxDataPoints": 1340, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(polkadot_parachain_availability_recovery_bytes_total{}[30s])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Bytes recovered", + "queryType": "randomWalk", + "refId": "B" + } + ], + "title": "Recovery throughtput", + "transformations": [], + "type": "timeseries" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateInferno", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(polkadot_parachain_availability_recovery_bytes_total{}[30s])", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "Bytes recovered", - "queryType": "randomWalk", - "refId": "B" + "scaleDistribution": { + "type": "linear" + } } - ], - "title": "Recovery throughtput", - "transformations": [], - "type": "timeseries" + }, + "overrides": [] }, - { - "cards": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 88 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 84, + "interval": "1s", + "legend": { + "show": true + }, + "maxDataPoints": 1340, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": { + "decimals": 0 + }, "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateInferno", "exponent": 0.5, - "mode": "spectrum" + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Inferno", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 }, - "dataFormat": "tsbuckets", - "datasource": { - "type": "prometheus", - "uid": "${data_source}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 88 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 84, - "interval": "1s", "legend": { "show": true }, - "maxDataPoints": 1340, - "options": { - "calculate": false, - "calculation": {}, - "cellGap": 2, - "cellValues": { - "decimals": 0 - }, - "color": { - "exponent": 0.5, - "fill": "#b4ff00", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Inferno", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "showValue": "never", - "tooltip": { - "show": true, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "decimals": 0, - "reverse": false, - "unit": "s" - } + "rowsFrame": { + "layout": "auto" }, - "pluginVersion": "10.1.1", - "reverseYBuckets": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${data_source}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(polkadot_parachain_availability_reencode_chunks_bucket{}[$__rate_interval])) by (le)", - "format": "heatmap", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "B" - } - ], - "title": "Re-encoding chunks timing", + "showValue": "never", "tooltip": { "show": true, - "showHistogram": true - }, - "tooltipDecimals": 0, - "transformations": [], - "type": "heatmap", - "xAxis": { - "show": true + "yHistogram": true }, "yAxis": { + "axisPlacement": "left", "decimals": 0, - "format": "s", - "logBase": 1, - "show": true + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.1.1", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(polkadot_parachain_availability_reencode_chunks_bucket{}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "B" + } + ], + "title": "Re-encoding chunks timing", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 0, + "transformations": [], + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateInferno", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } }, - "yBucketBound": "auto" + "overrides": [] }, - { - "cards": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 98 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 83, + "interval": "1s", + "legend": { + "show": true + }, + "maxDataPoints": 1340, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": { + "decimals": 0 + }, "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateInferno", "exponent": 0.5, - "mode": "spectrum" + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Inferno", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 }, - "dataFormat": "tsbuckets", - "datasource": { - "type": "prometheus", - "uid": "${data_source}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 98 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 83, - "interval": "1s", "legend": { "show": true }, - "maxDataPoints": 1340, - "options": { - "calculate": false, - "calculation": {}, - "cellGap": 2, - "cellValues": { - "decimals": 0 - }, - "color": { - "exponent": 0.5, - "fill": "#b4ff00", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Inferno", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "showValue": "never", - "tooltip": { - "show": true, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "decimals": 0, - "reverse": false, - "unit": "s" - } + "rowsFrame": { + "layout": "auto" }, - "pluginVersion": "10.1.1", - "reverseYBuckets": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${data_source}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(polkadot_parachain_availability_recovery_time_erasure_recovery_bucket{}[$__rate_interval])) by (le)", - "format": "heatmap", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "B" - } - ], - "title": "Erasure recovery (no I/O)", + "showValue": "never", "tooltip": { "show": true, - "showHistogram": true - }, - "tooltipDecimals": 0, - "transformations": [], - "type": "heatmap", - "xAxis": { - "show": true + "yHistogram": true }, "yAxis": { + "axisPlacement": "left", "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" + "reverse": false, + "unit": "s" + } }, - { - "datasource": { - "type": "prometheus", - "uid": "${data_source}" - }, - "description": "Number of erasure-encoded chunks of data belonging to candidate blocks. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic", - "seriesBy": "max" + "pluginVersion": "10.1.1", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(polkadot_parachain_availability_recovery_time_erasure_recovery_bucket{}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "B" + } + ], + "title": "Erasure recovery (no I/O)", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 0, + "transformations": [], + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "Number of erasure-encoded chunks of data belonging to candidate blocks. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic", + "seriesBy": "max" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "stepAfter", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" }, - "unit": "cps" + "thresholdsStyle": { + "mode": "off" + } }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 108 - }, - "id": 86, - "interval": "1s", - "maxDataPoints": 1340, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "unit": "cps" }, - "pluginVersion": "8.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${data_source}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(polkadot_parachain_availability_recovery_recoveries_finished{}[1s]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "Finished", - "queryType": "randomWalk", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${data_source}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(polkadot_parachain_availability_recovery_recovieries_started{}[1s]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "Started", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "Recoveries", - "transformations": [], - "type": "timeseries" + "overrides": [] }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 118 + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 108 + }, + "id": 86, + "interval": "1s", + "maxDataPoints": 1340, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "id": 2, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Approval voting", - "type": "row" - } - ], - "refresh": "5s", - "schemaVersion": 38, - "style": "dark", - "tags": [ - "subsystem", - "benchmark" - ], - "templating": { - "list": [ + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.2.2", + "targets": [ { - "current": { - "selected": false, - "text": "Prometheus", - "value": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" - }, - "hide": 0, - "includeAll": false, - "label": "Source of data", - "multi": false, - "name": "data_source", - "options": [], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(polkadot_parachain_availability_recovery_recoveries_finished{}[1s]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Finished", + "queryType": "randomWalk", + "refId": "B" }, { - "current": { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(polkadot_parachain_availability_recovery_recovieries_started{}[1s]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Started", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Recoveries", + "transformations": [], + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 118 + }, + "id": 2, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Approval voting", + "type": "row" + } + ], + "refresh": false, + "schemaVersion": 38, + "style": "dark", + "tags": [ + "subsystem", + "benchmark" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "e56e7dd2-a992-4eec-aa96-e47b21c9020b" + }, + "hide": 0, + "includeAll": false, + "label": "Source of data", + "multi": false, + "name": "data_source", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "task_name", + "value": "task_name" + }, + "description": "Sum CPU usage by task name or task group.", + "hide": 0, + "includeAll": false, + "label": "Group CPU usage", + "multi": false, + "name": "cpu_group_by", + "options": [ + { "selected": true, "text": "task_name", "value": "task_name" }, - "description": "Sum CPU usage by task name or task group.", - "hide": 0, - "includeAll": false, - "label": "Group CPU usage", - "multi": false, - "name": "cpu_group_by", - "options": [ - { - "selected": true, - "text": "task_name", - "value": "task_name" - }, - { - "selected": false, - "text": "task_group", - "value": "task_group" - } - ], - "query": "task_name, task_group", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s" - ] - }, - "timezone": "utc", - "title": "Data Availability Read", - "uid": "asdadasd1", - "version": 56, - "weekStart": "" - } \ No newline at end of file + { + "selected": false, + "text": "task_group", + "value": "task_group" + } + ], + "query": "task_name, task_group", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "2023-11-28T13:05:32.794Z", + "to": "2023-11-28T13:06:56.173Z" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s" + ] + }, + "timezone": "utc", + "title": "Data Availability Read", + "uid": "asdadasd1", + "version": 58, + "weekStart": "" +} \ No newline at end of file From b2490560da9d8924a393a31322de2ee59e31ca72 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 28 Nov 2023 15:51:16 +0200 Subject: [PATCH 109/192] remove unused Signed-off-by: Andrei Sandu --- polkadot/node/subsystem-bench/src/availability/cli.rs | 4 ---- polkadot/node/subsystem-bench/src/availability/mod.rs | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/cli.rs b/polkadot/node/subsystem-bench/src/availability/cli.rs index 8da4a59253c6..65df8c1552aa 100644 --- a/polkadot/node/subsystem-bench/src/availability/cli.rs +++ b/polkadot/node/subsystem-bench/src/availability/cli.rs @@ -15,10 +15,6 @@ // along with Polkadot. If not, see . use serde::{Deserialize, Serialize}; -#[derive(Debug, clap::Parser, Clone)] -#[clap(rename_all = "kebab-case")] -#[allow(missing_docs)] -pub struct NetworkOptions {} #[derive(clap::ValueEnum, Clone, Copy, Debug, PartialEq)] #[value(rename_all = "kebab-case")] diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index e5543e5d3904..cbd2f8287633 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -68,7 +68,7 @@ use sc_service::SpawnTaskHandle; mod cli; pub mod configuration; -pub use cli::{DataAvailabilityReadOptions, NetworkEmulation, NetworkOptions}; +pub use cli::{DataAvailabilityReadOptions, NetworkEmulation}; pub use configuration::AvailabilityRecoveryConfiguration; fn build_overseer( From fb34181c26cc58f84ffa687fbb0b823e8f5233fb Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 28 Nov 2023 15:58:24 +0200 Subject: [PATCH 110/192] revert unneeded changes Signed-off-by: Andrei Sandu --- .../node/subsystem-test-helpers/src/lib.rs | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/polkadot/node/subsystem-test-helpers/src/lib.rs b/polkadot/node/subsystem-test-helpers/src/lib.rs index 1c3c47150ac6..3f92513498c4 100644 --- a/polkadot/node/subsystem-test-helpers/src/lib.rs +++ b/polkadot/node/subsystem-test-helpers/src/lib.rs @@ -187,7 +187,6 @@ pub struct TestSubsystemContext { tx: TestSubsystemSender, rx: mpsc::Receiver>, spawn: S, - name: &'static str, } #[async_trait::async_trait] @@ -224,7 +223,7 @@ where name: &'static str, s: Pin + Send>>, ) -> SubsystemResult<()> { - self.spawn.spawn(name, Some(self.name), s); + self.spawn.spawn(name, None, s); Ok(()) } @@ -233,7 +232,7 @@ where name: &'static str, s: Pin + Send>>, ) -> SubsystemResult<()> { - self.spawn.spawn_blocking(name, Some(self.name), s); + self.spawn.spawn_blocking(name, None, s); Ok(()) } @@ -279,13 +278,6 @@ impl TestSubsystemContextHandle { .expect("Test subsystem no longer live") } - /// Receive the next message from the subsystem. - pub async fn maybe_recv(&mut self) -> Option { - self.try_recv() - .timeout(Self::TIMEOUT) - .await - .expect("`fn recv` does not timeout") - } /// Receive the next message from the subsystem, or `None` if the channel has been closed. pub async fn try_recv(&mut self) -> Option { self.rx @@ -300,9 +292,8 @@ impl TestSubsystemContextHandle { /// of the tests. pub fn make_subsystem_context( spawner: S, - name: &'static str, ) -> (TestSubsystemContext>, TestSubsystemContextHandle) { - make_buffered_subsystem_context(spawner, 0, name) + make_buffered_subsystem_context(spawner, 0) } /// Make a test subsystem context with buffered overseer channel. Some tests (e.g. @@ -311,7 +302,6 @@ pub fn make_subsystem_context( pub fn make_buffered_subsystem_context( spawner: S, buffer_size: usize, - name: &'static str, ) -> (TestSubsystemContext>, TestSubsystemContextHandle) { let (overseer_tx, overseer_rx) = mpsc::channel(buffer_size); let (all_messages_tx, all_messages_rx) = mpsc::unbounded(); @@ -321,7 +311,6 @@ pub fn make_buffered_subsystem_context( tx: TestSubsystemSender { tx: all_messages_tx }, rx: overseer_rx, spawn: SpawnGlue(spawner), - name, }, TestSubsystemContextHandle { tx: overseer_tx, rx: all_messages_rx }, ) @@ -343,7 +332,7 @@ pub fn subsystem_test_harness( Test: Future, { let pool = TaskExecutor::new(); - let (context, handle) = make_subsystem_context(pool, "default"); + let (context, handle) = make_subsystem_context(pool); let overseer = overseer_factory(handle); let test = test_factory(context); From 3a716a54830aac41d75b9e6f411829966de0a92f Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 29 Nov 2023 12:11:08 +0200 Subject: [PATCH 111/192] add missing comments and minor fixes Signed-off-by: Andrei Sandu --- .../subsystem-bench/src/availability/mod.rs | 31 ++++++++++--------- .../node/subsystem-bench/src/core/display.rs | 4 --- .../subsystem-bench/src/core/environment.rs | 18 +++++------ .../src/core/mock/network_bridge.rs | 20 ++++++------ .../node/subsystem-bench/src/core/network.rs | 16 +++++----- 5 files changed, 43 insertions(+), 46 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index cbd2f8287633..f4c39893215b 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -105,11 +105,6 @@ fn prepare_test_inner( state: &mut TestState, dependencies: TestEnvironmentDependencies, ) -> (TestEnvironment, ProtocolConfig) { - // We need to first create the high level test state object. - // This will then be decomposed into per subsystem states. - let candidate_count = config.n_cores * config.num_blocks; - state.generate_candidates(candidate_count); - // Generate test authorities. let test_authorities = config.generate_authorities(); @@ -173,6 +168,7 @@ fn prepare_test_inner( pub struct TestState { // Full test configuration config: TestConfiguration, + // A cycle iterator on all PoV sizes used in the test. pov_sizes: Cycle>, // Generated candidate receipts to be used in the test candidates: Cycle>, @@ -181,9 +177,11 @@ pub struct TestState { // Map from generated candidate hashes to candidate index in `available_data` // and `chunks`. candidate_hashes: HashMap, - - candidate_receipts: Vec, + // Per candidate index receipts. + candidate_receipt_templates: Vec, + // Per candidate index `AvailableData` available_data: Vec, + // Per candiadte index chunks chunks: Vec>, } @@ -200,7 +198,8 @@ impl TestState { } /// Generate candidates to be used in the test. - pub fn generate_candidates(&mut self, count: usize) { + fn generate_candidates(&mut self) { + let count = self.config.n_cores * self.config.num_blocks; gum::info!(target: LOG_TARGET,"{}", format!("Pre-generating {} candidates.", count).bright_blue()); // Generate all candidates @@ -211,7 +210,8 @@ impl TestState { .pov_size_to_candidate .get(&pov_size) .expect("pov_size always exists; qed"); - let mut candidate_receipt = self.candidate_receipts[candidate_index].clone(); + let mut candidate_receipt = + self.candidate_receipt_templates[candidate_index].clone(); // Make it unique. candidate_receipt.descriptor.relay_parent = Hash::from_low_u64_be(index as u64); @@ -232,7 +232,7 @@ impl TestState { let mut chunks = Vec::new(); let mut available_data = Vec::new(); - let mut candidate_receipts = Vec::new(); + let mut candidate_receipt_templates = Vec::new(); let mut pov_size_to_candidate = HashMap::new(); // we use it for all candidates. @@ -266,22 +266,25 @@ impl TestState { chunks.push(new_chunks); available_data.push(new_available_data); pov_size_to_candidate.insert(pov_size, index); - candidate_receipts.push(candidate_receipt); + candidate_receipt_templates.push(candidate_receipt); } let pov_sizes = config.pov_sizes().to_vec().into_iter().cycle(); gum::info!(target: LOG_TARGET, "{}","Created test environment.".bright_blue()); - Self { + let mut _self = Self { config, available_data, - candidate_receipts, + candidate_receipt_templates, chunks, pov_size_to_candidate, pov_sizes, candidate_hashes: HashMap::new(), candidates: Vec::new().into_iter().cycle(), - } + }; + + _self.generate_candidates(); + _self } } diff --git a/polkadot/node/subsystem-bench/src/core/display.rs b/polkadot/node/subsystem-bench/src/core/display.rs index 03a5c13aeb47..b9ff82d1c06a 100644 --- a/polkadot/node/subsystem-bench/src/core/display.rs +++ b/polkadot/node/subsystem-bench/src/core/display.rs @@ -36,10 +36,6 @@ impl From> for MetricCollection { } impl MetricCollection { - pub fn get(&self, name: &str) -> Vec<&TestMetric> { - self.all().into_iter().filter(|metric| &metric.name == name).collect() - } - pub fn all(&self) -> &Vec { &self.0 } diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 5c04071c442f..247596474078 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -20,6 +20,7 @@ use crate::{ }; use colored::Colorize; use core::time::Duration; +use futures::FutureExt; use polkadot_overseer::{BlockInfo, Handle as OverseerHandle}; use polkadot_node_subsystem::{messages::AllMessages, Overseer, SpawnGlue, TimeoutExt}; @@ -179,23 +180,22 @@ const MAX_TIME_OF_FLIGHT: Duration = Duration::from_millis(5000); /// ### CLI /// A subset of the Prometheus metrics are printed at the end of the test. pub struct TestEnvironment { - // Test dependencies + /// Test dependencies dependencies: TestEnvironmentDependencies, - // A runtime handle + /// A runtime handle runtime_handle: tokio::runtime::Handle, - // A handle to the lovely overseer + /// A handle to the lovely overseer overseer_handle: OverseerHandle, - // The test configuration. + /// The test configuration. config: TestConfiguration, - // A handle to the network emulator. + /// A handle to the network emulator. network: NetworkEmulator, - // Configuration/env metrics + /// Configuration/env metrics metrics: TestEnvironmentMetrics, } impl TestEnvironment { - // Create a new test environment with specified initial state and prometheus registry. - // We use prometheus metrics to collect per job task poll time and subsystem metrics. + /// Create a new test environment pub fn new( dependencies: TestEnvironmentDependencies, config: TestConfiguration, @@ -207,8 +207,8 @@ impl TestEnvironment { .expect("Metrics need to be registered"); let spawn_handle = dependencies.task_manager.spawn_handle(); + spawn_handle.spawn_blocking("overseer", "overseer", overseer.run().boxed()); - spawn_handle.spawn_blocking("overseer", "overseer", overseer.run()); let registry_clone = dependencies.registry.clone(); dependencies.task_manager.spawn_handle().spawn_blocking( "prometheus", diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index c14a3895e238..2bc8d22234b6 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -219,19 +219,18 @@ impl MockNetworkBridgeTx { // Initialize our node bandwidth limits. let mut rx_limiter = RateLimit::new(10, self.config.bandwidth); - // Get a handle to our node network emulation stats. - let our_network_stats = self.network.peer_stats(0); - // This task will handle receipt of messages on our simulated network of the node. + let our_network = self.network.clone(); + + // This task will handle node messages receipt from the simulated network. let _ = ctx .spawn_blocking( - "node0-rx", + "network-receive", async move { while let Some(action) = ingress_rx.recv().await { let size = action.size(); // account for our node receiving the data. - our_network_stats.inc_received(size); - + our_network.inc_received(size); rx_limiter.reap(size).await; action.run().await; } @@ -271,12 +270,11 @@ impl MockNetworkBridgeTx { } // A helper to determine the request payload size. -fn request_size(request: &Requests) -> u64 { +fn request_size(request: &Requests) -> usize { match request { - Requests::ChunkFetchingV1(outgoing_request) => - outgoing_request.payload.encoded_size() as u64, + Requests::ChunkFetchingV1(outgoing_request) => outgoing_request.payload.encoded_size(), Requests::AvailableDataFetchingV1(outgoing_request) => - outgoing_request.payload.encoded_size() as u64, - _ => panic!("received an unexpected request"), + outgoing_request.payload.encoded_size(), + _ => unimplemented!("received an unexpected request"), } } diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 40809ce36e8d..67dc0e0f267e 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -225,12 +225,12 @@ impl PeerEmulatorStats { pub fn inc_sent(&self, bytes: usize) { self.tx_bytes_total.fetch_add(bytes as u64, Ordering::Relaxed); - self.metrics.on_peer_sent(self.peer_index, bytes as u64); + self.metrics.on_peer_sent(self.peer_index, bytes); } pub fn inc_received(&self, bytes: usize) { self.rx_bytes_total.fetch_add(bytes as u64, Ordering::Relaxed); - self.metrics.on_peer_received(self.peer_index, bytes as u64); + self.metrics.on_peer_received(self.peer_index, bytes); } pub fn sent(&self) -> u64 { @@ -346,13 +346,13 @@ impl NetworkEmulator { } // Increment bytes sent by our node (the node that contains the subsystem under test) - pub fn inc_sent(&self, bytes: u64) { + pub fn inc_sent(&self, bytes: usize) { // Our node always is peer 0. self.metrics.on_peer_sent(0, bytes); } // Increment bytes received by our node (the node that contains the subsystem under test) - pub fn inc_received(&self, bytes: u64) { + pub fn inc_received(&self, bytes: usize) { // Our node always is peer 0. self.metrics.on_peer_received(0, bytes); } @@ -398,16 +398,16 @@ impl Metrics { } /// Increment total sent for a peer. - pub fn on_peer_sent(&self, peer_index: usize, bytes: u64) { + pub fn on_peer_sent(&self, peer_index: usize, bytes: usize) { self.peer_total_sent .with_label_values(vec![format!("node{}", peer_index).as_str()].as_slice()) - .inc_by(bytes); + .inc_by(bytes as u64); } /// Increment total receioved for a peer. - pub fn on_peer_received(&self, peer_index: usize, bytes: u64) { + pub fn on_peer_received(&self, peer_index: usize, bytes: usize) { self.peer_total_received .with_label_values(vec![format!("node{}", peer_index).as_str()].as_slice()) - .inc_by(bytes); + .inc_by(bytes as u64); } } From a092b764aad74194632d70224d1c2b53bd15dd63 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 29 Nov 2023 12:49:37 +0200 Subject: [PATCH 112/192] clippy Signed-off-by: Andrei Sandu --- polkadot/node/subsystem-bench/src/availability/mod.rs | 2 +- polkadot/node/subsystem-bench/src/core/network.rs | 2 -- polkadot/node/subsystem-bench/src/subsystem-bench.rs | 5 ----- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index f4c39893215b..7d6865fee958 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -120,7 +120,7 @@ fn prepare_test_inner( }; let network = NetworkEmulator::new( - config.n_validators.clone(), + config.n_validators, test_authorities.validator_authority_id, config.peer_bandwidth, dependencies.task_manager.spawn_handle(), diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 67dc0e0f267e..3d38a8f36b19 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -96,8 +96,6 @@ impl RateLimit { #[cfg(test)] mod tests { - use super::*; - use polkadot_node_metrics::metered::CoarseDuration; use std::time::Instant; use super::RateLimit; diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 5337a13e9729..0f3ae0f41417 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -184,8 +184,3 @@ fn main() -> eyre::Result<()> { cli.launch()?; Ok(()) } - -#[cfg(test)] -mod tests { - use super::*; -} From ca27370c275a0a5f3f10823b4e8ccf1dae3f1a36 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 29 Nov 2023 13:00:12 +0200 Subject: [PATCH 113/192] zepter format features --fix Signed-off-by: Andrei Sandu --- polkadot/node/network/availability-recovery/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/network/availability-recovery/Cargo.toml b/polkadot/node/network/availability-recovery/Cargo.toml index 4a3f5c26e7b9..3d77652acd03 100644 --- a/polkadot/node/network/availability-recovery/Cargo.toml +++ b/polkadot/node/network/availability-recovery/Cargo.toml @@ -40,4 +40,4 @@ polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } polkadot-primitives-test-helpers = { path = "../../../primitives/test-helpers" } [features] -subsystem-benchmarks = [] \ No newline at end of file +subsystem-benchmarks = [] From be814e554ae2445e1835e95ee7f9780519e643c8 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 29 Nov 2023 14:24:15 +0200 Subject: [PATCH 114/192] fix markdown Signed-off-by: Andrei Sandu --- polkadot/node/subsystem-bench/README.md | 82 +++++++++++++++++-------- 1 file changed, 57 insertions(+), 25 deletions(-) diff --git a/polkadot/node/subsystem-bench/README.md b/polkadot/node/subsystem-bench/README.md index 5b58dc3a5be4..351e07b6abca 100644 --- a/polkadot/node/subsystem-bench/README.md +++ b/polkadot/node/subsystem-bench/README.md @@ -3,9 +3,16 @@ Run parachain consensus stress and performance tests on your development machine. ## Motivation -The parachain consensus node implementation spans across many modules which we call subsystems. Each subsystem is responsible for a small part of logic of the parachain consensus pipeline, but in general the most load and performance issues are localized in just a few core subsystems like `availability-recovery`, `approval-voting` or `dispute-coordinator`. In the absence such a tool, we would run large test nets to load/stress test these parts of the system. Setting up and making sense of the amount of data produced by such a large test is very expensive, hard to orchestrate and is a huge development time sink. + +The parachain consensus node implementation spans across many modules which we call subsystems. Each subsystem is +responsible for a small part of logic of the parachain consensus pipeline, but in general the most load and +performance issues are localized in just a few core subsystems like `availability-recovery`, `approval-voting` or +`dispute-coordinator`. In the absence such a tool, we would run large test nets to load/stress test these parts of +the system. Setting up and making sense of the amount of data produced by such a large test is very expensive, hard +to orchestrate and is a huge development time sink. This tool aims to solve the problem by making it easy to: + - set up and run core subsystem load tests locally on your development machine - iterate and conclude faster when benchmarking new optimizations or comparing implementations - automate and keep track of performance regressions in CI runs @@ -18,17 +25,22 @@ This tool aims to solve the problem by making it easy to: The output binary will be placed in `target/testnet/subsystem-bench`. ### Test metrics + Subsystem, CPU usage and network metrics are exposed via a prometheus endpoint during the test execution. -A small subset of these collected metrics are displayed in the CLI, but for an in depth analysys of the test results, a local Grafana/Prometheus stack is needed. +A small subset of these collected metrics are displayed in the CLI, but for an in depth analysys of the test results, +a local Grafana/Prometheus stack is needed. ### Install Prometheus -Please follow the [official installation guide](https://prometheus.io/docs/prometheus/latest/installation/) for your platform/OS. + +Please follow the [official installation guide](https://prometheus.io/docs/prometheus/latest/installation/) for your +platform/OS. After succesfully installing and starting up Prometheus, we need to alter it's configuration such that it will scrape the benchmark prometheus endpoint `127.0.0.1:9999`. Please check the prometheus official documentation -regarding the location of `prometheus.yml`. On MacOS for example the full path `/opt/homebrew/etc/prometheus.yml` +regarding the location of `prometheus.yml`. On MacOS for example the full path `/opt/homebrew/etc/prometheus.yml` prometheus.yml: + ``` global: scrape_interval: 5s @@ -44,6 +56,7 @@ scrape_configs: ``` To complete this step restart Prometheus server such that it picks up the new configuration. + ### Install and setup Grafana Follow the [installation guide](https://grafana.com/docs/grafana/latest/setup-grafana/installation/) relevant @@ -54,7 +67,8 @@ Once you have the installation up and running, configure the local Prometheus as #### Import dashboards -Follow [this guide](https://grafana.com/docs/grafana/latest/dashboards/manage-dashboards/#export-and-import-dashboards) to import the dashboards from the repository `grafana` folder. +Follow [this guide](https://grafana.com/docs/grafana/latest/dashboards/manage-dashboards/#export-and-import-dashboards) +to import the dashboards from the repository `grafana` folder. ## How to run a test @@ -71,14 +85,15 @@ Commands: ``` -Note: `test-sequence` is a special test objective that wraps up an arbitrary number of test objectives. It is tipically used to run a suite of tests defined in a `yaml` file like in this [example](examples/availability_read.yaml). +Note: `test-sequence` is a special test objective that wraps up an arbitrary number of test objectives. It is tipically + used to run a suite of tests defined in a `yaml` file like in this [example](examples/availability_read.yaml). ### Standard test options ``` Options: - --network The type of network to be emulated [default: ideal] [possible values: ideal, - healthy, degraded] + --network The type of network to be emulated [default: ideal] [possible values: + ideal, healthy, degraded] --n-cores Number of cores to fetch availability for [default: 100] --n-validators Number of validators to fetch chunks from [default: 500] --min-pov-size The minimum pov size in KiB [default: 5120] @@ -96,9 +111,11 @@ Options: These apply to all test objectives, except `test-sequence` which relies on the values being specified in a file. ### Test objectives + Each test objective can have it's specific configuration options, in contrast with the standard test options. For `data-availability-read` the recovery strategy to be used is configurable. + ``` target/testnet/subsystem-bench data-availability-read --help Benchmark availability recovery strategies @@ -106,31 +123,38 @@ Benchmark availability recovery strategies Usage: subsystem-bench data-availability-read [OPTIONS] Options: - -f, --fetch-from-backers Turbo boost AD Read by fetching the full availability datafrom backers first. Saves CPU as we - don't need to re-construct from chunks. Tipically this is only faster if nodes have enough - bandwidth + -f, --fetch-from-backers Turbo boost AD Read by fetching the full availability datafrom backers first. Saves CPU + as we don't need to re-construct from chunks. Tipically this is only faster if nodes + have enough bandwidth -h, --help Print help ``` + ### Understanding the test configuration + A single test configuration `TestConfiguration` struct applies to a single run of a certain test objective. The configuration describes the following important parameters that influence the test duration and resource usage: + - how many validators are on the emulated network (`n_validators`) - how many cores per block the subsystem will have to do work on (`n_cores`) - for how many blocks the test should run (`num_blocks`) -From the perspective of the subsystem under test, this means that it will receive an `ActiveLeavesUpdate` signal -followed by an arbitrary amount of messages. This process repeats itself for `num_blocks`. The messages are generally test payloads pre-generated before the test run, or constructed on pre-genereated payloads. For example the `AvailabilityRecoveryMessage::RecoverAvailableData` message includes a `CandidateReceipt` which is generated before the test is started. +From the perspective of the subsystem under test, this means that it will receive an `ActiveLeavesUpdate` signal +followed by an arbitrary amount of messages. This process repeats itself for `num_blocks`. The messages are generally +test payloads pre-generated before the test run, or constructed on pre-genereated payloads. For example the +`AvailabilityRecoveryMessage::RecoverAvailableData` message includes a `CandidateReceipt` which is generated before +the test is started. -### Example run +### Example run Let's run an availabilty read test which will recover availability for 10 cores with max PoV size on a 500 node validator network. ``` target/testnet/subsystem-bench --n-cores 10 data-availability-read -[2023-11-28T09:01:59Z INFO subsystem_bench::core::display] n_validators = 500, n_cores = 10, pov_size = 5120 - 5120, error = 0, latency = None +[2023-11-28T09:01:59Z INFO subsystem_bench::core::display] n_validators = 500, n_cores = 10, pov_size = 5120 - 5120, + error = 0, latency = None [2023-11-28T09:01:59Z INFO subsystem-bench::availability] Generating template candidate index=0 pov_size=5242880 [2023-11-28T09:01:59Z INFO subsystem-bench::availability] Created test environment. [2023-11-28T09:01:59Z INFO subsystem-bench::availability] Pre-generating 10 candidates. @@ -153,32 +177,40 @@ node validator network. CPU usage per block 0.00s ``` -`Block time` in the context of `data-availability-read` has a different meaning. It measures the amount of time it took the subsystem to finish processing all of the messages sent in the context of the current test block. - +`Block time` in the context of `data-availability-read` has a different meaning. It measures the amount of time it +took the subsystem to finish processing all of the messages sent in the context of the current test block. ### Test logs -You can select log target, subtarget and verbosity just like with Polkadot node CLI, simply setting `RUST_LOOG="parachain=debug"` turns on debug logs for all parachain consensus subsystems in the test. +You can select log target, subtarget and verbosity just like with Polkadot node CLI, simply setting +`RUST_LOOG="parachain=debug"` turns on debug logs for all parachain consensus subsystems in the test. ### View test metrics -Assuming the Grafana/Prometheus stack installation steps completed succesfully, you should be able to +Assuming the Grafana/Prometheus stack installation steps completed succesfully, you should be able to view the test progress in real time by accessing [this link](http://localhost:3000/goto/SM5B8pNSR?orgId=1). -Now run `target/testnet/subsystem-bench test-sequence --path polkadot/node/subsystem-bench/examples/availability_read.yaml` and view the metrics in real time and spot differences between different `n_valiator` values. +Now run +`target/testnet/subsystem-bench test-sequence --path polkadot/node/subsystem-bench/examples/availability_read.yaml` +and view the metrics in real time and spot differences between different `n_valiator` values. ## Create new test objectives -This tool is intended to make it easy to write new test objectives that focus individual subsystems, + +This tool is intended to make it easy to write new test objectives that focus individual subsystems, or even multiple subsystems (for example `approval-distribution` and `approval-voting`). A special kind of test objectives are performance regression tests for the CI pipeline. These should be sequences -of tests that check the performance characteristics (such as CPU usage, speed) of the subsystem under test in both happy and negative scenarios (low bandwidth, network errors and low connectivity). +of tests that check the performance characteristics (such as CPU usage, speed) of the subsystem under test in both +happy and negative scenarios (low bandwidth, network errors and low connectivity). ### Reusable test components -To faster write a new test objective you need to use some higher level wrappers and logic: `TestEnvironment` `TestConfiguration`, `TestAuthorities`, `NetworkEmulator`. To create the `TestEnvironment` you will + +To faster write a new test objective you need to use some higher level wrappers and logic: `TestEnvironment`, +`TestConfiguration`, `TestAuthorities`, `NetworkEmulator`. To create the `TestEnvironment` you will need to also build an `Overseer`, but that should be easy using the mockups for subsystems in`core::mock`. ### Mocking -Ideally we want to have a single mock implementation for subsystems that can be minimally configured to -be used in different tests. A good example is `runtime-api` which currently only responds to session information requests based on static data. It can be easily extended to service other requests. +Ideally we want to have a single mock implementation for subsystems that can be minimally configured to +be used in different tests. A good example is `runtime-api` which currently only responds to session information +requests based on static data. It can be easily extended to service other requests. From 11ce8f5121ed5d6448a77e46647a4a0ffcfad066 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 29 Nov 2023 18:01:40 +0200 Subject: [PATCH 115/192] remove sleep till end of block Signed-off-by: Andrei Sandu --- polkadot/node/subsystem-bench/src/availability/mod.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 7d6865fee958..77888fa6058c 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -363,13 +363,9 @@ pub async fn benchmark_availability_read(env: &mut TestEnvironment, mut state: T let block_time = Instant::now().sub(block_start_ts).as_millis() as u64; env.metrics().set_block_time(block_time); - gum::info!("Block time {}", format!("{:?}ms", block_time).cyan()); - gum::info!(target: LOG_TARGET,"{}", format!("Sleeping till end of block ({}ms)", block_time_delta.as_millis()).bright_black()); - tokio::time::sleep(block_time_delta).await; + gum::info!("All work for block completed in {}", format!("{:?}ms", block_time).cyan()); } - env.stop().await; - let duration: u128 = start_marker.elapsed().as_millis(); let availability_bytes = availability_bytes / 1024; gum::info!("All blocks processed in {}", format!("{:?}ms", duration).cyan()); @@ -384,4 +380,5 @@ pub async fn benchmark_availability_read(env: &mut TestEnvironment, mut state: T ); gum::info!("{}", &env); + env.stop().await; } From 8d93abc6dd73a7cf82668b17c570235e3a4d7dbc Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 29 Nov 2023 20:04:37 +0200 Subject: [PATCH 116/192] review Signed-off-by: Andrei Sandu --- .../network/availability-recovery/src/lib.rs | 2 +- polkadot/node/subsystem-bench/README.md | 2 +- .../src/availability/configuration.rs | 24 ------------------- .../subsystem-bench/src/availability/mod.rs | 13 +--------- .../subsystem-bench/src/core/mock/av_store.rs | 2 +- .../src/core/mock/network_bridge.rs | 2 +- 6 files changed, 5 insertions(+), 40 deletions(-) delete mode 100644 polkadot/node/subsystem-bench/src/availability/configuration.rs diff --git a/polkadot/node/network/availability-recovery/src/lib.rs b/polkadot/node/network/availability-recovery/src/lib.rs index c454028b8650..d029bce04173 100644 --- a/polkadot/node/network/availability-recovery/src/lib.rs +++ b/polkadot/node/network/availability-recovery/src/lib.rs @@ -856,7 +856,7 @@ async fn erasure_task_thread( } // In benchmarks this is a very hot loop not yielding at all. - // To update promehteus metrics for the task we need to yield. + // To update CPU metrics for the task we need to yield. #[cfg(feature = "subsystem-benchmarks")] tokio::task::yield_now().await; } diff --git a/polkadot/node/subsystem-bench/README.md b/polkadot/node/subsystem-bench/README.md index 351e07b6abca..f4ea04662f9e 100644 --- a/polkadot/node/subsystem-bench/README.md +++ b/polkadot/node/subsystem-bench/README.md @@ -7,7 +7,7 @@ Run parachain consensus stress and performance tests on your development machine The parachain consensus node implementation spans across many modules which we call subsystems. Each subsystem is responsible for a small part of logic of the parachain consensus pipeline, but in general the most load and performance issues are localized in just a few core subsystems like `availability-recovery`, `approval-voting` or -`dispute-coordinator`. In the absence such a tool, we would run large test nets to load/stress test these parts of +`dispute-coordinator`. In the absence of such a tool, we would run large test nets to load/stress test these parts of the system. Setting up and making sense of the amount of data produced by such a large test is very expensive, hard to orchestrate and is a huge development time sink. diff --git a/polkadot/node/subsystem-bench/src/availability/configuration.rs b/polkadot/node/subsystem-bench/src/availability/configuration.rs deleted file mode 100644 index 1274862a8e4a..000000000000 --- a/polkadot/node/subsystem-bench/src/availability/configuration.rs +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use serde::{Deserialize, Serialize}; - -/// The test input parameters -#[derive(Clone, Default, Debug, Serialize, Deserialize)] -pub struct AvailabilityRecoveryConfiguration { - /// Prefer the fast path (try fetch from backers first) - pub use_fast_path: bool, -} diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 77888fa6058c..ca2e800d4c89 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -14,13 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . use itertools::Itertools; -use std::{ - collections::HashMap, - iter::Cycle, - ops::Sub, - sync::Arc, - time::{Duration, Instant}, -}; +use std::{collections::HashMap, iter::Cycle, ops::Sub, sync::Arc, time::Instant}; use crate::TestEnvironment; use polkadot_node_subsystem::{Overseer, OverseerConnector, SpawnGlue}; @@ -67,9 +61,7 @@ use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; use sc_service::SpawnTaskHandle; mod cli; -pub mod configuration; pub use cli::{DataAvailabilityReadOptions, NetworkEmulation}; -pub use configuration::AvailabilityRecoveryConfiguration; fn build_overseer( spawn_task_handle: SpawnTaskHandle, @@ -358,9 +350,6 @@ pub async fn benchmark_availability_read(env: &mut TestEnvironment, mut state: T availability_bytes += available_data.encoded_size() as u128; } - let block_time_delta = - Duration::from_secs(6).saturating_sub(Instant::now().sub(block_start_ts)); - let block_time = Instant::now().sub(block_start_ts).as_millis() as u64; env.metrics().set_block_time(block_time); gum::info!("All work for block completed in {}", format!("{:?}ms", block_time).cyan()); diff --git a/polkadot/node/subsystem-bench/src/core/mock/av_store.rs b/polkadot/node/subsystem-bench/src/core/mock/av_store.rs index 1ff7d1728af9..88747affc8c0 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/av_store.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/av_store.rs @@ -129,7 +129,7 @@ impl MockAvailabilityStore { let _ = tx.send(Some(chunk_size)); }, _ => { - unimplemented!("Unexpected runtime-api message") + unimplemented!("Unexpected av-store message") }, }, } diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index 2bc8d22234b6..53f4fb9631f2 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -261,7 +261,7 @@ impl MockNetworkBridgeTx { } }, _ => { - unimplemented!("Unexpected runtime-api message") + unimplemented!("Unexpected network bridge message") }, }, } From af141eefcc198926f7da7734228e208e8411deac Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 1 Dec 2023 15:28:09 +0200 Subject: [PATCH 117/192] Emulated network improvements Signed-off-by: Andrei Sandu --- .../subsystem-bench/src/availability/mod.rs | 8 +- .../subsystem-bench/src/core/configuration.rs | 33 ++++- .../src/core/mock/network_bridge.rs | 55 +++++++- .../node/subsystem-bench/src/core/network.rs | 128 ++++++++++++++---- 4 files changed, 184 insertions(+), 40 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index ca2e800d4c89..244119735966 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -111,13 +111,7 @@ fn prepare_test_inner( chunks: state.chunks.clone(), }; - let network = NetworkEmulator::new( - config.n_validators, - test_authorities.validator_authority_id, - config.peer_bandwidth, - dependencies.task_manager.spawn_handle(), - &dependencies.registry, - ); + let network = NetworkEmulator::new(&config, &dependencies, &test_authorities); let network_bridge_tx = network_bridge::MockNetworkBridgeTx::new( config.clone(), diff --git a/polkadot/node/subsystem-bench/src/core/configuration.rs b/polkadot/node/subsystem-bench/src/core/configuration.rs index 340b5c03ab84..adb5ce80c0d4 100644 --- a/polkadot/node/subsystem-bench/src/core/configuration.rs +++ b/polkadot/node/subsystem-bench/src/core/configuration.rs @@ -43,6 +43,21 @@ pub struct PeerLatency { pub max_latency: Duration, } +// Default PoV size in KiB. +fn default_pov_size() -> usize { + 5120 +} + +// Default bandwidth in bytes +fn default_bandwidth() -> usize { + 52428800 +} + +// Default connectivity percentage +fn default_connectivity() -> usize { + 100 +} + /// The test input parameters #[derive(Clone, Debug, Serialize, Deserialize)] pub struct TestConfiguration { @@ -53,22 +68,31 @@ pub struct TestConfiguration { /// Number of cores pub n_cores: usize, /// The min PoV size + #[serde(default = "default_pov_size")] pub min_pov_size: usize, /// The max PoV size, + #[serde(default = "default_pov_size")] pub max_pov_size: usize, /// Randomly sampled pov_sizes #[serde(skip)] pov_sizes: Vec, /// The amount of bandiwdth remote validators have. + #[serde(default = "default_bandwidth")] pub peer_bandwidth: usize, /// The amount of bandiwdth our node has. + #[serde(default = "default_bandwidth")] pub bandwidth: usize, /// Optional peer emulation latency + #[serde(default)] pub latency: Option, - /// Error probability + /// Error probability, applies to sending messages to the emulated network peers + #[serde(default)] pub error: usize, - /// Number of blocks - /// In one block `n_cores` candidates are recovered + /// Connectivity ratio, the percentage of peers we are not connected to, but ar part of + /// the topology. + #[serde(default = "default_connectivity")] + pub connectivity: usize, + /// Number of blocks to run the test for pub num_blocks: usize, } @@ -166,6 +190,7 @@ impl TestConfiguration { num_blocks, min_pov_size, max_pov_size, + connectivity: 100, } } @@ -192,6 +217,7 @@ impl TestConfiguration { num_blocks, min_pov_size, max_pov_size, + connectivity: 95, } } @@ -218,6 +244,7 @@ impl TestConfiguration { num_blocks, min_pov_size, max_pov_size, + connectivity: 67, } } } diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index 53f4fb9631f2..fa4730209183 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -16,10 +16,10 @@ //! //! A generic av store subsystem mockup suitable to be used in benchmarks. +use futures::Future; use parity_scale_codec::Encode; use polkadot_node_subsystem_types::OverseerSignal; - -use std::collections::HashMap; +use std::{collections::HashMap, pin::Pin}; use futures::FutureExt; @@ -35,6 +35,7 @@ use polkadot_node_subsystem::{ use polkadot_node_network_protocol::request_response::{ self as req_res, v1::ChunkResponse, Requests, }; +use polkadot_primitives::AuthorityDiscoveryId; use crate::core::{ configuration::{random_error, random_latency, TestConfiguration}, @@ -71,7 +72,24 @@ impl MockNetworkBridgeTx { Self { config, availabilty, network } } - pub fn respond_to_send_request( + fn not_connected_response( + &self, + authority_discovery_id: &AuthorityDiscoveryId, + future: Pin + Send>>, + ) -> NetworkAction { + // The network action will send the error after a random delay expires. + return NetworkAction::new( + authority_discovery_id.clone(), + future, + 0, + // Generate a random latency based on configuration. + random_latency(self.config.latency.as_ref()), + ) + } + /// Returns an `NetworkAction` corresponding to the peer sending the response. If + /// the peer is connected, the error is sent with a randomized latency as defined in + /// configuration. + fn respond_to_send_request( &mut self, request: Requests, ingress_tx: &mut tokio::sync::mpsc::UnboundedSender, @@ -86,9 +104,23 @@ impl MockNetworkBridgeTx { }; // Account our sent request bytes. self.network.peer_stats(0).inc_sent(outgoing_request.payload.encoded_size()); + + // If peer is disconnected return an error to the caller + if !self.network.is_peer_connected(&authority_discovery_id) { + // We always send `NotConnected` error and we ignore `IfDisconnected` value in + // the caller. + let future = async move { + let _ = outgoing_request + .pending_response + .send(Err(RequestFailure::NotConnected)); + } + .boxed(); + return self.not_connected_response(&authority_discovery_id, future) + } + // Account for remote received request bytes. self.network - .peer_stats_by_id(authority_discovery_id.clone()) + .peer_stats_by_id(&authority_discovery_id) .inc_received(outgoing_request.payload.encoded_size()); let validator_index: usize = outgoing_request.payload.index.0 as usize; @@ -153,11 +185,24 @@ impl MockNetworkBridgeTx { req_res::Recipient::Authority(authority_discovery_id) => authority_discovery_id, _ => unimplemented!("Peer recipient not supported yet"), }; + // Account our sent request bytes. self.network.peer_stats(0).inc_sent(outgoing_request.payload.encoded_size()); + + // If peer is disconnected return an error to the caller + if !self.network.is_peer_connected(&authority_discovery_id) { + let future = async move { + let _ = outgoing_request + .pending_response + .send(Err(RequestFailure::NotConnected)); + } + .boxed(); + return self.not_connected_response(&authority_discovery_id, future) + } + // Account for remote received request bytes. self.network - .peer_stats_by_id(authority_discovery_id.clone()) + .peer_stats_by_id(&authority_discovery_id) .inc_received(outgoing_request.payload.encoded_size()); let available_data = diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 3d38a8f36b19..09943becb65c 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -13,10 +13,15 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use super::*; +use super::{ + configuration::{TestAuthorities, TestConfiguration}, + environment::TestEnvironmentDependencies, + *, +}; use colored::Colorize; use polkadot_primitives::AuthorityDiscoveryId; use prometheus_endpoint::U64; +use rand::{seq::SliceRandom, thread_rng}; use sc_service::SpawnTaskHandle; use std::{ collections::HashMap, @@ -268,44 +273,97 @@ impl NetworkAction { } } +/// The state of a peer on the emulated network. +#[derive(Clone)] +enum Peer { + Connected(PeerEmulator), + Disconnected(PeerEmulator), +} + +impl Peer { + pub fn disconnect(&mut self) { + let new_self = match self { + Peer::Connected(peer) => Peer::Disconnected(peer.clone()), + _ => return, + }; + *self = new_self; + } + + pub fn is_connected(&self) -> bool { + if let Peer::Connected(_) = self { + true + } else { + false + } + } + + pub fn emulator(&mut self) -> &mut PeerEmulator { + match self { + Peer::Connected(ref mut emulator) => emulator, + Peer::Disconnected(ref mut emulator) => emulator, + } + } +} + /// Mocks the network bridge and an arbitrary number of connected peer nodes. /// Implements network latency, bandwidth and connection errors. #[derive(Clone)] pub struct NetworkEmulator { // Per peer network emulation. - peers: Vec, + peers: Vec, /// Per peer stats. stats: Vec>, - /// Network throughput metrics - metrics: Metrics, /// Each emulated peer is a validator. validator_authority_ids: HashMap, } impl NetworkEmulator { pub fn new( - n_peers: usize, - validator_authority_ids: Vec, - bandwidth: usize, - spawn_task_handle: SpawnTaskHandle, - registry: &Registry, + config: &TestConfiguration, + dependencies: &TestEnvironmentDependencies, + authorities: &TestAuthorities, ) -> Self { - gum::info!(target: LOG_TARGET, "{}",format!("Initializing network emulation for {} peers.", n_peers).bright_blue()); + let n_peers = config.n_validators; + gum::info!(target: LOG_TARGET, "{}",format!("Initializing emulation for a {} peer network.", n_peers).bright_blue()); + gum::info!(target: LOG_TARGET, "{}",format!("connectivity {}%, error {}%", config.connectivity, config.error).bright_black()); - let metrics = Metrics::new(®istry).expect("Metrics always register succesfully"); + let metrics = + Metrics::new(&dependencies.registry).expect("Metrics always register succesfully"); let mut validator_authority_id_mapping = HashMap::new(); // Create a `PeerEmulator` for each peer. - let (stats, peers) = (0..n_peers) - .zip(validator_authority_ids.into_iter()) + let (stats, mut peers): (_, Vec<_>) = (0..n_peers) + .zip(authorities.validator_authority_id.clone().into_iter()) .map(|(peer_index, authority_id)| { validator_authority_id_mapping.insert(authority_id, peer_index); let stats = Arc::new(PeerEmulatorStats::new(peer_index, metrics.clone())); - (stats.clone(), PeerEmulator::new(bandwidth, spawn_task_handle.clone(), stats)) + ( + stats.clone(), + Peer::Connected(PeerEmulator::new( + config.peer_bandwidth, + dependencies.task_manager.spawn_handle(), + stats, + )), + ) }) .unzip(); - Self { peers, stats, metrics, validator_authority_ids: validator_authority_id_mapping } + let connected_count = config.n_validators as f64 / (100.0 / config.connectivity as f64); + + let (_connected, to_disconnect) = + peers.partial_shuffle(&mut thread_rng(), connected_count as usize); + + for peer in to_disconnect { + peer.disconnect(); + } + + gum::info!(target: LOG_TARGET, "{}",format!("Network created, connected validator count {}", connected_count).bright_black()); + + Self { peers, stats, validator_authority_ids: validator_authority_id_mapping } + } + + pub fn is_peer_connected(&self, peer: &AuthorityDiscoveryId) -> bool { + self.peer(peer).is_connected() } pub fn submit_peer_action(&mut self, peer: AuthorityDiscoveryId, action: NetworkAction) { @@ -313,21 +371,41 @@ impl NetworkEmulator { .validator_authority_ids .get(&peer) .expect("all test authorities are valid; qed"); - self.peers[*index].send(action); + + let peer = self.peers.get_mut(*index).expect("We just retrieved the index above; qed"); + + // Only actions of size 0 are allowed on disconnected peers. + // Typically this are delayed error response sends. + if action.size() > 0 && !peer.is_connected() { + gum::warn!(target: LOG_TARGET, peer_index = index, "Attempted to send data from a disconnected peer, operation ignored"); + return + } + + peer.emulator().send(action); } // Returns the sent/received stats for `peer_index`. - pub fn peer_stats(&mut self, peer_index: usize) -> Arc { + pub fn peer_stats(&self, peer_index: usize) -> Arc { self.stats[peer_index].clone() } - // Returns the sent/received stats for `peer`. - pub fn peer_stats_by_id(&mut self, peer: AuthorityDiscoveryId) -> Arc { - let peer_index = self + // Helper to get peer index by `AuthorityDiscoveryId` + fn peer_index(&self, peer: &AuthorityDiscoveryId) -> usize { + *self .validator_authority_ids - .get(&peer) - .expect("all test authorities are valid; qed"); - self.stats[*peer_index].clone() + .get(peer) + .expect("all test authorities are valid; qed") + } + + // Return the Peer entry for a given `AuthorityDiscoveryId`. + fn peer(&self, peer: &AuthorityDiscoveryId) -> &Peer { + &self.peers[self.peer_index(peer)] + } + // Returns the sent/received stats for `peer`. + pub fn peer_stats_by_id(&mut self, peer: &AuthorityDiscoveryId) -> Arc { + let peer_index = self.peer_index(peer); + + self.stats[peer_index].clone() } // Returns the sent/received stats for all peers. @@ -346,13 +424,13 @@ impl NetworkEmulator { // Increment bytes sent by our node (the node that contains the subsystem under test) pub fn inc_sent(&self, bytes: usize) { // Our node always is peer 0. - self.metrics.on_peer_sent(0, bytes); + self.peer_stats(0).inc_sent(bytes); } // Increment bytes received by our node (the node that contains the subsystem under test) pub fn inc_received(&self, bytes: usize) { // Our node always is peer 0. - self.metrics.on_peer_received(0, bytes); + self.peer_stats(0).inc_received(bytes); } } From 29d80fa638ea4315319d7e96ec391e80d3a2350c Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 1 Dec 2023 16:21:47 +0200 Subject: [PATCH 118/192] fix comment Signed-off-by: Andrei Sandu --- polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index fa4730209183..c8140843b3b9 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -105,7 +105,7 @@ impl MockNetworkBridgeTx { // Account our sent request bytes. self.network.peer_stats(0).inc_sent(outgoing_request.payload.encoded_size()); - // If peer is disconnected return an error to the caller + // If peer is disconnected return an error if !self.network.is_peer_connected(&authority_discovery_id) { // We always send `NotConnected` error and we ignore `IfDisconnected` value in // the caller. @@ -189,7 +189,7 @@ impl MockNetworkBridgeTx { // Account our sent request bytes. self.network.peer_stats(0).inc_sent(outgoing_request.payload.encoded_size()); - // If peer is disconnected return an error to the caller + // If peer is disconnected return an error if !self.network.is_peer_connected(&authority_discovery_id) { let future = async move { let _ = outgoing_request From 84721eb4c423988bb4858e5cefd2630438a323fa Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 4 Dec 2023 12:55:17 +0200 Subject: [PATCH 119/192] Approval-voting subsytem draf1 Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 10 + .../node/core/approval-voting/src/criteria.rs | 8 +- polkadot/node/core/approval-voting/src/lib.rs | 4 +- .../node/core/approval-voting/src/time.rs | 4 +- .../network/approval-distribution/src/lib.rs | 2 +- .../approval-distribution/src/metrics.rs | 1 + polkadot/node/subsystem-bench/Cargo.toml | 15 +- .../examples/approvals_throughput.yaml | 22 + .../src/approval/mock_chain_api.rs | 80 + .../src/approval/mock_chain_selection.rs | 64 + .../src/approval/mock_runtime_api.rs | 83 + .../node/subsystem-bench/src/approval/mod.rs | 1334 +++++++++++++++++ .../subsystem-bench/src/availability/mod.rs | 11 +- polkadot/node/subsystem-bench/src/cli.rs | 4 + .../subsystem-bench/src/core/configuration.rs | 13 +- .../subsystem-bench/src/core/environment.rs | 3 +- .../node/subsystem-bench/src/core/keyring.rs | 6 +- .../node/subsystem-bench/src/core/mock/mod.rs | 18 +- .../src/core/mock/network_bridge.rs | 99 +- .../node/subsystem-bench/src/core/network.rs | 53 +- .../subsystem-bench/src/subsystem-bench.rs | 33 +- 21 files changed, 1818 insertions(+), 49 deletions(-) create mode 100644 polkadot/node/subsystem-bench/examples/approvals_throughput.yaml create mode 100644 polkadot/node/subsystem-bench/src/approval/mock_chain_api.rs create mode 100644 polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs create mode 100644 polkadot/node/subsystem-bench/src/approval/mock_runtime_api.rs create mode 100644 polkadot/node/subsystem-bench/src/approval/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 2cc35a275424..917237077aef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13445,12 +13445,15 @@ dependencies = [ "futures", "futures-timer", "itertools 0.11.0", + "kvdb-memorydb", "log", "orchestra", "parity-scale-codec", "paste", + "polkadot-approval-distribution", "polkadot-availability-recovery", "polkadot-erasure-coding", + "polkadot-node-core-approval-voting", "polkadot-node-metrics", "polkadot-node-network-protocol", "polkadot-node-primitives", @@ -13463,15 +13466,22 @@ dependencies = [ "polkadot-primitives-test-helpers", "prometheus", "rand 0.8.5", + "rand_chacha 0.3.1", + "rand_core 0.5.1", "sc-keystore", "sc-network", "sc-service", + "schnorrkel 0.9.1", "serde", "serde_yaml", "sp-application-crypto", + "sp-consensus", + "sp-consensus-babe", "sp-core", "sp-keyring", "sp-keystore", + "sp-runtime", + "sp-timestamp", "substrate-prometheus-endpoint", "tokio", "tracing-gum", diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 2bb5a151fe23..911645135512 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -53,11 +53,11 @@ pub struct OurAssignment { } impl OurAssignment { - pub(crate) fn cert(&self) -> &AssignmentCertV2 { + pub fn cert(&self) -> &AssignmentCertV2 { &self.cert } - pub(crate) fn tranche(&self) -> DelayTranche { + pub fn tranche(&self) -> DelayTranche { self.tranche } @@ -223,7 +223,7 @@ fn assigned_core_transcript(core_index: CoreIndex) -> Transcript { /// Information about the world assignments are being produced in. #[derive(Clone, Debug)] -pub(crate) struct Config { +pub struct Config { /// The assignment public keys for validators. assignment_keys: Vec, /// The groups of validators assigned to each core. @@ -317,7 +317,7 @@ impl AssignmentCriteria for RealAssignmentCriteria { /// different times. The idea is that most assignments are never triggered and fall by the wayside. /// /// This will not assign to anything the local validator was part of the backing group for. -pub(crate) fn compute_assignments( +pub fn compute_assignments( keystore: &LocalKeystore, relay_vrf_story: RelayVRFStory, config: &Config, diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index ef01727b7eb6..36cf3039308c 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -88,11 +88,11 @@ use time::{slot_number_to_tick, Clock, ClockExt, SystemClock, Tick}; mod approval_checking; pub mod approval_db; mod backend; -mod criteria; +pub mod criteria; mod import; mod ops; mod persisted_entries; -mod time; +pub mod time; use crate::{ approval_db::v2::{Config as DatabaseConfig, DbBackend}, diff --git a/polkadot/node/core/approval-voting/src/time.rs b/polkadot/node/core/approval-voting/src/time.rs index a45866402c82..248e5d8be0a5 100644 --- a/polkadot/node/core/approval-voting/src/time.rs +++ b/polkadot/node/core/approval-voting/src/time.rs @@ -40,7 +40,7 @@ pub(crate) trait Clock { } /// Extension methods for clocks. -pub(crate) trait ClockExt { +pub trait ClockExt { fn tranche_now(&self, slot_duration_millis: u64, base_slot: Slot) -> DelayTranche; } @@ -52,7 +52,7 @@ impl ClockExt for C { } /// A clock which uses the actual underlying system clock. -pub(crate) struct SystemClock; +pub struct SystemClock; impl Clock for SystemClock { /// Yields the current tick. diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 47482eef7640..603232cabf8c 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -58,7 +58,7 @@ use std::{ time::Duration, }; -mod metrics; +pub mod metrics; #[cfg(test)] mod tests; diff --git a/polkadot/node/network/approval-distribution/src/metrics.rs b/polkadot/node/network/approval-distribution/src/metrics.rs index 6864259e6fdb..252a4abd018c 100644 --- a/polkadot/node/network/approval-distribution/src/metrics.rs +++ b/polkadot/node/network/approval-distribution/src/metrics.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +//! [`ApprovalDistribution`] metrics implementation. use polkadot_node_metrics::metrics::{prometheus, Metrics as MetricsTrait}; use polkadot_node_primitives::approval::v2::AssignmentCertKindV2; diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index f775a1ff9efe..977427c68ea0 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -23,6 +23,7 @@ polkadot-node-primitives = { path = "../primitives" } polkadot-primitives = { path = "../../primitives" } polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-availability-recovery = { path = "../network/availability-recovery", features=["subsystem-benchmarks"]} + color-eyre = { version = "0.6.1", default-features = false } polkadot-overseer = { path = "../overseer" } colored = "2.0.4" @@ -42,7 +43,7 @@ rand = "0.8.5" parity-scale-codec = { version = "3.6.1", features = ["std", "derive"] } tokio = "1.24.2" clap-num = "1.0.2" -polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" } +polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers"} sp-keyring = { path = "../../../substrate/primitives/keyring" } sp-application-crypto = { path = "../../../substrate/primitives/application-crypto" } sc-network = { path = "../../../substrate/client/network" } @@ -55,7 +56,19 @@ prometheus = { version = "0.13.0", default-features = false } serde = "1.0.192" serde_yaml = "0.9" paste = "1.0.14" + +polkadot-node-core-approval-voting = { path = "../core/approval-voting"} +polkadot-approval-distribution = {path = "../network/approval-distribution"} +kvdb-memorydb = "0.13.0" +sp-consensus = { path = "../../../substrate/primitives/consensus/common", default-features = false } +sp-consensus-babe = { path = "../../../substrate/primitives/consensus/babe" } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-timestamp = { path = "../../../substrate/primitives/timestamp" } orchestra = { version = "0.3.3", default-features = false, features=["futures_channel"] } +schnorrkel = { version = "0.9.1", default-features = false } +rand_core = "0.5.1" # should match schnorrkel +rand_chacha = { version = "0.3.1" } + [features] default = [] diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml new file mode 100644 index 000000000000..9fe46aa03d8d --- /dev/null +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml @@ -0,0 +1,22 @@ +TestConfiguration: +# Test 1 +- objective: !ApprovalsTest + last_considered_tranche: 89 + min_coalesce: 1 + max_coalesce: 1 + n_validators: 500 + n_cores: 100 + n_included_candidates: 70 + min_pov_size: 1120 + max_pov_size: 5120 + peer_bandwidth: 524288000000 + bandwidth: 524288000000 + latency: + min_latency: + secs: 0 + nanos: 1000000 + max_latency: + secs: 0 + nanos: 100000000 + error: 3 + num_blocks: 10 diff --git a/polkadot/node/subsystem-bench/src/approval/mock_chain_api.rs b/polkadot/node/subsystem-bench/src/approval/mock_chain_api.rs new file mode 100644 index 000000000000..8c6c3ee780b3 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/approval/mock_chain_api.rs @@ -0,0 +1,80 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use super::ApprovalTestState; +use futures::FutureExt; +use itertools::Itertools; +use polkadot_node_subsystem::{overseer, SpawnedSubsystem, SubsystemError}; +use polkadot_node_subsystem_types::messages::ChainApiMessage; + +/// Mock ChainApi subsystem used to answer request made by the approval-voting subsystem, during +/// benchmark. All the necessary information to answer the requests is stored in the `state` +pub struct MockChainApi { + pub state: ApprovalTestState, +} +#[overseer::subsystem(ChainApi, error=SubsystemError, prefix=self::overseer)] +impl MockChainApi { + fn start(self, ctx: Context) -> SpawnedSubsystem { + let future = self.run(ctx).map(|_| Ok(())).boxed(); + + SpawnedSubsystem { name: "chain-api-subsystem", future } + } +} + +#[overseer::contextbounds(ChainApi, prefix = self::overseer)] +impl MockChainApi { + async fn run(self, mut ctx: Context) { + loop { + let msg = ctx.recv().await.expect("Should not fail"); + match msg { + orchestra::FromOrchestra::Signal(_) => {}, + orchestra::FromOrchestra::Communication { msg } => match msg { + ChainApiMessage::FinalizedBlockNumber(val) => { + val.send(Ok(0)).unwrap(); + }, + ChainApiMessage::BlockHeader(requested_hash, sender) => { + let info = self.state.get_info_by_hash(requested_hash); + sender.send(Ok(Some(info.header.clone()))).unwrap(); + }, + ChainApiMessage::FinalizedBlockHash(requested_number, sender) => { + let hash = self.state.get_info_by_number(requested_number).hash; + sender.send(Ok(Some(hash))).unwrap(); + }, + ChainApiMessage::BlockNumber(requested_hash, sender) => { + sender + .send(Ok(Some( + self.state.get_info_by_hash(requested_hash).block_number, + ))) + .unwrap(); + }, + ChainApiMessage::Ancestors { hash, k: _, response_channel } => { + let position = self + .state + .per_slot_heads + .iter() + .find_position(|block_info| block_info.hash == hash) + .unwrap(); + let (ancestors, _) = self.state.per_slot_heads.split_at(position.0); + + let ancestors = ancestors.iter().rev().map(|val| val.hash).collect_vec(); + response_channel.send(Ok(ancestors)).unwrap(); + }, + _ => {}, + }, + } + } + } +} diff --git a/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs b/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs new file mode 100644 index 000000000000..986bcf5db78f --- /dev/null +++ b/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs @@ -0,0 +1,64 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use crate::approval::{LOG_TARGET, SLOT_DURATION_MILLIS}; + +use super::ApprovalTestState; +use futures::FutureExt; +use polkadot_node_subsystem::{overseer, SpawnedSubsystem, SubsystemError}; +use polkadot_node_subsystem_types::messages::ChainSelectionMessage; +use sp_timestamp::Timestamp; + +/// Mock ChainSelection subsystem used to answer request made by the approval-voting subsystem, +/// during benchmark. All the necessary information to answer the requests is stored in the `state` +pub struct MockChainSelection { + pub state: ApprovalTestState, +} +#[overseer::subsystem(ChainSelection, error=SubsystemError, prefix=self::overseer)] +impl MockChainSelection { + fn start(self, ctx: Context) -> SpawnedSubsystem { + let future = self.run(ctx).map(|_| Ok(())).boxed(); + + SpawnedSubsystem { name: "mock-chain-subsystem", future } + } +} + +#[overseer::contextbounds(ChainSelection, prefix = self::overseer)] +impl MockChainSelection { + async fn run(self, mut ctx: Context) { + loop { + let msg = ctx.recv().await.expect("Should not fail"); + match msg { + orchestra::FromOrchestra::Signal(_) => {}, + orchestra::FromOrchestra::Communication { msg } => match msg { + ChainSelectionMessage::Approved(hash) => { + let block_info = self.state.get_info_by_hash(hash); + let approved_number = block_info.block_number; + + block_info.approved.store(true, std::sync::atomic::Ordering::SeqCst); + self.state + .last_approved_block + .store(approved_number, std::sync::atomic::Ordering::SeqCst); + let passed_since_slot_start = Timestamp::current().as_millis() - + *block_info.slot * SLOT_DURATION_MILLIS; + gum::info!(target: LOG_TARGET, ?hash, "Chain selection approved after {:} ms", passed_since_slot_start); + }, + _ => {}, + }, + } + } + } +} diff --git a/polkadot/node/subsystem-bench/src/approval/mock_runtime_api.rs b/polkadot/node/subsystem-bench/src/approval/mock_runtime_api.rs new file mode 100644 index 000000000000..7021aec68c37 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/approval/mock_runtime_api.rs @@ -0,0 +1,83 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use super::ApprovalTestState; +use futures::FutureExt; +use polkadot_node_subsystem::{overseer, SpawnedSubsystem, SubsystemError}; +use polkadot_node_subsystem_types::messages::{RuntimeApiMessage, RuntimeApiRequest}; +use polkadot_primitives::ExecutorParams; + +/// Mock RuntimeApi subsystem used to answer request made by the approval-voting subsystem, +/// during benchmark. All the necessary information to answer the requests is stored in the `state` +pub struct MockRuntimeApi { + pub state: ApprovalTestState, +} +#[overseer::subsystem(RuntimeApi, error=SubsystemError, prefix=self::overseer)] +impl MockRuntimeApi { + fn start(self, ctx: Context) -> SpawnedSubsystem { + let future = self.run(ctx).map(|_| Ok(())).boxed(); + + SpawnedSubsystem { name: "runtime-api-subsystem", future } + } +} + +#[overseer::contextbounds(RuntimeApi, prefix = self::overseer)] +impl MockRuntimeApi { + async fn run(self, mut ctx: Context) { + loop { + let msg = ctx.recv().await.expect("Should not fail"); + + match msg { + orchestra::FromOrchestra::Signal(_) => {}, + orchestra::FromOrchestra::Communication { msg } => match msg { + RuntimeApiMessage::Request( + request, + RuntimeApiRequest::CandidateEvents(sender), + ) => { + let candidate_events = + self.state.get_info_by_hash(request).candidates.clone(); + let _ = sender.send(Ok(candidate_events)); + }, + RuntimeApiMessage::Request( + _request, + RuntimeApiRequest::SessionIndexForChild(sender), + ) => { + let _ = sender.send(Ok(1)); + }, + RuntimeApiMessage::Request( + _request, + RuntimeApiRequest::SessionInfo(_session_index, sender), + ) => { + let _ = sender.send(Ok(Some(self.state.session_info.clone()))); + }, + RuntimeApiMessage::Request( + _request, + RuntimeApiRequest::SessionExecutorParams(_session_index, sender), + ) => { + let _ = sender.send(Ok(Some(ExecutorParams::default()))); + }, + RuntimeApiMessage::Request( + _request, + RuntimeApiRequest::CurrentBabeEpoch(sender), + ) => { + let _ = sender.send(Ok(self.state.babe_epoch.clone())); + }, + _ => {}, + }, + } + } + } +} diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs new file mode 100644 index 000000000000..220d05a08b1b --- /dev/null +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -0,0 +1,1334 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use serde::{Deserialize, Serialize}; +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + sync::{ + atomic::{AtomicBool, AtomicU32}, + Arc, + }, + time::{Duration, Instant}, +}; + +use colored::Colorize; +use futures::{channel::oneshot, FutureExt}; +use itertools::Itertools; +use orchestra::TimeoutExt; +use overseer::{metrics::Metrics as OverseerMetrics, MetricsTrait}; +use polkadot_approval_distribution::{ + metrics::Metrics as ApprovalDistributionMetrics, ApprovalDistribution, +}; +use polkadot_node_core_approval_voting::{ + criteria::{compute_assignments, Config}, + time::{ClockExt, SystemClock}, + ApprovalVotingSubsystem, Metrics as ApprovalVotingMetrics, +}; +use polkadot_node_primitives::approval::{ + self, + v1::{IndirectSignedApprovalVote, RelayVRFStory}, + v2::{CoreBitfield, IndirectAssignmentCertV2}, +}; + +use polkadot_node_network_protocol::{ + grid_topology::{ + GridNeighbors, RandomRouting, RequiredRouting, SessionGridTopology, TopologyPeerInfo, + }, + peer_set::{ProtocolVersion, ValidationVersion}, + vstaging as protocol_vstaging, ObservedRole, Versioned, View, +}; +use polkadot_node_subsystem::{overseer, AllMessages, Overseer, OverseerConnector, SpawnGlue}; +use polkadot_node_subsystem_test_helpers::mock::new_block_import_info; +use polkadot_overseer::Handle as OverseerHandleReal; + +use polkadot_node_core_approval_voting::Config as ApprovalVotingConfig; +use polkadot_node_subsystem_types::messages::{ + network_bridge_event::NewGossipTopology, ApprovalDistributionMessage, NetworkBridgeEvent, +}; + +use rand::{seq::SliceRandom, RngCore, SeedableRng}; +use rand_chacha::ChaCha20Rng; + +use polkadot_primitives::{ + ApprovalVote, BlockNumber, CandidateEvent, CandidateHash, CandidateIndex, CandidateReceipt, + CoreIndex, GroupIndex, Hash, Header, Id as ParaId, IndexedVec, SessionInfo, Slot, + ValidatorIndex, ValidatorPair, ASSIGNMENT_KEY_TYPE_ID, +}; +use polkadot_primitives_test_helpers::dummy_candidate_receipt_bad_sig; +use sc_keystore::LocalKeystore; +use sc_network::PeerId; +use sc_service::SpawnTaskHandle; +use sp_consensus_babe::{ + digests::{CompatibleDigestItem, PreDigest, SecondaryVRFPreDigest}, + AllowedSlots, BabeEpochConfiguration, Epoch as BabeEpoch, SlotDuration, VrfSignature, + VrfTranscript, +}; +use sp_core::{crypto::VrfSecret, Pair}; +use sp_keystore::Keystore; +use sp_runtime::{Digest, DigestItem}; +use std::ops::Sub; + +use sp_keyring::sr25519::Keyring as Sr25519Keyring; +use sp_timestamp::Timestamp; + +use crate::{ + approval::{ + mock_chain_api::MockChainApi, mock_chain_selection::MockChainSelection, + mock_runtime_api::MockRuntimeApi, + }, + core::{ + configuration::{TestAuthorities, TestConfiguration}, + environment::{TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT}, + keyring::Keyring, + mock::{dummy_builder, AlwaysSupportsParachains, MockNetworkBridgeTx, TestSyncOracle}, + network::{NetworkAction, NetworkEmulator}, + }, +}; + +use tokio::time::sleep; + +mod mock_chain_api; +mod mock_chain_selection; +mod mock_runtime_api; + +pub const LOG_TARGET: &str = "bench::approval"; +const DATA_COL: u32 = 0; +pub(crate) const NUM_COLUMNS: u32 = 1; +pub(crate) const SLOT_DURATION_MILLIS: u64 = 6000; +pub(crate) const TEST_CONFIG: ApprovalVotingConfig = ApprovalVotingConfig { + col_approval_data: DATA_COL, + slot_duration_millis: SLOT_DURATION_MILLIS, +}; + +pub const NODE_UNDER_TEST: u32 = 0; + +/// Start generating messages for a slot into the future, so that the +/// generation nevers falls behind the current slot. +const BUFFER_FOR_GENERATION_MILLIS: u64 = 6_000; + +/// Parameters specific to the approvals benchmark +#[derive(Debug, Clone, Serialize, Deserialize, clap::Parser)] +#[clap(rename_all = "kebab-case")] +#[allow(missing_docs)] +pub struct ApprovalsOptions { + #[clap(short, long, default_value_t = 89)] + /// The last considered tranche for which we should generate message, this does not + /// mean the message is sent, because if the block is approved no other message is sent + /// anymore. + pub last_considered_tranche: u32, + #[clap(short, long, default_value_t = 1)] + /// Min candidates to be signed in a single approval. + pub min_coalesce: u32, + #[clap(short, long, default_value_t = 89)] + /// Max candidate to be signed in a single approval. + pub max_coalesce: u32, +} + +/// Information about a block. It is part of test state and it is used by the mock +/// subsystems to be able to answer the calls approval-voting and approval-distribution +/// do into the outside world. +#[derive(Clone, Debug)] +struct BlockTestData { + /// The slot this block occupies, see implementer's guide to understand what a slot + /// is in the context of polkadot. + slot: Slot, + /// The hash of the block. + hash: Hash, + /// The block number. + block_number: BlockNumber, + /// The list of candidates included in this block. + candidates: Vec, + /// The block header. + header: Header, + /// The vrf story for the given block. + relay_vrf_story: RelayVRFStory, + /// If the block has been approved by the approval-voting subsystem. + /// This set on `true` when ChainSelectionMessage::Approved is received inside the chain + /// selection mock subsystem. + approved: Arc, +} + +/// Approval test state used by all mock subsystems to be able to answer messages emitted +/// by the approval-voting and approval-distribution-subystems. +/// +/// This gets cloned across all mock subsystems, so if there is any information that gets +/// updated between subsystems, they would have to be wrapped in Arc's. +#[derive(Clone, Debug)] +pub struct ApprovalTestState { + /// The generic test configuration passed when starting the benchmark. + configuration: TestConfiguration, + /// The specific test configurations passed when starting the benchmark. + options: ApprovalsOptions, + /// The list of blocks used for testing. + per_slot_heads: Vec, + /// The babe epoch used during testing. + babe_epoch: BabeEpoch, + /// The session info used during testing. + session_info: SessionInfo, + /// An array of pre-generated random samplings, that is used to determine, which nodes would + /// send a given assignment, to the node under test because of the random samplings. + /// As an optimization we generate this sampling at the begining of the test and just pick + /// one randomly, because always taking the samples would be too expensive for benchamrk. + random_samplings: Vec>, + /// The slot at which this benchamrk begins. + initial_slot: Slot, + /// The test authorities + test_authorities: TestAuthorities, + /// Last approved block number. + last_approved_block: Arc, +} + +impl ApprovalTestState { + /// Build a new `ApprovalTestState` object out of the configurations passed when the benchmark + /// was tested. + fn new(configuration: &TestConfiguration, options: ApprovalsOptions) -> Self { + let test_authorities = configuration.generate_authorities(); + + let random_samplings = random_samplings_to_node_patterns( + ValidatorIndex(NODE_UNDER_TEST), + test_authorities.keyrings.len(), + test_authorities.keyrings.len() as usize * 2, + ); + + let delta_to_first_slot_under_test = Timestamp::new(BUFFER_FOR_GENERATION_MILLIS); + let initial_slot = Slot::from_timestamp( + Timestamp::current() + delta_to_first_slot_under_test, + SlotDuration::from_millis(SLOT_DURATION_MILLIS), + ); + + let babe_epoch = generate_babe_epoch(initial_slot, test_authorities.clone()); + let session_info = session_info_for_peers(configuration, test_authorities.clone()); + + let mut state = ApprovalTestState { + per_slot_heads: Default::default(), + babe_epoch: babe_epoch.clone(), + session_info: session_info.clone(), + random_samplings, + configuration: configuration.clone(), + initial_slot, + test_authorities, + last_approved_block: Arc::new(AtomicU32::new(0)), + options, + }; + state.generate_blocks_information(); + gum::info!("Built testing state"); + + state + } + + /// Generates the blocks and the information about the blocks that will be used + /// to drive this test. + fn generate_blocks_information(&mut self) { + for block_number in 1..self.configuration.num_blocks + 1 { + let block_hash = Hash::repeat_byte(block_number as u8); + let parent_hash = self + .per_slot_heads + .last() + .map(|val| val.hash) + .unwrap_or(Hash::repeat_byte(0xde)); + let slot_for_block = self.initial_slot + (block_number as u64 - 1); + + let header = make_header(parent_hash, slot_for_block, block_number as u32); + + let unsafe_vrf = approval::v1::babe_unsafe_vrf_info(&header) + .expect("Can not continue without vrf generator"); + let relay_vrf_story = unsafe_vrf + .compute_randomness( + &self.babe_epoch.authorities, + &self.babe_epoch.randomness, + self.babe_epoch.epoch_index, + ) + .expect("Can not continue without vrf story"); + + let block_info = BlockTestData { + slot: slot_for_block, + block_number: block_number as BlockNumber, + hash: block_hash, + header, + candidates: make_candidates( + block_hash, + block_number as BlockNumber, + self.configuration.n_cores as u32, + self.configuration.n_included_candidates as u32, + ), + relay_vrf_story, + approved: Arc::new(AtomicBool::new(false)), + }; + self.per_slot_heads.push(block_info) + } + } + + /// Starts the generation of messages(Assignments & Approvals) needed for approving blocks. + fn start_message_generation( + &mut self, + network_emulator: &NetworkEmulator, + overseer_handle: OverseerHandleReal, + spawn_task_handle: &SpawnTaskHandle, + ) { + gum::info!(target: LOG_TARGET, "Start assignments/approvals generation"); + + let topology = generate_topology(&self.test_authorities); + + let topology_node_under_test = + topology.compute_grid_neighbors_for(ValidatorIndex(NODE_UNDER_TEST)).unwrap(); + + for current_validator_index in 1..self.test_authorities.keyrings.len() { + let peer_message_source = PeerMessagesGenerator { + topology_node_under_test: topology_node_under_test.clone(), + topology: topology.clone(), + validator_index: ValidatorIndex(current_validator_index as u32), + network: network_emulator.clone(), + overseer_handle: overseer_handle.clone(), + state: self.clone(), + options: self.options.clone(), + }; + + peer_message_source.generate_messages(&spawn_task_handle); + } + } +} + +impl ApprovalTestState { + /// Returns test data for the given hash + fn get_info_by_hash(&self, requested_hash: Hash) -> &BlockTestData { + self.per_slot_heads + .iter() + .filter(|block| block.hash == requested_hash) + .next() + .expect("Mocks should not use unknown hashes") + } + + /// Returns test data for the given block number + fn get_info_by_number(&self, requested_number: u32) -> &BlockTestData { + self.per_slot_heads + .iter() + .filter(|block| block.block_number == requested_number) + .next() + .expect("Mocks should not use unknown numbers") + } + + /// Returns test data for the given slot + fn get_info_by_slot(&self, slot: Slot) -> Option<&BlockTestData> { + self.per_slot_heads.iter().filter(|block| block.slot == slot).next() + } +} + +/// Type of generated messages. +#[derive(Debug, Copy, Clone)] +enum MessageType { + Approval, + Assignment, + Other, +} + +/// A test message generated by the `PeerMessagesGenerator` +struct TestMessage { + /// The actual message + msg: ApprovalDistributionMessage, + /// The list of peers that would sends this message in a real topology. + /// It includes both the peers that would send the message because of the topology + /// or because of randomly chosing so. + sent_by: HashSet<(ValidatorIndex, PeerId)>, + /// The tranche at which this message should be sent. + tranche: u32, + /// The block hash this message refers to. + block_hash: Hash, + /// The type of the message. + typ: MessageType, +} + +impl TestMessage { + /// Returns the lantency based on the message type. + fn get_latency(&self) -> Option { + match &self.typ { + // We want assignments to always arrive before approval, so + // we don't send them with a latency. + MessageType::Approval => Some(Duration::from_millis(300)), + MessageType::Assignment => None, + MessageType::Other => None, + } + } + + /// Splits a message into multiple messages based on what peers should send this message. + /// It build a HashMap of messages that should be sent by each peer. + fn split_by_peer_id(self) -> HashMap<(ValidatorIndex, PeerId), Vec> { + let mut result: HashMap<(ValidatorIndex, PeerId), Vec> = HashMap::new(); + + for peer in &self.sent_by { + match &self.msg { + ApprovalDistributionMessage::NetworkBridgeUpdate(msg) => { + result.entry(*peer).or_default().push(TestMessage { + msg: ApprovalDistributionMessage::NetworkBridgeUpdate(match msg { + NetworkBridgeEvent::PeerMessage(_, msg) => + NetworkBridgeEvent::PeerMessage(peer.1, msg.clone()), + NetworkBridgeEvent::OurViewChange(_) => todo!(), + _ => todo!(), + }), + sent_by: Default::default(), + tranche: self.tranche, + block_hash: self.block_hash, + typ: self.typ, + }); + }, + _ => {}, + } + } + result + } +} + +/// A generator of messages coming from a given Peer/Validator +struct PeerMessagesGenerator { + /// The state state used to know what messages to generate. + state: ApprovalTestState, + /// Configuration options, passed at the beginning of the test. + options: ApprovalsOptions, + /// The grid neighbors of the node under test. + topology_node_under_test: GridNeighbors, + /// The topology of the network for the epoch under test. + topology: SessionGridTopology, + /// The validator index for this object generates the messages. + validator_index: ValidatorIndex, + /// A reference to the network emulator + network: NetworkEmulator, + /// A handle to the overseer, used for sending messages to the node + /// under test. + overseer_handle: OverseerHandleReal, +} + +impl PeerMessagesGenerator { + /// Generates messages by spawning a blocking task in the background which begins creating + /// the assignments/approvals and peer view changes at the begining of each block. + fn generate_messages(mut self, spawn_task_handle: &SpawnTaskHandle) { + spawn_task_handle.spawn_blocking("generate-messages", "generate-messages", async move { + let mut messages_to_send = Vec::new(); + let mut already_generated = HashSet::new(); + let system_clock = SystemClock {}; + + loop { + sleep(Duration::from_millis(50)).await; + + let current_slot = Slot::from_timestamp( + Timestamp::current(), + SlotDuration::from_millis(SLOT_DURATION_MILLIS), + ); + + let block_info = + self.state.get_info_by_slot(current_slot).map(|block| block.clone()); + + if let Some(block_info) = block_info { + if already_generated.insert(block_info.hash) { + let (tx, rx) = oneshot::channel(); + self.overseer_handle.wait_for_activation(block_info.hash, tx).await; + rx.await + .expect("We should not fail waiting for block to be activated") + .expect("We should not fail waiting for block to be activated"); + + let peer_id = self + .state + .test_authorities + .peer_ids + .get(self.validator_index.0 as usize) + .unwrap(); + + let view_update = generate_peer_view_change_for( + block_info.hash, + *peer_id, + self.validator_index, + ); + + self.send_message(view_update, self.validator_index, None); + + let assignments = generate_assignments( + &block_info, + self.state + .test_authorities + .keyrings + .clone() + .into_iter() + .zip(self.state.test_authorities.peer_ids.clone().into_iter()) + .collect_vec(), + &self.state.session_info, + false, + &self.state.random_samplings, + self.validator_index.0, + &block_info.relay_vrf_story, + &self.topology_node_under_test, + &self.topology, + self.options.last_considered_tranche, + ); + + let approvals = issue_approvals( + &assignments, + block_info.hash, + self.state + .test_authorities + .keyrings + .clone() + .into_iter() + .zip(self.state.test_authorities.peer_ids.clone().into_iter()) + .collect_vec(), + block_info.candidates.clone(), + &self.options, + ); + + let generated_assignments = assignments.into_iter().peekable(); + let approvals = approvals.into_iter().peekable(); + + messages_to_send.push(generated_assignments); + messages_to_send.push(approvals); + } + } + + loop { + let mut at_least_one_sent = false; + // Messages are sorted per block and per tranches, so earlier blocks would be + // at the front of messages_to_send, so we always prefer to send all messages + // we can send for older blocks. + for message_to_send in messages_to_send.iter_mut() { + let current_slot = Slot::from_timestamp( + Timestamp::current(), + SlotDuration::from_millis(SLOT_DURATION_MILLIS), + ); + + if message_to_send + .peek() + .map(|val| { + let block_info = self.state.get_info_by_hash(val.block_hash); + let tranche_now = + system_clock.tranche_now(SLOT_DURATION_MILLIS, block_info.slot); + val.tranche <= tranche_now && current_slot >= block_info.slot + }) + .unwrap_or_default() + { + let message = message_to_send.next().unwrap(); + + let block_info = self.state.get_info_by_hash(message.block_hash); + if !block_info.approved.load(std::sync::atomic::Ordering::SeqCst) { + for (peer, messages) in message.split_by_peer_id() { + for message in messages { + let latency = message.get_latency(); + self.send_message(message, peer.0, latency) + } + } + } + at_least_one_sent = true; + break + } + } + if !at_least_one_sent { + break + } + } + } + }); + } + + /// Queues a message to be sent by the peer identified by the `sent_by` value. + fn send_message( + &mut self, + message: TestMessage, + sent_by: ValidatorIndex, + latency: Option, + ) { + let peer = self + .state + .test_authorities + .validator_authority_id + .get(sent_by.0 as usize) + .expect("We can't handle unknown peers") + .clone(); + + let mut overseer_handle = self.overseer_handle.clone(); + let network_action = NetworkAction::new( + peer.clone(), + async move { + overseer_handle + .send_msg(AllMessages::ApprovalDistribution(message.msg), LOG_TARGET) + .timeout(MAX_TIME_OF_FLIGHT) + .await + .unwrap_or_else(|| { + panic!( + "{} ms maximum time of flight breached", + MAX_TIME_OF_FLIGHT.as_millis() + ) + }); + } + .boxed(), + 200, + latency, + ); + self.network.submit_peer_action(peer, network_action); + } +} + +/// Helper function to create a a signature for the block header. +fn garbage_vrf_signature() -> VrfSignature { + let transcript = VrfTranscript::new(b"test-garbage", &[]); + Sr25519Keyring::Alice.pair().vrf_sign(&transcript.into()) +} + +/// Helper function to create a block header. +fn make_header(parent_hash: Hash, slot: Slot, number: u32) -> Header { + let digest = + { + let mut digest = Digest::default(); + let vrf_signature = garbage_vrf_signature(); + digest.push(DigestItem::babe_pre_digest(PreDigest::SecondaryVRF( + SecondaryVRFPreDigest { authority_index: 0, slot, vrf_signature }, + ))); + digest + }; + + Header { + digest, + extrinsics_root: Default::default(), + number, + state_root: Default::default(), + parent_hash, + } +} + +/// Helper function to create a candidate receipt. +fn make_candidate(para_id: ParaId, hash: &Hash) -> CandidateReceipt { + let mut r = dummy_candidate_receipt_bad_sig(*hash, Some(Default::default())); + r.descriptor.para_id = para_id; + r +} + +/// Helper function to create a list of candidates that are included in the block +fn make_candidates( + block_hash: Hash, + block_number: BlockNumber, + num_cores: u32, + num_candidates: u32, +) -> Vec { + let seed = [block_number as u8; 32]; + let mut rand_chacha = ChaCha20Rng::from_seed(seed); + let mut candidates = (0..num_cores) + .map(|core| { + CandidateEvent::CandidateIncluded( + make_candidate(ParaId::from(core), &block_hash), + Vec::new().into(), + CoreIndex(core), + GroupIndex(core), + ) + }) + .collect_vec(); + let (candidates, _) = candidates.partial_shuffle(&mut rand_chacha, num_candidates as usize); + candidates + .into_iter() + .map(|val| val.clone()) + .sorted_by(|a, b| match (a, b) { + ( + CandidateEvent::CandidateIncluded(_, _, core_a, _), + CandidateEvent::CandidateIncluded(_, _, core_b, _), + ) => core_a.0.cmp(&core_b.0), + (_, _) => todo!("Should not happen"), + }) + .collect_vec() +} + +/// Generates a test session info with all passed authorities as consensus validators. +fn session_info_for_peers( + configuration: &TestConfiguration, + authorities: TestAuthorities, +) -> SessionInfo { + let keys = authorities.keyrings.iter().zip(authorities.peer_ids.iter()); + SessionInfo { + validators: keys.clone().map(|v| v.0.clone().public().into()).collect(), + discovery_keys: keys.clone().map(|v| v.0.clone().public().into()).collect(), + assignment_keys: keys.clone().map(|v| v.0.clone().public().into()).collect(), + validator_groups: IndexedVec::>::from( + (0..authorities.keyrings.len()) + .map(|index| vec![ValidatorIndex(index as u32)]) + .collect_vec(), + ), + n_cores: configuration.n_cores as u32, + needed_approvals: 30, + zeroth_delay_tranche_width: 0, + relay_vrf_modulo_samples: 6, + n_delay_tranches: 89, + no_show_slots: 3, + active_validator_indices: (0..authorities.keyrings.len()) + .map(|index| ValidatorIndex(index as u32)) + .collect_vec(), + dispute_period: 6, + random_seed: [0u8; 32], + } +} + +/// Helper function to randomly determine how many approvals we coalesce together in a single +/// message. +fn coalesce_approvals_len(min_coalesce: u32, max_coalesce: u32) -> usize { + let seed = [7u8; 32]; + let mut rand_chacha = ChaCha20Rng::from_seed(seed); + let mut sampling: Vec = (min_coalesce as usize..max_coalesce as usize + 1).collect_vec(); + *(sampling.partial_shuffle(&mut rand_chacha, 1).0.first().unwrap()) +} + +/// Helper function to create approvals signatures for all assignments passed as arguments. +/// Returns a list of Approvals messages that need to be sent. +fn issue_approvals( + assignments: &Vec, + block_hash: Hash, + keyrings: Vec<(Keyring, PeerId)>, + candidates: Vec, + options: &ApprovalsOptions, +) -> Vec { + let mut to_sign: Vec = Vec::new(); + + let result = assignments + .iter() + .enumerate() + .map(|(_index, message)| match &message.msg { + ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage( + _, + Versioned::VStaging(protocol_vstaging::ApprovalDistributionMessage::Assignments( + assignments, + )), + )) => { + let mut approvals_to_create = Vec::new(); + + let current_validator_index = + to_sign.first().map(|msg| msg.validator_index).unwrap_or(ValidatorIndex(999)); + + // Invariant for this benchmark. + assert_eq!(assignments.len(), 1); + + let assignment = assignments.first().unwrap(); + + if to_sign.len() >= + coalesce_approvals_len(options.min_coalesce, options.max_coalesce) as usize || + (!to_sign.is_empty() && current_validator_index != assignment.0.validator) + { + approvals_to_create.push(sign_candidates(&mut to_sign, &keyrings, block_hash)) + } + + // If more that one candidate was in the assignment queue all of them. + for candidate_index in assignment.1.iter_ones() { + let candidate = candidates.get(candidate_index).unwrap(); + if let CandidateEvent::CandidateIncluded(candidate, _, _, _) = candidate { + to_sign.push(TestSignInfo { + candidate_hash: candidate.hash(), + candidate_index: candidate_index as CandidateIndex, + validator_index: assignment.0.validator, + sent_by: message.sent_by.clone(), + tranche: message.tranche, + }); + + if to_sign.len() >= + coalesce_approvals_len(options.min_coalesce, options.max_coalesce) + as usize + { + approvals_to_create.push(sign_candidates( + &mut to_sign, + &keyrings, + block_hash, + )) + } + } else { + todo!("Other enum variants are not used in this benchmark"); + } + } + approvals_to_create + }, + _ => { + todo!("Other enum variants are not used in this benchmark"); + }, + }) + .collect_vec(); + + let mut result = result.into_iter().flatten().collect_vec(); + + if !to_sign.is_empty() { + result.push(sign_candidates(&mut to_sign, &keyrings, block_hash)); + } + result +} + +/// Helper struct to gather information about more than one candidate an sign it in a single +/// approval message. +struct TestSignInfo { + candidate_hash: CandidateHash, + candidate_index: CandidateIndex, + validator_index: ValidatorIndex, + sent_by: HashSet<(ValidatorIndex, PeerId)>, + tranche: u32, +} + +/// Helper function to create a signture for all candidates in `to_sign` parameter. +/// Returns a TestMessage +fn sign_candidates( + to_sign: &mut Vec, + keyrings: &Vec<(Keyring, PeerId)>, + block_hash: Hash, +) -> TestMessage { + let current_validator_index = to_sign.first().map(|val| val.validator_index).unwrap(); + let tranche_trigger_timestamp = to_sign.iter().map(|val| val.tranche).max().unwrap(); + let keyring = keyrings.get(current_validator_index.0 as usize).unwrap().clone(); + + let to_sign = to_sign + .drain(..) + .sorted_by(|val1, val2| val1.candidate_index.cmp(&val2.candidate_index)) + .collect_vec(); + + let hashes = to_sign.iter().map(|val| val.candidate_hash).collect_vec(); + let candidate_indices = to_sign.iter().map(|val| val.candidate_index).collect_vec(); + let sent_by = to_sign + .iter() + .map(|val| val.sent_by.iter()) + .flatten() + .map(|peer| *peer) + .collect::>(); + + let payload = ApprovalVote(*hashes.first().unwrap()).signing_payload(1); + + let validator_key: ValidatorPair = keyring.0.pair().into(); + let signature = validator_key.sign(&payload[..]); + let indirect = IndirectSignedApprovalVote { + block_hash, + candidate_index: *candidate_indices.first().unwrap() as CandidateIndex, + validator: current_validator_index, + signature, + }; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![indirect]); + TestMessage { + msg: ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage( + keyring.1, + Versioned::VStaging(msg), + )), + sent_by, + tranche: tranche_trigger_timestamp, + block_hash, + typ: MessageType::Approval, + } +} + +fn neighbours_that_would_sent_message( + keyrings: &Vec<(Keyring, PeerId)>, + current_validator_index: u32, + topology_node_under_test: &GridNeighbors, + topology: &SessionGridTopology, +) -> Vec<(ValidatorIndex, PeerId)> { + let topology_originator = topology + .compute_grid_neighbors_for(ValidatorIndex(current_validator_index as u32)) + .unwrap(); + + let originator_y = topology_originator + .validator_indices_y + .iter() + .filter(|validator| { + topology_node_under_test.required_routing_by_index(**validator, false) == + RequiredRouting::GridY + }) + .next(); + let originator_x = topology_originator + .validator_indices_x + .iter() + .filter(|validator| { + topology_node_under_test.required_routing_by_index(**validator, false) == + RequiredRouting::GridX + }) + .next(); + + let is_neighbour = topology_originator + .validator_indices_x + .contains(&ValidatorIndex(NODE_UNDER_TEST)) || + topology_originator + .validator_indices_y + .contains(&ValidatorIndex(NODE_UNDER_TEST)); + + let mut to_be_sent_by = originator_y + .into_iter() + .chain(originator_x) + .map(|val| (*val, keyrings[val.0 as usize].1)) + .collect_vec(); + + if is_neighbour { + to_be_sent_by.push((ValidatorIndex(NODE_UNDER_TEST), keyrings[0].1)); + } + to_be_sent_by +} + +/// Generates assignments for the given `current_validator_index` +/// Returns a list of assignments to be sent sorted by tranche. +fn generate_assignments( + block_info: &BlockTestData, + keyrings: Vec<(Keyring, PeerId)>, + session_info: &SessionInfo, + generate_v2_assignments: bool, + random_samplings: &Vec>, + current_validator_index: u32, + relay_vrf_story: &RelayVRFStory, + topology_node_under_test: &GridNeighbors, + topology: &SessionGridTopology, + last_considered_tranche: u32, +) -> Vec { + let config = Config::from(session_info); + + let leaving_cores = block_info + .candidates + .clone() + .into_iter() + .map(|candidate_event| { + if let CandidateEvent::CandidateIncluded(candidate, _, core_index, group_index) = + candidate_event + { + (candidate.hash(), core_index, group_index) + } else { + todo!("Variant is never created in this benchmark") + } + }) + .collect_vec(); + + let mut assignments_by_tranche = BTreeMap::new(); + + let bytes = current_validator_index.to_be_bytes(); + let seed = [ + bytes[0], bytes[1], bytes[2], bytes[3], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + let mut rand_chacha = ChaCha20Rng::from_seed(seed); + + let to_be_sent_by = neighbours_that_would_sent_message( + &keyrings, + current_validator_index, + topology_node_under_test, + topology, + ); + + let store = LocalKeystore::in_memory(); + let _public = store + .sr25519_generate_new( + ASSIGNMENT_KEY_TYPE_ID, + Some(keyrings[current_validator_index as usize].0.seed().as_str()), + ) + .expect("should not fail"); + + let leaving_cores = leaving_cores + .clone() + .into_iter() + .filter(|(_, core_index, _group_index)| core_index.0 != current_validator_index) + .collect_vec(); + + let assignments = compute_assignments( + &store, + relay_vrf_story.clone(), + &config, + leaving_cores.clone(), + generate_v2_assignments, + ); + + let random_sending_nodes = random_samplings + .get(rand_chacha.next_u32() as usize % random_samplings.len()) + .unwrap(); + let random_sending_peer_ids = random_sending_nodes + .into_iter() + .map(|validator| (*validator, keyrings[validator.0 as usize].1)) + .collect_vec(); + + let mut no_duplicates = HashSet::new(); + for (core_index, assignment) in assignments { + let assigned_cores = match &assignment.cert().kind { + approval::v2::AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => + core_bitfield.iter_ones().map(|val| CoreIndex::from(val as u32)).collect_vec(), + approval::v2::AssignmentCertKindV2::RelayVRFDelay { core_index } => vec![*core_index], + approval::v2::AssignmentCertKindV2::RelayVRFModulo { sample: _ } => vec![core_index], + }; + if assignment.tranche() > last_considered_tranche { + continue + } + + let bitfiled: CoreBitfield = assigned_cores.clone().try_into().unwrap(); + + // For the cases where tranch0 assignments are in a single certificate we need to make + // sure we create a single message. + if no_duplicates.insert(bitfiled) { + if !assignments_by_tranche.contains_key(&assignment.tranche()) { + assignments_by_tranche.insert(assignment.tranche(), Vec::new()); + } + let this_tranche_assignments = + assignments_by_tranche.get_mut(&assignment.tranche()).unwrap(); + this_tranche_assignments.push(( + IndirectAssignmentCertV2 { + block_hash: block_info.hash, + validator: ValidatorIndex(current_validator_index), + cert: assignment.cert().clone(), + }, + block_info + .candidates + .iter() + .enumerate() + .filter(|(_index, candidate)| { + if let CandidateEvent::CandidateIncluded(_, _, core, _) = candidate { + assigned_cores.contains(core) + } else { + panic!("Should not happen"); + } + }) + .map(|(index, _)| index as u32) + .collect_vec() + .try_into() + .unwrap(), + to_be_sent_by + .iter() + .chain(random_sending_peer_ids.iter()) + .map(|peer| *peer) + .collect::>(), + assignment.tranche(), + )); + } + } + + let res = assignments_by_tranche + .into_values() + .map(|assignments| assignments.into_iter()) + .flatten() + .map(|indirect| { + let validator_index = indirect.0.validator.0; + let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(vec![( + indirect.0, indirect.1, + )]); + TestMessage { + msg: ApprovalDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::PeerMessage( + keyrings[validator_index as usize].1, + Versioned::VStaging(msg), + ), + ), + sent_by: indirect.2, + tranche: indirect.3, + block_hash: block_info.hash, + typ: MessageType::Assignment, + } + }) + .collect_vec(); + + res +} + +/// A list of random samplings that we use to determine which nodes should send a given message to +/// the node under test. +/// We can not sample every time for all the messages because that would be too expensive to +/// perform, so pre-generate a list of samples for a given network size. +fn random_samplings_to_node_patterns( + node_under_test: ValidatorIndex, + num_validators: usize, + num_patterns: usize, +) -> Vec> { + let seed = [7u8; 32]; + let mut rand_chacha = ChaCha20Rng::from_seed(seed); + + (0..num_patterns) + .map(|_| { + (0..num_validators) + .map(|sending_validator_index| { + let mut validators = (0..num_validators).map(|val| val).collect_vec(); + validators.shuffle(&mut rand_chacha); + + let mut random_routing = RandomRouting::default(); + validators + .into_iter() + .flat_map(|validator_to_send| { + if random_routing.sample(num_validators, &mut rand_chacha) { + random_routing.inc_sent(); + if validator_to_send == node_under_test.0 as usize { + Some(ValidatorIndex(sending_validator_index as u32)) + } else { + None + } + } else { + None + } + }) + .collect_vec() + }) + .flatten() + .collect_vec() + }) + .collect_vec() +} + +/// Generates a peer view change for the passed `block_hash` +fn generate_peer_view_change_for( + block_hash: Hash, + peer_id: PeerId, + validator_index: ValidatorIndex, +) -> TestMessage { + let network = + NetworkBridgeEvent::PeerViewChange(peer_id, View::new([block_hash].into_iter(), 0)); + + TestMessage { + msg: ApprovalDistributionMessage::NetworkBridgeUpdate(network), + sent_by: [(validator_index, peer_id)].into_iter().collect(), + tranche: 0, + block_hash, + typ: MessageType::Other, + } +} + +/// Generates peer_connected messages for all peers in `test_authorities` +fn generate_peer_connected( + test_authorities: &TestAuthorities, + block_hash: Hash, +) -> Vec { + let keyrings = test_authorities + .keyrings + .clone() + .into_iter() + .zip(test_authorities.peer_ids.clone().into_iter()) + .collect_vec(); + keyrings + .into_iter() + .map(|(_, peer_id)| { + let network = NetworkBridgeEvent::PeerConnected( + peer_id, + ObservedRole::Full, + ProtocolVersion::from(ValidationVersion::VStaging), + None, + ); + TestMessage { + msg: ApprovalDistributionMessage::NetworkBridgeUpdate(network), + sent_by: Default::default(), + tranche: 0, + block_hash, + typ: MessageType::Other, + } + }) + .collect_vec() +} + +/// Generates a topology to be used for this benchmark. +fn generate_topology(test_authorities: &TestAuthorities) -> SessionGridTopology { + let keyrings = test_authorities + .keyrings + .clone() + .into_iter() + .zip(test_authorities.peer_ids.clone().into_iter()) + .collect_vec(); + + let topology = keyrings + .clone() + .into_iter() + .enumerate() + .map(|(index, (keyring, peer_id))| TopologyPeerInfo { + peer_ids: vec![peer_id], + validator_index: ValidatorIndex(index as u32), + discovery_id: keyring.public().into(), + }) + .collect_vec(); + let shuffled = (0..keyrings.len()).collect_vec(); + + SessionGridTopology::new(shuffled, topology) +} + +/// Generates new session topology message. +fn generate_new_session_topology( + test_authorities: &TestAuthorities, + block_hash: Hash, +) -> Vec { + let topology = generate_topology(test_authorities); + + let event = NetworkBridgeEvent::NewGossipTopology(NewGossipTopology { + session: 1, + topology, + local_index: Some(ValidatorIndex(NODE_UNDER_TEST)), // TODO + }); + vec![TestMessage { + msg: ApprovalDistributionMessage::NetworkBridgeUpdate(event), + sent_by: Default::default(), + tranche: 0, + block_hash, + typ: MessageType::Other, + }] +} + +/// Helper function to generate a babe epoch for this benchmark. +/// It does not change for the duration of the test. +fn generate_babe_epoch(current_slot: Slot, authorities: TestAuthorities) -> BabeEpoch { + let authorities = authorities + .keyrings + .into_iter() + .enumerate() + .map(|(index, keyring)| (keyring.public().into(), index as u64)) + .collect_vec(); + BabeEpoch { + epoch_index: 1, + start_slot: current_slot.saturating_sub(1u64), + duration: 200, + authorities, + randomness: [0xde; 32], + config: BabeEpochConfiguration { c: (1, 4), allowed_slots: AllowedSlots::PrimarySlots }, + } +} + +/// Helper function to build an overseer with the real implementation for `ApprovalDistribution` and +/// `ApprovalVoting` subystems and mock subsytems for all others. +fn build_overseer( + state: &ApprovalTestState, + network: &NetworkEmulator, + config: &TestConfiguration, + dependencies: &TestEnvironmentDependencies, +) -> (Overseer, AlwaysSupportsParachains>, OverseerHandleReal) { + let overseer_connector = OverseerConnector::with_event_capacity(640000); + + let spawn_task_handle = dependencies.task_manager.spawn_handle(); + + let db = kvdb_memorydb::create(NUM_COLUMNS); + let db: polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter = + polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[]); + let keystore = LocalKeystore::in_memory(); + let approval_voting = ApprovalVotingSubsystem::with_config( + TEST_CONFIG, + Arc::new(db), + Arc::new(keystore), + Box::new(TestSyncOracle {}), + ApprovalVotingMetrics::try_register(&dependencies.registry).unwrap(), + ); + + let approval_distribution = ApprovalDistribution::new( + ApprovalDistributionMetrics::try_register(&dependencies.registry).unwrap(), + ); + let mock_chain_api = MockChainApi { state: state.clone() }; + let mock_chain_selection = MockChainSelection { state: state.clone() }; + let mock_runtime_api = MockRuntimeApi { state: state.clone() }; + let mock_tx_bridge = MockNetworkBridgeTx::new(config.clone(), state.clone(), network.clone()); + let overseer_metrics = OverseerMetrics::try_register(&dependencies.registry).unwrap(); + let dummy = dummy_builder!(spawn_task_handle, overseer_metrics) + .replace_approval_distribution(|_| approval_distribution) + .replace_approval_voting(|_| approval_voting) + .replace_chain_api(|_| mock_chain_api) + .replace_chain_selection(|_| mock_chain_selection) + .replace_runtime_api(|_| mock_runtime_api) + .replace_network_bridge_tx(|_| mock_tx_bridge); + + let (overseer, raw_handle) = + dummy.build_with_connector(overseer_connector).expect("Should not fail"); + + let overseer_handle = OverseerHandleReal::new(raw_handle); + (overseer, overseer_handle) +} + +/// Takes a test configuration and uses it to creates the `TestEnvironment`. +pub fn prepare_test( + config: TestConfiguration, + options: ApprovalsOptions, +) -> (TestEnvironment, ApprovalTestState) { + prepare_test_inner(config, TestEnvironmentDependencies::default(), options) +} + +/// Build the test environment for an Approval benchmark. +fn prepare_test_inner( + config: TestConfiguration, + dependencies: TestEnvironmentDependencies, + options: ApprovalsOptions, +) -> (TestEnvironment, ApprovalTestState) { + gum::info!("Prepare test state"); + let mut state = ApprovalTestState::new(&config, options); + + gum::info!("Build network emulator"); + + let network = NetworkEmulator::new(&config, &dependencies, &state.test_authorities); + gum::info!("Build overseer"); + + let (overseer, overseer_handle) = build_overseer(&state, &network, &config, &dependencies); + + state.start_message_generation( + &network, + overseer_handle.clone(), + &dependencies.task_manager.spawn_handle(), + ); + + (TestEnvironment::new(dependencies, config, network, overseer, overseer_handle), state) +} + +/// Runs the approval benchmark. +pub async fn bench_approvals(env: &mut TestEnvironment, state: ApprovalTestState) { + let config = env.config().clone(); + + env.metrics().set_n_validators(config.n_validators); + env.metrics().set_n_cores(config.n_cores); + + // First create the initialization messages that make sure that then node under + // tests receives notifications about the topology used and the connected peers. + let mut initialization_messages = generate_peer_connected( + &state.test_authorities, + state.per_slot_heads.first().unwrap().hash, + ); + initialization_messages.extend(generate_new_session_topology( + &state.test_authorities, + state.per_slot_heads.first().unwrap().hash, + )); + for message in initialization_messages { + env.send_message(AllMessages::ApprovalDistribution(message.msg)).await; + } + + let start_marker = Instant::now(); + + for block_num in 0..env.config().num_blocks { + let mut current_slot = Slot::from_timestamp( + Timestamp::current(), + SlotDuration::from_millis(SLOT_DURATION_MILLIS), + ); + + // Wait untill the time arrieves at the first slot under test. + while current_slot < state.initial_slot { + sleep(Duration::from_millis(5)).await; + current_slot = Slot::from_timestamp( + Timestamp::current(), + SlotDuration::from_millis(SLOT_DURATION_MILLIS), + ); + } + + gum::info!(target: LOG_TARGET, "Current block {}/{}", block_num + 1, env.config().num_blocks); + env.metrics().set_current_block(block_num); + let block_start_ts = Instant::now(); + + if let Some(block_info) = state.get_info_by_slot(current_slot) { + env.import_block(new_block_import_info(block_info.hash, block_info.block_number)) + .await; + } + + let block_time_delta = Duration::from_millis( + (*current_slot + 1) * SLOT_DURATION_MILLIS - Timestamp::current().as_millis(), + ); + let block_time = Instant::now().sub(block_start_ts).as_millis() as u64; + env.metrics().set_block_time(block_time); + gum::info!("Block time {}", format!("{:?}ms", block_time).cyan()); + gum::info!(target: LOG_TARGET,"{}", format!("Sleeping till end of block ({}ms)", block_time_delta.as_millis()).bright_black()); + tokio::time::sleep(block_time_delta).await; + } + + // Wait for all blocks to be approved before exiting. + // This is an invariant of the benchmark, if this does not happen something went teribbly wrong. + while state.last_approved_block.load(std::sync::atomic::Ordering::SeqCst) < + env.config().num_blocks as u32 + { + gum::info!( + "Waiting for all blocks to be approved current approved {:}", + state.last_approved_block.load(std::sync::atomic::Ordering::SeqCst) + ); + tokio::time::sleep(Duration::from_secs(6)).await; + } + + env.stop().await; + + let duration: u128 = start_marker.elapsed().as_millis(); + gum::info!("All blocks processed in {}", format!("{:?}ms", duration).cyan()); + + gum::info!("{}", &env); +} diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 244119735966..91997fb8be4d 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -18,8 +18,7 @@ use std::{collections::HashMap, iter::Cycle, ops::Sub, sync::Arc, time::Instant} use crate::TestEnvironment; use polkadot_node_subsystem::{Overseer, OverseerConnector, SpawnGlue}; -use polkadot_overseer::Handle as OverseerHandle; - +use polkadot_overseer::{metrics::Metrics as OverseerMetrics, Handle as OverseerHandle}; use sc_network::request_responses::ProtocolConfig; use colored::Colorize; @@ -67,11 +66,14 @@ fn build_overseer( spawn_task_handle: SpawnTaskHandle, runtime_api: MockRuntimeApi, av_store: MockAvailabilityStore, - network_bridge: MockNetworkBridgeTx, + network_bridge: MockNetworkBridgeTx, availability_recovery: AvailabilityRecoverySubsystem, + dependencies: &TestEnvironmentDependencies, ) -> (Overseer, AlwaysSupportsParachains>, OverseerHandle) { let overseer_connector = OverseerConnector::with_event_capacity(64000); - let dummy = dummy_builder!(spawn_task_handle); + let overseer_metrics = OverseerMetrics::try_register(&dependencies.registry).unwrap(); + + let dummy = dummy_builder!(spawn_task_handle, overseer_metrics); let builder = dummy .replace_runtime_api(|_| runtime_api) .replace_availability_store(|_| av_store) @@ -145,6 +147,7 @@ fn prepare_test_inner( av_store, network_bridge_tx, subsystem, + &dependencies, ); (TestEnvironment::new(dependencies, config, network, overseer, overseer_handle), req_cfg) diff --git a/polkadot/node/subsystem-bench/src/cli.rs b/polkadot/node/subsystem-bench/src/cli.rs index 3352f33a3503..a5d6fa7fec09 100644 --- a/polkadot/node/subsystem-bench/src/cli.rs +++ b/polkadot/node/subsystem-bench/src/cli.rs @@ -11,6 +11,8 @@ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. +use crate::approval::ApprovalsOptions; + // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . use super::availability::DataAvailabilityReadOptions; @@ -32,6 +34,8 @@ pub enum TestObjective { DataAvailabilityRead(DataAvailabilityReadOptions), /// Run a test sequence specified in a file TestSequence(TestSequenceOptions), + /// + ApprovalsTest(ApprovalsOptions), } #[derive(Debug, clap::Parser)] diff --git a/polkadot/node/subsystem-bench/src/core/configuration.rs b/polkadot/node/subsystem-bench/src/core/configuration.rs index adb5ce80c0d4..80f916a3e786 100644 --- a/polkadot/node/subsystem-bench/src/core/configuration.rs +++ b/polkadot/node/subsystem-bench/src/core/configuration.rs @@ -17,6 +17,7 @@ //! Test configuration definition and helpers. use super::*; use keyring::Keyring; +use sc_network::PeerId; use std::{path::Path, time::Duration}; pub use crate::cli::TestObjective; @@ -67,6 +68,8 @@ pub struct TestConfiguration { pub n_validators: usize, /// Number of cores pub n_cores: usize, + /// Number of included candidates + pub n_included_candidates: usize, /// The min PoV size #[serde(default = "default_pov_size")] pub min_pov_size: usize, @@ -127,11 +130,12 @@ impl TestSequence { } /// Helper struct for authority related state. -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct TestAuthorities { pub keyrings: Vec, pub validator_public: Vec, pub validator_authority_id: Vec, + pub peer_ids: Vec, } impl TestConfiguration { @@ -165,7 +169,9 @@ impl TestConfiguration { .collect::>() .into(); - TestAuthorities { keyrings, validator_public, validator_authority_id } + let peer_ids: Vec = + keyrings.iter().map(|_| PeerId::random()).collect::>().into(); + TestAuthorities { keyrings, validator_public, validator_authority_id, peer_ids } } /// An unconstrained standard configuration matching Polkadot/Kusama @@ -180,6 +186,7 @@ impl TestConfiguration { Self { objective, n_cores, + n_included_candidates: n_cores, n_validators, pov_sizes: generate_pov_sizes(n_cores, min_pov_size, max_pov_size), bandwidth: 50 * 1024 * 1024, @@ -205,6 +212,7 @@ impl TestConfiguration { Self { objective, n_cores, + n_included_candidates: n_cores, n_validators, pov_sizes: generate_pov_sizes(n_cores, min_pov_size, max_pov_size), bandwidth: 50 * 1024 * 1024, @@ -232,6 +240,7 @@ impl TestConfiguration { Self { objective, n_cores, + n_included_candidates: n_cores, n_validators, pov_sizes: generate_pov_sizes(n_cores, min_pov_size, max_pov_size), bandwidth: 50 * 1024 * 1024, diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 247596474078..b5dc2c5510e0 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -150,7 +150,7 @@ pub const GENESIS_HASH: Hash = Hash::repeat_byte(0xff); // We use this to bail out sending messages to the subsystem if it is overloaded such that // the time of flight is breaches 5s. // This should eventually be a test parameter. -const MAX_TIME_OF_FLIGHT: Duration = Duration::from_millis(5000); +pub const MAX_TIME_OF_FLIGHT: Duration = Duration::from_millis(5000); /// The test environment is the high level wrapper of all things required to test /// a certain subsystem. @@ -294,6 +294,7 @@ impl Display for TestEnvironment { stats .iter() .enumerate() + .filter(|(index, _)| *index != 0) .map(|(_index, stats)| stats.tx_bytes_total as u128) .sum::() / (1024 * 1024) ) diff --git a/polkadot/node/subsystem-bench/src/core/keyring.rs b/polkadot/node/subsystem-bench/src/core/keyring.rs index 2d9aa348a922..bc8da8c8ebb6 100644 --- a/polkadot/node/subsystem-bench/src/core/keyring.rs +++ b/polkadot/node/subsystem-bench/src/core/keyring.rs @@ -22,7 +22,7 @@ use sp_core::{ /// Set of test accounts. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Keyring { - name: String, + pub name: String, } impl Keyring { @@ -30,6 +30,10 @@ impl Keyring { Self { name } } + pub fn seed(&self) -> String { + format!("//{}", self.name) + } + pub fn pair(self) -> Pair { Pair::from_string(&format!("//{}", self.name), None).expect("input is always good; qed") } diff --git a/polkadot/node/subsystem-bench/src/core/mock/mod.rs b/polkadot/node/subsystem-bench/src/core/mock/mod.rs index d59642e96058..199f08587b0b 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/mod.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/mod.rs @@ -36,7 +36,7 @@ impl HeadSupportsParachains for AlwaysSupportsParachains { // An orchestra with dummy subsystems macro_rules! dummy_builder { - ($spawn_task_handle: ident) => {{ + ($spawn_task_handle: ident, $metrics: ident) => {{ use super::core::mock::dummy::*; // Initialize a mock overseer. @@ -68,10 +68,24 @@ macro_rules! dummy_builder { .activation_external_listeners(Default::default()) .span_per_active_leaf(Default::default()) .active_leaves(Default::default()) - .metrics(Default::default()) + .metrics($metrics) .supports_parachains(AlwaysSupportsParachains {}) .spawner(SpawnGlue($spawn_task_handle)) }}; } pub(crate) use dummy_builder; +use sp_consensus::SyncOracle; + +#[derive(Clone)] +pub struct TestSyncOracle {} + +impl SyncOracle for TestSyncOracle { + fn is_major_syncing(&self) -> bool { + false + } + + fn is_offline(&self) -> bool { + unimplemented!("should not be used by bridge") + } +} diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index fa4730209183..b674ee2cbca1 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -37,9 +37,12 @@ use polkadot_node_network_protocol::request_response::{ }; use polkadot_primitives::AuthorityDiscoveryId; -use crate::core::{ - configuration::{random_error, random_latency, TestConfiguration}, - network::{NetworkAction, NetworkEmulator, RateLimit}, +use crate::{ + approval::ApprovalTestState, + core::{ + configuration::{random_error, random_latency, TestConfiguration}, + network::{NetworkAction, NetworkEmulator, RateLimit}, + }, }; /// The availability store state of all emulated peers. @@ -54,22 +57,57 @@ pub struct NetworkAvailabilityState { const LOG_TARGET: &str = "subsystem-bench::network-bridge-tx-mock"; /// A mock of the network bridge tx subsystem. -pub struct MockNetworkBridgeTx { +pub struct MockNetworkBridgeTx { /// The test configurationg config: TestConfiguration, /// The network availability state - availabilty: NetworkAvailabilityState, + state: State, /// A network emulator instance network: NetworkEmulator, } -impl MockNetworkBridgeTx { +/// Trait to be implemented by benchamrks that need to respond to requests info. +pub trait RespondToRequestsInfo { + fn candidate_hashes(&self) -> &HashMap; + fn available_data(&self) -> &Vec; + fn chunks(&self) -> &Vec>; +} + +impl RespondToRequestsInfo for NetworkAvailabilityState { + fn candidate_hashes(&self) -> &HashMap { + &self.candidate_hashes + } + + fn available_data(&self) -> &Vec { + &self.available_data + } + + fn chunks(&self) -> &Vec> { + &self.chunks + } +} + +impl RespondToRequestsInfo for ApprovalTestState { + fn candidate_hashes(&self) -> &HashMap { + todo!() + } + + fn available_data(&self) -> &Vec { + todo!() + } + + fn chunks(&self) -> &Vec> { + todo!() + } +} + +impl MockNetworkBridgeTx { pub fn new( config: TestConfiguration, - availabilty: NetworkAvailabilityState, + state: State, network: NetworkEmulator, - ) -> MockNetworkBridgeTx { - Self { config, availabilty, network } + ) -> MockNetworkBridgeTx { + Self { config, state, network } } fn not_connected_response( @@ -127,15 +165,14 @@ impl MockNetworkBridgeTx { let candidate_hash = outgoing_request.payload.candidate_hash; let candidate_index = self - .availabilty - .candidate_hashes + .state + .candidate_hashes() .get(&candidate_hash) .expect("candidate was generated previously; qed"); gum::warn!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); let chunk: ChunkResponse = - self.availabilty.chunks.get(*candidate_index as usize).unwrap() - [validator_index] + self.state.chunks().get(*candidate_index as usize).unwrap()[validator_index] .clone() .into(); let mut size = chunk.encoded_size(); @@ -175,8 +212,8 @@ impl MockNetworkBridgeTx { Requests::AvailableDataFetchingV1(outgoing_request) => { let candidate_hash = outgoing_request.payload.candidate_hash; let candidate_index = self - .availabilty - .candidate_hashes + .state + .candidate_hashes() .get(&candidate_hash) .expect("candidate was generated previously; qed"); gum::debug!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); @@ -206,7 +243,7 @@ impl MockNetworkBridgeTx { .inc_received(outgoing_request.payload.encoded_size()); let available_data = - self.availabilty.available_data.get(*candidate_index as usize).unwrap().clone(); + self.state.available_data().get(*candidate_index as usize).unwrap().clone(); let size = available_data.encoded_size(); @@ -247,7 +284,9 @@ impl MockNetworkBridgeTx { } #[overseer::subsystem(NetworkBridgeTx, error=SubsystemError, prefix=self::overseer)] -impl MockNetworkBridgeTx { +impl + MockNetworkBridgeTx +{ fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx).map(|_| Ok(())).boxed(); @@ -256,7 +295,7 @@ impl MockNetworkBridgeTx { } #[overseer::contextbounds(NetworkBridgeTx, prefix = self::overseer)] -impl MockNetworkBridgeTx { +impl MockNetworkBridgeTx { async fn run(mut self, mut ctx: Context) { let (mut ingress_tx, mut ingress_rx) = tokio::sync::mpsc::unbounded_channel::(); @@ -267,6 +306,8 @@ impl MockNetworkBridgeTx { let our_network = self.network.clone(); // This task will handle node messages receipt from the simulated network. + // TODO: Probably this is not needed here anymore, because it is handled in + // start_node_under_test_receiver. let _ = ctx .spawn_blocking( "network-receive", @@ -285,6 +326,8 @@ impl MockNetworkBridgeTx { .expect("We never fail to spawn tasks"); // Main subsystem loop. + let our_network_stats = self.network.peer_stats(0); + loop { let msg = ctx.recv().await.expect("Overseer never fails us"); @@ -305,9 +348,25 @@ impl MockNetworkBridgeTx { self.network.submit_peer_action(action.peer(), action); } }, - _ => { - unimplemented!("Unexpected network bridge message") + NetworkBridgeTxMessage::SendValidationMessage(peers, _message) => { + for _peer in peers { + our_network_stats.inc_sent(200); + } }, + NetworkBridgeTxMessage::ReportPeer(_) => {}, + NetworkBridgeTxMessage::DisconnectPeer(_, _) => todo!(), + NetworkBridgeTxMessage::SendCollationMessage(_, _) => todo!(), + NetworkBridgeTxMessage::SendValidationMessages(_) => todo!(), + NetworkBridgeTxMessage::SendCollationMessages(_) => todo!(), + NetworkBridgeTxMessage::ConnectToValidators { + validator_ids: _, + peer_set: _, + failed: _, + } => todo!(), + NetworkBridgeTxMessage::ConnectToResolvedValidators { + validator_addrs: _, + peer_set: _, + } => todo!(), }, } } diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 09943becb65c..dc254db48786 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -19,6 +19,7 @@ use super::{ *, }; use colored::Colorize; +use futures::FutureExt; use polkadot_primitives::AuthorityDiscoveryId; use prometheus_endpoint::U64; use rand::{seq::SliceRandom, thread_rng}; @@ -31,7 +32,7 @@ use std::{ }, time::{Duration, Instant}, }; -use tokio::sync::mpsc::UnboundedSender; +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; // An emulated node egress traffic rate_limiter. #[derive(Debug)] @@ -150,6 +151,7 @@ impl PeerEmulator { bandwidth: usize, spawn_task_handle: SpawnTaskHandle, stats: Arc, + to_node_under_test: UnboundedSender, ) -> Self { let (actions_tx, mut actions_rx) = tokio::sync::mpsc::unbounded_channel(); @@ -161,6 +163,7 @@ impl PeerEmulator { loop { let stats_clone = stats.clone(); let maybe_action: Option = actions_rx.recv().await; + let to_node_under_test_clone = to_node_under_test.clone(); if let Some(action) = maybe_action { let size = action.size(); rate_limiter.reap(size).await; @@ -170,12 +173,18 @@ impl PeerEmulator { "test-environment", async move { tokio::time::sleep(latency).await; - action.run().await; + // action.run().await; + to_node_under_test_clone + .send(action) + .expect("Should not fail unbounded channel"); stats_clone.inc_sent(size); }, ) } else { - action.run().await; + // action.run().await; + to_node_under_test + .send(action) + .expect("Should not fail unbounded channel"); stats_clone.inc_sent(size); } } else { @@ -331,6 +340,8 @@ impl NetworkEmulator { Metrics::new(&dependencies.registry).expect("Metrics always register succesfully"); let mut validator_authority_id_mapping = HashMap::new(); + let (ingress_tx, ingress_rx) = tokio::sync::mpsc::unbounded_channel::(); + // Create a `PeerEmulator` for each peer. let (stats, mut peers): (_, Vec<_>) = (0..n_peers) .zip(authorities.validator_authority_id.clone().into_iter()) @@ -343,6 +354,7 @@ impl NetworkEmulator { config.peer_bandwidth, dependencies.task_manager.spawn_handle(), stats, + ingress_tx.clone(), )), ) }) @@ -358,8 +370,41 @@ impl NetworkEmulator { } gum::info!(target: LOG_TARGET, "{}",format!("Network created, connected validator count {}", connected_count).bright_black()); + let our_network = + Self { peers, stats, validator_authority_ids: validator_authority_id_mapping }; + our_network.start_node_under_test_receiver( + config, + dependencies.task_manager.spawn_handle(), + ingress_rx, + ); + our_network + } - Self { peers, stats, validator_authority_ids: validator_authority_id_mapping } + pub fn start_node_under_test_receiver( + &self, + config: &TestConfiguration, + spawn_task_handle: SpawnTaskHandle, + mut ingress_rx: UnboundedReceiver, + ) { + let mut rx_limiter = RateLimit::new(10, config.bandwidth); + + let our_network = self.clone(); + // This task will handle node messages receipt from the simulated network. + let _ = spawn_task_handle.spawn_blocking( + "network-receiver-0", + "network-receiver", + async move { + while let Some(action) = ingress_rx.recv().await { + let size = action.size(); + + // account for our node receiving the data. + our_network.inc_received(size); + rx_limiter.reap(size).await; + action.run().await; + } + } + .boxed(), + ); } pub fn is_peer_connected(&self, peer: &AuthorityDiscoveryId) -> bool { diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 0f3ae0f41417..07b2c1bd1f4d 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -22,6 +22,7 @@ use color_eyre::eyre; use colored::Colorize; use std::{path::Path, time::Duration}; +pub(crate) mod approval; pub(crate) mod availability; pub(crate) mod cli; pub(crate) mod core; @@ -36,6 +37,8 @@ use core::{ use clap_num::number_range; +use crate::approval::bench_approvals; +// const LOG_TARGET: &str = "subsystem-bench"; use crate::core::display::display_configuration; fn le_100(s: &str) -> Result { @@ -83,6 +86,7 @@ struct BenchCli { impl BenchCli { fn launch(self) -> eyre::Result<()> { let configuration = self.standard_configuration; + let mut test_config = match self.objective { TestObjective::TestSequence(options) => { let test_sequence = @@ -98,10 +102,25 @@ impl BenchCli { gum::info!("{}", format!("Step {}/{}", index + 1, num_steps).bright_purple(),); display_configuration(&test_config); - let mut state = TestState::new(&test_config); - let (mut env, _protocol_config) = prepare_test(test_config, &mut state); - env.runtime() - .block_on(availability::benchmark_availability_read(&mut env, state)); + match test_config.objective { + TestObjective::DataAvailabilityRead(_) => { + let mut state = TestState::new(&test_config); + let (mut env, _protocol_config) = prepare_test(test_config, &mut state); + + env.runtime().block_on(availability::benchmark_availability_read( + &mut env, state, + )); + }, + TestObjective::ApprovalsTest(ref options) => { + let (mut env, state) = + approval::prepare_test(test_config.clone(), options.clone()); + + env.runtime().block_on(async { + bench_approvals(&mut env, state).await; + }); + }, + TestObjective::TestSequence(_) => todo!(), + } } return Ok(()) }, @@ -131,6 +150,9 @@ impl BenchCli { configuration.max_pov_size, ), }, + TestObjective::ApprovalsTest(ref options) => { + todo!("Not implemented"); + }, }; let mut latency_config = test_config.latency.clone().unwrap_or_default(); @@ -176,11 +198,12 @@ fn main() -> eyre::Result<()> { // Avoid `Terminating due to subsystem exit subsystem` warnings .filter(Some("polkadot_overseer"), log::LevelFilter::Error) .filter(None, log::LevelFilter::Info) - // .filter(None, log::LevelFilter::Trace) + .format_timestamp(Some(env_logger::TimestampPrecision::Millis)) .try_init() .unwrap(); let cli: BenchCli = BenchCli::parse(); cli.launch()?; + Ok(()) } From 4f9d8fde074da01a6eb6a3e2823e7ed663affaff Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 6 Dec 2023 10:59:07 +0200 Subject: [PATCH 120/192] Subsystem improvements Signed-off-by: Alexandru Gheorghe --- .../examples/approvals_throughput.yaml | 1 + .../node/subsystem-bench/src/approval/mod.rs | 83 ++++++++++++++----- .../subsystem-bench/src/core/environment.rs | 22 ++++- 3 files changed, 81 insertions(+), 25 deletions(-) diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml index 9fe46aa03d8d..18f099656f7a 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml @@ -4,6 +4,7 @@ TestConfiguration: last_considered_tranche: 89 min_coalesce: 1 max_coalesce: 1 + enable_assignments_v2: false n_validators: 500 n_cores: 100 n_included_candidates: 70 diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 220d05a08b1b..e4b31bdca047 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -18,7 +18,7 @@ use serde::{Deserialize, Serialize}; use std::{ collections::{BTreeMap, HashMap, HashSet}, sync::{ - atomic::{AtomicBool, AtomicU32}, + atomic::{AtomicBool, AtomicU32, AtomicU64}, Arc, }, time::{Duration, Instant}, @@ -39,8 +39,8 @@ use polkadot_node_core_approval_voting::{ }; use polkadot_node_primitives::approval::{ self, - v1::{IndirectSignedApprovalVote, RelayVRFStory}, - v2::{CoreBitfield, IndirectAssignmentCertV2}, + v1::RelayVRFStory, + v2::{CoreBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2}, }; use polkadot_node_network_protocol::{ @@ -63,9 +63,10 @@ use rand::{seq::SliceRandom, RngCore, SeedableRng}; use rand_chacha::ChaCha20Rng; use polkadot_primitives::{ - ApprovalVote, BlockNumber, CandidateEvent, CandidateHash, CandidateIndex, CandidateReceipt, - CoreIndex, GroupIndex, Hash, Header, Id as ParaId, IndexedVec, SessionInfo, Slot, - ValidatorIndex, ValidatorPair, ASSIGNMENT_KEY_TYPE_ID, + vstaging::ApprovalVoteMultipleCandidates, ApprovalVote, BlockNumber, CandidateEvent, + CandidateHash, CandidateIndex, CandidateReceipt, CoreIndex, GroupIndex, Hash, Header, + Id as ParaId, IndexedVec, SessionInfo, Slot, ValidatorIndex, ValidatorPair, + ASSIGNMENT_KEY_TYPE_ID, }; use polkadot_primitives_test_helpers::dummy_candidate_receipt_bad_sig; use sc_keystore::LocalKeystore; @@ -135,6 +136,9 @@ pub struct ApprovalsOptions { #[clap(short, long, default_value_t = 89)] /// Max candidate to be signed in a single approval. pub max_coalesce: u32, + #[clap(short, long, default_value_t = false)] + /// Enable assignments v2. + pub enable_assignments_v2: bool, } /// Information about a block. It is part of test state and it is used by the mock @@ -159,6 +163,7 @@ struct BlockTestData { /// This set on `true` when ChainSelectionMessage::Approved is received inside the chain /// selection mock subsystem. approved: Arc, + prev_candidates: u64, } /// Approval test state used by all mock subsystems to be able to answer messages emitted @@ -166,7 +171,7 @@ struct BlockTestData { /// /// This gets cloned across all mock subsystems, so if there is any information that gets /// updated between subsystems, they would have to be wrapped in Arc's. -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct ApprovalTestState { /// The generic test configuration passed when starting the benchmark. configuration: TestConfiguration, @@ -189,12 +194,20 @@ pub struct ApprovalTestState { test_authorities: TestAuthorities, /// Last approved block number. last_approved_block: Arc, + /// Total sent messages. + total_sent_messages: Arc, + /// Approval voting metrics. + approval_voting_metrics: ApprovalVotingMetrics, } impl ApprovalTestState { /// Build a new `ApprovalTestState` object out of the configurations passed when the benchmark /// was tested. - fn new(configuration: &TestConfiguration, options: ApprovalsOptions) -> Self { + fn new( + configuration: &TestConfiguration, + options: ApprovalsOptions, + dependencies: &TestEnvironmentDependencies, + ) -> Self { let test_authorities = configuration.generate_authorities(); let random_samplings = random_samplings_to_node_patterns( @@ -221,7 +234,10 @@ impl ApprovalTestState { initial_slot, test_authorities, last_approved_block: Arc::new(AtomicU32::new(0)), + total_sent_messages: Arc::new(AtomicU64::new(0)), options, + approval_voting_metrics: ApprovalVotingMetrics::try_register(&dependencies.registry) + .unwrap(), }; state.generate_blocks_information(); gum::info!("Built testing state"); @@ -232,6 +248,7 @@ impl ApprovalTestState { /// Generates the blocks and the information about the blocks that will be used /// to drive this test. fn generate_blocks_information(&mut self) { + let mut prev_candidates = 0; for block_number in 1..self.configuration.num_blocks + 1 { let block_hash = Hash::repeat_byte(block_number as u8); let parent_hash = self @@ -266,7 +283,9 @@ impl ApprovalTestState { ), relay_vrf_story, approved: Arc::new(AtomicBool::new(false)), + prev_candidates, }; + prev_candidates += block_info.candidates.len() as u64; self.per_slot_heads.push(block_info) } } @@ -356,7 +375,7 @@ impl TestMessage { match &self.typ { // We want assignments to always arrive before approval, so // we don't send them with a latency. - MessageType::Approval => Some(Duration::from_millis(300)), + MessageType::Approval => None, MessageType::Assignment => None, MessageType::Other => None, } @@ -430,9 +449,14 @@ impl PeerMessagesGenerator { self.state.get_info_by_slot(current_slot).map(|block| block.clone()); if let Some(block_info) = block_info { - if already_generated.insert(block_info.hash) { + let candidates = self.state.approval_voting_metrics.candidates_imported(); + + if candidates >= block_info.prev_candidates + block_info.candidates.len() as u64 && + already_generated.insert(block_info.hash) + { let (tx, rx) = oneshot::channel(); self.overseer_handle.wait_for_activation(block_info.hash, tx).await; + rx.await .expect("We should not fail waiting for block to be activated") .expect("We should not fail waiting for block to be activated"); @@ -462,7 +486,7 @@ impl PeerMessagesGenerator { .zip(self.state.test_authorities.peer_ids.clone().into_iter()) .collect_vec(), &self.state.session_info, - false, + self.options.enable_assignments_v2, &self.state.random_samplings, self.validator_index.0, &block_info.relay_vrf_story, @@ -498,19 +522,24 @@ impl PeerMessagesGenerator { // Messages are sorted per block and per tranches, so earlier blocks would be // at the front of messages_to_send, so we always prefer to send all messages // we can send for older blocks. - for message_to_send in messages_to_send.iter_mut() { - let current_slot = Slot::from_timestamp( - Timestamp::current(), - SlotDuration::from_millis(SLOT_DURATION_MILLIS), - ); + let current_slot = Slot::from_timestamp( + Timestamp::current(), + SlotDuration::from_millis(SLOT_DURATION_MILLIS), + ); + for message_to_send in messages_to_send.iter_mut() { if message_to_send .peek() .map(|val| { let block_info = self.state.get_info_by_hash(val.block_hash); let tranche_now = system_clock.tranche_now(SLOT_DURATION_MILLIS, block_info.slot); - val.tranche <= tranche_now && current_slot >= block_info.slot + let tranche_to_send = match val.typ { + MessageType::Approval => val.tranche + 1, + MessageType::Assignment => val.tranche, + MessageType::Other => val.tranche, + }; + tranche_to_send <= tranche_now && current_slot >= block_info.slot }) .unwrap_or_default() { @@ -521,6 +550,10 @@ impl PeerMessagesGenerator { for (peer, messages) in message.split_by_peer_id() { for message in messages { let latency = message.get_latency(); + self.state + .total_sent_messages + .as_ref() + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); self.send_message(message, peer.0, latency) } } @@ -795,13 +828,13 @@ fn sign_candidates( .map(|peer| *peer) .collect::>(); - let payload = ApprovalVote(*hashes.first().unwrap()).signing_payload(1); + let payload = ApprovalVoteMultipleCandidates(&hashes).signing_payload(1); let validator_key: ValidatorPair = keyring.0.pair().into(); let signature = validator_key.sign(&payload[..]); - let indirect = IndirectSignedApprovalVote { + let indirect = IndirectSignedApprovalVoteV2 { block_hash, - candidate_index: *candidate_indices.first().unwrap() as CandidateIndex, + candidate_indices: candidate_indices.try_into().unwrap(), validator: current_validator_index, signature, }; @@ -1197,7 +1230,7 @@ fn build_overseer( Arc::new(db), Arc::new(keystore), Box::new(TestSyncOracle {}), - ApprovalVotingMetrics::try_register(&dependencies.registry).unwrap(), + state.approval_voting_metrics.clone(), ); let approval_distribution = ApprovalDistribution::new( @@ -1238,7 +1271,7 @@ fn prepare_test_inner( options: ApprovalsOptions, ) -> (TestEnvironment, ApprovalTestState) { gum::info!("Prepare test state"); - let mut state = ApprovalTestState::new(&config, options); + let mut state = ApprovalTestState::new(&config, options, &dependencies); gum::info!("Build network emulator"); @@ -1328,7 +1361,11 @@ pub async fn bench_approvals(env: &mut TestEnvironment, state: ApprovalTestState env.stop().await; let duration: u128 = start_marker.elapsed().as_millis(); - gum::info!("All blocks processed in {}", format!("{:?}ms", duration).cyan()); + gum::info!( + "All blocks processed in {} num_messages {}", + format!("{:?}ms", duration).cyan(), + state.total_sent_messages.load(std::sync::atomic::Ordering::SeqCst) + ); gum::info!("{}", &env); } diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index b5dc2c5510e0..e4fcb3922cc8 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -308,9 +308,27 @@ impl Display for TestEnvironment { let test_metrics = super::display::parse_metrics(self.registry()); let subsystem_cpu_metrics = - test_metrics.subset_with_label_value("task_group", "availability-recovery"); + test_metrics.subset_with_label_value("task_group", "approval-distribution"); let total_cpu = subsystem_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); - writeln!(f, "Total subsystem CPU usage {}", format!("{:.2}s", total_cpu).bright_purple())?; + writeln!( + f, + "Total approval-distribution CPU usage {}", + format!("{:.2}s", total_cpu).bright_purple() + )?; + writeln!( + f, + "CPU usage per block {}", + format!("{:.2}s", total_cpu / self.config().num_blocks as f64).bright_purple() + )?; + + let subsystem_cpu_metrics = + test_metrics.subset_with_label_value("task_group", "approval-voting"); + let total_cpu = subsystem_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); + writeln!( + f, + "Total approval-voting CPU usage {}", + format!("{:.2}s", total_cpu).bright_purple() + )?; writeln!( f, "CPU usage per block {}", From b39f65fa065965130234b62a9edb306ead0ca08f Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 6 Dec 2023 10:59:37 +0200 Subject: [PATCH 121/192] approvals optimisation Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 7 ++ .../network/approval-distribution/src/lib.rs | 76 +++++++++++-------- 2 files changed, 53 insertions(+), 30 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 669e1b05c395..f415551f0f21 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -194,6 +194,13 @@ impl Metrics { } } + pub fn candidates_imported(&self) -> u64 { + self.0 + .as_ref() + .map(|metrics| metrics.imported_candidates_total.get()) + .unwrap_or_default() + } + fn on_assignment_produced(&self, tranche: DelayTranche) { if let Some(metrics) = &self.0 { metrics.assignments_produced.observe(tranche as f64); diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index cc0145fe7797..3754e2903441 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -114,6 +114,7 @@ struct ApprovalRouting { required_routing: RequiredRouting, local: bool, random_routing: RandomRouting, + random_peers: Vec, } // This struct is responsible for tracking the full state of an assignment and grid routing @@ -646,8 +647,9 @@ impl BlockEntry { pub fn note_approval( &mut self, approval: IndirectSignedApprovalVoteV2, - ) -> Result { + ) -> Result<(RequiredRouting, HashSet), ApprovalEntryError> { let mut required_routing = None; + let mut random_peer_ids = HashSet::new(); if self.candidates.len() < approval.candidate_indices.len() as usize { return Err(ApprovalEntryError::CandidateIndexOutOfBounds) @@ -670,6 +672,7 @@ impl BlockEntry { self.approval_entries.get_mut(&(approval.validator, assignment_bitfield)) { approval_entry.note_approval(approval.clone())?; + random_peer_ids.extend(approval_entry.routing_info().random_peers.iter()); if let Some(required_routing) = required_routing { if required_routing != approval_entry.routing_info().required_routing { @@ -688,7 +691,7 @@ impl BlockEntry { } if let Some(required_routing) = required_routing { - Ok(required_routing) + Ok((required_routing, random_peer_ids)) } else { Err(ApprovalEntryError::UnknownAssignment) } @@ -1259,7 +1262,7 @@ impl State { Some(entry) => entry, None => { if let Some(peer_id) = source.peer_id() { - gum::trace!( + gum::info!( target: LOG_TARGET, ?peer_id, hash = ?block_hash, @@ -1278,6 +1281,12 @@ impl State { metrics.on_assignment_recent_outdated(); } } + gum::info!( + target: LOG_TARGET, + hash = ?block_hash, + ?validator_index, + "Unexpected assignment invalid", + ); metrics.on_assignment_invalid_block(); return }, @@ -1489,7 +1498,12 @@ impl State { let approval_entry = entry.insert_approval_entry(ApprovalEntry::new( assignment.clone(), claimed_candidate_indices.clone(), - ApprovalRouting { required_routing, local, random_routing: Default::default() }, + ApprovalRouting { + required_routing, + local, + random_routing: Default::default(), + random_peers: Default::default(), + }, )); // Dispatch the message to all peers in the routing set which @@ -1531,6 +1545,7 @@ impl State { if route_random { approval_entry.routing_info_mut().random_routing.inc_sent(); + approval_entry.routing_info_mut().random_peers.push(peer); peers.push(peer); } } @@ -1577,7 +1592,7 @@ impl State { ) -> bool { for message_subject in assignments_knowledge_key { if !entry.knowledge.contains(&message_subject.0, message_subject.1) { - gum::trace!( + gum::info!( target: LOG_TARGET, ?peer_id, ?message_subject, @@ -1797,7 +1812,7 @@ impl State { } } - let required_routing = match entry.note_approval(vote.clone()) { + let (required_routing, random_peer_ids) = match entry.note_approval(vote.clone()) { Ok(required_routing) => required_routing, Err(err) => { gum::warn!( @@ -1840,7 +1855,7 @@ impl State { // 3. Any randomly selected peers have been sent the assignment already. let in_topology = topology .map_or(false, |t| t.local_grid_neighbors().route_to_peer(required_routing, peer)); - in_topology || knowledge.sent.contains_any(&assignments_knowledge_keys) + in_topology || random_peer_ids.contains(peer) }; let peers = entry @@ -1856,25 +1871,25 @@ impl State { if let Some(entry) = entry.known_by.get_mut(&peer.0) { // A random assignment could have been sent to the peer, so we need to make sure // we send all the other assignments, before we can send the corresponding approval. - let (sent_to_peer, notknown_by_peer) = entry.partition_sent_notknown(&assignments); - if !notknown_by_peer.is_empty() { - let notknown_by_peer = notknown_by_peer - .into_iter() - .map(|(assignment, bitfield)| (assignment.clone(), bitfield.clone())) - .collect_vec(); - gum::trace!( - target: LOG_TARGET, - ?block_hash, - ?peer, - missing = notknown_by_peer.len(), - part1 = sent_to_peer.len(), - "Sending missing assignments", - ); - - entry.mark_sent(¬known_by_peer); - send_assignments_batched(ctx.sender(), notknown_by_peer, &[(peer.0, peer.1)]) - .await; - } + // let (sent_to_peer, notknown_by_peer) = + // entry.partition_sent_notknown(&assignments); if !notknown_by_peer.is_empty() { + // let notknown_by_peer = notknown_by_peer + // .into_iter() + // .map(|(assignment, bitfield)| (assignment.clone(), bitfield.clone())) + // .collect_vec(); + // gum::trace!( + // target: LOG_TARGET, + // ?block_hash, + // ?peer, + // missing = notknown_by_peer.len(), + // part1 = sent_to_peer.len(), + // "Sending missing assignments", + // ); + + // entry.mark_sent(¬known_by_peer); + // send_assignments_batched(ctx.sender(), notknown_by_peer, &[(peer.0, peer.1)]) + // .await; + // } entry.sent.insert(approval_knwowledge_key.0.clone(), approval_knwowledge_key.1); } @@ -1985,8 +2000,7 @@ impl State { // randomly sample peers. { let required_routing = approval_entry.routing_info().required_routing; - let random_routing = - &mut approval_entry.routing_info_mut().random_routing; + let random_routing = &mut approval_entry.routing_info_mut(); let rng = &mut *rng; let mut peer_filter = move |peer_id| { let in_topology = topology.as_ref().map_or(false, |t| { @@ -2001,9 +2015,11 @@ impl State { return false } - let route_random = random_routing.sample(total_peers, rng); + let route_random = + random_routing.random_routing.sample(total_peers, rng); if route_random { - random_routing.inc_sent(); + random_routing.random_routing.inc_sent(); + random_routing.random_peers.push(*peer_id); } route_random From 5644f81e80dcaf3b129f12ffe27baadf682f1efe Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 6 Dec 2023 12:23:48 +0200 Subject: [PATCH 122/192] Add fix number of tranches Signed-off-by: Alexandru Gheorghe --- polkadot/node/subsystem-bench/src/approval/mod.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index e4b31bdca047..5f637bf56901 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -133,12 +133,18 @@ pub struct ApprovalsOptions { #[clap(short, long, default_value_t = 1)] /// Min candidates to be signed in a single approval. pub min_coalesce: u32, - #[clap(short, long, default_value_t = 89)] + #[clap(short, long, default_value_t = 1)] /// Max candidate to be signed in a single approval. pub max_coalesce: u32, #[clap(short, long, default_value_t = false)] /// Enable assignments v2. pub enable_assignments_v2: bool, + #[clap(short, long, default_value_t = 89)] + /// Send messages till tranche + pub send_till_tranche: u32, + #[clap(short, long, default_value_t = true)] + /// Sends messages only till block is approved. + pub stop_when_approved: bool, } /// Information about a block. It is part of test state and it is used by the mock @@ -546,7 +552,10 @@ impl PeerMessagesGenerator { let message = message_to_send.next().unwrap(); let block_info = self.state.get_info_by_hash(message.block_hash); - if !block_info.approved.load(std::sync::atomic::Ordering::SeqCst) { + if !block_info.approved.load(std::sync::atomic::Ordering::SeqCst) || + (!self.options.stop_when_approved && + message.tranche <= self.options.send_till_tranche) + { for (peer, messages) in message.split_by_peer_id() { for message in messages { let latency = message.get_latency(); From c5a61431547560d958259796aa2508d10fd1a82c Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 7 Dec 2023 13:10:52 +0200 Subject: [PATCH 123/192] availability distribution + bitfield distribution Signed-off-by: Andrei Sandu --- Cargo.lock | 9 +- polkadot/node/subsystem-bench/Cargo.toml | 7 + .../src/availability/av_store_helpers.rs | 66 +++++ .../subsystem-bench/src/availability/mod.rs | 231 +++++++++++++++--- polkadot/node/subsystem-bench/src/cli.rs | 4 +- .../subsystem-bench/src/core/configuration.rs | 34 ++- .../subsystem-bench/src/core/environment.rs | 58 ++--- .../node/subsystem-bench/src/core/keyring.rs | 31 ++- .../subsystem-bench/src/core/mock/av_store.rs | 4 + .../src/core/mock/chain_api.rs | 94 +++++++ .../subsystem-bench/src/core/mock/dummy.rs | 1 + .../node/subsystem-bench/src/core/mock/mod.rs | 2 + .../src/core/mock/network_bridge.rs | 1 - .../src/core/mock/runtime_api.rs | 89 ++++++- .../node/subsystem-bench/src/core/network.rs | 1 + .../subsystem-bench/src/subsystem-bench.rs | 78 +++--- 16 files changed, 581 insertions(+), 129 deletions(-) create mode 100644 polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs create mode 100644 polkadot/node/subsystem-bench/src/core/mock/chain_api.rs diff --git a/Cargo.lock b/Cargo.lock index 039257e67872..2b35faa3ba5a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13245,7 +13245,8 @@ version = "1.0.0" dependencies = [ "assert_matches", "async-trait", - "clap 4.4.6", + "bitvec", + "clap 4.4.10", "clap-num", "color-eyre", "colored", @@ -13253,12 +13254,17 @@ dependencies = [ "futures", "futures-timer", "itertools 0.11.0", + "kvdb-memorydb", "log", "orchestra", "parity-scale-codec", "paste", + "polkadot-availability-bitfield-distribution", + "polkadot-availability-distribution", "polkadot-availability-recovery", "polkadot-erasure-coding", + "polkadot-node-core-av-store", + "polkadot-node-core-chain-api", "polkadot-node-metrics", "polkadot-node-network-protocol", "polkadot-node-primitives", @@ -13277,6 +13283,7 @@ dependencies = [ "serde", "serde_yaml", "sp-application-crypto", + "sp-consensus", "sp-core", "sp-keyring", "sp-keystore", diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index f775a1ff9efe..3f4e724dca11 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -23,6 +23,10 @@ polkadot-node-primitives = { path = "../primitives" } polkadot-primitives = { path = "../../primitives" } polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-availability-recovery = { path = "../network/availability-recovery", features=["subsystem-benchmarks"]} +polkadot-availability-distribution = { path = "../network/availability-distribution"} +polkadot-node-core-av-store = { path = "../core/av-store"} +polkadot-node-core-chain-api = { path = "../core/chain-api"} +polkadot-availability-bitfield-distribution = { path = "../network/bitfield-distribution"} color-eyre = { version = "0.6.1", default-features = false } polkadot-overseer = { path = "../overseer" } colored = "2.0.4" @@ -56,6 +60,9 @@ serde = "1.0.192" serde_yaml = "0.9" paste = "1.0.14" orchestra = { version = "0.3.3", default-features = false, features=["futures_channel"] } +kvdb-memorydb = "0.13.0" +sp-consensus = { path = "../../../substrate/primitives/consensus/common", default-features = false } +bitvec = "1.0.0" [features] default = [] diff --git a/polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs b/polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs new file mode 100644 index 000000000000..e8b38a829a56 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs @@ -0,0 +1,66 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use super::*; + +use polkadot_node_metrics::metrics::Metrics; + +use polkadot_node_core_av_store::Config; +use polkadot_node_subsystem_test_helpers as test_helpers; +use polkadot_node_subsystem_util::{database::Database, TimeoutExt}; +use polkadot_primitives::{ + CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, HeadData, Header, + PersistedValidationData, ValidatorId, +}; + +use polkadot_node_core_av_store::AvailabilityStoreSubsystem; +use sp_keyring::Sr25519Keyring; +use std::time::Duration; + +mod columns { + pub const DATA: u32 = 0; + pub const META: u32 = 1; + pub const NUM_COLUMNS: u32 = 2; +} + +const TEST_CONFIG: Config = Config { col_data: columns::DATA, col_meta: columns::META }; + +struct DumbOracle; + +impl sp_consensus::SyncOracle for DumbOracle { + fn is_major_syncing(&self) -> bool { + false + } + + fn is_offline(&self) -> bool { + unimplemented!("oh no!") + } +} + +pub fn new_av_store(dependencies: &TestEnvironmentDependencies) -> AvailabilityStoreSubsystem { + let metrics = Metrics::try_register(&dependencies.registry).unwrap(); + + AvailabilityStoreSubsystem::new(test_store(), TEST_CONFIG, Box::new(DumbOracle), metrics) +} + +const TIMEOUT: Duration = Duration::from_millis(100); + +fn test_store() -> Arc { + let db = kvdb_memorydb::create(columns::NUM_COLUMNS); + let db = + polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[columns::META]); + Arc::new(db) +} diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 244119735966..47d09dc59f51 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -13,12 +13,14 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use crate::{core::mock::ChainApiState, TestEnvironment}; use itertools::Itertools; -use std::{collections::HashMap, iter::Cycle, ops::Sub, sync::Arc, time::Instant}; - -use crate::TestEnvironment; +use polkadot_availability_bitfield_distribution::BitfieldDistribution; +use polkadot_node_core_av_store::AvailabilityStoreSubsystem; use polkadot_node_subsystem::{Overseer, OverseerConnector, SpawnGlue}; use polkadot_overseer::Handle as OverseerHandle; +use sp_core::H256; +use std::{collections::HashMap, iter::Cycle, ops::Sub, sync::Arc, time::Instant}; use sc_network::request_responses::ProtocolConfig; @@ -27,6 +29,11 @@ use colored::Colorize; use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; use polkadot_node_metrics::metrics::Metrics; +use av_store_helpers::new_av_store; +use polkadot_availability_distribution::{ + AvailabilityDistributionSubsystem, IncomingRequestReceivers, +}; + use polkadot_availability_recovery::AvailabilityRecoverySubsystem; use crate::GENESIS_HASH; @@ -41,7 +48,7 @@ use crate::core::{ mock::{ av_store, network_bridge::{self, MockNetworkBridgeTx, NetworkAvailabilityState}, - runtime_api, MockAvailabilityStore, MockRuntimeApi, + runtime_api, MockAvailabilityStore, MockChainApi, MockRuntimeApi, }, }; @@ -52,18 +59,19 @@ const LOG_TARGET: &str = "subsystem-bench::availability"; use polkadot_node_primitives::{AvailableData, ErasureChunk}; use super::{cli::TestObjective, core::mock::AlwaysSupportsParachains}; -use polkadot_node_subsystem_test_helpers::mock::new_block_import_info; +use polkadot_node_subsystem_test_helpers::mock::{make_ferdie_keystore, new_block_import_info}; use polkadot_primitives::{ - CandidateHash, CandidateReceipt, GroupIndex, Hash, HeadData, PersistedValidationData, - ValidatorIndex, + BlockNumber, CandidateHash, CandidateReceipt, GroupIndex, Hash, HeadData, Header, + PersistedValidationData, ValidatorIndex, }; use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; use sc_service::SpawnTaskHandle; +mod av_store_helpers; mod cli; pub use cli::{DataAvailabilityReadOptions, NetworkEmulation}; -fn build_overseer( +fn build_overseer_for_availability_read( spawn_task_handle: SpawnTaskHandle, runtime_api: MockRuntimeApi, av_store: MockAvailabilityStore, @@ -84,11 +92,38 @@ fn build_overseer( (overseer, OverseerHandle::new(raw_handle)) } +fn build_overseer_for_availability_write( + spawn_task_handle: SpawnTaskHandle, + runtime_api: MockRuntimeApi, + av_store: MockAvailabilityStore, + network_bridge: MockNetworkBridgeTx, + availability_distribution: AvailabilityDistributionSubsystem, + chain_api: MockChainApi, + availability_store: AvailabilityStoreSubsystem, + bitfield_distribution: BitfieldDistribution, +) -> (Overseer, AlwaysSupportsParachains>, OverseerHandle) { + let overseer_connector = OverseerConnector::with_event_capacity(64000); + let dummy = dummy_builder!(spawn_task_handle); + let builder = dummy + .replace_runtime_api(|_| runtime_api) + .replace_availability_store(|_| availability_store) + .replace_network_bridge_tx(|_| network_bridge) + .replace_chain_api(|_| chain_api) + .replace_bitfield_distribution(|_| bitfield_distribution) + // This is needed to test own chunk recovery for `n_cores`. + .replace_availability_distribution(|_| availability_distribution); + + let (overseer, raw_handle) = + builder.build_with_connector(overseer_connector).expect("Should not fail"); + + (overseer, OverseerHandle::new(raw_handle)) +} + /// Takes a test configuration and uses it to creates the `TestEnvironment`. pub fn prepare_test( config: TestConfiguration, state: &mut TestState, -) -> (TestEnvironment, ProtocolConfig) { +) -> (TestEnvironment, Vec) { prepare_test_inner(config, state, TestEnvironmentDependencies::default()) } @@ -96,11 +131,27 @@ fn prepare_test_inner( config: TestConfiguration, state: &mut TestState, dependencies: TestEnvironmentDependencies, -) -> (TestEnvironment, ProtocolConfig) { +) -> (TestEnvironment, Vec) { // Generate test authorities. let test_authorities = config.generate_authorities(); - let runtime_api = runtime_api::MockRuntimeApi::new(config.clone(), test_authorities.clone()); + let mut candidate_hashes: HashMap> = HashMap::new(); + + // Prepare per block candidates. + for block_num in 0..config.num_blocks { + for _ in 0..config.n_cores { + candidate_hashes + .entry(Hash::repeat_byte(block_num as u8)) + .or_default() + .push(state.next_candidate().expect("Cycle iterator")) + } + } + + let runtime_api = runtime_api::MockRuntimeApi::new( + config.clone(), + test_authorities.clone(), + candidate_hashes, + ); let av_store = av_store::MockAvailabilityStore::new(state.chunks.clone(), state.candidate_hashes.clone()); @@ -119,35 +170,91 @@ fn prepare_test_inner( network.clone(), ); - let use_fast_path = match &state.config().objective { - TestObjective::DataAvailabilityRead(options) => options.fetch_from_backers, - _ => panic!("Unexpected objective"), - }; + let mut req_cfgs = Vec::new(); - let (collation_req_receiver, req_cfg) = - IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); - - let subsystem = if use_fast_path { - AvailabilityRecoverySubsystem::with_fast_path( - collation_req_receiver, - Metrics::try_register(&dependencies.registry).unwrap(), - ) - } else { - AvailabilityRecoverySubsystem::with_chunks_only( - collation_req_receiver, - Metrics::try_register(&dependencies.registry).unwrap(), - ) - }; + let (overseer, overseer_handle) = match &state.config().objective { + TestObjective::DataAvailabilityRead(_options) => { + let use_fast_path = match &state.config().objective { + TestObjective::DataAvailabilityRead(options) => options.fetch_from_backers, + _ => panic!("Unexpected objective"), + }; - let (overseer, overseer_handle) = build_overseer( - dependencies.task_manager.spawn_handle(), - runtime_api, - av_store, - network_bridge_tx, - subsystem, - ); + let (collation_req_receiver, req_cfg) = + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); + req_cfgs.push(req_cfg); + + let subsystem = if use_fast_path { + AvailabilityRecoverySubsystem::with_fast_path( + collation_req_receiver, + Metrics::try_register(&dependencies.registry).unwrap(), + ) + } else { + AvailabilityRecoverySubsystem::with_chunks_only( + collation_req_receiver, + Metrics::try_register(&dependencies.registry).unwrap(), + ) + }; + + build_overseer_for_availability_read( + dependencies.task_manager.spawn_handle(), + runtime_api, + av_store, + network_bridge_tx, + subsystem, + ) + }, + TestObjective::DataAvailabilityWrite => { + let (pov_req_receiver, pov_req_cfg) = + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); + let (chunk_req_receiver, chunk_req_cfg) = + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); + req_cfgs.push(pov_req_cfg); + req_cfgs.push(chunk_req_cfg); + + let availability_distribution = AvailabilityDistributionSubsystem::new( + test_authorities.keyring.keystore(), + IncomingRequestReceivers { pov_req_receiver, chunk_req_receiver }, + Metrics::try_register(&dependencies.registry).unwrap(), + ); + let availability_store = new_av_store(&dependencies); + + let block_headers = (0..config.num_blocks) + .map(|block_number| { + ( + Hash::repeat_byte(block_number as u8), + Header { + digest: Default::default(), + number: block_number as BlockNumber, + parent_hash: Default::default(), + extrinsics_root: Default::default(), + state_root: Default::default(), + }, + ) + }) + .collect::>(); + + let chain_api_state = ChainApiState { block_headers }; + let chain_api = MockChainApi::new(config.clone(), chain_api_state); + + let bitfield_distribution = + BitfieldDistribution::new(Metrics::try_register(&dependencies.registry).unwrap()); + build_overseer_for_availability_write( + dependencies.task_manager.spawn_handle(), + runtime_api, + av_store, + network_bridge_tx, + availability_distribution, + chain_api, + availability_store, + bitfield_distribution, + ) + }, + _ => { + unimplemented!("Invalid test objective") + }, + }; - (TestEnvironment::new(dependencies, config, network, overseer, overseer_handle), req_cfg) + (TestEnvironment::new(dependencies, config, network, overseer, overseer_handle), req_cfgs) } #[derive(Clone)] @@ -362,6 +469,54 @@ pub async fn benchmark_availability_read(env: &mut TestEnvironment, mut state: T .red() ); - gum::info!("{}", &env); + env.display_cpu_usage(&["availability-recovery"]); + env.stop().await; +} + +pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: TestState) { + let config = env.config().clone(); + let start_marker = Instant::now(); + let mut availability_bytes = 0u128; + + env.metrics().set_n_validators(config.n_validators); + env.metrics().set_n_cores(config.n_cores); + + for block_num in 0..env.config().num_blocks { + gum::info!(target: LOG_TARGET, "Current block {}/{}", block_num + 1, env.config().num_blocks); + env.metrics().set_current_block(block_num); + + let block_start_ts = Instant::now(); + + env.import_block(new_block_import_info( + Hash::repeat_byte(block_num as u8), + block_num as BlockNumber, + )) + .await; + + tokio::time::sleep(std::time::Duration::from_secs(6)).await; + + let block_time = Instant::now().sub(block_start_ts).as_millis() as u64; + env.metrics().set_block_time(block_time); + gum::info!("All work for block completed in {}", format!("{:?}ms", block_time).cyan()); + } + + let duration: u128 = start_marker.elapsed().as_millis(); + let availability_bytes = availability_bytes / 1024; + gum::info!("All blocks processed in {}", format!("{:?}ms", duration).cyan()); + gum::info!( + "Throughput: {}", + format!("{} KiB/block", availability_bytes / env.config().num_blocks as u128).bright_red() + ); + gum::info!( + "Block time: {}", + format!("{} ms", start_marker.elapsed().as_millis() / env.config().num_blocks as u128) + .red() + ); + + env.display_cpu_usage(&[ + "availability-distribution", + "bitfield-distribution", + "availability-store", + ]); env.stop().await; } diff --git a/polkadot/node/subsystem-bench/src/cli.rs b/polkadot/node/subsystem-bench/src/cli.rs index 3352f33a3503..7213713eb6ba 100644 --- a/polkadot/node/subsystem-bench/src/cli.rs +++ b/polkadot/node/subsystem-bench/src/cli.rs @@ -24,12 +24,14 @@ pub struct TestSequenceOptions { pub path: String, } -/// Define the supported benchmarks targets +/// Supported test objectives #[derive(Debug, Clone, clap::Parser, Serialize, Deserialize)] #[command(rename_all = "kebab-case")] pub enum TestObjective { /// Benchmark availability recovery strategies. DataAvailabilityRead(DataAvailabilityReadOptions), + /// Benchmark availability and bitfield distribution. + DataAvailabilityWrite, /// Run a test sequence specified in a file TestSequence(TestSequenceOptions), } diff --git a/polkadot/node/subsystem-bench/src/core/configuration.rs b/polkadot/node/subsystem-bench/src/core/configuration.rs index adb5ce80c0d4..fe2e045fb0b2 100644 --- a/polkadot/node/subsystem-bench/src/core/configuration.rs +++ b/polkadot/node/subsystem-bench/src/core/configuration.rs @@ -58,6 +58,11 @@ fn default_connectivity() -> usize { 100 } +// Default backing group size +fn default_backing_group_size() -> usize { + 5 +} + /// The test input parameters #[derive(Clone, Debug, Serialize, Deserialize)] pub struct TestConfiguration { @@ -67,6 +72,9 @@ pub struct TestConfiguration { pub n_validators: usize, /// Number of cores pub n_cores: usize, + /// Maximum backing group size + #[serde(default = "default_backing_group_size")] + pub max_validators_per_core: usize, /// The min PoV size #[serde(default = "default_pov_size")] pub min_pov_size: usize, @@ -129,7 +137,7 @@ impl TestSequence { /// Helper struct for authority related state. #[derive(Clone)] pub struct TestAuthorities { - pub keyrings: Vec, + pub keyring: Keyring, pub validator_public: Vec, pub validator_authority_id: Vec, } @@ -149,23 +157,20 @@ impl TestConfiguration { /// Generates the authority keys we need for the network emulation. pub fn generate_authorities(&self) -> TestAuthorities { - let keyrings = (0..self.n_validators) - .map(|peer_index| Keyring::new(format!("Node{}", peer_index).into())) + let keyring = Keyring::default(); + + let keys = (0..self.n_validators) + .map(|peer_index| keyring.sr25519_new(format!("Node{}", peer_index).into())) .collect::>(); // Generate `AuthorityDiscoveryId`` for each peer - let validator_public: Vec = keyrings - .iter() - .map(|keyring: &Keyring| keyring.clone().public().into()) - .collect::>(); + let validator_public: Vec = + keys.iter().map(|key| key.clone().into()).collect::>(); - let validator_authority_id: Vec = keyrings - .iter() - .map(|keyring| keyring.clone().public().into()) - .collect::>() - .into(); + let validator_authority_id: Vec = + keys.iter().map(|key| key.clone().into()).collect::>().into(); - TestAuthorities { keyrings, validator_public, validator_authority_id } + TestAuthorities { keyring, validator_public, validator_authority_id } } /// An unconstrained standard configuration matching Polkadot/Kusama @@ -181,6 +186,7 @@ impl TestConfiguration { objective, n_cores, n_validators, + max_validators_per_core: 5, pov_sizes: generate_pov_sizes(n_cores, min_pov_size, max_pov_size), bandwidth: 50 * 1024 * 1024, peer_bandwidth: 50 * 1024 * 1024, @@ -206,6 +212,7 @@ impl TestConfiguration { objective, n_cores, n_validators, + max_validators_per_core: 5, pov_sizes: generate_pov_sizes(n_cores, min_pov_size, max_pov_size), bandwidth: 50 * 1024 * 1024, peer_bandwidth: 50 * 1024 * 1024, @@ -233,6 +240,7 @@ impl TestConfiguration { objective, n_cores, n_validators, + max_validators_per_core: 5, pov_sizes: generate_pov_sizes(n_cores, min_pov_size, max_pov_size), bandwidth: 50 * 1024 * 1024, peer_bandwidth: 50 * 1024 * 1024, diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 247596474078..7b0ad6b1257b 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -279,16 +279,12 @@ impl TestEnvironment { pub async fn stop(&mut self) { self.overseer_handle.stop().await; } -} -impl Display for TestEnvironment { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Print CPU usage stats in the CLI. + pub fn display_cpu_usage(&self, subsystems_under_test: &[&str]) { let stats = self.network().stats(); - - writeln!(f, "\n")?; - writeln!( - f, - "Total received from network: {}", + println!( + "\nTotal received from network: {}", format!( "{} MiB", stats @@ -298,36 +294,40 @@ impl Display for TestEnvironment { .sum::() / (1024 * 1024) ) .cyan() - )?; - writeln!( - f, + ); + println!( "Total sent to network: {}", format!("{} KiB", stats[0].tx_bytes_total / (1024)).cyan() - )?; + ); let test_metrics = super::display::parse_metrics(self.registry()); - let subsystem_cpu_metrics = - test_metrics.subset_with_label_value("task_group", "availability-recovery"); - let total_cpu = subsystem_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); - writeln!(f, "Total subsystem CPU usage {}", format!("{:.2}s", total_cpu).bright_purple())?; - writeln!( - f, - "CPU usage per block {}", - format!("{:.2}s", total_cpu / self.config().num_blocks as f64).bright_purple() - )?; + + for subsystem in subsystems_under_test.into_iter() { + let subsystem_cpu_metrics = + test_metrics.subset_with_label_value("task_group", subsystem); + let total_cpu = subsystem_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); + println!( + "{} CPU usage {}", + format!("{}", subsystem).bright_green(), + format!("{:.3}s", total_cpu).bright_purple() + ); + println!( + "{} CPU usage per block {}", + format!("{}", subsystem).bright_green(), + format!("{:.3}s", total_cpu / self.config().num_blocks as f64).bright_purple() + ); + } let test_env_cpu_metrics = test_metrics.subset_with_label_value("task_group", "test-environment"); let total_cpu = test_env_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); - writeln!( - f, + println!( "Total test environment CPU usage {}", - format!("{:.2}s", total_cpu).bright_purple() - )?; - writeln!( - f, - "CPU usage per block {}", - format!("{:.2}s", total_cpu / self.config().num_blocks as f64).bright_purple() + format!("{:.3}s", total_cpu).bright_purple() + ); + println!( + "Test environment CPU usage per block {}", + format!("{:.3}s", total_cpu / self.config().num_blocks as f64).bright_purple() ) } } diff --git a/polkadot/node/subsystem-bench/src/core/keyring.rs b/polkadot/node/subsystem-bench/src/core/keyring.rs index 2d9aa348a922..2997fff69509 100644 --- a/polkadot/node/subsystem-bench/src/core/keyring.rs +++ b/polkadot/node/subsystem-bench/src/core/keyring.rs @@ -14,27 +14,38 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use polkadot_primitives::ValidatorId; +use sc_keystore::LocalKeystore; +use sp_application_crypto::AppCrypto; pub use sp_core::sr25519; use sp_core::{ sr25519::{Pair, Public}, Pair as PairT, }; -/// Set of test accounts. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +use sp_keyring::Sr25519Keyring; +use sp_keystore::{Keystore, KeystorePtr}; +use std::sync::Arc; + +/// Set of test accounts generated and kept safe by a keystore. +#[derive(Clone)] pub struct Keyring { - name: String, + keystore: Arc, } -impl Keyring { - pub fn new(name: String) -> Keyring { - Self { name } +impl Default for Keyring { + fn default() -> Self { + Self { keystore: Arc::new(LocalKeystore::in_memory()) } } +} - pub fn pair(self) -> Pair { - Pair::from_string(&format!("//{}", self.name), None).expect("input is always good; qed") +impl Keyring { + pub fn sr25519_new(&self, name: String) -> Public { + self.keystore + .sr25519_generate_new(ValidatorId::ID, Some(&format!("//{}", name))) + .expect("Insert key into keystore") } - pub fn public(self) -> Public { - self.pair().public() + pub fn keystore(&self) -> Arc { + self.keystore.clone() } } diff --git a/polkadot/node/subsystem-bench/src/core/mock/av_store.rs b/polkadot/node/subsystem-bench/src/core/mock/av_store.rs index 88747affc8c0..e5bcdc6685d6 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/av_store.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/av_store.rs @@ -128,6 +128,10 @@ impl MockAvailabilityStore { .encoded_size(); let _ = tx.send(Some(chunk_size)); }, + AvailabilityStoreMessage::StoreChunk { candidate_hash, chunk, tx } => { + gum::debug!(target: LOG_TARGET, chunk_index = ?chunk.index ,candidate_hash = ?candidate_hash, "Responding to StoreChunk"); + let _ = tx.send(Ok(())); + }, _ => { unimplemented!("Unexpected av-store message") }, diff --git a/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs b/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs new file mode 100644 index 000000000000..655276dc624f --- /dev/null +++ b/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs @@ -0,0 +1,94 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . +//! +//! A generic runtime api subsystem mockup suitable to be used in benchmarks. + +use polkadot_primitives::{GroupIndex, Header, IndexedVec, SessionInfo, ValidatorIndex}; + +use polkadot_node_subsystem::{ + messages::ChainApiMessage, overseer, SpawnedSubsystem, SubsystemError, +}; +use polkadot_node_subsystem_types::OverseerSignal; +use sp_core::H256; +use std::collections::HashMap; + +use crate::core::configuration::{TestAuthorities, TestConfiguration}; +use futures::FutureExt; + +const LOG_TARGET: &str = "subsystem-bench::chain-api-mock"; + +/// State used to respond to `BlockHeader` requests. +pub struct ChainApiState { + pub block_headers: HashMap, +} + +pub struct MockChainApi { + state: ChainApiState, + config: TestConfiguration, +} + +impl MockChainApi { + pub fn new(config: TestConfiguration, state: ChainApiState) -> MockChainApi { + Self { state, config } + } +} + +#[overseer::subsystem(ChainApi, error=SubsystemError, prefix=self::overseer)] +impl MockChainApi { + fn start(self, ctx: Context) -> SpawnedSubsystem { + let future = self.run(ctx).map(|_| Ok(())).boxed(); + + SpawnedSubsystem { name: "test-environment", future } + } +} + +#[overseer::contextbounds(ChainApi, prefix = self::overseer)] +impl MockChainApi { + async fn run(self, mut ctx: Context) { + loop { + let msg = ctx.recv().await.expect("Overseer never fails us"); + + match msg { + orchestra::FromOrchestra::Signal(signal) => match signal { + OverseerSignal::Conclude => return, + _ => {}, + }, + orchestra::FromOrchestra::Communication { msg } => { + gum::debug!(target: LOG_TARGET, msg=?msg, "recv message"); + + match msg { + ChainApiMessage::BlockHeader(hash, response_channel) => { + let _ = response_channel.send(Ok(Some( + self.state + .block_headers + .get(&hash) + .cloned() + .expect("Relay chain block hashes are known"), + ))); + }, + ChainApiMessage::Ancestors { hash, k, response_channel } => { + // For our purposes, no ancestors is fine. + let _ = response_channel.send(Ok(Vec::new())); + }, + _ => { + unimplemented!("Unexpected chain-api message") + }, + } + }, + } + } + } +} diff --git a/polkadot/node/subsystem-bench/src/core/mock/dummy.rs b/polkadot/node/subsystem-bench/src/core/mock/dummy.rs index 0628368a49c0..a0a908750c51 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/dummy.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/dummy.rs @@ -73,6 +73,7 @@ macro_rules! mock { }; } +// Generate dummy implementation for all subsystems mock!(AvailabilityStore); mock!(StatementDistribution); mock!(BitfieldSigning); diff --git a/polkadot/node/subsystem-bench/src/core/mock/mod.rs b/polkadot/node/subsystem-bench/src/core/mock/mod.rs index d59642e96058..2bcc0c08c57b 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/mod.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/mod.rs @@ -18,11 +18,13 @@ use polkadot_node_subsystem::HeadSupportsParachains; use polkadot_node_subsystem_types::Hash; pub mod av_store; +pub mod chain_api; pub mod dummy; pub mod network_bridge; pub mod runtime_api; pub use av_store::*; +pub use chain_api::*; pub use network_bridge::*; pub use runtime_api::*; diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index c8140843b3b9..22abd0625c51 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -273,7 +273,6 @@ impl MockNetworkBridgeTx { async move { while let Some(action) = ingress_rx.recv().await { let size = action.size(); - // account for our node receiving the data. our_network.inc_received(size); rx_limiter.reap(size).await; diff --git a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs index 9cbe025ae806..dba9c0e66f4b 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs @@ -16,31 +16,46 @@ //! //! A generic runtime api subsystem mockup suitable to be used in benchmarks. -use polkadot_primitives::{GroupIndex, IndexedVec, SessionInfo, ValidatorIndex}; +use polkadot_primitives::{ + CandidateHash, CandidateReceipt, CoreState, GroupIndex, IndexedVec, OccupiedCore, SessionInfo, + ValidatorIndex, +}; +use bitvec::prelude::BitVec; use polkadot_node_subsystem::{ messages::{RuntimeApiMessage, RuntimeApiRequest}, overseer, SpawnedSubsystem, SubsystemError, }; use polkadot_node_subsystem_types::OverseerSignal; +use sp_core::H256; +use std::collections::HashMap; use crate::core::configuration::{TestAuthorities, TestConfiguration}; use futures::FutureExt; const LOG_TARGET: &str = "subsystem-bench::runtime-api-mock"; +/// Minimal state to answer requests. pub struct RuntimeApiState { + // All authorities in the test, authorities: TestAuthorities, + // Candidate + candidate_hashes: HashMap>, } +/// A mocked `runtime-api` subsystem. pub struct MockRuntimeApi { state: RuntimeApiState, config: TestConfiguration, } impl MockRuntimeApi { - pub fn new(config: TestConfiguration, authorities: TestAuthorities) -> MockRuntimeApi { - Self { state: RuntimeApiState { authorities }, config } + pub fn new( + config: TestConfiguration, + authorities: TestAuthorities, + candidate_hashes: HashMap>, + ) -> MockRuntimeApi { + Self { state: RuntimeApiState { authorities, candidate_hashes }, config } } fn session_info(&self) -> SessionInfo { @@ -48,7 +63,10 @@ impl MockRuntimeApi { .map(|i| ValidatorIndex(i as _)) .collect::>(); - let validator_groups = all_validators.chunks(5).map(|x| Vec::from(x)).collect::>(); + let validator_groups = all_validators + .chunks(self.config.max_validators_per_core) + .map(|x| Vec::from(x)) + .collect::>(); SessionInfo { validators: self.state.authorities.validator_public.clone().into(), @@ -80,6 +98,8 @@ impl MockRuntimeApi { #[overseer::contextbounds(RuntimeApi, prefix = self::overseer)] impl MockRuntimeApi { async fn run(self, mut ctx: Context) { + let validator_group_count = self.session_info().validator_groups.len(); + loop { let msg = ctx.recv().await.expect("Overseer never fails us"); @@ -93,11 +113,70 @@ impl MockRuntimeApi { match msg { RuntimeApiMessage::Request( - _request, + _block_hash, RuntimeApiRequest::SessionInfo(_session_index, sender), ) => { let _ = sender.send(Ok(Some(self.session_info()))); }, + RuntimeApiMessage::Request( + _block_hash, + RuntimeApiRequest::SessionExecutorParams(_session_index, sender), + ) => { + let _ = sender.send(Ok(Some(Default::default()))); + }, + RuntimeApiMessage::Request( + _block_hash, + RuntimeApiRequest::Validators(sender), + ) => { + let _ = + sender.send(Ok(self.state.authorities.validator_public.clone())); + }, + RuntimeApiMessage::Request( + _block_hash, + RuntimeApiRequest::CandidateEvents(sender), + ) => { + let _ = sender.send(Ok(Default::default())); + }, + RuntimeApiMessage::Request( + _block_hash, + RuntimeApiRequest::SessionIndexForChild(sender), + ) => { + // Session is always the same. + let _ = sender.send(Ok(0)); + }, + RuntimeApiMessage::Request( + block_hash, + RuntimeApiRequest::AvailabilityCores(sender), + ) => { + let candidate_hashes = self + .state + .candidate_hashes + .get(&block_hash) + .expect("Relay chain block hashes are generated at test start"); + + // All cores are always occupied. + let cores = candidate_hashes + .into_iter() + .enumerate() + .map(|(index, candidate_receipt)| { + // Ensure test breaks if badly configured. + assert!(index < validator_group_count); + + CoreState::Occupied(OccupiedCore { + next_up_on_available: None, + occupied_since: 0, + time_out_at: 0, + next_up_on_time_out: None, + availability: BitVec::default(), + group_responsible: GroupIndex(index as u32), + candidate_hash: candidate_receipt.hash(), + candidate_descriptor: candidate_receipt.descriptor.clone(), + }) + }) + .collect::>(); + + let _ = sender.send(Ok(cores)); + }, // Long term TODO: implement more as needed. _ => { unimplemented!("Unexpected runtime-api message") diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 09943becb65c..9d7098d4f3df 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -194,6 +194,7 @@ impl PeerEmulator { } pub type ActionFuture = std::pin::Pin + std::marker::Send>>; + /// An network action to be completed by the emulator task. pub struct NetworkAction { // The function that performs the action diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 0f3ae0f41417..423c0f5193ef 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -26,7 +26,7 @@ pub(crate) mod availability; pub(crate) mod cli; pub(crate) mod core; -use availability::{prepare_test, NetworkEmulation, TestState}; +use crate::availability::{prepare_test, NetworkEmulation, TestState}; use cli::TestObjective; use core::{ @@ -81,8 +81,39 @@ struct BenchCli { } impl BenchCli { + fn create_test_configuration(&self) -> TestConfiguration { + let configuration = &self.standard_configuration; + + match self.network { + NetworkEmulation::Healthy => TestConfiguration::healthy_network( + self.objective.clone(), + configuration.num_blocks, + configuration.n_validators, + configuration.n_cores, + configuration.min_pov_size, + configuration.max_pov_size, + ), + NetworkEmulation::Degraded => TestConfiguration::degraded_network( + self.objective.clone(), + configuration.num_blocks, + configuration.n_validators, + configuration.n_cores, + configuration.min_pov_size, + configuration.max_pov_size, + ), + NetworkEmulation::Ideal => TestConfiguration::ideal_network( + self.objective.clone(), + configuration.num_blocks, + configuration.n_validators, + configuration.n_cores, + configuration.min_pov_size, + configuration.max_pov_size, + ), + } + } + fn launch(self) -> eyre::Result<()> { - let configuration = self.standard_configuration; + let configuration = &self.standard_configuration; let mut test_config = match self.objective { TestObjective::TestSequence(options) => { let test_sequence = @@ -105,32 +136,8 @@ impl BenchCli { } return Ok(()) }, - TestObjective::DataAvailabilityRead(ref _options) => match self.network { - NetworkEmulation::Healthy => TestConfiguration::healthy_network( - self.objective, - configuration.num_blocks, - configuration.n_validators, - configuration.n_cores, - configuration.min_pov_size, - configuration.max_pov_size, - ), - NetworkEmulation::Degraded => TestConfiguration::degraded_network( - self.objective, - configuration.num_blocks, - configuration.n_validators, - configuration.n_cores, - configuration.min_pov_size, - configuration.max_pov_size, - ), - NetworkEmulation::Ideal => TestConfiguration::ideal_network( - self.objective, - configuration.num_blocks, - configuration.n_validators, - configuration.n_cores, - configuration.min_pov_size, - configuration.max_pov_size, - ), - }, + TestObjective::DataAvailabilityRead(ref _options) => self.create_test_configuration(), + TestObjective::DataAvailabilityWrite => self.create_test_configuration(), }; let mut latency_config = test_config.latency.clone().unwrap_or_default(); @@ -161,9 +168,18 @@ impl BenchCli { let mut state = TestState::new(&test_config); let (mut env, _protocol_config) = prepare_test(test_config, &mut state); - // test_config.write_to_disk(); - env.runtime() - .block_on(availability::benchmark_availability_read(&mut env, state)); + + match self.objective { + TestObjective::DataAvailabilityRead(_options) => { + env.runtime() + .block_on(availability::benchmark_availability_read(&mut env, state)); + }, + TestObjective::DataAvailabilityWrite => { + env.runtime() + .block_on(availability::benchmark_availability_write(&mut env, state)); + }, + TestObjective::TestSequence(_options) => {}, + } Ok(()) } From 5e781278227c62354a63c54d2b0a625cfed6c760 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 7 Dec 2023 15:01:00 +0200 Subject: [PATCH 124/192] Revert a2e602444662bd0e2443219b438d609da62a995d ... it is actually not needed and gets us a preformance penalty which I discovered while working on: https://github.com/paritytech/polkadot-sdk/pull/2621 Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 357 ++++-------------- 1 file changed, 82 insertions(+), 275 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 2ebf050849f1..edba3e1af2da 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -114,6 +114,14 @@ struct ApprovalRouting { required_routing: RequiredRouting, local: bool, random_routing: RandomRouting, + peers_randomly_routed: Vec, +} + +impl ApprovalRouting { + fn mark_randomly_sent(&mut self, peer: PeerId) { + self.random_routing.inc_sent(); + self.peers_randomly_routed.push(peer); + } } // This struct is responsible for tracking the full state of an assignment and grid routing @@ -412,18 +420,6 @@ impl Knowledge { } success } - - // Tells if all keys are contained by this peer_knowledge - pub fn contains_all_keys(&self, keys: &Vec<(MessageSubject, MessageKind)>) -> bool { - keys.iter() - .all(|assignment_key| self.contains(&assignment_key.0, assignment_key.1)) - } - - // Tells if all keys are contained by this peer_knowledge - pub fn contains_any(&self, keys: &Vec<(MessageSubject, MessageKind)>) -> bool { - keys.iter() - .any(|assignment_key| self.contains(&assignment_key.0, assignment_key.1)) - } } /// Information that has been circulated to and from a peer. @@ -440,73 +436,6 @@ impl PeerKnowledge { self.sent.contains(message, kind) || self.received.contains(message, kind) } - /// Partition a list of assignments into two lists. - /// - /// The the first one contains the list of assignments that we had - /// sent to the peer and the second one the list of assignments that - /// we have no knowledge if the peer has it or not. - fn partition_sent_notknown<'a>( - &self, - assignments: &'a Vec<(IndirectAssignmentCertV2, CandidateBitfield)>, - ) -> ( - Vec<&'a (IndirectAssignmentCertV2, CandidateBitfield)>, - Vec<&'a (IndirectAssignmentCertV2, CandidateBitfield)>, - ) { - let (sent, not_sent): ( - Vec<&'a (IndirectAssignmentCertV2, CandidateBitfield)>, - Vec<&'a (IndirectAssignmentCertV2, CandidateBitfield)>, - ) = assignments.iter().partition(|assignment| { - self.sent.contains( - &MessageSubject( - assignment.0.block_hash, - assignment.1.clone(), - assignment.0.validator, - ), - MessageKind::Assignment, - ) - }); - - let notknown_by_peer = not_sent - .into_iter() - .filter(|(assignment, candidate_bitfield)| { - !self.contains( - &MessageSubject( - assignment.block_hash, - candidate_bitfield.clone(), - assignment.validator, - ), - MessageKind::Assignment, - ) - }) - .collect_vec(); - - (sent, notknown_by_peer) - } - - /// Marks a list of assignments as sent to the peer - fn mark_sent(&mut self, assignments: &Vec<(IndirectAssignmentCertV2, CandidateBitfield)>) { - for assignment in assignments { - self.sent.insert( - MessageSubject( - assignment.0.block_hash, - assignment.1.clone(), - assignment.0.validator, - ), - MessageKind::Assignment, - ); - } - } - - // Tells if all assignments for a given approval are included in the knowledge of the peer - fn contains_assignments_for_approval(&self, approval: &IndirectSignedApprovalVoteV2) -> bool { - self.contains_all_keys(&Self::generate_assignments_keys(&approval)) - } - - // Tells if all keys are contained by this peer_knowledge - pub fn contains_all_keys(&self, keys: &Vec<(MessageSubject, MessageKind)>) -> bool { - self.sent.contains_all_keys(keys) || self.received.contains_all_keys(keys) - } - // Generate the knowledge keys for querying if all assignments of an approval are known // by this peer. fn generate_assignments_keys( @@ -606,48 +535,17 @@ impl BlockEntry { .all(|candidate_index| self.candidates.get(candidate_index as usize).is_some()) } - // Returns all assignments covering the candidates in a given `approval` - pub fn assignments_for_approval( - &self, - approval: &IndirectSignedApprovalVoteV2, - ) -> Result, ApprovalEntryError> { - let mut assignments = Vec::new(); - - if self.candidates.len() < approval.candidate_indices.len() as usize { - return Err(ApprovalEntryError::CandidateIndexOutOfBounds) - } - - // First determine all assignments bitfields that might be covered by this approval - let covered_assignments_bitfields: HashSet = approval - .candidate_indices - .iter_ones() - .filter_map(|candidate_index| { - self.candidates.get(candidate_index).map_or(None, |candidate_entry| { - candidate_entry.assignments.get(&approval.validator).cloned() - }) - }) - .collect(); - - for assignment_bitfield in covered_assignments_bitfields { - if let Some(approval_entry) = - self.approval_entries.get(&(approval.validator, assignment_bitfield.clone())) - { - assignments.push((approval_entry.assignment.clone(), assignment_bitfield.clone())) - } - } - - Ok(assignments) - } - // Saves the given approval in all ApprovalEntries that contain an assignment for any of the // candidates in the approval. // - // Returns the required routing needed for this approval. + // Returns the required routing needed for this approval and the lit of random peers the + // covering assignments were sent. pub fn note_approval( &mut self, approval: IndirectSignedApprovalVoteV2, - ) -> Result { + ) -> Result<(RequiredRouting, HashSet), ApprovalEntryError> { let mut required_routing = None; + let mut peers_randomly_routed_to = HashSet::new(); if self.candidates.len() < approval.candidate_indices.len() as usize { return Err(ApprovalEntryError::CandidateIndexOutOfBounds) @@ -670,6 +568,8 @@ impl BlockEntry { self.approval_entries.get_mut(&(approval.validator, assignment_bitfield)) { approval_entry.note_approval(approval.clone())?; + peers_randomly_routed_to + .extend(approval_entry.routing_info().peers_randomly_routed.iter()); if let Some(required_routing) = required_routing { if required_routing != approval_entry.routing_info().required_routing { @@ -688,7 +588,7 @@ impl BlockEntry { } if let Some(required_routing) = required_routing { - Ok(required_routing) + Ok((required_routing, peers_randomly_routed_to)) } else { Err(ApprovalEntryError::UnknownAssignment) } @@ -1489,7 +1389,12 @@ impl State { let approval_entry = entry.insert_approval_entry(ApprovalEntry::new( assignment.clone(), claimed_candidate_indices.clone(), - ApprovalRouting { required_routing, local, random_routing: Default::default() }, + ApprovalRouting { + required_routing, + local, + random_routing: Default::default(), + peers_randomly_routed: Default::default(), + }, )); // Dispatch the message to all peers in the routing set which @@ -1530,7 +1435,7 @@ impl State { approval_entry.routing_info().random_routing.sample(n_peers_total, rng); if route_random { - approval_entry.routing_info_mut().random_routing.inc_sent(); + approval_entry.routing_info_mut().mark_randomly_sent(peer); peers.push(peer); } } @@ -1797,7 +1702,7 @@ impl State { } } - let required_routing = match entry.note_approval(vote.clone()) { + let (required_routing, peers_randomly_routed_to) = match entry.note_approval(vote.clone()) { Ok(required_routing) => required_routing, Err(err) => { gum::warn!( @@ -1813,8 +1718,6 @@ impl State { }, }; - let assignments = entry.assignments_for_approval(&vote).unwrap_or_default(); - // Invariant: to our knowledge, none of the peers except for the `source` know about the // approval. metrics.on_approval_imported(); @@ -1824,7 +1727,7 @@ impl State { let topology = self.topologies.get_topology(entry.session); let source_peer = source.peer_id(); - let peer_filter = move |peer, knowledge: &PeerKnowledge| { + let peer_filter = move |peer| { if Some(peer) == source_peer.as_ref() { return false } @@ -1840,13 +1743,13 @@ impl State { // 3. Any randomly selected peers have been sent the assignment already. let in_topology = topology .map_or(false, |t| t.local_grid_neighbors().route_to_peer(required_routing, peer)); - in_topology || knowledge.sent.contains_any(&assignments_knowledge_keys) + in_topology || peers_randomly_routed_to.contains(peer) }; let peers = entry .known_by .iter() - .filter(|(p, k)| peer_filter(p, k)) + .filter(|(p, _)| peer_filter(p)) .filter_map(|(p, _)| self.peer_views.get(p).map(|entry| (*p, entry.version))) .collect::>(); @@ -1854,28 +1757,6 @@ impl State { for peer in peers.iter() { // we already filtered peers above, so this should always be Some if let Some(entry) = entry.known_by.get_mut(&peer.0) { - // A random assignment could have been sent to the peer, so we need to make sure - // we send all the other assignments, before we can send the corresponding approval. - let (sent_to_peer, notknown_by_peer) = entry.partition_sent_notknown(&assignments); - if !notknown_by_peer.is_empty() { - let notknown_by_peer = notknown_by_peer - .into_iter() - .map(|(assignment, bitfield)| (assignment.clone(), bitfield.clone())) - .collect_vec(); - gum::trace!( - target: LOG_TARGET, - ?block_hash, - ?peer, - missing = notknown_by_peer.len(), - part1 = sent_to_peer.len(), - "Sending missing assignments", - ); - - entry.mark_sent(¬known_by_peer); - send_assignments_batched(ctx.sender(), notknown_by_peer, &[(peer.0, peer.1)]) - .await; - } - entry.sent.insert(approval_knwowledge_key.0.clone(), approval_knwowledge_key.1); } } @@ -1973,105 +1854,70 @@ impl State { if entry.known_by.contains_key(&peer_id) { break } - let approvals_to_send_this_block = { - let peer_knowledge = entry.known_by.entry(peer_id).or_default(); - let topology = topologies.get_topology(entry.session); - - let mut approvals_to_send_this_block = HashMap::new(); - // We want to iterate the `approval_entries` of the block entry as these contain - // all assignments that also link all approval votes. - for approval_entry in entry.approval_entries.values_mut() { - // Propagate the message to all peers in the required routing set OR - // randomly sample peers. - { - let required_routing = approval_entry.routing_info().required_routing; - let random_routing = - &mut approval_entry.routing_info_mut().random_routing; - let rng = &mut *rng; - let mut peer_filter = move |peer_id| { - let in_topology = topology.as_ref().map_or(false, |t| { - t.local_grid_neighbors() - .route_to_peer(required_routing, peer_id) - }); - in_topology || { - if !topology - .map(|topology| topology.is_validator(peer_id)) - .unwrap_or(false) - { - return false - } - - let route_random = random_routing.sample(total_peers, rng); - if route_random { - random_routing.inc_sent(); - } - - route_random - } - }; - - if !peer_filter(&peer_id) { - continue - } - } - - let assignment_message = approval_entry.assignment(); - let approval_messages = approval_entry.approvals(); - let (assignment_knowledge, message_kind) = - approval_entry.create_assignment_knowledge(block); - - // Only send stuff a peer doesn't know in the context of a relay chain - // block. - if !peer_knowledge.contains(&assignment_knowledge, message_kind) { - peer_knowledge.sent.insert(assignment_knowledge, message_kind); - assignments_to_send.push(assignment_message); - } - // Filter approval votes. - for approval_message in approval_messages { - let approval_knowledge = - PeerKnowledge::generate_approval_key(&approval_message); + let peer_knowledge = entry.known_by.entry(peer_id).or_default(); + let topology = topologies.get_topology(entry.session); - if !peer_knowledge.contains(&approval_knowledge.0, approval_knowledge.1) - { - if !approvals_to_send_this_block.contains_key(&approval_knowledge.0) + // We want to iterate the `approval_entries` of the block entry as these contain + // all assignments that also link all approval votes. + for approval_entry in entry.approval_entries.values_mut() { + // Propagate the message to all peers in the required routing set OR + // randomly sample peers. + { + let required_routing = approval_entry.routing_info().required_routing; + let routing_info = &mut approval_entry.routing_info_mut(); + let rng = &mut *rng; + let mut peer_filter = move |peer_id| { + let in_topology = topology.as_ref().map_or(false, |t| { + t.local_grid_neighbors().route_to_peer(required_routing, peer_id) + }); + in_topology || { + if !topology + .map(|topology| topology.is_validator(peer_id)) + .unwrap_or(false) { - approvals_to_send_this_block - .insert(approval_knowledge.0.clone(), approval_message); + return false + } + + let route_random = + routing_info.random_routing.sample(total_peers, rng); + if route_random { + routing_info.mark_randomly_sent(*peer_id); } + + route_random } + }; + + if !peer_filter(&peer_id) { + continue } } - approvals_to_send_this_block - }; - // An approval can span multiple assignments/ApprovalEntries, so after we processed - // all of the assignments decide which of the approvals we can safely send, because - // all of assignments are already sent or about to be sent. - for (approval_key, approval) in approvals_to_send_this_block { - let assignments = entry.assignments_for_approval(&approval).unwrap_or_default(); - let peer_knowledge = entry.known_by.entry(peer_id).or_default(); - let (sent_to_peer, notknown_by_peer) = - peer_knowledge.partition_sent_notknown(&assignments); - if !sent_to_peer.is_empty() { - let notknown_by_peer = notknown_by_peer - .into_iter() - .map(|(assignment, bitfield)| (assignment.clone(), bitfield.clone())) - .collect_vec(); - gum::trace!( - target: LOG_TARGET, - ?notknown_by_peer, - part1 = sent_to_peer.len(), - "Sending missing assignment unify_with_peer", - ); - peer_knowledge.mark_sent(¬known_by_peer); - assignments_to_send.extend(notknown_by_peer); + let assignment_message = approval_entry.assignment(); + let approval_messages = approval_entry.approvals(); + let (assignment_knowledge, message_kind) = + approval_entry.create_assignment_knowledge(block); + + // Only send stuff a peer doesn't know in the context of a relay chain + // block. + if !peer_knowledge.contains(&assignment_knowledge, message_kind) { + peer_knowledge.sent.insert(assignment_knowledge, message_kind); + assignments_to_send.push(assignment_message); } - if peer_knowledge.contains_assignments_for_approval(&approval) { - approvals_to_send.push(approval); - peer_knowledge.sent.insert(approval_key, MessageKind::Approval); + + // Filter approval votes. + for approval_message in approval_messages { + let approval_knowledge = + PeerKnowledge::generate_approval_key(&approval_message); + + if !peer_knowledge.contains(&approval_knowledge.0, approval_knowledge.1) { + approvals_to_send.push(approval_message); + peer_knowledge.sent.insert(approval_knowledge.0, approval_knowledge.1); + } } } + block = entry.parent_hash; } } @@ -2334,8 +2180,6 @@ async fn adjust_required_routing_and_propagate continue, }; - let mut approvals_to_send_for_this_block = HashMap::new(); - // We just need to iterate the `approval_entries` of the block entry as these contain all // assignments that also link all approval votes. for approval_entry in block_entry.approval_entries.values_mut() { @@ -2372,8 +2216,6 @@ async fn adjust_required_routing_and_propagate Date: Thu, 7 Dec 2023 17:48:16 +0200 Subject: [PATCH 125/192] Address review feedback Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 29 ++++++++++--------- .../approval-voting/src/persisted_entries.rs | 13 ++++----- polkadot/primitives/src/v6/mod.rs | 2 +- polkadot/primitives/src/vstaging/mod.rs | 6 ++++ .../src/protocol-approval.md | 6 ++-- .../0007-approval-voting-coalescing.zndsl | 2 +- 6 files changed, 33 insertions(+), 25 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index f0356e6ded35..af76b576d7ca 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -174,7 +174,7 @@ struct MetricsInner { approved_by_one_third: prometheus::Counter, wakeups_triggered_total: prometheus::Counter, coalesced_approvals_buckets: prometheus::Histogram, - coalesced_approvals_waiting_times: prometheus::Histogram, + coalesced_approvals_delay: prometheus::Histogram, candidate_approval_time_ticks: prometheus::Histogram, block_approval_time_ticks: prometheus::Histogram, time_db_transaction: prometheus::Histogram, @@ -212,7 +212,7 @@ impl Metrics { fn on_delayed_approval(&self, delayed_ticks: u64) { if let Some(metrics) = &self.0 { - metrics.coalesced_approvals_waiting_times.observe(delayed_ticks as f64) + metrics.coalesced_approvals_delay.observe(delayed_ticks as f64) } } @@ -373,10 +373,10 @@ impl metrics::Metrics for Metrics { )?, registry, )?, - coalesced_approvals_waiting_times: prometheus::register( + coalesced_approvals_delay: prometheus::register( prometheus::Histogram::with_opts( prometheus::HistogramOpts::new( - "polkadot_parachain_approvals_coalesced_approvals_waiting_times", + "polkadot_parachain_approvals_coalescing_delay", "Number of ticks we delay the sending of a candidate approval", ).buckets(vec![1.1, 2.1, 3.1, 4.1, 6.1, 8.1, 12.1, 20.1, 32.1]), )?, @@ -840,7 +840,7 @@ impl State { ctx: &mut Context, session_index: SessionIndex, block_hash: Hash, - ) -> ApprovalVotingParams { + ) -> Option { let (s_tx, s_rx) = oneshot::channel(); ctx.send_message(RuntimeApiMessage::Request( @@ -857,23 +857,23 @@ impl State { session = ?session_index, "Using the following subsystem params" ); - params + Some(params) }, Ok(Err(err)) => { gum::debug!( target: LOG_TARGET, ?err, - "Could not request approval voting params from runtime using defaults" + "Could not request approval voting params from runtime" ); - ApprovalVotingParams { max_approval_coalesce_count: 1 } + None }, Err(err) => { gum::debug!( target: LOG_TARGET, ?err, - "Could not request approval voting params from runtime using defaults" + "Could not request approval voting params from runtime" ); - ApprovalVotingParams { max_approval_coalesce_count: 1 } + None }, } } @@ -3411,7 +3411,8 @@ async fn maybe_create_signature( let approval_params = state .get_approval_voting_params_or_default(ctx, block_entry.session(), block_hash) - .await; + .await + .unwrap_or_default(); gum::trace!( target: LOG_TARGET, @@ -3464,7 +3465,7 @@ async fn maybe_create_signature( let signature = match sign_approval( &state.keystore, &validator_pubkey, - candidates_hashes.clone(), + &candidates_hashes, block_entry.session(), ) { Some(sig) => sig, @@ -3534,12 +3535,12 @@ async fn maybe_create_signature( fn sign_approval( keystore: &LocalKeystore, public: &ValidatorId, - candidate_hashes: Vec, + candidate_hashes: &[CandidateHash], session_index: SessionIndex, ) -> Option { let key = keystore.key_pair::(public).ok().flatten()?; - let payload = ApprovalVoteMultipleCandidates(&candidate_hashes).signing_payload(session_index); + let payload = ApprovalVoteMultipleCandidates(candidate_hashes).signing_payload(session_index); Some(key.sign(&payload[..])) } diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index 39c1fffc2ee7..ef47bdb2213a 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -602,7 +602,8 @@ impl BlockEntry { .keys() .map(|val| *val) .collect_vec() - .try_into().ok() + .try_into() + .ok() } /// Returns a list of candidates hashes that need need signature created at the current tick: @@ -628,11 +629,9 @@ impl BlockEntry { self.num_candidates_pending_signature() >= max_approval_coalesce_count as usize { ( - self.candidate_indices_pending_signature().and_then(|candidate_indices| - Some(( - self.candidate_hashes_pending_signature(), - candidate_indices, - ))), + self.candidate_indices_pending_signature().and_then(|candidate_indices| { + Some((self.candidate_hashes_pending_signature(), candidate_indices)) + }), Some(sign_no_later_than_tick), ) } else { @@ -648,7 +647,7 @@ impl BlockEntry { } } - /// Signals the approval was issued for the candidates pending signature + /// Clears the candidates pending signature because the approval was issued. pub fn issued_approval(&mut self) { self.candidates_pending_signature.clear(); } diff --git a/polkadot/primitives/src/v6/mod.rs b/polkadot/primitives/src/v6/mod.rs index 6b3920f48ff3..c3a947644fff 100644 --- a/polkadot/primitives/src/v6/mod.rs +++ b/polkadot/primitives/src/v6/mod.rs @@ -1072,7 +1072,7 @@ impl ApprovalVote { /// A vote of approval for multiple candidates. #[derive(Clone, RuntimeDebug)] -pub struct ApprovalVoteMultipleCandidates<'a>(pub &'a Vec); +pub struct ApprovalVoteMultipleCandidates<'a>(pub &'a [CandidateHash]); impl<'a> ApprovalVoteMultipleCandidates<'a> { /// Yields the signing payload for this approval vote. diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 1fe72eaa2b8f..7d213b8e7a73 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -44,6 +44,12 @@ pub struct ApprovalVotingParams { pub max_approval_coalesce_count: u32, } +impl Default for ApprovalVotingParams { + fn default() -> Self { + Self { max_approval_coalesce_count: 1 } + } +} + use bitvec::vec::BitVec; /// Bit indices in the `HostConfiguration.node_features` that correspond to different node features. diff --git a/polkadot/roadmap/implementers-guide/src/protocol-approval.md b/polkadot/roadmap/implementers-guide/src/protocol-approval.md index f3009f5fb44a..ebe4894f0b3e 100644 --- a/polkadot/roadmap/implementers-guide/src/protocol-approval.md +++ b/polkadot/roadmap/implementers-guide/src/protocol-approval.md @@ -301,8 +301,10 @@ To reduce the necessary network bandwidth and cpu time when a validator has more doing our best effort to send a single message that approves all available candidates with a single signature. The implemented heuristic, is that each time we are ready to create a signature and send a vote for a candidate we delay sending it until one of three things happen: -- We gathered a maximum of `MAX_APPROVAL_COALESCE_COUNT` candidates that we are ready to vote for. -- `MAX_APPROVAL_COALESCE_WAIT_TICKS` have passed since the we were ready to approve the candidate. +- We gathered a maximum of `MAX_APPROVAL_COALESCE_COUNT` candidates that we have already checked and we are + ready to sign approval for. +- `MAX_APPROVAL_COALESCE_WAIT_TICKS` have passed since checking oldest candidate and we were ready to sign + and send the approval message. - We are already in the last third of the no-show period in order to avoid creating accidental no-shows, which in turn might trigger other assignments. diff --git a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl index 07ef30d546a1..43b2c5e13a80 100644 --- a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl +++ b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl @@ -29,4 +29,4 @@ alice: reports substrate_block_height{status="finalized"} is at least 30 within alice: reports polkadot_parachain_approval_checking_finality_lag < 3 -alice: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds +alice: reports polkadot_parachain_approvals_no_shows_total < 3 within 10 seconds From 1913b63e6dca44b496b41c53d54dfabe1bd23780 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 7 Dec 2023 18:24:00 +0200 Subject: [PATCH 126/192] protocol: rename vstaging into v3 Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 51 +++--- .../approval-distribution/src/tests.rs | 146 +++++++++--------- .../network/bitfield-distribution/src/lib.rs | 18 +-- polkadot/node/network/bridge/src/network.rs | 12 +- polkadot/node/network/bridge/src/rx/mod.rs | 97 ++++++------ polkadot/node/network/bridge/src/rx/tests.rs | 6 +- polkadot/node/network/bridge/src/tx/mod.rs | 10 +- .../src/collator_side/mod.rs | 6 +- .../src/validator_side/mod.rs | 6 +- .../node/network/gossip-support/src/lib.rs | 2 +- polkadot/node/network/protocol/src/lib.rs | 54 ++++--- .../node/network/protocol/src/peer_set.rs | 11 +- .../src/legacy_v1/mod.rs | 30 ++-- .../network/statement-distribution/src/lib.rs | 8 +- .../statement-distribution/src/v2/mod.rs | 90 ++++++----- .../src/v2/tests/grid.rs | 2 +- 16 files changed, 264 insertions(+), 285 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index edba3e1af2da..08a5e57a9fb1 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -32,7 +32,7 @@ use polkadot_node_network_protocol::{ self as net_protocol, filter_by_peer_version, grid_topology::{RandomRouting, RequiredRouting, SessionGridTopologies, SessionGridTopology}, peer_set::MAX_NOTIFICATION_SIZE, - v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, PeerId, + v1 as protocol_v1, v2 as protocol_v2, v3 as protocol_v3, PeerId, UnifiedReputationChange as Rep, Versioned, View, }; use polkadot_node_primitives::approval::{ @@ -966,16 +966,14 @@ impl State { msg: Versioned< protocol_v1::ApprovalDistributionMessage, protocol_v2::ApprovalDistributionMessage, - protocol_vstaging::ApprovalDistributionMessage, + protocol_v3::ApprovalDistributionMessage, >, rng: &mut R, ) where R: CryptoRng + Rng, { match msg { - Versioned::VStaging(protocol_vstaging::ApprovalDistributionMessage::Assignments( - assignments, - )) => { + Versioned::V3(protocol_v3::ApprovalDistributionMessage::Assignments(assignments)) => { gum::trace!( target: LOG_TARGET, peer_id = %peer_id, @@ -1015,9 +1013,7 @@ impl State { ) .await; }, - Versioned::VStaging(protocol_vstaging::ApprovalDistributionMessage::Approvals( - approvals, - )) => { + Versioned::V3(protocol_v3::ApprovalDistributionMessage::Approvals(approvals)) => { self.process_incoming_approvals(ctx, metrics, peer_id, approvals).await; }, Versioned::V1(protocol_v1::ApprovalDistributionMessage::Approvals(approvals)) | @@ -2458,12 +2454,12 @@ async fn send_assignments_batched_inner( peers: Vec, peer_version: ValidationVersion, ) { - if peer_version == ValidationVersion::VStaging { + if peer_version == ValidationVersion::V3 { sender .send_message(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments( + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments( batch.into_iter().collect(), ), )), @@ -2514,7 +2510,7 @@ pub(crate) async fn send_assignments_batched( ) { let v1_peers = filter_by_peer_version(peers, ValidationVersion::V1.into()); let v2_peers = filter_by_peer_version(peers, ValidationVersion::V2.into()); - let vstaging_peers = filter_by_peer_version(peers, ValidationVersion::VStaging.into()); + let v3_peers = filter_by_peer_version(peers, ValidationVersion::V3.into()); // V1 and V2 validation protocol do not have any changes with regard to // ApprovalDistributionMessage so they can be treated the same. @@ -2552,18 +2548,13 @@ pub(crate) async fn send_assignments_batched( } } - if !vstaging_peers.is_empty() { - let mut vstaging = v2_assignments.into_iter().peekable(); + if !v3_peers.is_empty() { + let mut v3 = v2_assignments.into_iter().peekable(); - while vstaging.peek().is_some() { - let batch = vstaging.by_ref().take(MAX_ASSIGNMENT_BATCH_SIZE).collect::>(); - send_assignments_batched_inner( - sender, - batch, - vstaging_peers.clone(), - ValidationVersion::VStaging, - ) - .await; + while v3.peek().is_some() { + let batch = v3.by_ref().take(MAX_ASSIGNMENT_BATCH_SIZE).collect::>(); + send_assignments_batched_inner(sender, batch, v3_peers.clone(), ValidationVersion::V3) + .await; } } } @@ -2576,7 +2567,7 @@ pub(crate) async fn send_approvals_batched( ) { let v1_peers = filter_by_peer_version(peers, ValidationVersion::V1.into()); let v2_peers = filter_by_peer_version(peers, ValidationVersion::V2.into()); - let vstaging_peers = filter_by_peer_version(peers, ValidationVersion::VStaging.into()); + let v3_peers = filter_by_peer_version(peers, ValidationVersion::V3.into()); if !v1_peers.is_empty() || !v2_peers.is_empty() { let mut batches = approvals @@ -2613,7 +2604,7 @@ pub(crate) async fn send_approvals_batched( } } - if !vstaging_peers.is_empty() { + if !v3_peers.is_empty() { let mut batches = approvals.into_iter().peekable(); while batches.peek().is_some() { @@ -2621,12 +2612,10 @@ pub(crate) async fn send_approvals_batched( sender .send_message(NetworkBridgeTxMessage::SendValidationMessage( - vstaging_peers.clone(), - Versioned::VStaging( - protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(batch), - ), - ), + v3_peers.clone(), + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Approvals(batch), + )), )) .await; } diff --git a/polkadot/node/network/approval-distribution/src/tests.rs b/polkadot/node/network/approval-distribution/src/tests.rs index b221d6aeef76..ad5d0bb0a9c5 100644 --- a/polkadot/node/network/approval-distribution/src/tests.rs +++ b/polkadot/node/network/approval-distribution/src/tests.rs @@ -276,16 +276,16 @@ async fn send_message_from_peer_v2( .await; } -async fn send_message_from_peer_vstaging( +async fn send_message_from_peer_v3( virtual_overseer: &mut VirtualOverseer, peer_id: &PeerId, - msg: protocol_vstaging::ApprovalDistributionMessage, + msg: protocol_v3::ApprovalDistributionMessage, ) { overseer_send( virtual_overseer, ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage( *peer_id, - Versioned::VStaging(msg), + Versioned::V3(msg), )), ) .await; @@ -450,7 +450,7 @@ fn try_import_the_same_assignment() { ); // setup new peer with V2 - setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::V3).await; // send the same assignment from peer_d let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments); @@ -479,9 +479,9 @@ fn try_import_the_same_assignment_v2() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; // setup peers - setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::VStaging).await; - setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; - setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V3).await; // Set up a gossip topology, where a, b, c and d are topology neighboors to the node under // testing. @@ -512,8 +512,8 @@ fn try_import_the_same_assignment_v2() { let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfield.clone()); let assignments = vec![(cert.clone(), cores.clone().try_into().unwrap())]; - let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer_vstaging(overseer, &peer_a, msg).await; + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer_v3(overseer, &peer_a, msg).await; expect_reputation_change(overseer, &peer_a, COST_UNEXPECTED_MESSAGE).await; @@ -537,8 +537,8 @@ fn try_import_the_same_assignment_v2() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) )) )) => { assert_eq!(peers.len(), 2); @@ -547,11 +547,11 @@ fn try_import_the_same_assignment_v2() { ); // setup new peer - setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::V3).await; // send the same assignment from peer_d - let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments); - send_message_from_peer_vstaging(overseer, &peer_d, msg).await; + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(assignments); + send_message_from_peer_v3(overseer, &peer_d, msg).await; expect_reputation_change(overseer, &peer_d, COST_UNEXPECTED_MESSAGE).await; expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE).await; @@ -807,7 +807,7 @@ fn import_approval_happy_path_v1_v2_peers() { let overseer = &mut virtual_overseer; // setup peers with V1 and V2 protocol versions setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; - setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates @@ -857,8 +857,8 @@ fn import_approval_happy_path_v1_v2_peers() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) )) )) => { assert_eq!(peers.len(), 1); @@ -873,9 +873,9 @@ fn import_approval_happy_path_v1_v2_peers() { validator: validator_index, signature: dummy_signature(), }; - let msg: protocol_vstaging::ApprovalDistributionMessage = - protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_vstaging(overseer, &peer_b, msg).await; + let msg: protocol_v3::ApprovalDistributionMessage = + protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -920,9 +920,9 @@ fn import_approval_happy_path_v2() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; // setup peers with V2 protocol versions - setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::VStaging).await; - setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; - setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V3).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -959,8 +959,8 @@ fn import_approval_happy_path_v2() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) )) )) => { assert_eq!(peers.len(), 2); @@ -975,8 +975,8 @@ fn import_approval_happy_path_v2() { validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_vstaging(overseer, &peer_b, msg).await; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -995,8 +995,8 @@ fn import_approval_happy_path_v2() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) )) )) => { assert_eq!(peers.len(), 1); @@ -1022,10 +1022,10 @@ fn multiple_assignments_covered_with_one_approval_vote() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; // setup peers with V2 protocol versions - setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::VStaging).await; - setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; - setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::VStaging).await; - setup_peer_with_view(overseer, &peer_d, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_d, view![hash], ValidationVersion::V3).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -1056,11 +1056,11 @@ fn multiple_assignments_covered_with_one_approval_vote() { validator: validator_index, cert: cert.cert, }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(vec![( + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( assignment, (0 as CandidateIndex).into(), )]); - send_message_from_peer_vstaging(overseer, &peer_d, msg).await; + send_message_from_peer_v3(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1077,8 +1077,8 @@ fn multiple_assignments_covered_with_one_approval_vote() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) )) )) => { assert!(peers.len() >= 2); @@ -1097,12 +1097,12 @@ fn multiple_assignments_covered_with_one_approval_vote() { validator: validator_index, cert: cert.cert, }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(vec![( + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( assignment, (1 as CandidateIndex).into(), )]); - send_message_from_peer_vstaging(overseer, &peer_c, msg).await; + send_message_from_peer_v3(overseer, &peer_c, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1119,8 +1119,8 @@ fn multiple_assignments_covered_with_one_approval_vote() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) )) )) => { assert!(peers.len() >= 2); @@ -1137,8 +1137,8 @@ fn multiple_assignments_covered_with_one_approval_vote() { validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_vstaging(overseer, &peer_d, msg).await; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1157,8 +1157,8 @@ fn multiple_assignments_covered_with_one_approval_vote() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) )) )) => { assert!(peers.len() >= 2); @@ -1207,7 +1207,7 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; - setup_peer_with_view(overseer, &peer_d, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_d, view![hash], ValidationVersion::V3).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -1238,11 +1238,11 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { validator: validator_index, cert: cert.cert, }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(vec![( + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( assignment, (0 as CandidateIndex).into(), )]); - send_message_from_peer_vstaging(overseer, &peer_d, msg).await; + send_message_from_peer_v3(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1264,12 +1264,12 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { validator: validator_index, cert: cert.cert, }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(vec![( + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( assignment, (1 as CandidateIndex).into(), )]); - send_message_from_peer_vstaging(overseer, &peer_d, msg).await; + send_message_from_peer_v3(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1289,8 +1289,8 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_vstaging(overseer, &peer_d, msg).await; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1306,16 +1306,16 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; // setup peers with V2 protocol versions - setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::VStaging).await; - setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; let mut expected_peers_assignments = vec![peer_a, peer_b]; let mut expected_peers_approvals = vec![peer_a, peer_b]; assert_matches!( overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) )) )) => { assert!(peers.len() == 1); @@ -1329,8 +1329,8 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) )) )) => { assert!(peers.len() == 1); @@ -1344,8 +1344,8 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) )) )) => { assert!(peers.len() == 1); @@ -1359,8 +1359,8 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) )) )) => { assert!(peers.len() == 1); @@ -1410,8 +1410,8 @@ fn import_approval_bad() { validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_vstaging(overseer, &peer_b, msg).await; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_b, msg).await; expect_reputation_change(overseer, &peer_b, COST_UNEXPECTED_MESSAGE).await; @@ -1436,8 +1436,8 @@ fn import_approval_bad() { expect_reputation_change(overseer, &peer_b, BENEFIT_VALID_MESSAGE_FIRST).await; // and try again - let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_vstaging(overseer, &peer_b, msg).await; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1762,8 +1762,8 @@ fn import_remotely_then_locally() { validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_vstaging(overseer, peer, msg).await; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, peer, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1945,7 +1945,7 @@ fn sends_assignments_even_when_state_is_approved_v2() { } // connect the peer. - setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V3).await; let assignments = vec![(cert.clone(), candidate_bitfield.clone())]; @@ -1953,8 +1953,8 @@ fn sends_assignments_even_when_state_is_approved_v2() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(sent_assignments) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(sent_assignments) )) )) => { assert_eq!(peers, vec![*peer]); @@ -1966,8 +1966,8 @@ fn sends_assignments_even_when_state_is_approved_v2() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(sent_approvals) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Approvals(sent_approvals) )) )) => { // Construct a hashmaps of approvals for comparison. Approval distribution reorders messages because they are kept in a diff --git a/polkadot/node/network/bitfield-distribution/src/lib.rs b/polkadot/node/network/bitfield-distribution/src/lib.rs index 9cc79aee8490..76baf499cad7 100644 --- a/polkadot/node/network/bitfield-distribution/src/lib.rs +++ b/polkadot/node/network/bitfield-distribution/src/lib.rs @@ -32,7 +32,7 @@ use polkadot_node_network_protocol::{ GridNeighbors, RandomRouting, RequiredRouting, SessionBoundGridTopologyStorage, }, peer_set::{ProtocolVersion, ValidationVersion}, - v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, OurView, PeerId, + v1 as protocol_v1, v2 as protocol_v2, v3 as protocol_v3, OurView, PeerId, UnifiedReputationChange as Rep, Versioned, View, }; use polkadot_node_subsystem::{ @@ -102,8 +102,8 @@ impl BitfieldGossipMessage { self.relay_parent, self.signed_availability.into(), )), - Some(ValidationVersion::VStaging) => - Versioned::VStaging(protocol_vstaging::BitfieldDistributionMessage::Bitfield( + Some(ValidationVersion::V3) => + Versioned::V3(protocol_v3::BitfieldDistributionMessage::Bitfield( self.relay_parent, self.signed_availability.into(), )), @@ -503,8 +503,8 @@ async fn relay_message( let v2_interested_peers = filter_by_peer_version(&interested_peers, ValidationVersion::V2.into()); - let vstaging_interested_peers = - filter_by_peer_version(&interested_peers, ValidationVersion::VStaging.into()); + let v3_interested_peers = + filter_by_peer_version(&interested_peers, ValidationVersion::V3.into()); if !v1_interested_peers.is_empty() { ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( @@ -522,10 +522,10 @@ async fn relay_message( .await } - if !vstaging_interested_peers.is_empty() { + if !v3_interested_peers.is_empty() { ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - vstaging_interested_peers, - message.into_validation_protocol(ValidationVersion::VStaging.into()), + v3_interested_peers, + message.into_validation_protocol(ValidationVersion::V3.into()), )) .await } @@ -551,7 +551,7 @@ async fn process_incoming_peer_message( relay_parent, bitfield, )) | - Versioned::VStaging(protocol_vstaging::BitfieldDistributionMessage::Bitfield( + Versioned::V3(protocol_v3::BitfieldDistributionMessage::Bitfield( relay_parent, bitfield, )) => (relay_parent, bitfield), diff --git a/polkadot/node/network/bridge/src/network.rs b/polkadot/node/network/bridge/src/network.rs index a9339a5c443c..2fcf5cec489d 100644 --- a/polkadot/node/network/bridge/src/network.rs +++ b/polkadot/node/network/bridge/src/network.rs @@ -33,7 +33,7 @@ use sc_network::{ use polkadot_node_network_protocol::{ peer_set::{CollationVersion, PeerSet, ProtocolVersion, ValidationVersion}, request_response::{OutgoingRequest, Recipient, ReqProtocolNames, Requests}, - v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, PeerId, + v1 as protocol_v1, v2 as protocol_v2, v3 as protocol_v3, PeerId, }; use polkadot_primitives::{AuthorityDiscoveryId, Block, Hash}; @@ -62,20 +62,20 @@ pub(crate) fn send_validation_message_v1( ); } -// Helper function to send a validation vstaging message to a list of peers. +// Helper function to send a validation v3 message to a list of peers. // Messages are always sent via the main protocol, even legacy protocol messages. -pub(crate) fn send_validation_message_vstaging( +pub(crate) fn send_validation_message_v3( peers: Vec, - message: WireMessage, + message: WireMessage, metrics: &Metrics, notification_sinks: &Arc>>>, ) { - gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation vstaging message to peers",); + gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation v3 message to peers",); send_message( peers, PeerSet::Validation, - ValidationVersion::VStaging.into(), + ValidationVersion::V3.into(), message, metrics, notification_sinks, diff --git a/polkadot/node/network/bridge/src/rx/mod.rs b/polkadot/node/network/bridge/src/rx/mod.rs index 40cd167a968b..49d81aea76af 100644 --- a/polkadot/node/network/bridge/src/rx/mod.rs +++ b/polkadot/node/network/bridge/src/rx/mod.rs @@ -37,8 +37,8 @@ use polkadot_node_network_protocol::{ CollationVersion, PeerSet, PeerSetProtocolNames, PerPeerSet, ProtocolVersion, ValidationVersion, }, - v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, ObservedRole, OurView, - PeerId, UnifiedReputationChange as Rep, View, + v1 as protocol_v1, v2 as protocol_v2, v3 as protocol_v3, ObservedRole, OurView, PeerId, + UnifiedReputationChange as Rep, View, }; use polkadot_node_subsystem::{ @@ -70,7 +70,7 @@ use super::validator_discovery; /// Defines the `Network` trait with an implementation for an `Arc`. use crate::network::{ send_collation_message_v1, send_collation_message_v2, send_validation_message_v1, - send_validation_message_v2, send_validation_message_vstaging, Network, + send_validation_message_v2, send_validation_message_v3, Network, }; use crate::{network::get_peer_id_by_authority_id, WireMessage}; @@ -294,9 +294,9 @@ async fn handle_validation_message( metrics, notification_sinks, ), - ValidationVersion::VStaging => send_validation_message_vstaging( + ValidationVersion::V3 => send_validation_message_v3( vec![peer], - WireMessage::::ViewUpdate(local_view), + WireMessage::::ViewUpdate(local_view), metrics, notification_sinks, ), @@ -360,48 +360,47 @@ async fn handle_validation_message( ?peer, ); - let (events, reports) = - if expected_versions[PeerSet::Validation] == Some(ValidationVersion::V1.into()) { - handle_peer_messages::( - peer, - PeerSet::Validation, - &mut shared.0.lock().validation_peers, - vec![notification.into()], - metrics, - ) - } else if expected_versions[PeerSet::Validation] == - Some(ValidationVersion::V2.into()) - { - handle_peer_messages::( - peer, - PeerSet::Validation, - &mut shared.0.lock().validation_peers, - vec![notification.into()], - metrics, - ) - } else if expected_versions[PeerSet::Validation] == - Some(ValidationVersion::VStaging.into()) - { - handle_peer_messages::( - peer, - PeerSet::Validation, - &mut shared.0.lock().validation_peers, - vec![notification.into()], - metrics, - ) - } else { - gum::warn!( - target: LOG_TARGET, - version = ?expected_versions[PeerSet::Validation], - "Major logic bug. Peer somehow has unsupported validation protocol version." - ); + let (events, reports) = if expected_versions[PeerSet::Validation] == + Some(ValidationVersion::V1.into()) + { + handle_peer_messages::( + peer, + PeerSet::Validation, + &mut shared.0.lock().validation_peers, + vec![notification.into()], + metrics, + ) + } else if expected_versions[PeerSet::Validation] == Some(ValidationVersion::V2.into()) { + handle_peer_messages::( + peer, + PeerSet::Validation, + &mut shared.0.lock().validation_peers, + vec![notification.into()], + metrics, + ) + } else if expected_versions[PeerSet::Validation] == Some(ValidationVersion::V3.into()) { + handle_peer_messages::( + peer, + PeerSet::Validation, + &mut shared.0.lock().validation_peers, + vec![notification.into()], + metrics, + ) + } else { + gum::warn!( + target: LOG_TARGET, + version = ?expected_versions[PeerSet::Validation], + "Major logic bug. Peer somehow has unsupported validation protocol version." + ); - never!("Only versions 1 and 2 are supported; peer set connection checked above; qed"); + never!( + "Only versions 1 and 2 are supported; peer set connection checked above; qed" + ); - // If a peer somehow triggers this, we'll disconnect them - // eventually. - (Vec::new(), vec![UNCONNECTED_PEERSET_COST]) - }; + // If a peer somehow triggers this, we'll disconnect them + // eventually. + (Vec::new(), vec![UNCONNECTED_PEERSET_COST]) + }; for report in reports { network_service.report_peer(peer, report.into()); @@ -980,8 +979,8 @@ fn update_our_view( filter_by_peer_version(&validation_peers, ValidationVersion::V2.into()); let v2_collation_peers = filter_by_peer_version(&collation_peers, CollationVersion::V2.into()); - let vstaging_validation_peers = - filter_by_peer_version(&validation_peers, ValidationVersion::VStaging.into()); + let v3_validation_peers = + filter_by_peer_version(&validation_peers, ValidationVersion::V3.into()); send_validation_message_v1( v1_validation_peers, @@ -1011,8 +1010,8 @@ fn update_our_view( notification_sinks, ); - send_validation_message_vstaging( - vstaging_validation_peers, + send_validation_message_v3( + v3_validation_peers, WireMessage::ViewUpdate(new_view.clone()), metrics, notification_sinks, diff --git a/polkadot/node/network/bridge/src/rx/tests.rs b/polkadot/node/network/bridge/src/rx/tests.rs index e0b86feb6442..ad503ee21f53 100644 --- a/polkadot/node/network/bridge/src/rx/tests.rs +++ b/polkadot/node/network/bridge/src/rx/tests.rs @@ -224,7 +224,7 @@ impl TestNetworkHandle { PeerSet::Validation => Some(ProtocolName::from("/polkadot/validation/1")), PeerSet::Collation => Some(ProtocolName::from("/polkadot/collation/1")), }, - ValidationVersion::VStaging => match peer_set { + ValidationVersion::V3 => match peer_set { PeerSet::Validation => Some(ProtocolName::from("/polkadot/validation/3")), PeerSet::Collation => unreachable!(), }, @@ -1433,8 +1433,8 @@ fn network_protocol_versioning_view_update() { ValidationVersion::V2 => WireMessage::::ViewUpdate(view.clone()) .encode(), - ValidationVersion::VStaging => - WireMessage::::ViewUpdate(view.clone()) + ValidationVersion::V3 => + WireMessage::::ViewUpdate(view.clone()) .encode(), }; assert_network_actions_contains( diff --git a/polkadot/node/network/bridge/src/tx/mod.rs b/polkadot/node/network/bridge/src/tx/mod.rs index bdcd1574e335..22802608e1d5 100644 --- a/polkadot/node/network/bridge/src/tx/mod.rs +++ b/polkadot/node/network/bridge/src/tx/mod.rs @@ -41,7 +41,7 @@ use crate::validator_discovery; /// Defines the `Network` trait with an implementation for an `Arc`. use crate::network::{ send_collation_message_v1, send_collation_message_v2, send_validation_message_v1, - send_validation_message_v2, send_validation_message_vstaging, Network, + send_validation_message_v2, send_validation_message_v3, Network, }; use crate::metrics::Metrics; @@ -205,7 +205,7 @@ where &metrics, notification_sinks, ), - Versioned::VStaging(msg) => send_validation_message_vstaging( + Versioned::V3(msg) => send_validation_message_v3( peers, WireMessage::ProtocolMessage(msg), &metrics, @@ -235,7 +235,7 @@ where &metrics, notification_sinks, ), - Versioned::VStaging(msg) => send_validation_message_vstaging( + Versioned::V3(msg) => send_validation_message_v3( peers, WireMessage::ProtocolMessage(msg), &metrics, @@ -264,7 +264,7 @@ where &metrics, notification_sinks, ), - Versioned::V2(msg) | Versioned::VStaging(msg) => send_collation_message_v2( + Versioned::V2(msg) | Versioned::V3(msg) => send_collation_message_v2( peers, WireMessage::ProtocolMessage(msg), &metrics, @@ -287,7 +287,7 @@ where &metrics, notification_sinks, ), - Versioned::V2(msg) | Versioned::VStaging(msg) => send_collation_message_v2( + Versioned::V2(msg) | Versioned::V3(msg) => send_collation_message_v2( peers, WireMessage::ProtocolMessage(msg), &metrics, diff --git a/polkadot/node/network/collator-protocol/src/collator_side/mod.rs b/polkadot/node/network/collator-protocol/src/collator_side/mod.rs index b3a396e1be34..8fb0bb215444 100644 --- a/polkadot/node/network/collator-protocol/src/collator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/collator_side/mod.rs @@ -882,7 +882,7 @@ async fn handle_incoming_peer_message( match msg { Versioned::V1(V1::Declare(..)) | Versioned::V2(V2::Declare(..)) | - Versioned::VStaging(V2::Declare(..)) => { + Versioned::V3(V2::Declare(..)) => { gum::trace!( target: LOG_TARGET, ?origin, @@ -895,7 +895,7 @@ async fn handle_incoming_peer_message( }, Versioned::V1(V1::AdvertiseCollation(_)) | Versioned::V2(V2::AdvertiseCollation { .. }) | - Versioned::VStaging(V2::AdvertiseCollation { .. }) => { + Versioned::V3(V2::AdvertiseCollation { .. }) => { gum::trace!( target: LOG_TARGET, ?origin, @@ -911,7 +911,7 @@ async fn handle_incoming_peer_message( }, Versioned::V1(V1::CollationSeconded(relay_parent, statement)) | Versioned::V2(V2::CollationSeconded(relay_parent, statement)) | - Versioned::VStaging(V2::CollationSeconded(relay_parent, statement)) => { + Versioned::V3(V2::CollationSeconded(relay_parent, statement)) => { if !matches!(statement.unchecked_payload(), Statement::Seconded(_)) { gum::warn!( target: LOG_TARGET, diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 20b3b9ea1d26..48ad3c711a6d 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -777,7 +777,7 @@ async fn process_incoming_peer_message( match msg { Versioned::V1(V1::Declare(collator_id, para_id, signature)) | Versioned::V2(V2::Declare(collator_id, para_id, signature)) | - Versioned::VStaging(V2::Declare(collator_id, para_id, signature)) => { + Versioned::V3(V2::Declare(collator_id, para_id, signature)) => { if collator_peer_id(&state.peer_data, &collator_id).is_some() { modify_reputation( &mut state.reputation, @@ -894,7 +894,7 @@ async fn process_incoming_peer_message( candidate_hash, parent_head_data_hash, }) | - Versioned::VStaging(V2::AdvertiseCollation { + Versioned::V3(V2::AdvertiseCollation { relay_parent, candidate_hash, parent_head_data_hash, @@ -923,7 +923,7 @@ async fn process_incoming_peer_message( }, Versioned::V1(V1::CollationSeconded(..)) | Versioned::V2(V2::CollationSeconded(..)) | - Versioned::VStaging(V2::CollationSeconded(..)) => { + Versioned::V3(V2::CollationSeconded(..)) => { gum::warn!( target: LOG_TARGET, peer_id = ?origin, diff --git a/polkadot/node/network/gossip-support/src/lib.rs b/polkadot/node/network/gossip-support/src/lib.rs index 0d1b04f2ba23..22417795d5ea 100644 --- a/polkadot/node/network/gossip-support/src/lib.rs +++ b/polkadot/node/network/gossip-support/src/lib.rs @@ -477,7 +477,7 @@ where match message { Versioned::V1(m) => match m {}, Versioned::V2(m) => match m {}, - Versioned::VStaging(m) => match m {}, + Versioned::V3(m) => match m {}, } }, } diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index b0e30701208b..ae72230ee43d 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -253,29 +253,29 @@ impl View { /// A protocol-versioned type. #[derive(Debug, Clone, PartialEq, Eq)] -pub enum Versioned { +pub enum Versioned { /// V1 type. V1(V1), /// V2 type. V2(V2), - /// VStaging type - VStaging(VStaging), + /// V3 type + V3(V3), } -impl Versioned<&'_ V1, &'_ V2, &'_ VStaging> { +impl Versioned<&'_ V1, &'_ V2, &'_ V3> { /// Convert to a fully-owned version of the message. - pub fn clone_inner(&self) -> Versioned { + pub fn clone_inner(&self) -> Versioned { match *self { Versioned::V1(inner) => Versioned::V1(inner.clone()), Versioned::V2(inner) => Versioned::V2(inner.clone()), - Versioned::VStaging(inner) => Versioned::VStaging(inner.clone()), + Versioned::V3(inner) => Versioned::V3(inner.clone()), } } } /// All supported versions of the validation protocol message. pub type VersionedValidationProtocol = - Versioned; + Versioned; impl From for VersionedValidationProtocol { fn from(v1: v1::ValidationProtocol) -> Self { @@ -289,9 +289,9 @@ impl From for VersionedValidationProtocol { } } -impl From for VersionedValidationProtocol { - fn from(vstaging: vstaging::ValidationProtocol) -> Self { - VersionedValidationProtocol::VStaging(vstaging) +impl From for VersionedValidationProtocol { + fn from(v3: v3::ValidationProtocol) -> Self { + VersionedValidationProtocol::V3(v3) } } @@ -317,7 +317,7 @@ macro_rules! impl_versioned_full_protocol_from { match versioned_from { Versioned::V1(x) => Versioned::V1(x.into()), Versioned::V2(x) => Versioned::V2(x.into()), - Versioned::VStaging(x) => Versioned::VStaging(x.into()), + Versioned::V3(x) => Versioned::V3(x.into()), } } } @@ -331,7 +331,7 @@ macro_rules! impl_versioned_try_from { $out:ty, $v1_pat:pat => $v1_out:expr, $v2_pat:pat => $v2_out:expr, - $vstaging_pat:pat => $vstaging_out:expr + $v3_pat:pat => $v3_out:expr ) => { impl TryFrom<$from> for $out { type Error = crate::WrongVariant; @@ -341,7 +341,7 @@ macro_rules! impl_versioned_try_from { match x { Versioned::V1($v1_pat) => Ok(Versioned::V1($v1_out)), Versioned::V2($v2_pat) => Ok(Versioned::V2($v2_out)), - Versioned::VStaging($vstaging_pat) => Ok(Versioned::VStaging($vstaging_out)), + Versioned::V3($v3_pat) => Ok(Versioned::V3($v3_out)), _ => Err(crate::WrongVariant), } } @@ -355,8 +355,7 @@ macro_rules! impl_versioned_try_from { match x { Versioned::V1($v1_pat) => Ok(Versioned::V1($v1_out.clone())), Versioned::V2($v2_pat) => Ok(Versioned::V2($v2_out.clone())), - Versioned::VStaging($vstaging_pat) => - Ok(Versioned::VStaging($vstaging_out.clone())), + Versioned::V3($v3_pat) => Ok(Versioned::V3($v3_out.clone())), _ => Err(crate::WrongVariant), } } @@ -368,7 +367,7 @@ macro_rules! impl_versioned_try_from { pub type BitfieldDistributionMessage = Versioned< v1::BitfieldDistributionMessage, v2::BitfieldDistributionMessage, - vstaging::BitfieldDistributionMessage, + v3::BitfieldDistributionMessage, >; impl_versioned_full_protocol_from!( BitfieldDistributionMessage, @@ -380,14 +379,14 @@ impl_versioned_try_from!( BitfieldDistributionMessage, v1::ValidationProtocol::BitfieldDistribution(x) => x, v2::ValidationProtocol::BitfieldDistribution(x) => x, - vstaging::ValidationProtocol::BitfieldDistribution(x) => x + v3::ValidationProtocol::BitfieldDistribution(x) => x ); /// Version-annotated messages used by the statement distribution subsystem. pub type StatementDistributionMessage = Versioned< v1::StatementDistributionMessage, v2::StatementDistributionMessage, - vstaging::StatementDistributionMessage, + v3::StatementDistributionMessage, >; impl_versioned_full_protocol_from!( StatementDistributionMessage, @@ -399,14 +398,14 @@ impl_versioned_try_from!( StatementDistributionMessage, v1::ValidationProtocol::StatementDistribution(x) => x, v2::ValidationProtocol::StatementDistribution(x) => x, - vstaging::ValidationProtocol::StatementDistribution(x) => x + v3::ValidationProtocol::StatementDistribution(x) => x ); /// Version-annotated messages used by the approval distribution subsystem. pub type ApprovalDistributionMessage = Versioned< v1::ApprovalDistributionMessage, v2::ApprovalDistributionMessage, - vstaging::ApprovalDistributionMessage, + v3::ApprovalDistributionMessage, >; impl_versioned_full_protocol_from!( ApprovalDistributionMessage, @@ -418,7 +417,7 @@ impl_versioned_try_from!( ApprovalDistributionMessage, v1::ValidationProtocol::ApprovalDistribution(x) => x, v2::ValidationProtocol::ApprovalDistribution(x) => x, - vstaging::ValidationProtocol::ApprovalDistribution(x) => x + v3::ValidationProtocol::ApprovalDistribution(x) => x ); @@ -426,7 +425,7 @@ impl_versioned_try_from!( pub type GossipSupportNetworkMessage = Versioned< v1::GossipSupportNetworkMessage, v2::GossipSupportNetworkMessage, - vstaging::GossipSupportNetworkMessage, + v3::GossipSupportNetworkMessage, >; // This is a void enum placeholder, so never gets sent over the wire. @@ -871,18 +870,17 @@ pub mod v2 { } } -/// vstaging network protocol types, intended to become v3. -/// Initial purpose is for chaning ApprovalDistributionMessage to -/// include more than one assignment in the message. -pub mod vstaging { +/// v3 network protocol types. +/// Purpose is for chaning ApprovalDistributionMessage to +/// include more than one assignment and approval in a message. +pub mod v3 { use parity_scale_codec::{Decode, Encode}; use polkadot_node_primitives::approval::v2::{ CandidateBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2, }; - /// This parts of the protocol did not change from v2, so just alias them in vstaging, - /// no reason why they can't be change untill vstaging becomes v3 and is released. + /// This parts of the protocol did not change from v2, so just alias them in v3. pub use super::v2::{ declare_signature_payload, BackedCandidateAcknowledgement, BackedCandidateManifest, BitfieldDistributionMessage, GossipSupportNetworkMessage, StatementDistributionMessage, diff --git a/polkadot/node/network/protocol/src/peer_set.rs b/polkadot/node/network/protocol/src/peer_set.rs index 7e257d508b5b..fb2e2d77f6ff 100644 --- a/polkadot/node/network/protocol/src/peer_set.rs +++ b/polkadot/node/network/protocol/src/peer_set.rs @@ -135,7 +135,7 @@ impl PeerSet { #[cfg(feature = "network-protocol-staging")] match self { - PeerSet::Validation => ValidationVersion::VStaging.into(), + PeerSet::Validation => ValidationVersion::V3.into(), PeerSet::Collation => CollationVersion::V2.into(), } } @@ -163,7 +163,7 @@ impl PeerSet { Some("validation/1") } else if version == ValidationVersion::V2.into() { Some("validation/2") - } else if version == ValidationVersion::VStaging.into() { + } else if version == ValidationVersion::V3.into() { Some("validation/3") } else { None @@ -236,9 +236,10 @@ pub enum ValidationVersion { V1 = 1, /// The second version. V2 = 2, - /// The staging version to gather changes - /// that before the release become v3. - VStaging = 3, + /// The third version where changes to ApprovalDistributionMessage had been made. + /// The changes are translatable to V2 format untill assignments v2 and approvals + /// coalescing is enabled through a runtime upgrade. + V3 = 3, } /// Supported collation protocol versions. Only versions defined here must be used in the codebase. diff --git a/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs b/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs index d9866af1ee23..93f97fe1dd6e 100644 --- a/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs +++ b/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs @@ -22,8 +22,8 @@ use polkadot_node_network_protocol::{ grid_topology::{GridNeighbors, RequiredRouting, SessionBoundGridTopologyStorage}, peer_set::{IsAuthority, PeerSet, ValidationVersion}, v1::{self as protocol_v1, StatementMetadata}, - v2 as protocol_v2, vstaging as protocol_vstaging, IfDisconnected, PeerId, - UnifiedReputationChange as Rep, Versioned, View, + v2 as protocol_v2, v3 as protocol_v3, IfDisconnected, PeerId, UnifiedReputationChange as Rep, + Versioned, View, }; use polkadot_node_primitives::{ SignedFullStatement, Statement, StatementWithPVD, UncheckedSignedFullStatement, @@ -1075,7 +1075,7 @@ async fn circulate_statement<'a, Context>( }) .partition::, _>(|(_, _, version)| match version { ValidationVersion::V1 => true, - ValidationVersion::V2 | ValidationVersion::VStaging => false, + ValidationVersion::V2 | ValidationVersion::V3 => false, }); // partition is handy here but not if we add more protocol versions let payload = v1_statement_message(relay_parent, stored.statement.clone(), metrics); @@ -1108,8 +1108,7 @@ async fn circulate_statement<'a, Context>( .collect(); let v2_peers_to_send = filter_by_peer_version(&peers_to_send, ValidationVersion::V2.into()); - let vstaging_to_send = - filter_by_peer_version(&peers_to_send, ValidationVersion::VStaging.into()); + let v3_to_send = filter_by_peer_version(&peers_to_send, ValidationVersion::V3.into()); if !v2_peers_to_send.is_empty() { gum::trace!( @@ -1126,17 +1125,17 @@ async fn circulate_statement<'a, Context>( .await; } - if !vstaging_to_send.is_empty() { + if !v3_to_send.is_empty() { gum::trace!( target: LOG_TARGET, - ?vstaging_to_send, + ?v3_to_send, ?relay_parent, statement = ?stored.statement, - "Sending statement to vstaging peers", + "Sending statement to v3 peers", ); ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - vstaging_to_send, - compatible_v1_message(ValidationVersion::VStaging, payload.clone()).into(), + v3_to_send, + compatible_v1_message(ValidationVersion::V3, payload.clone()).into(), )) .await; } @@ -1472,10 +1471,8 @@ async fn handle_incoming_message<'a, Context>( let message = match message { Versioned::V1(m) => m, Versioned::V2(protocol_v2::StatementDistributionMessage::V1Compatibility(m)) | - Versioned::VStaging(protocol_vstaging::StatementDistributionMessage::V1Compatibility( - m, - )) => m, - Versioned::V2(_) | Versioned::VStaging(_) => { + Versioned::V3(protocol_v3::StatementDistributionMessage::V1Compatibility(m)) => m, + Versioned::V2(_) | Versioned::V3(_) => { // The higher-level subsystem code is supposed to filter out // all non v1 messages. gum::debug!( @@ -2201,8 +2198,7 @@ fn compatible_v1_message( ValidationVersion::V1 => Versioned::V1(message), ValidationVersion::V2 => Versioned::V2(protocol_v2::StatementDistributionMessage::V1Compatibility(message)), - ValidationVersion::VStaging => Versioned::VStaging( - protocol_vstaging::StatementDistributionMessage::V1Compatibility(message), - ), + ValidationVersion::V3 => + Versioned::V3(protocol_v3::StatementDistributionMessage::V1Compatibility(message)), } } diff --git a/polkadot/node/network/statement-distribution/src/lib.rs b/polkadot/node/network/statement-distribution/src/lib.rs index ef1fc7cd78b5..a1ba1137b5ac 100644 --- a/polkadot/node/network/statement-distribution/src/lib.rs +++ b/polkadot/node/network/statement-distribution/src/lib.rs @@ -27,7 +27,7 @@ use std::time::Duration; use polkadot_node_network_protocol::{ request_response::{v1 as request_v1, v2::AttestedCandidateRequest, IncomingRequestReceiver}, - v2 as protocol_v2, vstaging as protocol_vstaging, Versioned, + v2 as protocol_v2, v3 as protocol_v3, Versioned, }; use polkadot_node_primitives::StatementWithPVD; use polkadot_node_subsystem::{ @@ -400,11 +400,11 @@ impl StatementDistributionSubsystem { Versioned::V2( protocol_v2::StatementDistributionMessage::V1Compatibility(_), ) | - Versioned::VStaging( - protocol_vstaging::StatementDistributionMessage::V1Compatibility(_), + Versioned::V3( + protocol_v3::StatementDistributionMessage::V1Compatibility(_), ) => VersionTarget::Legacy, Versioned::V1(_) => VersionTarget::Legacy, - Versioned::V2(_) | Versioned::VStaging(_) => VersionTarget::Current, + Versioned::V2(_) | Versioned::V3(_) => VersionTarget::Current, }, _ => VersionTarget::Both, }; diff --git a/polkadot/node/network/statement-distribution/src/v2/mod.rs b/polkadot/node/network/statement-distribution/src/v2/mod.rs index 406f11305909..2f06d3685b81 100644 --- a/polkadot/node/network/statement-distribution/src/v2/mod.rs +++ b/polkadot/node/network/statement-distribution/src/v2/mod.rs @@ -29,8 +29,7 @@ use polkadot_node_network_protocol::{ MAX_PARALLEL_ATTESTED_CANDIDATE_REQUESTS, }, v2::{self as protocol_v2, StatementFilter}, - vstaging as protocol_vstaging, IfDisconnected, PeerId, UnifiedReputationChange as Rep, - Versioned, View, + v3 as protocol_v3, IfDisconnected, PeerId, UnifiedReputationChange as Rep, Versioned, View, }; use polkadot_node_primitives::{ SignedFullStatementWithPVD, StatementWithPVD as FullStatementWithPVD, @@ -366,7 +365,7 @@ pub(crate) async fn handle_network_update( gum::trace!(target: LOG_TARGET, ?peer_id, ?role, ?protocol_version, "Peer connected"); let versioned_protocol = if protocol_version != ValidationVersion::V2.into() && - protocol_version != ValidationVersion::VStaging.into() + protocol_version != ValidationVersion::V3.into() { return } else { @@ -432,28 +431,28 @@ pub(crate) async fn handle_network_update( net_protocol::StatementDistributionMessage::V2( protocol_v2::StatementDistributionMessage::V1Compatibility(_), ) | - net_protocol::StatementDistributionMessage::VStaging( - protocol_vstaging::StatementDistributionMessage::V1Compatibility(_), + net_protocol::StatementDistributionMessage::V3( + protocol_v3::StatementDistributionMessage::V1Compatibility(_), ) => return, net_protocol::StatementDistributionMessage::V2( protocol_v2::StatementDistributionMessage::Statement(relay_parent, statement), ) | - net_protocol::StatementDistributionMessage::VStaging( - protocol_vstaging::StatementDistributionMessage::Statement(relay_parent, statement), + net_protocol::StatementDistributionMessage::V3( + protocol_v3::StatementDistributionMessage::Statement(relay_parent, statement), ) => handle_incoming_statement(ctx, state, peer_id, relay_parent, statement, reputation) .await, net_protocol::StatementDistributionMessage::V2( protocol_v2::StatementDistributionMessage::BackedCandidateManifest(inner), ) | - net_protocol::StatementDistributionMessage::VStaging( - protocol_vstaging::StatementDistributionMessage::BackedCandidateManifest(inner), + net_protocol::StatementDistributionMessage::V3( + protocol_v3::StatementDistributionMessage::BackedCandidateManifest(inner), ) => handle_incoming_manifest(ctx, state, peer_id, inner, reputation).await, net_protocol::StatementDistributionMessage::V2( protocol_v2::StatementDistributionMessage::BackedCandidateKnown(inner), ) | - net_protocol::StatementDistributionMessage::VStaging( - protocol_vstaging::StatementDistributionMessage::BackedCandidateKnown(inner), + net_protocol::StatementDistributionMessage::V3( + protocol_v3::StatementDistributionMessage::BackedCandidateKnown(inner), ) => handle_incoming_acknowledgement(ctx, state, peer_id, inner, reputation).await, }, NetworkBridgeEvent::PeerViewChange(peer_id, view) => @@ -806,13 +805,13 @@ fn pending_statement_network_message( protocol_v2::StatementDistributionMessage::Statement(relay_parent, signed) }) .map(|msg| (vec![peer.0], Versioned::V2(msg).into())), - ValidationVersion::VStaging => statement_store + ValidationVersion::V3 => statement_store .validator_statement(originator, compact) .map(|s| s.as_unchecked().clone()) .map(|signed| { - protocol_vstaging::StatementDistributionMessage::Statement(relay_parent, signed) + protocol_v3::StatementDistributionMessage::Statement(relay_parent, signed) }) - .map(|msg| (vec![peer.0], Versioned::VStaging(msg).into())), + .map(|msg| (vec![peer.0], Versioned::V3(msg).into())), ValidationVersion::V1 => { gum::error!( target: LOG_TARGET, @@ -945,10 +944,10 @@ async fn send_pending_grid_messages( ) .into(), )), - ValidationVersion::VStaging => messages.push(( + ValidationVersion::V3 => messages.push(( vec![peer_id.0], - Versioned::VStaging( - protocol_vstaging::StatementDistributionMessage::BackedCandidateManifest( + Versioned::V3( + protocol_v3::StatementDistributionMessage::BackedCandidateManifest( manifest, ), ) @@ -960,7 +959,7 @@ async fn send_pending_grid_messages( "Bug ValidationVersion::V1 should not be used in statement-distribution v2, legacy should have handled this" ); - } + }, }; }, grid::ManifestKind::Acknowledgement => { @@ -1308,8 +1307,8 @@ async fn circulate_statement( let statement_to_v2_peers = filter_by_peer_version(&statement_to_peers, ValidationVersion::V2.into()); - let statement_to_vstaging_peers = - filter_by_peer_version(&statement_to_peers, ValidationVersion::VStaging.into()); + let statement_to_v3_peers = + filter_by_peer_version(&statement_to_peers, ValidationVersion::V3.into()); // ship off the network messages to the network bridge. if !statement_to_v2_peers.is_empty() { @@ -1331,17 +1330,17 @@ async fn circulate_statement( .await; } - if !statement_to_vstaging_peers.is_empty() { + if !statement_to_v3_peers.is_empty() { gum::debug!( target: LOG_TARGET, ?compact_statement, n_peers = ?statement_to_peers.len(), - "Sending statement to vstaging peers", + "Sending statement to v3 peers", ); ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - statement_to_vstaging_peers, - Versioned::VStaging(protocol_vstaging::StatementDistributionMessage::Statement( + statement_to_v3_peers, + Versioned::V3(protocol_v3::StatementDistributionMessage::Statement( relay_parent, statement.as_unchecked().clone(), )) @@ -1887,8 +1886,7 @@ async fn provide_candidate_to_grid( } let manifest_peers_v2 = filter_by_peer_version(&manifest_peers, ValidationVersion::V2.into()); - let manifest_peers_vstaging = - filter_by_peer_version(&manifest_peers, ValidationVersion::VStaging.into()); + let manifest_peers_v3 = filter_by_peer_version(&manifest_peers, ValidationVersion::V3.into()); if !manifest_peers_v2.is_empty() { gum::debug!( target: LOG_TARGET, @@ -1908,27 +1906,27 @@ async fn provide_candidate_to_grid( .await; } - if !manifest_peers_vstaging.is_empty() { + if !manifest_peers_v3.is_empty() { gum::debug!( target: LOG_TARGET, ?candidate_hash, local_validator = ?per_session.local_validator, - n_peers = manifest_peers_vstaging.len(), - "Sending manifest to vstaging peers" + n_peers = manifest_peers_v3.len(), + "Sending manifest to v3 peers" ); ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - manifest_peers_vstaging, - Versioned::VStaging( - protocol_vstaging::StatementDistributionMessage::BackedCandidateManifest(manifest), - ) + manifest_peers_v3, + Versioned::V3(protocol_v3::StatementDistributionMessage::BackedCandidateManifest( + manifest, + )) .into(), )) .await; } let ack_peers_v2 = filter_by_peer_version(&ack_peers, ValidationVersion::V2.into()); - let ack_peers_vstaging = filter_by_peer_version(&ack_peers, ValidationVersion::VStaging.into()); + let ack_peers_v3 = filter_by_peer_version(&ack_peers, ValidationVersion::V3.into()); if !ack_peers_v2.is_empty() { gum::debug!( target: LOG_TARGET, @@ -1948,22 +1946,20 @@ async fn provide_candidate_to_grid( .await; } - if !ack_peers_vstaging.is_empty() { + if !ack_peers_v3.is_empty() { gum::debug!( target: LOG_TARGET, ?candidate_hash, local_validator = ?per_session.local_validator, - n_peers = ack_peers_vstaging.len(), - "Sending acknowledgement to vstaging peers" + n_peers = ack_peers_v3.len(), + "Sending acknowledgement to v3 peers" ); ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - ack_peers_vstaging, - Versioned::VStaging( - protocol_vstaging::StatementDistributionMessage::BackedCandidateKnown( - acknowledgement, - ), - ) + ack_peers_v3, + Versioned::V3(protocol_v3::StatementDistributionMessage::BackedCandidateKnown( + acknowledgement, + )) .into(), )) .await; @@ -2293,8 +2289,8 @@ fn post_acknowledgement_statement_messages( ) .into(), )), - ValidationVersion::VStaging => messages.push(Versioned::VStaging( - protocol_vstaging::StatementDistributionMessage::Statement( + ValidationVersion::V3 => messages.push(Versioned::V3( + protocol_v3::StatementDistributionMessage::Statement( relay_parent, statement.as_unchecked().clone(), ) @@ -2441,9 +2437,9 @@ fn acknowledgement_and_statement_messages( let mut messages = match peer.1 { ValidationVersion::V2 => vec![(vec![peer.0], msg_v2.into())], - ValidationVersion::VStaging => vec![( + ValidationVersion::V3 => vec![( vec![peer.0], - Versioned::VStaging(protocol_v2::StatementDistributionMessage::BackedCandidateKnown( + Versioned::V3(protocol_v2::StatementDistributionMessage::BackedCandidateKnown( acknowledgement, )) .into(), diff --git a/polkadot/node/network/statement-distribution/src/v2/tests/grid.rs b/polkadot/node/network/statement-distribution/src/v2/tests/grid.rs index 116116659cb1..aa1a473b833f 100644 --- a/polkadot/node/network/statement-distribution/src/v2/tests/grid.rs +++ b/polkadot/node/network/statement-distribution/src/v2/tests/grid.rs @@ -2830,7 +2830,7 @@ fn inactive_local_participates_in_grid() { send_peer_message( &mut overseer, peer_a.clone(), - protocol_vstaging::StatementDistributionMessage::BackedCandidateManifest(manifest), + protocol_v3::StatementDistributionMessage::BackedCandidateManifest(manifest), ) .await; From 54e86981da6312586e38e8d59a66679d3f89d805 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 7 Dec 2023 22:59:21 +0200 Subject: [PATCH 127/192] WIP Signed-off-by: Andrei Sandu --- .../src/availability/av_store_helpers.rs | 2 - .../subsystem-bench/src/availability/mod.rs | 261 ++++++++++++++++-- .../subsystem-bench/src/core/environment.rs | 59 ++-- .../node/subsystem-bench/src/core/keyring.rs | 2 +- .../src/core/mock/chain_api.rs | 6 +- .../node/subsystem-bench/src/core/network.rs | 43 +-- .../subsystem-bench/src/subsystem-bench.rs | 1 - 7 files changed, 282 insertions(+), 92 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs b/polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs index e8b38a829a56..12c86aa7e546 100644 --- a/polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs +++ b/polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs @@ -56,8 +56,6 @@ pub fn new_av_store(dependencies: &TestEnvironmentDependencies) -> AvailabilityS AvailabilityStoreSubsystem::new(test_store(), TEST_CONFIG, Box::new(DumbOracle), metrics) } -const TIMEOUT: Duration = Duration::from_millis(100); - fn test_store() -> Arc { let db = kvdb_memorydb::create(columns::NUM_COLUMNS); let db = diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 47d09dc59f51..01510505d22a 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -13,33 +13,43 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use super::core::environment::MAX_TIME_OF_FLIGHT; use crate::{core::mock::ChainApiState, TestEnvironment}; +use bitvec::bitvec; +use colored::Colorize; use itertools::Itertools; use polkadot_availability_bitfield_distribution::BitfieldDistribution; use polkadot_node_core_av_store::AvailabilityStoreSubsystem; use polkadot_node_subsystem::{Overseer, OverseerConnector, SpawnGlue}; +use polkadot_node_subsystem_types::{ + messages::{AvailabilityStoreMessage, NetworkBridgeEvent}, + Span, +}; use polkadot_overseer::Handle as OverseerHandle; +use sc_network::{request_responses::ProtocolConfig, PeerId}; use sp_core::H256; -use std::{collections::HashMap, iter::Cycle, ops::Sub, sync::Arc, time::Instant}; - -use sc_network::request_responses::ProtocolConfig; - -use colored::Colorize; - -use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; -use polkadot_node_metrics::metrics::Metrics; +use std::{collections::HashMap, iter::Cycle, ops::Sub, pin::Pin, sync::Arc, time::Instant}; use av_store_helpers::new_av_store; +use futures::{channel::oneshot, stream::FuturesUnordered, Future, StreamExt}; use polkadot_availability_distribution::{ AvailabilityDistributionSubsystem, IncomingRequestReceivers, }; +use polkadot_node_metrics::metrics::Metrics; +use polkadot_node_subsystem::TimeoutExt; use polkadot_availability_recovery::AvailabilityRecoverySubsystem; use crate::GENESIS_HASH; +use futures::FutureExt; use parity_scale_codec::Encode; use polkadot_erasure_coding::{branches, obtain_chunks_v1 as obtain_chunks}; -use polkadot_node_network_protocol::request_response::{IncomingRequest, ReqProtocolNames}; +use polkadot_node_network_protocol::{ + request_response::{ + v1::ChunkFetchingRequest, IncomingRequest, OutgoingRequest, ReqProtocolNames, Requests, + }, + BitfieldDistributionMessage, OurView, Versioned, View, +}; use polkadot_node_primitives::{BlockData, PoV, Proof}; use polkadot_node_subsystem::messages::{AllMessages, AvailabilityRecoveryMessage}; @@ -59,10 +69,10 @@ const LOG_TARGET: &str = "subsystem-bench::availability"; use polkadot_node_primitives::{AvailableData, ErasureChunk}; use super::{cli::TestObjective, core::mock::AlwaysSupportsParachains}; -use polkadot_node_subsystem_test_helpers::mock::{make_ferdie_keystore, new_block_import_info}; +use polkadot_node_subsystem_test_helpers::mock::new_block_import_info; use polkadot_primitives::{ - BlockNumber, CandidateHash, CandidateReceipt, GroupIndex, Hash, HeadData, Header, - PersistedValidationData, ValidatorIndex, + AvailabilityBitfield, BlockNumber, CandidateHash, CandidateReceipt, GroupIndex, Hash, HeadData, + Header, PersistedValidationData, Signed, SigningContext, ValidatorIndex, }; use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; use sc_service::SpawnTaskHandle; @@ -95,7 +105,6 @@ fn build_overseer_for_availability_read( fn build_overseer_for_availability_write( spawn_task_handle: SpawnTaskHandle, runtime_api: MockRuntimeApi, - av_store: MockAvailabilityStore, network_bridge: MockNetworkBridgeTx, availability_distribution: AvailabilityDistributionSubsystem, chain_api: MockChainApi, @@ -145,6 +154,16 @@ fn prepare_test_inner( .or_default() .push(state.next_candidate().expect("Cycle iterator")) } + + // First candidate is our backed candidate. + state.backed_candidates.push( + candidate_hashes + .get(&Hash::repeat_byte(block_num as u8)) + .expect("just inserted above") + .get(0) + .expect("just inserted above") + .clone(), + ); } let runtime_api = runtime_api::MockRuntimeApi::new( @@ -153,9 +172,6 @@ fn prepare_test_inner( candidate_hashes, ); - let av_store = - av_store::MockAvailabilityStore::new(state.chunks.clone(), state.candidate_hashes.clone()); - let availability_state = NetworkAvailabilityState { candidate_hashes: state.candidate_hashes.clone(), available_data: state.available_data.clone(), @@ -195,6 +211,13 @@ fn prepare_test_inner( ) }; + // Use a mocked av-store. + // TODO: switch to real av-store. + let av_store = av_store::MockAvailabilityStore::new( + state.chunks.clone(), + state.candidate_hashes.clone(), + ); + build_overseer_for_availability_read( dependencies.task_manager.spawn_handle(), runtime_api, @@ -209,14 +232,14 @@ fn prepare_test_inner( let (chunk_req_receiver, chunk_req_cfg) = IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); req_cfgs.push(pov_req_cfg); - req_cfgs.push(chunk_req_cfg); + + state.set_chunk_request_protocol(chunk_req_cfg); let availability_distribution = AvailabilityDistributionSubsystem::new( test_authorities.keyring.keystore(), IncomingRequestReceivers { pov_req_receiver, chunk_req_receiver }, Metrics::try_register(&dependencies.registry).unwrap(), ); - let availability_store = new_av_store(&dependencies); let block_headers = (0..config.num_blocks) .map(|block_number| { @@ -234,18 +257,16 @@ fn prepare_test_inner( .collect::>(); let chain_api_state = ChainApiState { block_headers }; - let chain_api = MockChainApi::new(config.clone(), chain_api_state); - + let chain_api = MockChainApi::new(chain_api_state); let bitfield_distribution = BitfieldDistribution::new(Metrics::try_register(&dependencies.registry).unwrap()); build_overseer_for_availability_write( dependencies.task_manager.spawn_handle(), runtime_api, - av_store, network_bridge_tx, availability_distribution, chain_api, - availability_store, + new_av_store(&dependencies), bitfield_distribution, ) }, @@ -254,7 +275,17 @@ fn prepare_test_inner( }, }; - (TestEnvironment::new(dependencies, config, network, overseer, overseer_handle), req_cfgs) + ( + TestEnvironment::new( + dependencies, + config, + network, + overseer, + overseer_handle, + test_authorities, + ), + req_cfgs, + ) } #[derive(Clone)] @@ -276,6 +307,12 @@ pub struct TestState { available_data: Vec, // Per candiadte index chunks chunks: Vec>, + // Availability distribution + chunk_request_protocol: Option, + // Availability distribution. + pov_request_protocol: Option, + // Per relay chain block - our backed candidate + backed_candidates: Vec, } impl TestState { @@ -374,11 +411,30 @@ impl TestState { pov_sizes, candidate_hashes: HashMap::new(), candidates: Vec::new().into_iter().cycle(), + chunk_request_protocol: None, + pov_request_protocol: None, + backed_candidates: Vec::new(), }; _self.generate_candidates(); _self } + + pub fn backed_candidates(&mut self) -> &mut Vec { + &mut self.backed_candidates + } + + pub fn set_chunk_request_protocol(&mut self, config: ProtocolConfig) { + self.chunk_request_protocol = Some(config); + } + + pub fn set_pov_request_protocol(&mut self, config: ProtocolConfig) { + self.pov_request_protocol = Some(config); + } + + pub fn chunk_request_protocol(&self) -> Option { + self.chunk_request_protocol.clone() + } } fn derive_erasure_chunks_with_proofs_and_root( @@ -469,6 +525,7 @@ pub async fn benchmark_availability_read(env: &mut TestEnvironment, mut state: T .red() ); + env.display_network_usage(); env.display_cpu_usage(&["availability-recovery"]); env.stop().await; } @@ -481,19 +538,131 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: env.metrics().set_n_validators(config.n_validators); env.metrics().set_n_cores(config.n_cores); + gum::info!("Seeding availability store with candidates ..."); + for backed_candidate in state.backed_candidates().clone() { + let candidate_index = *state.candidate_hashes.get(&backed_candidate.hash()).unwrap(); + let available_data = state.available_data[candidate_index].clone(); + let (tx, rx) = oneshot::channel(); + env.send_message(AllMessages::AvailabilityStore( + AvailabilityStoreMessage::StoreAvailableData { + candidate_hash: backed_candidate.hash(), + n_validators: config.n_validators as u32, + available_data, + expected_erasure_root: backed_candidate.descriptor().erasure_root, + tx, + }, + )) + .await; + + let _ = rx + .await + .unwrap() + .expect("Test candidates are stored nicely in availability store"); + } + + gum::info!("Done"); + for block_num in 0..env.config().num_blocks { gum::info!(target: LOG_TARGET, "Current block {}/{}", block_num + 1, env.config().num_blocks); env.metrics().set_current_block(block_num); let block_start_ts = Instant::now(); + let relay_block_hash = Hash::repeat_byte(block_num as u8); + env.import_block(new_block_import_info(relay_block_hash, block_num as BlockNumber)) + .await; + + let mut chunk_request_protocol = + state.chunk_request_protocol().expect("No chunk fetching protocol configured"); + + // Inform bitfield distribution about our view of current test block + let message = polkadot_node_subsystem_types::messages::BitfieldDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::OurViewChange(OurView::new(vec![(relay_block_hash, Arc::new(Span::Disabled))], 0)) + ); + env.send_message(AllMessages::BitfieldDistribution(message)).await; + + // Request chunks of backed candidate from all validators + let mut receivers = Vec::new(); + for index in 1..config.n_validators { + let (pending_response, pending_response_receiver) = oneshot::channel(); + + // Our backed candidate is first in candidate hashes entry for current block. + let payload = ChunkFetchingRequest { + candidate_hash: state.backed_candidates()[block_num].hash(), + index: ValidatorIndex(index as u32), + }; + // We don't really care. + let peer = PeerId::random(); + + // They sent it. + env.network().peer_stats(index).inc_sent(payload.encoded_size()); + // We received it. + env.network().inc_received(payload.encoded_size()); + + // TODO: implement TX rate limiter + if let Some(sender) = chunk_request_protocol.inbound_queue.clone() { + receivers.push(pending_response_receiver); + let _ = sender + .send( + IncomingRequest::new(PeerId::random(), payload, pending_response) + .into_raw(), + ) + .await; + } + } - env.import_block(new_block_import_info( - Hash::repeat_byte(block_num as u8), - block_num as BlockNumber, - )) - .await; + gum::info!("Waiting for all emulated peers to receive their chunk from us ..."); + for (index, receiver) in receivers.into_iter().enumerate() { + let response = receiver.await.expect("Chunk is always served succesfully"); + assert!(response.result.is_ok()); + env.network().peer_stats(index).inc_received(response.result.encoded_size()); + env.network().inc_sent(response.result.encoded_size()); + } - tokio::time::sleep(std::time::Duration::from_secs(6)).await; + gum::info!("All chunks sent"); + + // This reflects the bitfield sign timer, we expect bitfields to come in from the network + // after it expires. + tokio::time::sleep(std::time::Duration::from_millis(1500)).await; + let signing_context = SigningContext { session_index: 0, parent_hash: relay_block_hash }; + + // Generate `n_validator` - 1 messages and inject them to the subsystem via overseer. + for index in 1..config.n_validators { + let validator_public = env + .authorities() + .validator_public + .get(index) + .expect("All validator keys are known"); + + // Node has all the chunks in the world. + let payload: AvailabilityBitfield = + AvailabilityBitfield(bitvec![u8, bitvec::order::Lsb0; 1u8; 32]); + let signed_bitfield = Signed::::sign( + &env.authorities().keyring.keystore(), + payload, + &signing_context, + ValidatorIndex(index as u32), + &validator_public.clone().into(), + ) + .ok() + .flatten() + .expect("should be signed"); + + let overseer_handle = env.overseer_handle(); + + let (run, size) = + send_peer_bitfield(overseer_handle, relay_block_hash, signed_bitfield); + let network_action = NetworkAction::new( + env.authorities() + .validator_authority_id + .get(index) + .cloned() + .expect("All validator keys are known"), + run, + size, + None, + ); + env.network_mut().submit_peer_action(network_action.peer(), network_action); + } let block_time = Instant::now().sub(block_start_ts).as_millis() as u64; env.metrics().set_block_time(block_time); @@ -513,10 +682,44 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: .red() ); + env.display_network_usage(); + env.display_cpu_usage(&[ "availability-distribution", "bitfield-distribution", "availability-store", ]); + env.stop().await; } + +pub fn send_peer_bitfield( + mut overseer_handle: OverseerHandle, + relay_hash: H256, + signed_bitfield: Signed, +) -> (Pin + std::marker::Send + 'static)>>, usize) { + let bitfield = polkadot_node_network_protocol::v2::BitfieldDistributionMessage::Bitfield( + relay_hash, + signed_bitfield.into(), + ); + let payload_size = bitfield.encoded_size(); + + let message = + polkadot_node_subsystem_types::messages::BitfieldDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::PeerMessage(PeerId::random(), Versioned::V2(bitfield)), + ); + + ( + async move { + overseer_handle + .send_msg(AllMessages::BitfieldDistribution(message), LOG_TARGET) + .timeout(MAX_TIME_OF_FLIGHT) + .await + .unwrap_or_else(|| { + panic!("{}ms maximum time of flight breached", MAX_TIME_OF_FLIGHT.as_millis()) + }); + } + .boxed(), + payload_size, + ) +} diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 7b0ad6b1257b..0f1c58d1aee0 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -37,6 +37,8 @@ use std::{ }; use tokio::runtime::Handle; +use super::configuration::TestAuthorities; + const MIB: f64 = 1024.0 * 1024.0; /// Test environment/configuration metrics @@ -150,7 +152,7 @@ pub const GENESIS_HASH: Hash = Hash::repeat_byte(0xff); // We use this to bail out sending messages to the subsystem if it is overloaded such that // the time of flight is breaches 5s. // This should eventually be a test parameter. -const MAX_TIME_OF_FLIGHT: Duration = Duration::from_millis(5000); +pub const MAX_TIME_OF_FLIGHT: Duration = Duration::from_millis(5000); /// The test environment is the high level wrapper of all things required to test /// a certain subsystem. @@ -192,6 +194,8 @@ pub struct TestEnvironment { network: NetworkEmulator, /// Configuration/env metrics metrics: TestEnvironmentMetrics, + /// Test authorities generated from the configuration. + authorities: TestAuthorities, } impl TestEnvironment { @@ -202,6 +206,7 @@ impl TestEnvironment { network: NetworkEmulator, overseer: Overseer, AlwaysSupportsParachains>, overseer_handle: OverseerHandle, + authorities: TestAuthorities, ) -> Self { let metrics = TestEnvironmentMetrics::new(&dependencies.registry) .expect("Metrics need to be registered"); @@ -230,6 +235,7 @@ impl TestEnvironment { config, network, metrics, + authorities, } } @@ -241,6 +247,10 @@ impl TestEnvironment { &self.network } + pub fn network_mut(&mut self) -> &mut NetworkEmulator { + &mut self.network + } + pub fn registry(&self) -> &Registry { &self.dependencies.registry } @@ -253,6 +263,14 @@ impl TestEnvironment { self.runtime_handle.clone() } + pub fn authorities(&self) -> &TestAuthorities { + &self.authorities + } + + pub fn overseer_handle(&self) -> OverseerHandle { + self.overseer_handle.clone() + } + // Send a message to the subsystem under test environment. pub async fn send_message(&mut self, msg: AllMessages) { self.overseer_handle @@ -280,26 +298,35 @@ impl TestEnvironment { self.overseer_handle.stop().await; } - // Print CPU usage stats in the CLI. - pub fn display_cpu_usage(&self, subsystems_under_test: &[&str]) { - let stats = self.network().stats(); + pub fn display_network_usage(&self) { + let test_metrics = super::display::parse_metrics(self.registry()); + + let node_receive_metrics = test_metrics.subset_with_label_value("peer", "node0"); + let total_node_received = + node_receive_metrics.sum_by("subsystem_benchmark_network_peer_total_bytes_received"); + + let node_send_metrics = test_metrics.subset_with_label_value("peer", "node0"); + let total_node_sent = + node_send_metrics.sum_by("subsystem_benchmark_network_peer_total_bytes_sent"); + + let total_node_received = total_node_received / MIB; + let total_node_sent = total_node_sent / MIB; + println!( - "\nTotal received from network: {}", - format!( - "{} MiB", - stats - .iter() - .enumerate() - .map(|(_index, stats)| stats.tx_bytes_total as u128) - .sum::() / (1024 * 1024) - ) - .cyan() + "\nPayload bytes received from peers: {}, {}", + format!("{:.2} MiB total", total_node_received).blue(), + format!("{:.2} MiB/block", total_node_received / self.config().num_blocks as f64).bright_blue() ); + println!( - "Total sent to network: {}", - format!("{} KiB", stats[0].tx_bytes_total / (1024)).cyan() + "Payload bytes sent to peers: {}, {}", + format!("{:.2} MiB total", total_node_sent).blue(), + format!("{:.2} MiB/block", total_node_sent / self.config().num_blocks as f64).bright_blue() ); + } + // Print CPU usage stats in the CLI. + pub fn display_cpu_usage(&self, subsystems_under_test: &[&str]) { let test_metrics = super::display::parse_metrics(self.registry()); for subsystem in subsystems_under_test.into_iter() { diff --git a/polkadot/node/subsystem-bench/src/core/keyring.rs b/polkadot/node/subsystem-bench/src/core/keyring.rs index 2997fff69509..a8aba1aec850 100644 --- a/polkadot/node/subsystem-bench/src/core/keyring.rs +++ b/polkadot/node/subsystem-bench/src/core/keyring.rs @@ -45,7 +45,7 @@ impl Keyring { .expect("Insert key into keystore") } - pub fn keystore(&self) -> Arc { + pub fn keystore(&self) -> Arc { self.keystore.clone() } } diff --git a/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs b/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs index 655276dc624f..ba6d53366bc5 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs @@ -25,7 +25,6 @@ use polkadot_node_subsystem_types::OverseerSignal; use sp_core::H256; use std::collections::HashMap; -use crate::core::configuration::{TestAuthorities, TestConfiguration}; use futures::FutureExt; const LOG_TARGET: &str = "subsystem-bench::chain-api-mock"; @@ -37,12 +36,11 @@ pub struct ChainApiState { pub struct MockChainApi { state: ChainApiState, - config: TestConfiguration, } impl MockChainApi { - pub fn new(config: TestConfiguration, state: ChainApiState) -> MockChainApi { - Self { state, config } + pub fn new(state: ChainApiState) -> MockChainApi { + Self { state } } } diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 9d7098d4f3df..e340dba8ed92 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -195,7 +195,7 @@ impl PeerEmulator { pub type ActionFuture = std::pin::Pin + std::marker::Send>>; -/// An network action to be completed by the emulator task. +/// A network action to be completed by the emulator task. pub struct NetworkAction { // The function that performs the action run: ActionFuture, @@ -211,46 +211,24 @@ unsafe impl Send for NetworkAction {} /// Book keeping of sent and received bytes. pub struct PeerEmulatorStats { - rx_bytes_total: AtomicU64, - tx_bytes_total: AtomicU64, metrics: Metrics, peer_index: usize, } impl PeerEmulatorStats { pub(crate) fn new(peer_index: usize, metrics: Metrics) -> Self { - Self { - metrics, - rx_bytes_total: AtomicU64::from(0), - tx_bytes_total: AtomicU64::from(0), - peer_index, - } + Self { metrics, peer_index } } pub fn inc_sent(&self, bytes: usize) { - self.tx_bytes_total.fetch_add(bytes as u64, Ordering::Relaxed); self.metrics.on_peer_sent(self.peer_index, bytes); } pub fn inc_received(&self, bytes: usize) { - self.rx_bytes_total.fetch_add(bytes as u64, Ordering::Relaxed); self.metrics.on_peer_received(self.peer_index, bytes); } - - pub fn sent(&self) -> u64 { - self.tx_bytes_total.load(Ordering::Relaxed) - } - - pub fn received(&self) -> u64 { - self.rx_bytes_total.load(Ordering::Relaxed) - } } -#[derive(Debug, Default)] -pub struct PeerStats { - pub rx_bytes_total: u64, - pub tx_bytes_total: u64, -} impl NetworkAction { pub fn new( peer: AuthorityDiscoveryId, @@ -409,28 +387,15 @@ impl NetworkEmulator { self.stats[peer_index].clone() } - // Returns the sent/received stats for all peers. - pub fn stats(&self) -> Vec { - let r = self - .stats - .iter() - .map(|stats| PeerStats { - rx_bytes_total: stats.received(), - tx_bytes_total: stats.sent(), - }) - .collect::>(); - r - } - // Increment bytes sent by our node (the node that contains the subsystem under test) pub fn inc_sent(&self, bytes: usize) { - // Our node always is peer 0. + // Our node is always peer 0. self.peer_stats(0).inc_sent(bytes); } // Increment bytes received by our node (the node that contains the subsystem under test) pub fn inc_received(&self, bytes: usize) { - // Our node always is peer 0. + // Our node is always peer 0. self.peer_stats(0).inc_received(bytes); } } diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 423c0f5193ef..342540bc29b8 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -113,7 +113,6 @@ impl BenchCli { } fn launch(self) -> eyre::Result<()> { - let configuration = &self.standard_configuration; let mut test_config = match self.objective { TestObjective::TestSequence(options) => { let test_sequence = From 871e9cfd63dff3ee638210db46a33af41ff005ed Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 7 Dec 2023 18:39:51 +0200 Subject: [PATCH 128/192] Remove network-protocol-staging Signed-off-by: Alexandru Gheorghe --- polkadot/Cargo.toml | 1 - polkadot/cli/Cargo.toml | 4 +--- polkadot/node/network/protocol/Cargo.toml | 3 --- polkadot/node/network/protocol/src/peer_set.rs | 7 ------- polkadot/node/service/Cargo.toml | 6 +----- .../parachain/test-parachains/undying/collator/Cargo.toml | 3 --- 6 files changed, 2 insertions(+), 22 deletions(-) diff --git a/polkadot/Cargo.toml b/polkadot/Cargo.toml index c0227823c6b2..e5fe654f31fe 100644 --- a/polkadot/Cargo.toml +++ b/polkadot/Cargo.toml @@ -64,7 +64,6 @@ jemalloc-allocator = [ "polkadot-node-core-pvf/jemalloc-allocator", "polkadot-overseer/jemalloc-allocator", ] -network-protocol-staging = ["polkadot-cli/network-protocol-staging"] # Enables timeout-based tests supposed to be run only in CI environment as they may be flaky diff --git a/polkadot/cli/Cargo.toml b/polkadot/cli/Cargo.toml index 72b2a18f36b3..3f877923923b 100644 --- a/polkadot/cli/Cargo.toml +++ b/polkadot/cli/Cargo.toml @@ -74,6 +74,4 @@ malus = ["full-node", "service/malus"] runtime-metrics = [ "polkadot-node-metrics/runtime-metrics", "service/runtime-metrics", -] - -network-protocol-staging = ["service/network-protocol-staging"] +] \ No newline at end of file diff --git a/polkadot/node/network/protocol/Cargo.toml b/polkadot/node/network/protocol/Cargo.toml index c33b9eae3252..379334ded24a 100644 --- a/polkadot/node/network/protocol/Cargo.toml +++ b/polkadot/node/network/protocol/Cargo.toml @@ -27,6 +27,3 @@ bitvec = "1" [dev-dependencies] rand_chacha = "0.3.1" - -[features] -network-protocol-staging = [] diff --git a/polkadot/node/network/protocol/src/peer_set.rs b/polkadot/node/network/protocol/src/peer_set.rs index fb2e2d77f6ff..3a3c3edbbd0b 100644 --- a/polkadot/node/network/protocol/src/peer_set.rs +++ b/polkadot/node/network/protocol/src/peer_set.rs @@ -127,13 +127,6 @@ impl PeerSet { /// Networking layer relies on `get_main_version()` being the version /// of the main protocol name reported by [`PeerSetProtocolNames::get_main_name()`]. pub fn get_main_version(self) -> ProtocolVersion { - #[cfg(not(feature = "network-protocol-staging"))] - match self { - PeerSet::Validation => ValidationVersion::V2.into(), - PeerSet::Collation => CollationVersion::V2.into(), - } - - #[cfg(feature = "network-protocol-staging")] match self { PeerSet::Validation => ValidationVersion::V3.into(), PeerSet::Collation => CollationVersion::V2.into(), diff --git a/polkadot/node/service/Cargo.toml b/polkadot/node/service/Cargo.toml index 448ab605aa92..23a7bace30eb 100644 --- a/polkadot/node/service/Cargo.toml +++ b/polkadot/node/service/Cargo.toml @@ -224,8 +224,4 @@ runtime-metrics = [ "polkadot-runtime-parachains/runtime-metrics", "rococo-runtime?/runtime-metrics", "westend-runtime?/runtime-metrics", -] - -network-protocol-staging = [ - "polkadot-node-network-protocol/network-protocol-staging", -] +] \ No newline at end of file diff --git a/polkadot/parachain/test-parachains/undying/collator/Cargo.toml b/polkadot/parachain/test-parachains/undying/collator/Cargo.toml index 0de349eac011..fac05dada74e 100644 --- a/polkadot/parachain/test-parachains/undying/collator/Cargo.toml +++ b/polkadot/parachain/test-parachains/undying/collator/Cargo.toml @@ -39,6 +39,3 @@ sc-service = { path = "../../../../../substrate/client/service" } sp-keyring = { path = "../../../../../substrate/primitives/keyring" } tokio = { version = "1.24.2", features = ["macros"] } - -[features] -network-protocol-staging = ["polkadot-cli/network-protocol-staging"] From a364b6495a76fcdb405926841445b708b20dd5d7 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 8 Dec 2023 10:41:26 +0200 Subject: [PATCH 129/192] Rename zombienet Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/zombienet/polkadot.yml | 12 ++++++------ ...ing.toml => 0009-approval-voting-coalescing.toml} | 0 ...g.zndsl => 0009-approval-voting-coalescing.zndsl} | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) rename polkadot/zombienet_tests/functional/{0007-approval-voting-coalescing.toml => 0009-approval-voting-coalescing.toml} (100%) rename polkadot/zombienet_tests/functional/{0007-approval-voting-coalescing.zndsl => 0009-approval-voting-coalescing.zndsl} (96%) diff --git a/.gitlab/pipeline/zombienet/polkadot.yml b/.gitlab/pipeline/zombienet/polkadot.yml index 9a5a6d671cbb..356abaa93cdd 100644 --- a/.gitlab/pipeline/zombienet/polkadot.yml +++ b/.gitlab/pipeline/zombienet/polkadot.yml @@ -115,29 +115,29 @@ zombienet-polkadot-functional-0006-parachains-max-tranche0: --local-dir="${LOCAL_DIR}/functional" --test="0006-parachains-max-tranche0.zndsl" -zombienet-polkadot-functional-0007-approval-voting-coalescing: +zombienet-polkadot-functional-0007-dispute-freshly-finalized: extends: - .zombienet-polkadot-common script: - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh --local-dir="${LOCAL_DIR}/functional" - --test="0007-approval-voting-coalescing.zndsl" + --test="0007-dispute-freshly-finalized.zndsl" -zombienet-polkadot-functional-0007-dispute-freshly-finalized: +zombienet-polkadot-functional-0008-dispute-old-finalized: extends: - .zombienet-polkadot-common script: - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh --local-dir="${LOCAL_DIR}/functional" - --test="0007-dispute-freshly-finalized.zndsl" + --test="0008-dispute-old-finalized.zndsl" -zombienet-polkadot-functional-0008-dispute-old-finalized: +zombienet-polkadot-functional-0009-approval-voting-coalescing: extends: - .zombienet-polkadot-common script: - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh --local-dir="${LOCAL_DIR}/functional" - --test="0008-dispute-old-finalized.zndsl" + --test="0009-approval-voting-coalescing.zndsl" zombienet-polkadot-smoke-0001-parachains-smoke-test: extends: diff --git a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml b/polkadot/zombienet_tests/functional/0009-approval-voting-coalescing.toml similarity index 100% rename from polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml rename to polkadot/zombienet_tests/functional/0009-approval-voting-coalescing.toml diff --git a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl b/polkadot/zombienet_tests/functional/0009-approval-voting-coalescing.zndsl similarity index 96% rename from polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl rename to polkadot/zombienet_tests/functional/0009-approval-voting-coalescing.zndsl index 43b2c5e13a80..1fc4f6784460 100644 --- a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl +++ b/polkadot/zombienet_tests/functional/0009-approval-voting-coalescing.zndsl @@ -1,5 +1,5 @@ Description: Approval voting coalescing does not lag finality -Network: ./0007-approval-voting-coalescing.toml +Network: ./0009-approval-voting-coalescing.toml Creds: config # Check authority status. From 5bfe2d91913c6db45161e07c796470923b2cab1a Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 8 Dec 2023 13:31:08 +0200 Subject: [PATCH 130/192] add fn to wait on metric and CLI output changes Signed-off-by: Andrei Sandu --- .../subsystem-bench/src/availability/mod.rs | 17 +++++++------ .../subsystem-bench/src/core/environment.rs | 25 ++++++++++++++++--- .../subsystem-bench/src/subsystem-bench.rs | 1 + 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 01510505d22a..2763e3a269a0 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -533,7 +533,6 @@ pub async fn benchmark_availability_read(env: &mut TestEnvironment, mut state: T pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: TestState) { let config = env.config().clone(); let start_marker = Instant::now(); - let mut availability_bytes = 0u128; env.metrics().set_n_validators(config.n_validators); env.metrics().set_n_cores(config.n_cores); @@ -563,7 +562,7 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: gum::info!("Done"); for block_num in 0..env.config().num_blocks { - gum::info!(target: LOG_TARGET, "Current block {}/{}", block_num + 1, env.config().num_blocks); + gum::info!(target: LOG_TARGET, "Current block #{}", block_num); env.metrics().set_current_block(block_num); let block_start_ts = Instant::now(); @@ -610,7 +609,7 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: } } - gum::info!("Waiting for all emulated peers to receive their chunk from us ..."); + gum::info!(target: LOG_TARGET, "Waiting for all emulated peers to receive their chunk from us ..."); for (index, receiver) in receivers.into_iter().enumerate() { let response = receiver.await.expect("Chunk is always served succesfully"); assert!(response.result.is_ok()); @@ -664,18 +663,20 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: env.network_mut().submit_peer_action(network_action.peer(), network_action); } + // Wait for all bitfields to be processed. + env.wait_until_metric_ge( + "polkadot_parachain_received_availabilty_bitfields_total", + (config.n_validators - 1) * (block_num + 1), + ) + .await; + let block_time = Instant::now().sub(block_start_ts).as_millis() as u64; env.metrics().set_block_time(block_time); gum::info!("All work for block completed in {}", format!("{:?}ms", block_time).cyan()); } let duration: u128 = start_marker.elapsed().as_millis(); - let availability_bytes = availability_bytes / 1024; gum::info!("All blocks processed in {}", format!("{:?}ms", duration).cyan()); - gum::info!( - "Throughput: {}", - format!("{} KiB/block", availability_bytes / env.config().num_blocks as u128).bright_red() - ); gum::info!( "Block time: {}", format!("{} ms", start_marker.elapsed().as_millis() / env.config().num_blocks as u128) diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 0f1c58d1aee0..ec61f715cd6b 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -29,7 +29,6 @@ use polkadot_node_subsystem_util::metrics::prometheus::{ self, Gauge, Histogram, PrometheusError, Registry, U64, }; -use sc_network::peer_store::LOG_TARGET; use sc_service::{SpawnTaskHandle, TaskManager}; use std::{ fmt::Display, @@ -37,6 +36,7 @@ use std::{ }; use tokio::runtime::Handle; +const LOG_TARGET: &str = "subsystem-bench::environment"; use super::configuration::TestAuthorities; const MIB: f64 = 1024.0 * 1024.0; @@ -298,6 +298,23 @@ impl TestEnvironment { self.overseer_handle.stop().await; } + // Blocks until `metric_name` >= `value` + pub async fn wait_until_metric_ge(&self, metric_name: &str, value: usize) { + let value = value as f64; + loop { + let test_metrics = super::display::parse_metrics(self.registry()); + let current_value = test_metrics.sum_by(metric_name); + + gum::debug!(target: LOG_TARGET, metric_name, current_value, value, "Waiting for metric"); + if current_value >= value { + break + } + + // Check value every 50ms. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + } + pub fn display_network_usage(&self) { let test_metrics = super::display::parse_metrics(self.registry()); @@ -315,13 +332,15 @@ impl TestEnvironment { println!( "\nPayload bytes received from peers: {}, {}", format!("{:.2} MiB total", total_node_received).blue(), - format!("{:.2} MiB/block", total_node_received / self.config().num_blocks as f64).bright_blue() + format!("{:.2} MiB/block", total_node_received / self.config().num_blocks as f64) + .bright_blue() ); println!( "Payload bytes sent to peers: {}, {}", format!("{:.2} MiB total", total_node_sent).blue(), - format!("{:.2} MiB/block", total_node_sent / self.config().num_blocks as f64).bright_blue() + format!("{:.2} MiB/block", total_node_sent / self.config().num_blocks as f64) + .bright_blue() ); } diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 342540bc29b8..195316871050 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -191,6 +191,7 @@ fn main() -> eyre::Result<()> { // Avoid `Terminating due to subsystem exit subsystem` warnings .filter(Some("polkadot_overseer"), log::LevelFilter::Error) .filter(None, log::LevelFilter::Info) + .format_timestamp_millis() // .filter(None, log::LevelFilter::Trace) .try_init() .unwrap(); From 4d21e5bae1baf8f225a31b831dd47cc4ad4e646c Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 8 Dec 2023 13:35:43 +0200 Subject: [PATCH 131/192] cargo lock Signed-off-by: Andrei Sandu --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 74438749859e..f31b1091500e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13279,7 +13279,7 @@ version = "1.0.0" dependencies = [ "assert_matches", "async-trait", - "clap 4.4.6", + "clap 4.4.10", "clap-num", "color-eyre", "colored", From 3e25fdc1d5df30b4c61961e58eb9f694519a1d5b Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 8 Dec 2023 13:47:17 +0200 Subject: [PATCH 132/192] more review feedback Signed-off-by: Andrei Sandu --- Cargo.lock | 2 ++ .../availability-recovery/src/tests.rs | 28 +--------------- .../subsystem-bench/src/availability/mod.rs | 33 ++----------------- .../node/subsystem-test-helpers/Cargo.toml | 3 ++ .../node/subsystem-test-helpers/src/lib.rs | 31 +++++++++++++++++ .../node/subsystem-test-helpers/src/mock.rs | 9 ++--- 6 files changed, 41 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f31b1091500e..f51bcb279d3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12658,6 +12658,8 @@ dependencies = [ "async-trait", "futures", "parking_lot 0.12.1", + "polkadot-erasure-coding", + "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-util", "polkadot-primitives", diff --git a/polkadot/node/network/availability-recovery/src/tests.rs b/polkadot/node/network/availability-recovery/src/tests.rs index 63ccf0e94f91..18d3d41d88d5 100644 --- a/polkadot/node/network/availability-recovery/src/tests.rs +++ b/polkadot/node/network/availability-recovery/src/tests.rs @@ -24,6 +24,7 @@ use parity_scale_codec::Encode; use polkadot_node_network_protocol::request_response::{ self as req_res, IncomingRequest, Recipient, ReqProtocolNames, Requests, }; +use polkadot_node_subsystem_test_helpers::derive_erasure_chunks_with_proofs_and_root; use super::*; @@ -456,33 +457,6 @@ fn validator_authority_id(val_ids: &[Sr25519Keyring]) -> Vec), -) -> (Vec, Hash) { - let mut chunks: Vec> = obtain_chunks(n_validators, available_data).unwrap(); - - for (i, chunk) in chunks.iter_mut().enumerate() { - alter_chunk(i, chunk) - } - - // create proofs for each erasure chunk - let branches = branches(chunks.as_ref()); - - let root = branches.root(); - let erasure_chunks = branches - .enumerate() - .map(|(index, (proof, chunk))| ErasureChunk { - chunk: chunk.to_vec(), - index: ValidatorIndex(index as _), - proof: Proof::try_from(proof).unwrap(), - }) - .collect::>(); - - (erasure_chunks, root) -} - impl Default for TestState { fn default() -> Self { let validators = vec![ diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 244119735966..40b8f4abceeb 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -18,8 +18,8 @@ use std::{collections::HashMap, iter::Cycle, ops::Sub, sync::Arc, time::Instant} use crate::TestEnvironment; use polkadot_node_subsystem::{Overseer, OverseerConnector, SpawnGlue}; +use polkadot_node_subsystem_test_helpers::derive_erasure_chunks_with_proofs_and_root; use polkadot_overseer::Handle as OverseerHandle; - use sc_network::request_responses::ProtocolConfig; use colored::Colorize; @@ -31,9 +31,8 @@ use polkadot_availability_recovery::AvailabilityRecoverySubsystem; use crate::GENESIS_HASH; use parity_scale_codec::Encode; -use polkadot_erasure_coding::{branches, obtain_chunks_v1 as obtain_chunks}; use polkadot_node_network_protocol::request_response::{IncomingRequest, ReqProtocolNames}; -use polkadot_node_primitives::{BlockData, PoV, Proof}; +use polkadot_node_primitives::{BlockData, PoV}; use polkadot_node_subsystem::messages::{AllMessages, AvailabilityRecoveryMessage}; use crate::core::{ @@ -55,7 +54,6 @@ use super::{cli::TestObjective, core::mock::AlwaysSupportsParachains}; use polkadot_node_subsystem_test_helpers::mock::new_block_import_info; use polkadot_primitives::{ CandidateHash, CandidateReceipt, GroupIndex, Hash, HeadData, PersistedValidationData, - ValidatorIndex, }; use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; use sc_service::SpawnTaskHandle; @@ -274,33 +272,6 @@ impl TestState { } } -fn derive_erasure_chunks_with_proofs_and_root( - n_validators: usize, - available_data: &AvailableData, - alter_chunk: impl Fn(usize, &mut Vec), -) -> (Vec, Hash) { - let mut chunks: Vec> = obtain_chunks(n_validators, available_data).unwrap(); - - for (i, chunk) in chunks.iter_mut().enumerate() { - alter_chunk(i, chunk) - } - - // create proofs for each erasure chunk - let branches = branches(chunks.as_ref()); - - let root = branches.root(); - let erasure_chunks = branches - .enumerate() - .map(|(index, (proof, chunk))| ErasureChunk { - chunk: chunk.to_vec(), - index: ValidatorIndex(index as _), - proof: Proof::try_from(proof).unwrap(), - }) - .collect::>(); - - (erasure_chunks, root) -} - pub async fn benchmark_availability_read(env: &mut TestEnvironment, mut state: TestState) { let config = env.config().clone(); diff --git a/polkadot/node/subsystem-test-helpers/Cargo.toml b/polkadot/node/subsystem-test-helpers/Cargo.toml index 9087ca11f5d2..ccbca1c3ea1d 100644 --- a/polkadot/node/subsystem-test-helpers/Cargo.toml +++ b/polkadot/node/subsystem-test-helpers/Cargo.toml @@ -12,8 +12,11 @@ async-trait = "0.1.57" futures = "0.3.21" parking_lot = "0.12.0" polkadot-node-subsystem = { path = "../subsystem" } +polkadot-erasure-coding = { path = "../../erasure-coding" } polkadot-node-subsystem-util = { path = "../subsystem-util" } polkadot-primitives = { path = "../../primitives" } +polkadot-node-primitives = { path = "../primitives" } + sc-client-api = { path = "../../../substrate/client/api" } sc-utils = { path = "../../../substrate/client/utils" } sp-core = { path = "../../../substrate/primitives/core" } diff --git a/polkadot/node/subsystem-test-helpers/src/lib.rs b/polkadot/node/subsystem-test-helpers/src/lib.rs index 3f92513498c4..daa8a10e6171 100644 --- a/polkadot/node/subsystem-test-helpers/src/lib.rs +++ b/polkadot/node/subsystem-test-helpers/src/lib.rs @@ -18,11 +18,14 @@ #![warn(missing_docs)] +use polkadot_erasure_coding::{branches, obtain_chunks_v1 as obtain_chunks}; +use polkadot_node_primitives::{AvailableData, BlockData, ErasureChunk, PoV, Proof}; use polkadot_node_subsystem::{ messages::AllMessages, overseer, FromOrchestra, OverseerSignal, SpawnGlue, SpawnedSubsystem, SubsystemError, SubsystemResult, TrySendError, }; use polkadot_node_subsystem_util::TimeoutExt; +use polkadot_primitives::{Hash, ValidatorIndex}; use futures::{channel::mpsc, poll, prelude::*}; use parking_lot::Mutex; @@ -440,6 +443,34 @@ impl Future for Yield { } } +// Helper for chunking available data. +pub fn derive_erasure_chunks_with_proofs_and_root( + n_validators: usize, + available_data: &AvailableData, + alter_chunk: impl Fn(usize, &mut Vec), +) -> (Vec, Hash) { + let mut chunks: Vec> = obtain_chunks(n_validators, available_data).unwrap(); + + for (i, chunk) in chunks.iter_mut().enumerate() { + alter_chunk(i, chunk) + } + + // create proofs for each erasure chunk + let branches = branches(chunks.as_ref()); + + let root = branches.root(); + let erasure_chunks = branches + .enumerate() + .map(|(index, (proof, chunk))| ErasureChunk { + chunk: chunk.to_vec(), + index: ValidatorIndex(index as _), + proof: Proof::try_from(proof).unwrap(), + }) + .collect::>(); + + (erasure_chunks, root) +} + #[cfg(test)] mod tests { use super::*; diff --git a/polkadot/node/subsystem-test-helpers/src/mock.rs b/polkadot/node/subsystem-test-helpers/src/mock.rs index fc2dd6a4e34e..14026960ac13 100644 --- a/polkadot/node/subsystem-test-helpers/src/mock.rs +++ b/polkadot/node/subsystem-test-helpers/src/mock.rs @@ -16,7 +16,7 @@ use std::sync::Arc; -use polkadot_node_subsystem::{jaeger, ActivatedLeaf,BlockInfo}; +use polkadot_node_subsystem::{jaeger, ActivatedLeaf, BlockInfo}; use sc_client_api::UnpinHandle; use sc_keystore::LocalKeystore; use sc_utils::mpsc::tracing_unbounded; @@ -62,10 +62,5 @@ pub fn new_leaf(hash: Hash, number: BlockNumber) -> ActivatedLeaf { /// Create a new leaf with the given hash and number. pub fn new_block_import_info(hash: Hash, number: BlockNumber) -> BlockInfo { - BlockInfo { - hash, - parent_hash: Hash::default(), - number, - unpin_handle: dummy_unpin_handle(hash), - } + BlockInfo { hash, parent_hash: Hash::default(), number, unpin_handle: dummy_unpin_handle(hash) } } From 1458a73dbc2b96741de3cdadc22d381ac9dccb84 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 8 Dec 2023 13:51:35 +0200 Subject: [PATCH 133/192] change back to debug Signed-off-by: Andrei Sandu --- polkadot/node/network/availability-recovery/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/network/availability-recovery/src/lib.rs b/polkadot/node/network/availability-recovery/src/lib.rs index d029bce04173..fb8064878f4f 100644 --- a/polkadot/node/network/availability-recovery/src/lib.rs +++ b/polkadot/node/network/availability-recovery/src/lib.rs @@ -667,7 +667,7 @@ impl AvailabilityRecoverySubsystem { } }, None => { - gum::trace!( + gum::debug!( target: LOG_TARGET, "Erasure task channel closed", ); From de7b5c006eead38c8694e4ce09c754c75b0a9e82 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 8 Dec 2023 15:46:02 +0200 Subject: [PATCH 134/192] Add migration v11 HostConfiguration, missed during rebasing Signed-off-by: Alexandru Gheorghe --- polkadot/runtime/rococo/src/lib.rs | 1 + polkadot/runtime/westend/src/lib.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 6d6e54f1a97e..5f918119ca6d 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -1631,6 +1631,7 @@ pub mod migrations { // Remove `im-online` pallet on-chain storage frame_support::migrations::RemovePallet::DbWeight>, + parachains_configuration::migration::v11::MigrateToV11, ); } diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index b20e164fedf7..4d2807b45b7a 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -1648,6 +1648,7 @@ pub mod migrations { ImOnlinePalletName, ::DbWeight, >, + parachains_configuration::migration::v11::MigrateToV11, ); } From 5cf522aac9db8658722663042f57edeef8ba1652 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 8 Dec 2023 17:22:12 +0200 Subject: [PATCH 135/192] Fix network_protocol_versioning_subsystem_msg Signed-off-by: Alexandru Gheorghe --- polkadot/node/network/bridge/src/rx/tests.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/polkadot/node/network/bridge/src/rx/tests.rs b/polkadot/node/network/bridge/src/rx/tests.rs index ad503ee21f53..6847b8a7e24d 100644 --- a/polkadot/node/network/bridge/src/rx/tests.rs +++ b/polkadot/node/network/bridge/src/rx/tests.rs @@ -1469,7 +1469,7 @@ fn network_protocol_versioning_subsystem_msg() { NetworkBridgeEvent::PeerConnected( peer, ObservedRole::Full, - ValidationVersion::V2.into(), + ValidationVersion::V3.into(), None, ), &mut virtual_overseer, @@ -1484,9 +1484,9 @@ fn network_protocol_versioning_subsystem_msg() { } let approval_distribution_message = - protocol_v2::ApprovalDistributionMessage::Approvals(Vec::new()); + protocol_v3::ApprovalDistributionMessage::Approvals(Vec::new()); - let msg = protocol_v2::ValidationProtocol::ApprovalDistribution( + let msg = protocol_v3::ValidationProtocol::ApprovalDistribution( approval_distribution_message.clone(), ); @@ -1502,7 +1502,7 @@ fn network_protocol_versioning_subsystem_msg() { virtual_overseer.recv().await, AllMessages::ApprovalDistribution( ApprovalDistributionMessage::NetworkBridgeUpdate( - NetworkBridgeEvent::PeerMessage(p, Versioned::V2(m)) + NetworkBridgeEvent::PeerMessage(p, Versioned::V3(m)) ) ) => { assert_eq!(p, peer); @@ -1536,7 +1536,7 @@ fn network_protocol_versioning_subsystem_msg() { virtual_overseer.recv().await, AllMessages::StatementDistribution( StatementDistributionMessage::NetworkBridgeUpdate( - NetworkBridgeEvent::PeerMessage(p, Versioned::V2(m)) + NetworkBridgeEvent::PeerMessage(p, Versioned::V3(m)) ) ) => { assert_eq!(p, peer); From 673940cfecfbb5647216dad41b30af62eb347190 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 11 Dec 2023 12:22:19 +0200 Subject: [PATCH 136/192] Latest master Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 1079 +++++++---------- .../node/subsystem-bench/src/approval/mod.rs | 16 +- 2 files changed, 470 insertions(+), 625 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c87275787699..3017ebdba715 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -541,10 +541,24 @@ dependencies = [ "scale-info", ] +[[package]] +name = "ark-scale" +version = "0.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f69c00b3b529be29528a6f2fd5fa7b1790f8bed81b9cdca17e326538545a179" +dependencies = [ + "ark-ec", + "ark-ff", + "ark-serialize", + "ark-std", + "parity-scale-codec", + "scale-info", +] + [[package]] name = "ark-secret-scalar" version = "0.0.2" -source = "git+https://github.com/w3f/ring-vrf?rev=3ddc205#3ddc2051066c4b3f0eadd0ba5700df12500d9754" +source = "git+https://github.com/w3f/ring-vrf?rev=e9782f9#e9782f938629c90f3adb3fff2358bc8d1386af3e" dependencies = [ "ark-ec", "ark-ff", @@ -552,7 +566,7 @@ dependencies = [ "ark-std", "ark-transcript", "digest 0.10.7", - "rand_core 0.6.4", + "getrandom_or_panic", "zeroize", ] @@ -593,7 +607,7 @@ dependencies = [ [[package]] name = "ark-transcript" version = "0.0.2" -source = "git+https://github.com/w3f/ring-vrf?rev=3ddc205#3ddc2051066c4b3f0eadd0ba5700df12500d9754" +source = "git+https://github.com/w3f/ring-vrf?rev=e9782f9#e9782f938629c90f3adb3fff2358bc8d1386af3e" dependencies = [ "ark-ff", "ark-serialize", @@ -730,150 +744,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" -[[package]] -name = "asset-hub-kusama-runtime" -version = "0.9.420" -dependencies = [ - "asset-test-utils", - "assets-common", - "cumulus-pallet-aura-ext", - "cumulus-pallet-dmp-queue", - "cumulus-pallet-parachain-system", - "cumulus-pallet-session-benchmarking", - "cumulus-pallet-xcm", - "cumulus-pallet-xcmp-queue", - "cumulus-primitives-core", - "cumulus-primitives-utility", - "frame-benchmarking", - "frame-executive", - "frame-support", - "frame-system", - "frame-system-benchmarking", - "frame-system-rpc-runtime-api", - "frame-try-runtime", - "hex-literal", - "log", - "pallet-asset-conversion", - "pallet-asset-conversion-tx-payment", - "pallet-assets", - "pallet-aura", - "pallet-authorship", - "pallet-balances", - "pallet-collator-selection", - "pallet-message-queue", - "pallet-multisig", - "pallet-nft-fractionalization", - "pallet-nfts", - "pallet-nfts-runtime-api", - "pallet-proxy", - "pallet-session", - "pallet-state-trie-migration", - "pallet-timestamp", - "pallet-transaction-payment", - "pallet-transaction-payment-rpc-runtime-api", - "pallet-uniques", - "pallet-utility", - "pallet-xcm", - "pallet-xcm-benchmarks", - "parachains-common", - "parity-scale-codec", - "polkadot-core-primitives", - "polkadot-parachain-primitives", - "polkadot-runtime-common", - "primitive-types", - "scale-info", - "smallvec", - "sp-api", - "sp-block-builder", - "sp-consensus-aura", - "sp-core", - "sp-genesis-builder", - "sp-inherents", - "sp-offchain", - "sp-runtime", - "sp-session", - "sp-std 8.0.0", - "sp-storage 13.0.0", - "sp-transaction-pool", - "sp-version", - "sp-weights", - "staging-parachain-info", - "staging-xcm", - "staging-xcm-builder", - "staging-xcm-executor", - "substrate-wasm-builder", -] - -[[package]] -name = "asset-hub-polkadot-runtime" -version = "0.9.420" -dependencies = [ - "asset-test-utils", - "assets-common", - "cumulus-pallet-aura-ext", - "cumulus-pallet-dmp-queue", - "cumulus-pallet-parachain-system", - "cumulus-pallet-session-benchmarking", - "cumulus-pallet-xcm", - "cumulus-pallet-xcmp-queue", - "cumulus-primitives-core", - "cumulus-primitives-utility", - "frame-benchmarking", - "frame-executive", - "frame-support", - "frame-system", - "frame-system-benchmarking", - "frame-system-rpc-runtime-api", - "frame-try-runtime", - "hex-literal", - "log", - "pallet-asset-tx-payment", - "pallet-assets", - "pallet-aura", - "pallet-authorship", - "pallet-balances", - "pallet-collator-selection", - "pallet-message-queue", - "pallet-multisig", - "pallet-nfts", - "pallet-nfts-runtime-api", - "pallet-proxy", - "pallet-session", - "pallet-timestamp", - "pallet-transaction-payment", - "pallet-transaction-payment-rpc-runtime-api", - "pallet-uniques", - "pallet-utility", - "pallet-xcm", - "pallet-xcm-benchmarks", - "parachains-common", - "parity-scale-codec", - "polkadot-core-primitives", - "polkadot-parachain-primitives", - "polkadot-runtime-common", - "scale-info", - "smallvec", - "sp-api", - "sp-block-builder", - "sp-consensus-aura", - "sp-core", - "sp-genesis-builder", - "sp-inherents", - "sp-offchain", - "sp-runtime", - "sp-session", - "sp-std 8.0.0", - "sp-storage 13.0.0", - "sp-transaction-pool", - "sp-version", - "sp-weights", - "staging-parachain-info", - "staging-xcm", - "staging-xcm-builder", - "staging-xcm-executor", - "substrate-wasm-builder", -] - [[package]] name = "asset-hub-rococo-emulated-chain" version = "0.0.0" @@ -1015,6 +885,7 @@ dependencies = [ "asset-test-utils", "cumulus-pallet-dmp-queue", "cumulus-pallet-parachain-system", + "cumulus-pallet-xcmp-queue", "emulated-integration-tests-common", "frame-support", "frame-system", @@ -1048,7 +919,6 @@ dependencies = [ "bp-bridge-hub-rococo", "bp-bridge-hub-westend", "cumulus-pallet-aura-ext", - "cumulus-pallet-dmp-queue", "cumulus-pallet-parachain-system", "cumulus-pallet-session-benchmarking", "cumulus-pallet-xcm", @@ -1288,7 +1158,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -1305,7 +1175,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -1368,7 +1238,7 @@ dependencies = [ [[package]] name = "bandersnatch_vrfs" version = "0.0.4" -source = "git+https://github.com/w3f/ring-vrf?rev=3ddc205#3ddc2051066c4b3f0eadd0ba5700df12500d9754" +source = "git+https://github.com/w3f/ring-vrf?rev=e9782f9#e9782f938629c90f3adb3fff2358bc8d1386af3e" dependencies = [ "ark-bls12-381", "ark-ec", @@ -1481,7 +1351,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -1980,134 +1850,6 @@ dependencies = [ "sp-runtime", ] -[[package]] -name = "bridge-hub-kusama-runtime" -version = "0.1.0" -dependencies = [ - "bridge-hub-test-utils", - "cumulus-pallet-aura-ext", - "cumulus-pallet-dmp-queue", - "cumulus-pallet-parachain-system", - "cumulus-pallet-session-benchmarking", - "cumulus-pallet-xcm", - "cumulus-pallet-xcmp-queue", - "cumulus-primitives-core", - "cumulus-primitives-utility", - "frame-benchmarking", - "frame-executive", - "frame-support", - "frame-system", - "frame-system-benchmarking", - "frame-system-rpc-runtime-api", - "frame-try-runtime", - "hex-literal", - "log", - "pallet-aura", - "pallet-authorship", - "pallet-balances", - "pallet-collator-selection", - "pallet-message-queue", - "pallet-multisig", - "pallet-session", - "pallet-timestamp", - "pallet-transaction-payment", - "pallet-transaction-payment-rpc-runtime-api", - "pallet-utility", - "pallet-xcm", - "pallet-xcm-benchmarks", - "parachains-common", - "parity-scale-codec", - "polkadot-core-primitives", - "polkadot-parachain-primitives", - "polkadot-runtime-common", - "scale-info", - "serde", - "smallvec", - "sp-api", - "sp-block-builder", - "sp-consensus-aura", - "sp-core", - "sp-genesis-builder", - "sp-inherents", - "sp-io", - "sp-offchain", - "sp-runtime", - "sp-session", - "sp-std 8.0.0", - "sp-storage 13.0.0", - "sp-transaction-pool", - "sp-version", - "staging-parachain-info", - "staging-xcm", - "staging-xcm-builder", - "staging-xcm-executor", - "substrate-wasm-builder", -] - -[[package]] -name = "bridge-hub-polkadot-runtime" -version = "0.1.0" -dependencies = [ - "bridge-hub-test-utils", - "cumulus-pallet-aura-ext", - "cumulus-pallet-dmp-queue", - "cumulus-pallet-parachain-system", - "cumulus-pallet-session-benchmarking", - "cumulus-pallet-xcm", - "cumulus-pallet-xcmp-queue", - "cumulus-primitives-core", - "cumulus-primitives-utility", - "frame-benchmarking", - "frame-executive", - "frame-support", - "frame-system", - "frame-system-benchmarking", - "frame-system-rpc-runtime-api", - "frame-try-runtime", - "hex-literal", - "log", - "pallet-aura", - "pallet-authorship", - "pallet-balances", - "pallet-collator-selection", - "pallet-message-queue", - "pallet-multisig", - "pallet-session", - "pallet-timestamp", - "pallet-transaction-payment", - "pallet-transaction-payment-rpc-runtime-api", - "pallet-utility", - "pallet-xcm", - "pallet-xcm-benchmarks", - "parachains-common", - "parity-scale-codec", - "polkadot-core-primitives", - "polkadot-parachain-primitives", - "polkadot-runtime-common", - "scale-info", - "serde", - "smallvec", - "sp-api", - "sp-block-builder", - "sp-consensus-aura", - "sp-core", - "sp-genesis-builder", - "sp-inherents", - "sp-io", - "sp-offchain", - "sp-runtime", - "sp-session", - "sp-std 8.0.0", - "sp-storage 13.0.0", - "sp-transaction-pool", - "sp-version", - "staging-parachain-info", - "staging-xcm", - "staging-xcm-builder", - "staging-xcm-executor", - "substrate-wasm-builder", -] - [[package]] name = "bridge-hub-rococo-emulated-chain" version = "0.0.0" @@ -2329,7 +2071,6 @@ dependencies = [ "bridge-hub-test-utils", "bridge-runtime-common", "cumulus-pallet-aura-ext", - "cumulus-pallet-dmp-queue", "cumulus-pallet-parachain-system", "cumulus-pallet-session-benchmarking", "cumulus-pallet-xcm", @@ -2709,7 +2450,7 @@ checksum = "b9b68e3193982cd54187d71afdb2a271ad4cf8af157858e9cb911b91321de143" dependencies = [ "core2", "multibase", - "multihash", + "multihash 0.17.0", "serde", "unsigned-varint", ] @@ -2781,12 +2522,12 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.6" +version = "4.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" +checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" dependencies = [ "clap_builder", - "clap_derive 4.4.2", + "clap_derive 4.4.7", ] [[package]] @@ -2800,13 +2541,13 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.6" +version = "4.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" +checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" dependencies = [ "anstream", "anstyle", - "clap_lex 0.5.1", + "clap_lex 0.6.0", "strsim", "terminal_size", ] @@ -2817,7 +2558,7 @@ version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "586a385f7ef2f8b4d86bddaa0c094794e7ccbfe5ffef1f434fe928143fc783a5" dependencies = [ - "clap 4.4.6", + "clap 4.4.11", ] [[package]] @@ -2835,14 +2576,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.2" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -2856,9 +2597,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "coarsetime" @@ -2883,75 +2624,18 @@ dependencies = [ ] [[package]] -name = "collectives-polkadot-runtime" -version = "1.0.0" +name = "collectives-westend-emulated-chain" +version = "0.0.0" dependencies = [ - "cumulus-pallet-aura-ext", - "cumulus-pallet-dmp-queue", - "cumulus-pallet-parachain-system", - "cumulus-pallet-session-benchmarking", - "cumulus-pallet-xcm", - "cumulus-pallet-xcmp-queue", + "collectives-westend-runtime", "cumulus-primitives-core", - "cumulus-primitives-utility", - "frame-benchmarking", - "frame-executive", + "emulated-integration-tests-common", "frame-support", - "frame-system", - "frame-system-benchmarking", - "frame-system-rpc-runtime-api", - "frame-try-runtime", - "hex-literal", - "log", - "pallet-alliance", - "pallet-aura", - "pallet-authorship", - "pallet-balances", - "pallet-collator-selection", - "pallet-collective", - "pallet-collective-content", - "pallet-core-fellowship", - "pallet-message-queue", - "pallet-multisig", - "pallet-preimage", - "pallet-proxy", - "pallet-ranked-collective", - "pallet-referenda", - "pallet-salary", - "pallet-scheduler", - "pallet-session", - "pallet-timestamp", - "pallet-transaction-payment", - "pallet-transaction-payment-rpc-runtime-api", - "pallet-utility", - "pallet-xcm", "parachains-common", - "parity-scale-codec", - "polkadot-core-primitives", - "polkadot-parachain-primitives", - "polkadot-runtime-common", - "scale-info", - "smallvec", - "sp-api", - "sp-arithmetic", - "sp-block-builder", - "sp-consensus-aura", + "serde_json", "sp-core", - "sp-genesis-builder", - "sp-inherents", - "sp-io", - "sp-offchain", "sp-runtime", - "sp-session", - "sp-std 8.0.0", - "sp-storage 13.0.0", - "sp-transaction-pool", - "sp-version", - "staging-parachain-info", - "staging-xcm", - "staging-xcm-builder", - "staging-xcm-executor", - "substrate-wasm-builder", + "westend-emulated-chain", ] [[package]] @@ -2959,7 +2643,6 @@ name = "collectives-westend-runtime" version = "1.0.0" dependencies = [ "cumulus-pallet-aura-ext", - "cumulus-pallet-dmp-queue", "cumulus-pallet-parachain-system", "cumulus-pallet-session-benchmarking", "cumulus-pallet-xcm", @@ -2976,6 +2659,7 @@ dependencies = [ "hex-literal", "log", "pallet-alliance", + "pallet-asset-rate", "pallet-aura", "pallet-authorship", "pallet-balances", @@ -2995,6 +2679,7 @@ dependencies = [ "pallet-timestamp", "pallet-transaction-payment", "pallet-transaction-payment-rpc-runtime-api", + "pallet-treasury", "pallet-utility", "pallet-xcm", "parachains-common", @@ -3092,7 +2777,7 @@ dependencies = [ [[package]] name = "common" version = "0.1.0" -source = "git+https://github.com/burdges/ring-proof?branch=patch-1#05a756076cb20f981a52afea3a620168de49f95f" +source = "git+https://github.com/w3f/ring-proof#b273d33f9981e2bb3375ab45faeb537f7ee35224" dependencies = [ "ark-ec", "ark-ff", @@ -3100,6 +2785,7 @@ dependencies = [ "ark-serialize", "ark-std", "fflonk", + "getrandom_or_panic", "merlin 3.0.0", "rand_chacha 0.3.1", ] @@ -3210,7 +2896,6 @@ dependencies = [ "pallet-balances", "pallet-collator-selection", "pallet-contracts", - "pallet-contracts-primitives", "pallet-insecure-randomness-collective-flip", "pallet-message-queue", "pallet-multisig", @@ -3476,7 +3161,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.4.6", + "clap 4.4.11", "criterion-plot", "futures", "is-terminal", @@ -3651,7 +3336,7 @@ dependencies = [ name = "cumulus-client-cli" version = "0.1.0" dependencies = [ - "clap 4.4.6", + "clap 4.4.11", "parity-scale-codec", "sc-chain-spec", "sc-cli", @@ -3872,6 +3557,7 @@ dependencies = [ "cumulus-client-network", "cumulus-client-pov-recovery", "cumulus-primitives-core", + "cumulus-primitives-proof-size-hostfunction", "cumulus-relay-chain-inprocess-interface", "cumulus-relay-chain-interface", "cumulus-relay-chain-minimal-node", @@ -3941,6 +3627,7 @@ dependencies = [ "cumulus-pallet-parachain-system-proc-macro", "cumulus-primitives-core", "cumulus-primitives-parachain-inherent", + "cumulus-primitives-proof-size-hostfunction", "cumulus-test-client", "cumulus-test-relay-sproof-builder", "environmental", @@ -3972,16 +3659,17 @@ dependencies = [ "sp-version", "staging-xcm", "trie-db", + "trie-standardmap", ] [[package]] name = "cumulus-pallet-parachain-system-proc-macro" version = "0.1.0" dependencies = [ - "proc-macro-crate", + "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -4120,6 +3808,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "cumulus-primitives-proof-size-hostfunction" +version = "0.1.0" +dependencies = [ + "sp-core", + "sp-externalities 0.19.0", + "sp-io", + "sp-runtime-interface 17.0.0", + "sp-state-machine", + "sp-trie", +] + [[package]] name = "cumulus-primitives-timestamp" version = "0.1.0" @@ -4206,6 +3906,7 @@ dependencies = [ "cumulus-relay-chain-interface", "cumulus-relay-chain-rpc-interface", "futures", + "parking_lot 0.12.1", "polkadot-availability-recovery", "polkadot-collator-protocol", "polkadot-core-primitives", @@ -4279,6 +3980,7 @@ version = "0.1.0" dependencies = [ "cumulus-primitives-core", "cumulus-primitives-parachain-inherent", + "cumulus-primitives-proof-size-hostfunction", "cumulus-test-relay-sproof-builder", "cumulus-test-runtime", "cumulus-test-service", @@ -4355,7 +4057,7 @@ name = "cumulus-test-service" version = "0.1.0" dependencies = [ "async-trait", - "clap 4.4.6", + "clap 4.4.11", "criterion 0.5.1", "cumulus-client-cli", "cumulus-client-consensus-common", @@ -4478,7 +4180,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -4518,7 +4220,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -4535,7 +4237,7 @@ checksum = "50c49547d73ba8dcfd4ad7325d64c6d5391ff4224d498fc39a6f3f49825a530d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -4834,7 +4536,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -4846,17 +4548,16 @@ checksum = "86e3bdc80eee6e16b2b6b0f87fbc98c04bee3455e35174c0de1a125d0688c632" [[package]] name = "dleq_vrf" version = "0.0.2" -source = "git+https://github.com/w3f/ring-vrf?rev=3ddc205#3ddc2051066c4b3f0eadd0ba5700df12500d9754" +source = "git+https://github.com/w3f/ring-vrf?rev=e9782f9#e9782f938629c90f3adb3fff2358bc8d1386af3e" dependencies = [ "ark-ec", "ark-ff", - "ark-scale", + "ark-scale 0.0.12", "ark-secret-scalar", "ark-serialize", "ark-std", "ark-transcript", "arrayvec 0.7.4", - "rand_core 0.6.4", "zeroize", ] @@ -4896,7 +4597,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.38", + "syn 2.0.40", "termcolor", "toml 0.7.6", "walkdir", @@ -4985,15 +4686,16 @@ dependencies = [ [[package]] name = "ed25519-dalek" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980" +checksum = "1f628eaec48bfd21b865dc2950cfa014450c01d2fa2b69a86c2fd5844ec523c0" dependencies = [ "curve25519-dalek 4.0.0", "ed25519", "rand_core 0.6.4", "serde", "sha2 0.10.7", + "subtle 2.4.1", "zeroize", ] @@ -5151,7 +4853,7 @@ checksum = "5e9a1f9f7d83e59740248a6e14ecf93929ade55027844dfcea78beafccc15745" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -5162,7 +4864,17 @@ checksum = "c2ad8cef1d801a4686bfd8919f0b30eac4c8e48968c437a6405ded4fb5272d2b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", +] + +[[package]] +name = "env_logger" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" +dependencies = [ + "log", + "regex", ] [[package]] @@ -5307,7 +5019,7 @@ dependencies = [ "fs-err", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -5365,7 +5077,7 @@ checksum = "f5aa1e3ae159e592ad222dc90c5acbad632b527779ba88486abe92782ab268bd" dependencies = [ "expander 0.0.4", "indexmap 1.9.3", - "proc-macro-crate", + "proc-macro-crate 1.3.1", "proc-macro2", "quote", "syn 1.0.109", @@ -5577,7 +5289,7 @@ dependencies = [ "pallet-examples", "parity-scale-codec", "scale-info", - "simple-mermaid", + "simple-mermaid 0.1.0 (git+https://github.com/kianenigma/simple-mermaid.git?rev=e48b187bcfd5cc75111acd9d241f1bd36604344b)", "sp-api", "sp-arithmetic", "sp-block-builder", @@ -5628,7 +5340,7 @@ dependencies = [ "Inflector", "array-bytes 6.1.0", "chrono", - "clap 4.4.6", + "clap 4.4.11", "comfy-table", "frame-benchmarking", "frame-support", @@ -5689,12 +5401,12 @@ dependencies = [ "frame-election-provider-support", "frame-support", "parity-scale-codec", - "proc-macro-crate", + "proc-macro-crate 2.0.1", "proc-macro2", "quote", "scale-info", "sp-arithmetic", - "syn 2.0.38", + "syn 2.0.40", "trybuild", ] @@ -5720,7 +5432,7 @@ dependencies = [ name = "frame-election-solution-type-fuzzer" version = "2.0.0-alpha.5" dependencies = [ - "clap 4.4.6", + "clap 4.4.11", "frame-election-provider-solution-type", "frame-election-provider-support", "frame-support", @@ -5847,7 +5559,7 @@ dependencies = [ "quote", "regex", "sp-core-hashing", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -5855,10 +5567,10 @@ name = "frame-support-procedural-tools" version = "4.0.0-dev" dependencies = [ "frame-support-procedural-tools-derive", - "proc-macro-crate", + "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -5867,7 +5579,7 @@ version = "3.0.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -6008,9 +5720,9 @@ dependencies = [ [[package]] name = "fs4" -version = "0.6.6" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eeb4ed9e12f43b7fa0baae3f9cdda28352770132ef2e09a23760c29cae8bd47" +checksum = "29f9df8a11882c4e3335eb2d18a0137c505d9ca927470b0cac9c6f0ae07d28f7" dependencies = [ "rustix 0.38.21", "windows-sys 0.48.0", @@ -6100,7 +5812,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -6224,6 +5936,16 @@ dependencies = [ "wasi 0.11.0+wasi-snapshot-preview1", ] +[[package]] +name = "getrandom_or_panic" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea1015b5a70616b688dc230cfe50c8af89d972cb132d5a622814d29773b10b9" +dependencies = [ + "rand 0.8.5", + "rand_core 0.6.4", +] + [[package]] name = "ghash" version = "0.4.4" @@ -6280,51 +6002,6 @@ dependencies = [ "regex", ] -[[package]] -name = "glutton-runtime" -version = "1.0.0" -dependencies = [ - "cumulus-pallet-aura-ext", - "cumulus-pallet-parachain-system", - "cumulus-pallet-xcm", - "cumulus-primitives-aura", - "cumulus-primitives-core", - "cumulus-primitives-timestamp", - "frame-benchmarking", - "frame-executive", - "frame-support", - "frame-system", - "frame-system-benchmarking", - "frame-system-rpc-runtime-api", - "frame-try-runtime", - "pallet-aura", - "pallet-glutton", - "pallet-message-queue", - "pallet-sudo", - "pallet-timestamp", - "parachains-common", - "parity-scale-codec", - "scale-info", - "sp-api", - "sp-block-builder", - "sp-consensus-aura", - "sp-core", - "sp-genesis-builder", - "sp-inherents", - "sp-offchain", - "sp-runtime", - "sp-session", - "sp-std 8.0.0", - "sp-storage 13.0.0", - "sp-transaction-pool", - "sp-version", - "staging-parachain-info", - "staging-xcm", - "staging-xcm-builder", - "staging-xcm-executor", - "substrate-wasm-builder", -] - [[package]] name = "glutton-westend-runtime" version = "1.0.0" @@ -7125,7 +6802,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44e8ab85614a08792b9bff6c8feee23be78c98d0182d4c622c05256ab553892a" dependencies = [ "heck", - "proc-macro-crate", + "proc-macro-crate 1.3.1", "proc-macro2", "quote", "syn 1.0.109", @@ -7249,7 +6926,6 @@ dependencies = [ "pallet-child-bounties", "pallet-collective", "pallet-contracts", - "pallet-contracts-primitives", "pallet-conviction-voting", "pallet-core-fellowship", "pallet-democracy", @@ -7542,7 +7218,7 @@ dependencies = [ "libp2p-identity", "log", "multiaddr", - "multihash", + "multihash 0.17.0", "multistream-select", "once_cell", "parking_lot 0.12.1", @@ -7602,7 +7278,7 @@ dependencies = [ "ed25519-dalek", "log", "multiaddr", - "multihash", + "multihash 0.17.0", "quick-protobuf", "rand 0.8.5", "sha2 0.10.7", @@ -7849,7 +7525,7 @@ dependencies = [ "libp2p-identity", "libp2p-noise", "log", - "multihash", + "multihash 0.17.0", "quick-protobuf", "quick-protobuf-codec", "rand 0.8.5", @@ -8141,7 +7817,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -8155,7 +7831,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -8166,7 +7842,7 @@ checksum = "9ea73aa640dc01d62a590d48c0c3521ed739d53b27f919b25c3551e233481654" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -8177,7 +7853,7 @@ checksum = "ef9d79ae96aaba821963320eb2b6e34d17df1e5a83d8a1985c29cc5be59577b3" dependencies = [ "macro_magic_core", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -8346,7 +8022,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" name = "minimal-node" version = "4.0.0-dev" dependencies = [ - "clap 4.4.6", + "clap 4.4.11", "frame", "futures", "futures-timer", @@ -8516,7 +8192,7 @@ dependencies = [ "data-encoding", "log", "multibase", - "multihash", + "multihash 0.17.0", "percent-encoding", "serde", "static_assertions", @@ -8536,20 +8212,63 @@ dependencies = [ ] [[package]] -name = "multihash" -version = "0.17.0" +name = "multihash" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "835d6ff01d610179fbce3de1694d007e500bf33a7f29689838941d6bf783ae40" +dependencies = [ + "blake2b_simd", + "blake2s_simd", + "blake3", + "core2", + "digest 0.10.7", + "multihash-derive 0.8.0", + "sha2 0.10.7", + "sha3", + "unsigned-varint", +] + +[[package]] +name = "multihash" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd8a792c1694c6da4f68db0a9d707c72bd260994da179e6030a5dcee00bb815" +dependencies = [ + "core2", + "digest 0.10.7", + "multihash-derive 0.8.0", + "sha2 0.10.7", + "unsigned-varint", +] + +[[package]] +name = "multihash" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076d548d76a0e2a0d4ab471d0b1c36c577786dfc4471242035d97a12a735c492" +dependencies = [ + "core2", + "unsigned-varint", +] + +[[package]] +name = "multihash-codetable" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "835d6ff01d610179fbce3de1694d007e500bf33a7f29689838941d6bf783ae40" +checksum = "f6d815ecb3c8238d00647f8630ede7060a642c9f704761cd6082cb4028af6935" dependencies = [ "blake2b_simd", "blake2s_simd", "blake3", "core2", "digest 0.10.7", - "multihash-derive", + "multihash-derive 0.9.0", + "ripemd", + "serde", + "sha1", "sha2 0.10.7", "sha3", - "unsigned-varint", + "strobe-rs", ] [[package]] @@ -8558,7 +8277,32 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc076939022111618a5026d3be019fd8b366e76314538ff9a1b59ffbcbf98bcd" dependencies = [ - "proc-macro-crate", + "proc-macro-crate 1.3.1", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure", +] + +[[package]] +name = "multihash-derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "890e72cb7396cb99ed98c1246a97b243cc16394470d94e0bc8b0c2c11d84290e" +dependencies = [ + "core2", + "multihash 0.19.1", + "multihash-derive-impl", +] + +[[package]] +name = "multihash-derive-impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38685e08adb338659871ecfc6ee47ba9b22dcc8abcf6975d379cc49145c3040" +dependencies = [ + "proc-macro-crate 1.3.1", "proc-macro-error", "proc-macro2", "quote", @@ -8752,7 +8496,7 @@ name = "node-bench" version = "0.9.0-dev" dependencies = [ "array-bytes 6.1.0", - "clap 4.4.6", + "clap 4.4.11", "derive_more", "fs_extra", "futures", @@ -8827,7 +8571,7 @@ dependencies = [ name = "node-runtime-generate-bags" version = "3.0.0" dependencies = [ - "clap 4.4.6", + "clap 4.4.11", "generate-bags", "kitchensink-runtime", ] @@ -8836,7 +8580,7 @@ dependencies = [ name = "node-template" version = "4.0.0-dev" dependencies = [ - "clap 4.4.6", + "clap 4.4.11", "frame-benchmarking", "frame-benchmarking-cli", "frame-system", @@ -8880,14 +8624,14 @@ dependencies = [ name = "node-template-release" version = "3.0.0" dependencies = [ - "clap 4.4.6", + "clap 4.4.11", "flate2", "fs_extra", "glob", "itertools 0.10.5", "tar", "tempfile", - "toml_edit", + "toml_edit 0.19.14", ] [[package]] @@ -8961,7 +8705,7 @@ dependencies = [ "sp-keyring", "sp-runtime", "sp-timestamp", - "staging-node-executor", + "staging-node-cli", "substrate-test-client", "tempfile", ] @@ -9202,7 +8946,7 @@ dependencies = [ "itertools 0.11.0", "layout-rs", "petgraph", - "proc-macro-crate", + "proc-macro-crate 1.3.1", "proc-macro2", "quote", "syn 1.0.109", @@ -9802,8 +9546,8 @@ dependencies = [ "pallet-assets", "pallet-balances", "pallet-contracts-fixtures", - "pallet-contracts-primitives", "pallet-contracts-proc-macro", + "pallet-contracts-uapi", "pallet-insecure-randomness-collective-flip", "pallet-message-queue", "pallet-proxy", @@ -9834,11 +9578,21 @@ dependencies = [ name = "pallet-contracts-fixtures" version = "1.0.0" dependencies = [ + "anyhow", + "cfg-if", "frame-system", + "parity-wasm", "sp-runtime", + "tempfile", + "toml 0.8.2", + "twox-hash", "wat", ] +[[package]] +name = "pallet-contracts-fixtures-common" +version = "1.0.0" + [[package]] name = "pallet-contracts-mock-network" version = "1.0.0" @@ -9850,8 +9604,8 @@ dependencies = [ "pallet-balances", "pallet-contracts", "pallet-contracts-fixtures", - "pallet-contracts-primitives", "pallet-contracts-proc-macro", + "pallet-contracts-uapi", "pallet-insecure-randomness-collective-flip", "pallet-message-queue", "pallet-proxy", @@ -9878,24 +9632,22 @@ dependencies = [ ] [[package]] -name = "pallet-contracts-primitives" -version = "24.0.0" +name = "pallet-contracts-proc-macro" +version = "4.0.0-dev" dependencies = [ - "bitflags 1.3.2", - "parity-scale-codec", - "scale-info", - "sp-runtime", - "sp-std 8.0.0", - "sp-weights", + "proc-macro2", + "quote", + "syn 2.0.40", ] [[package]] -name = "pallet-contracts-proc-macro" +name = "pallet-contracts-uapi" version = "4.0.0-dev" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.38", + "bitflags 1.3.2", + "parity-scale-codec", + "paste", + "scale-info", ] [[package]] @@ -10878,6 +10630,24 @@ dependencies = [ "sp-std 8.0.0", ] +[[package]] +name = "pallet-sassafras" +version = "0.3.5-dev" +dependencies = [ + "array-bytes 6.1.0", + "frame-benchmarking", + "frame-support", + "frame-system", + "log", + "parity-scale-codec", + "scale-info", + "sp-consensus-sassafras", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std 8.0.0", +] + [[package]] name = "pallet-scheduler" version = "4.0.0-dev" @@ -11023,11 +10793,11 @@ dependencies = [ name = "pallet-staking-reward-curve" version = "4.0.0-dev" dependencies = [ - "proc-macro-crate", + "proc-macro-crate 2.0.1", "proc-macro2", "quote", "sp-runtime", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -11433,7 +11203,7 @@ dependencies = [ name = "parachain-template-node" version = "0.1.0" dependencies = [ - "clap 4.4.6", + "clap 4.4.11", "color-print", "cumulus-client-cli", "cumulus-client-collator", @@ -11634,9 +11404,9 @@ dependencies = [ [[package]] name = "parity-scale-codec" -version = "3.6.4" +version = "3.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8e946cc0cc711189c0b0249fb8b599cbeeab9784d83c415719368bb8d4ac64" +checksum = "881331e34fa842a2fb61cc2db9643a8fedc615e47cfcc52597d1af0db9a7e8fe" dependencies = [ "arrayvec 0.7.4", "bitvec", @@ -11649,11 +11419,11 @@ dependencies = [ [[package]] name = "parity-scale-codec-derive" -version = "3.6.4" +version = "3.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a296c3079b5fefbc499e1de58dc26c09b1b9a5952d26694ee89f04a43ebbb3e" +checksum = "be30eaf4b0a9fba5336683b38de57bb86d179a35862ba6bfcf57625d006bde5b" dependencies = [ - "proc-macro-crate", + "proc-macro-crate 2.0.1", "proc-macro2", "quote", "syn 1.0.109", @@ -11921,7 +11691,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -11962,7 +11732,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -12184,7 +11954,8 @@ dependencies = [ name = "polkadot-cli" version = "1.1.0" dependencies = [ - "clap 4.4.6", + "cfg-if", + "clap 4.4.11", "frame-benchmarking-cli", "futures", "log", @@ -12310,6 +12081,7 @@ dependencies = [ "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-util", "polkadot-primitives", + "quickcheck", "rand 0.8.5", "rand_chacha 0.3.1", "sc-network", @@ -12683,6 +12455,7 @@ dependencies = [ "tempfile", "test-parachain-adder", "test-parachain-halt", + "thiserror", "tokio", "tracing-gum", ] @@ -13016,17 +12789,12 @@ name = "polkadot-parachain-bin" version = "1.1.0" dependencies = [ "assert_cmd", - "asset-hub-kusama-runtime", - "asset-hub-polkadot-runtime", "asset-hub-rococo-runtime", "asset-hub-westend-runtime", "async-trait", - "bridge-hub-kusama-runtime", - "bridge-hub-polkadot-runtime", "bridge-hub-rococo-runtime", "bridge-hub-westend-runtime", - "clap 4.4.6", - "collectives-polkadot-runtime", + "clap 4.4.11", "collectives-westend-runtime", "color-print", "contracts-rococo-runtime", @@ -13047,7 +12815,6 @@ dependencies = [ "frame-system-rpc-runtime-api", "frame-try-runtime", "futures", - "glutton-runtime", "glutton-westend-runtime", "hex-literal", "jsonrpsee", @@ -13317,6 +13084,45 @@ dependencies = [ "thousands", ] +[[package]] +name = "polkadot-sdk-docs" +version = "0.0.1" +dependencies = [ + "cumulus-pallet-aura-ext", + "cumulus-pallet-parachain-system", + "docify", + "frame", + "kitchensink-runtime", + "pallet-aura", + "pallet-default-config-example", + "pallet-examples", + "pallet-timestamp", + "parity-scale-codec", + "sc-cli", + "sc-client-db", + "sc-consensus-aura", + "sc-consensus-babe", + "sc-consensus-beefy", + "sc-consensus-grandpa", + "sc-consensus-manual-seal", + "sc-consensus-pow", + "sc-network", + "sc-rpc", + "sc-rpc-api", + "scale-info", + "simple-mermaid 0.1.0 (git+https://github.com/kianenigma/simple-mermaid.git?branch=main)", + "sp-api", + "sp-core", + "sp-io", + "sp-keyring", + "sp-runtime", + "staging-chain-spec-builder", + "staging-node-cli", + "staging-parachain-info", + "subkey", + "substrate-wasm-builder", +] + [[package]] name = "polkadot-service" version = "1.0.0" @@ -13343,6 +13149,7 @@ dependencies = [ "pallet-transaction-payment-rpc-runtime-api", "parity-db", "parity-scale-codec", + "parking_lot 0.12.1", "polkadot-approval-distribution", "polkadot-availability-bitfield-distribution", "polkadot-availability-distribution", @@ -13458,7 +13265,6 @@ dependencies = [ "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", - "polkadot-node-subsystem-types", "polkadot-node-subsystem-util", "polkadot-primitives", "polkadot-primitives-test-helpers", @@ -13491,7 +13297,7 @@ version = "1.0.0" dependencies = [ "assert_matches", "async-trait", - "clap 4.4.6", + "clap 4.4.11", "clap-num", "color-eyre", "colored", @@ -13576,7 +13382,7 @@ version = "1.0.0" dependencies = [ "assert_matches", "async-trait", - "clap 4.4.6", + "clap 4.4.11", "color-eyre", "futures", "futures-timer", @@ -13723,7 +13529,7 @@ dependencies = [ name = "polkadot-voter-bags" version = "1.0.0" dependencies = [ - "clap 4.4.6", + "clap 4.4.11", "generate-bags", "sp-io", "westend-runtime", @@ -13901,7 +13707,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" dependencies = [ "proc-macro2", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -13942,7 +13748,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" dependencies = [ "once_cell", - "toml_edit", + "toml_edit 0.19.14", +] + +[[package]] +name = "proc-macro-crate" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97dc5fea232fc28d2f597b37c4876b348a40e33f3b02cc975c8d006d78d94b1a" +dependencies = [ + "toml_datetime", + "toml_edit 0.20.2", ] [[package]] @@ -13983,14 +13799,14 @@ checksum = "9b698b0b09d40e9b7c1a47b132d66a8b54bcd20583d9b6d06e4535e383b4405c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" dependencies = [ "unicode-ident", ] @@ -14055,7 +13871,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -14197,6 +14013,8 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" dependencies = [ + "env_logger 0.8.4", + "log", "rand 0.8.5", ] @@ -14405,6 +14223,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_users" version = "0.4.3" @@ -14446,7 +14273,7 @@ checksum = "7f7473c2cfcf90008193dd0e3e16599455cb601a9fce322b5bb55de799664925" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -14515,7 +14342,7 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" name = "remote-ext-tests-bags-list" version = "1.0.0" dependencies = [ - "clap 4.4.6", + "clap 4.4.11", "frame-system", "log", "pallet-bags-list-remote-tests", @@ -14599,7 +14426,7 @@ dependencies = [ [[package]] name = "ring" version = "0.1.0" -source = "git+https://github.com/burdges/ring-proof?branch=patch-1#05a756076cb20f981a52afea3a620168de49f95f" +source = "git+https://github.com/w3f/ring-proof#b273d33f9981e2bb3375ab45faeb537f7ee35224" dependencies = [ "ark-ec", "ark-ff", @@ -14627,6 +14454,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "ripemd" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd124222d17ad93a644ed9d011a40f4fb64aa54275c08cc216524a9ea82fb09f" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "rle-decode-fast" version = "1.0.3" @@ -15183,7 +15019,8 @@ dependencies = [ "ip_network", "libp2p", "log", - "multihash", + "multihash 0.18.1", + "multihash-codetable", "parity-scale-codec", "prost", "prost-build", @@ -15240,6 +15077,7 @@ dependencies = [ "sp-inherents", "sp-runtime", "sp-state-machine", + "sp-trie", "substrate-test-runtime-client", ] @@ -15275,10 +15113,10 @@ dependencies = [ name = "sc-chain-spec-derive" version = "4.0.0-dev" dependencies = [ - "proc-macro-crate", + "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -15288,7 +15126,7 @@ dependencies = [ "array-bytes 6.1.0", "bip39", "chrono", - "clap 4.4.6", + "clap 4.4.11", "fdlimit", "futures", "futures-timer", @@ -15860,6 +15698,7 @@ dependencies = [ "array-bytes 4.2.0", "arrayvec 0.7.4", "blake2 0.10.6", + "bytes", "futures", "futures-timer", "libp2p-identity", @@ -15925,6 +15764,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "tokio-stream", "tokio-test", "tokio-util", "unsigned-varint", @@ -15980,10 +15820,12 @@ name = "sc-network-gossip" version = "0.10.0-dev" dependencies = [ "ahash 0.8.3", + "async-trait", "futures", "futures-timer", "libp2p", "log", + "parity-scale-codec", "quickcheck", "sc-network", "sc-network-common", @@ -16428,7 +16270,7 @@ dependencies = [ name = "sc-storage-monitor" version = "0.1.0" dependencies = [ - "clap 4.4.6", + "clap 4.4.11", "fs4", "log", "sc-client-db", @@ -16527,10 +16369,10 @@ dependencies = [ name = "sc-tracing-proc-macro" version = "4.0.0-dev" dependencies = [ - "proc-macro-crate", + "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -16617,7 +16459,7 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abf2c68b89cafb3b8d918dd07b42be0da66ff202cf1155c5739a4e0c1ea0dc19" dependencies = [ - "proc-macro-crate", + "proc-macro-crate 1.3.1", "proc-macro2", "quote", "syn 1.0.109", @@ -16908,22 +16750,22 @@ checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" [[package]] name = "serde" -version = "1.0.192" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.192" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -17013,7 +16855,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -17201,6 +17043,11 @@ dependencies = [ "wide", ] +[[package]] +name = "simple-mermaid" +version = "0.1.0" +source = "git+https://github.com/kianenigma/simple-mermaid.git?branch=main#e48b187bcfd5cc75111acd9d241f1bd36604344b" + [[package]] name = "simple-mermaid" version = "0.1.0" @@ -17449,10 +17296,10 @@ dependencies = [ "assert_matches", "blake2 0.10.6", "expander 2.0.0", - "proc-macro-crate", + "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -17787,7 +17634,7 @@ version = "9.0.0" dependencies = [ "quote", "sp-core-hashing", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -17805,7 +17652,7 @@ dependencies = [ "ark-ed-on-bls12-377-ext", "ark-ed-on-bls12-381-bandersnatch", "ark-ed-on-bls12-381-bandersnatch-ext", - "ark-scale", + "ark-scale 0.0.12", "sp-runtime-interface 17.0.0", "sp-std 8.0.0", ] @@ -17826,7 +17673,7 @@ dependencies = [ "ark-ed-on-bls12-377-ext", "ark-ed-on-bls12-381-bandersnatch", "ark-ed-on-bls12-381-bandersnatch-ext", - "ark-scale", + "ark-scale 0.0.11", "sp-runtime-interface 17.0.0 (git+https://github.com/paritytech/polkadot-sdk)", "sp-std 8.0.0 (git+https://github.com/paritytech/polkadot-sdk)", ] @@ -17845,7 +17692,7 @@ version = "8.0.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -17855,7 +17702,7 @@ source = "git+https://github.com/paritytech/polkadot-sdk#82912acb33a9030c0ef3bf5 dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -18015,7 +17862,7 @@ dependencies = [ name = "sp-npos-elections-fuzzer" version = "2.0.0-alpha.5" dependencies = [ - "clap 4.4.6", + "clap 4.4.11", "honggfuzz", "rand 0.8.5", "sp-npos-elections", @@ -18054,6 +17901,7 @@ dependencies = [ name = "sp-runtime" version = "24.0.0" dependencies = [ + "docify", "either", "hash256-std-hasher", "impl-trait-for-tuples", @@ -18064,6 +17912,7 @@ dependencies = [ "scale-info", "serde", "serde_json", + "simple-mermaid 0.1.0 (git+https://github.com/kianenigma/simple-mermaid.git?branch=main)", "sp-api", "sp-application-crypto", "sp-arithmetic", @@ -18123,10 +17972,11 @@ name = "sp-runtime-interface-proc-macro" version = "11.0.0" dependencies = [ "Inflector", - "proc-macro-crate", + "expander 2.0.0", + "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -18135,10 +17985,10 @@ version = "11.0.0" source = "git+https://github.com/paritytech/polkadot-sdk#82912acb33a9030c0ef3bf590a34fca09b72dc5f" dependencies = [ "Inflector", - "proc-macro-crate", + "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -18374,6 +18224,7 @@ dependencies = [ "scale-info", "schnellru", "sp-core", + "sp-externalities 0.19.0", "sp-runtime", "sp-std 8.0.0", "thiserror", @@ -18408,7 +18259,7 @@ dependencies = [ "proc-macro2", "quote", "sp-version", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -18519,7 +18370,7 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" name = "staging-chain-spec-builder" version = "2.0.0" dependencies = [ - "clap 4.4.6", + "clap 4.4.11", "log", "sc-chain-spec", "serde_json", @@ -18532,10 +18383,12 @@ version = "3.0.0-dev" dependencies = [ "array-bytes 6.1.0", "assert_cmd", - "clap 4.4.6", + "clap 4.4.11", "clap_complete", "criterion 0.4.0", + "frame-benchmarking", "frame-benchmarking-cli", + "frame-support", "frame-system", "frame-system-rpc-runtime-api", "futures", @@ -18545,13 +18398,20 @@ dependencies = [ "nix 0.26.2", "node-primitives", "node-rpc", + "node-testing", "pallet-asset-conversion-tx-payment", "pallet-asset-tx-payment", "pallet-assets", "pallet-balances", + "pallet-contracts", + "pallet-glutton", "pallet-im-online", + "pallet-root-testing", "pallet-skip-feeless-payment", + "pallet-sudo", "pallet-timestamp", + "pallet-transaction-payment", + "pallet-treasury", "parity-scale-codec", "platforms", "rand 0.8.5", @@ -18586,27 +18446,31 @@ dependencies = [ "sc-telemetry", "sc-transaction-pool", "sc-transaction-pool-api", + "scale-info", "serde", "serde_json", "soketto", "sp-api", + "sp-application-crypto", "sp-authority-discovery", "sp-blockchain", "sp-consensus", "sp-consensus-babe", "sp-consensus-grandpa", "sp-core", + "sp-externalities 0.19.0", "sp-inherents", "sp-io", "sp-keyring", "sp-keystore", "sp-mixnet", "sp-runtime", + "sp-state-machine", "sp-statement-store", "sp-timestamp", "sp-tracing 10.0.0", "sp-transaction-storage-proof", - "staging-node-executor", + "sp-trie", "staging-node-inspect", "substrate-build-script-utils", "substrate-cli-test-utils", @@ -18617,44 +18481,6 @@ dependencies = [ "tokio-util", "try-runtime-cli", "wait-timeout", -] - -[[package]] -name = "staging-node-executor" -version = "3.0.0-dev" -dependencies = [ - "criterion 0.4.0", - "frame-benchmarking", - "frame-support", - "frame-system", - "futures", - "kitchensink-runtime", - "node-primitives", - "node-testing", - "pallet-balances", - "pallet-contracts", - "pallet-glutton", - "pallet-im-online", - "pallet-root-testing", - "pallet-sudo", - "pallet-timestamp", - "pallet-transaction-payment", - "pallet-treasury", - "parity-scale-codec", - "sc-executor", - "scale-info", - "serde_json", - "sp-application-crypto", - "sp-consensus-babe", - "sp-core", - "sp-externalities 0.19.0", - "sp-keyring", - "sp-keystore", - "sp-runtime", - "sp-state-machine", - "sp-statement-store", - "sp-tracing 10.0.0", - "sp-trie", "wat", ] @@ -18662,14 +18488,16 @@ dependencies = [ name = "staging-node-inspect" version = "0.9.0-dev" dependencies = [ - "clap 4.4.6", + "clap 4.4.11", "parity-scale-codec", "sc-cli", "sc-client-api", "sc-service", "sp-blockchain", "sp-core", + "sp-io", "sp-runtime", + "sp-statement-store", "thiserror", ] @@ -18819,6 +18647,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "strobe-rs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabb238a1cccccfa4c4fb703670c0d157e1256c1ba695abf1b93bd2bb14bab2d" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "keccak", + "subtle 2.4.1", + "zeroize", +] + [[package]] name = "strsim" version = "0.10.0" @@ -18870,28 +18711,8 @@ dependencies = [ name = "subkey" version = "3.0.0" dependencies = [ - "clap 4.4.6", - "sc-cli", -] - -[[package]] -name = "substrate" -version = "1.0.0" -dependencies = [ - "frame-support", - "sc-chain-spec", + "clap 4.4.11", "sc-cli", - "sc-consensus-aura", - "sc-consensus-babe", - "sc-consensus-beefy", - "sc-consensus-grandpa", - "sc-consensus-manual-seal", - "sc-consensus-pow", - "sc-service", - "simple-mermaid", - "sp-runtime", - "staging-chain-spec-builder", - "subkey", ] [[package]] @@ -18932,7 +18753,7 @@ dependencies = [ name = "substrate-frame-cli" version = "4.0.0-dev" dependencies = [ - "clap 4.4.6", + "clap 4.4.11", "frame-support", "frame-system", "sc-cli", @@ -19287,9 +19108,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.38" +version = "2.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "13fa70a4ee923979ffb522cacce59d34421ebdea5625e1073c4326ef9d2dd42e" dependencies = [ "proc-macro2", "quote", @@ -19354,13 +19175,13 @@ checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a" [[package]] name = "tempfile" -version = "3.8.0" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if", "fastrand 2.0.0", - "redox_syscall 0.3.5", + "redox_syscall 0.4.1", "rustix 0.38.21", "windows-sys 0.48.0", ] @@ -19407,7 +19228,7 @@ dependencies = [ name = "test-parachain-adder-collator" version = "1.0.0" dependencies = [ - "clap 4.4.6", + "clap 4.4.11", "futures", "futures-timer", "log", @@ -19455,7 +19276,7 @@ dependencies = [ name = "test-parachain-undying-collator" version = "1.0.0" dependencies = [ - "clap 4.4.6", + "clap 4.4.11", "futures", "futures-timer", "log", @@ -19544,7 +19365,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -19716,7 +19537,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -19810,14 +19631,26 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_edit 0.19.14", +] + +[[package]] +name = "toml" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "185d8ab0dfbb35cf1399a6344d8484209c088f75f8f68230da55d48d95d43e3d" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit 0.20.2", ] [[package]] name = "toml_datetime" -version = "0.6.5" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" dependencies = [ "serde", ] @@ -19835,6 +19668,19 @@ dependencies = [ "winnow", ] +[[package]] +name = "toml_edit" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338" +dependencies = [ + "indexmap 2.0.0", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -19897,7 +19743,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -19936,10 +19782,10 @@ version = "1.0.0" dependencies = [ "assert_matches", "expander 2.0.0", - "proc-macro-crate", + "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] @@ -20092,7 +19938,7 @@ version = "0.10.0-dev" dependencies = [ "assert_cmd", "async-trait", - "clap 4.4.6", + "clap 4.4.11", "frame-remote-externalities", "frame-try-runtime", "hex", @@ -20500,7 +20346,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", "wasm-bindgen-shared", ] @@ -20534,7 +20380,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -21307,6 +21153,7 @@ version = "0.0.0" dependencies = [ "asset-hub-westend-emulated-chain", "bridge-hub-westend-emulated-chain", + "collectives-westend-emulated-chain", "emulated-integration-tests-common", "penpal-emulated-chain", "westend-emulated-chain", @@ -21713,7 +21560,7 @@ dependencies = [ "proc-macro2", "quote", "staging-xcm", - "syn 2.0.38", + "syn 2.0.40", "trybuild", ] @@ -21833,7 +21680,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.40", ] [[package]] diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 5f637bf56901..3452c5fdec91 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -48,7 +48,7 @@ use polkadot_node_network_protocol::{ GridNeighbors, RandomRouting, RequiredRouting, SessionGridTopology, TopologyPeerInfo, }, peer_set::{ProtocolVersion, ValidationVersion}, - vstaging as protocol_vstaging, ObservedRole, Versioned, View, + v3 as protocol_v3, ObservedRole, Versioned, View, }; use polkadot_node_subsystem::{overseer, AllMessages, Overseer, OverseerConnector, SpawnGlue}; use polkadot_node_subsystem_test_helpers::mock::new_block_import_info; @@ -739,9 +739,7 @@ fn issue_approvals( .map(|(_index, message)| match &message.msg { ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage( _, - Versioned::VStaging(protocol_vstaging::ApprovalDistributionMessage::Assignments( - assignments, - )), + Versioned::V3(protocol_v3::ApprovalDistributionMessage::Assignments(assignments)), )) => { let mut approvals_to_create = Vec::new(); @@ -847,11 +845,11 @@ fn sign_candidates( validator: current_validator_index, signature, }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![indirect]); + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![indirect]); TestMessage { msg: ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage( keyring.1, - Versioned::VStaging(msg), + Versioned::V3(msg), )), sent_by, tranche: tranche_trigger_timestamp, @@ -1042,14 +1040,14 @@ fn generate_assignments( .flatten() .map(|indirect| { let validator_index = indirect.0.validator.0; - let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(vec![( + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( indirect.0, indirect.1, )]); TestMessage { msg: ApprovalDistributionMessage::NetworkBridgeUpdate( NetworkBridgeEvent::PeerMessage( keyrings[validator_index as usize].1, - Versioned::VStaging(msg), + Versioned::V3(msg), ), ), sent_by: indirect.2, @@ -1140,7 +1138,7 @@ fn generate_peer_connected( let network = NetworkBridgeEvent::PeerConnected( peer_id, ObservedRole::Full, - ProtocolVersion::from(ValidationVersion::VStaging), + ProtocolVersion::from(ValidationVersion::V3), None, ); TestMessage { From dc5a72cac0560d6728aab43ee86e2f918e7f9f77 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 11 Dec 2023 13:03:34 +0200 Subject: [PATCH 137/192] use prometheus network stats Signed-off-by: Andrei Sandu --- .../subsystem-bench/src/availability/mod.rs | 4 +++- .../subsystem-bench/src/core/environment.rs | 20 ++++++------------- .../node/subsystem-test-helpers/src/lib.rs | 4 ++-- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index febe1e7d2023..1130a417db71 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -68,7 +68,9 @@ const LOG_TARGET: &str = "subsystem-bench::availability"; use polkadot_node_primitives::{AvailableData, ErasureChunk}; use super::{cli::TestObjective, core::mock::AlwaysSupportsParachains}; -use polkadot_node_subsystem_test_helpers::mock::new_block_import_info; +use polkadot_node_subsystem_test_helpers::{ + derive_erasure_chunks_with_proofs_and_root, mock::new_block_import_info, +}; use polkadot_primitives::{ AvailabilityBitfield, BlockNumber, CandidateHash, CandidateReceipt, GroupIndex, Hash, HeadData, Header, PersistedValidationData, Signed, SigningContext, ValidatorIndex, diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index ec61f715cd6b..4a965d44d2b9 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -39,7 +39,7 @@ use tokio::runtime::Handle; const LOG_TARGET: &str = "subsystem-bench::environment"; use super::configuration::TestAuthorities; -const MIB: f64 = 1024.0 * 1024.0; +const MIB: usize = 1024 * 1024; /// Test environment/configuration metrics #[derive(Clone)] @@ -60,7 +60,6 @@ impl TestEnvironmentMetrics { pub fn new(registry: &Registry) -> Result { let mut buckets = prometheus::exponential_buckets(16384.0, 2.0, 9) .expect("arguments are always valid; qed"); - buckets.extend(vec![5.0 * MIB, 6.0 * MIB, 7.0 * MIB, 8.0 * MIB, 9.0 * MIB, 10.0 * MIB]); Ok(Self { n_validators: prometheus::register( @@ -318,29 +317,22 @@ impl TestEnvironment { pub fn display_network_usage(&self) { let test_metrics = super::display::parse_metrics(self.registry()); - let node_receive_metrics = test_metrics.subset_with_label_value("peer", "node0"); - let total_node_received = - node_receive_metrics.sum_by("subsystem_benchmark_network_peer_total_bytes_received"); + let stats = self.network().peer_stats(0); - let node_send_metrics = test_metrics.subset_with_label_value("peer", "node0"); - let total_node_sent = - node_send_metrics.sum_by("subsystem_benchmark_network_peer_total_bytes_sent"); - - let total_node_received = total_node_received / MIB; - let total_node_sent = total_node_sent / MIB; + let total_node_received = stats.received() / MIB; + let total_node_sent = stats.sent() / MIB; println!( "\nPayload bytes received from peers: {}, {}", format!("{:.2} MiB total", total_node_received).blue(), - format!("{:.2} MiB/block", total_node_received / self.config().num_blocks as f64) + format!("{:.2} MiB/block", total_node_received / self.config().num_blocks) .bright_blue() ); println!( "Payload bytes sent to peers: {}, {}", format!("{:.2} MiB total", total_node_sent).blue(), - format!("{:.2} MiB/block", total_node_sent / self.config().num_blocks as f64) - .bright_blue() + format!("{:.2} MiB/block", total_node_sent / self.config().num_blocks).bright_blue() ); } diff --git a/polkadot/node/subsystem-test-helpers/src/lib.rs b/polkadot/node/subsystem-test-helpers/src/lib.rs index daa8a10e6171..dfa78e04b8c9 100644 --- a/polkadot/node/subsystem-test-helpers/src/lib.rs +++ b/polkadot/node/subsystem-test-helpers/src/lib.rs @@ -19,7 +19,7 @@ #![warn(missing_docs)] use polkadot_erasure_coding::{branches, obtain_chunks_v1 as obtain_chunks}; -use polkadot_node_primitives::{AvailableData, BlockData, ErasureChunk, PoV, Proof}; +use polkadot_node_primitives::{AvailableData, ErasureChunk, Proof}; use polkadot_node_subsystem::{ messages::AllMessages, overseer, FromOrchestra, OverseerSignal, SpawnGlue, SpawnedSubsystem, SubsystemError, SubsystemResult, TrySendError, @@ -443,7 +443,7 @@ impl Future for Yield { } } -// Helper for chunking available data. +/// Helper for chunking available data. pub fn derive_erasure_chunks_with_proofs_and_root( n_validators: usize, available_data: &AvailableData, From 3cf90e27f6bc4a58912a17b8a158afa098b7a3a5 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 11 Dec 2023 16:19:56 +0200 Subject: [PATCH 138/192] Fix GetApprovalSignatures Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 24bd12cafcf8..d386146ecfb0 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -599,17 +599,29 @@ impl BlockEntry { &self, candidate_index: CandidateIndex, ) -> Vec { - let result: Option> = - self.candidates.get(candidate_index as usize).map(|candidate_entry| { - candidate_entry - .assignments - .iter() - .filter_map(|(validator, assignment_bitfield)| { - self.approval_entries.get(&(*validator, assignment_bitfield.clone())) - }) - .flat_map(|approval_entry| approval_entry.approvals.clone().into_iter()) - .collect() - }); + let result: Option< + HashMap<(ValidatorIndex, CandidateBitfield), IndirectSignedApprovalVoteV2>, + > = self.candidates.get(candidate_index as usize).map(|candidate_entry| { + candidate_entry + .assignments + .iter() + .filter_map(|(validator, assignment_bitfield)| { + self.approval_entries.get(&(*validator, assignment_bitfield.clone())) + }) + .flat_map(|approval_entry| { + approval_entry + .approvals + .clone() + .into_iter() + .filter(|(approved_candidates, _)| { + approved_candidates.bit_at(candidate_index.as_bit_index()) + }) + .map(|(approved_candidates, vote)| { + ((approval_entry.validator_index, approved_candidates), vote) + }) + }) + .collect() + }); result.map(|result| result.into_values().collect_vec()).unwrap_or_default() } From 60516d0136a59ddfbcd6427fdbc39f39fc145c46 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 11 Dec 2023 16:30:15 +0200 Subject: [PATCH 139/192] Add hacks for getapproval signatures Signed-off-by: Alexandru Gheorghe --- .../node/subsystem-bench/src/approval/mod.rs | 104 +++++++++++++++++- 1 file changed, 101 insertions(+), 3 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 3452c5fdec91..a72ebfc75ebe 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -25,7 +25,7 @@ use std::{ }; use colored::Colorize; -use futures::{channel::oneshot, FutureExt}; +use futures::{channel::oneshot, lock::Mutex, FutureExt}; use itertools::Itertools; use orchestra::TimeoutExt; use overseer::{metrics::Metrics as OverseerMetrics, MetricsTrait}; @@ -56,7 +56,8 @@ use polkadot_overseer::Handle as OverseerHandleReal; use polkadot_node_core_approval_voting::Config as ApprovalVotingConfig; use polkadot_node_subsystem_types::messages::{ - network_bridge_event::NewGossipTopology, ApprovalDistributionMessage, NetworkBridgeEvent, + network_bridge_event::NewGossipTopology, ApprovalDistributionMessage, ApprovalVotingMessage, + NetworkBridgeEvent, }; use rand::{seq::SliceRandom, RngCore, SeedableRng}; @@ -94,7 +95,10 @@ use crate::{ configuration::{TestAuthorities, TestConfiguration}, environment::{TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT}, keyring::Keyring, - mock::{dummy_builder, AlwaysSupportsParachains, MockNetworkBridgeTx, TestSyncOracle}, + mock::{ + dummy_builder, AlwaysSupportsParachains, MockNetworkBridgeTx, RespondToRequestsInfo, + TestSyncOracle, + }, network::{NetworkAction, NetworkEmulator}, }, }; @@ -170,6 +174,7 @@ struct BlockTestData { /// selection mock subsystem. approved: Arc, prev_candidates: u64, + votes: Arc>>, } /// Approval test state used by all mock subsystems to be able to answer messages emitted @@ -290,6 +295,15 @@ impl ApprovalTestState { relay_vrf_story, approved: Arc::new(AtomicBool::new(false)), prev_candidates, + votes: Arc::new( + (0..self.configuration.n_validators) + .map(|_| { + (0..self.configuration.n_cores) + .map(|_| AtomicBool::new(false)) + .collect_vec() + }) + .collect_vec(), + ), }; prev_candidates += block_info.candidates.len() as u64; self.per_slot_heads.push(block_info) @@ -387,6 +401,45 @@ impl TestMessage { } } + fn record_vote(&self, state: &BlockTestData) { + if let MessageType::Approval = self.typ { + match &self.msg { + ApprovalDistributionMessage::NewBlocks(_) => todo!(), + ApprovalDistributionMessage::DistributeAssignment(_, _) => todo!(), + ApprovalDistributionMessage::DistributeApproval(_) => todo!(), + ApprovalDistributionMessage::NetworkBridgeUpdate(msg) => match msg { + NetworkBridgeEvent::PeerConnected(_, _, _, _) => todo!(), + NetworkBridgeEvent::PeerDisconnected(_) => todo!(), + NetworkBridgeEvent::NewGossipTopology(_) => todo!(), + NetworkBridgeEvent::PeerMessage(peer, msg) => match msg { + Versioned::V1(_) => todo!(), + Versioned::V2(_) => todo!(), + Versioned::V3(msg) => match msg { + protocol_v3::ApprovalDistributionMessage::Assignments(_) => todo!(), + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => + for approval in approvals { + for candidate_index in approval.candidate_indices.iter_ones() { + state + .votes + .get(approval.validator.0 as usize) + .unwrap() + .get(candidate_index) + .unwrap() + .store(true, std::sync::atomic::Ordering::SeqCst); + } + }, + }, + }, + NetworkBridgeEvent::PeerViewChange(_, _) => todo!(), + NetworkBridgeEvent::OurViewChange(_) => todo!(), + NetworkBridgeEvent::UpdatedAuthorityIds(_, _) => todo!(), + }, + ApprovalDistributionMessage::GetApprovalSignatures(_, _) => todo!(), + ApprovalDistributionMessage::ApprovalCheckingLagUpdate(_) => todo!(), + } + } + } + /// Splits a message into multiple messages based on what peers should send this message. /// It build a HashMap of messages that should be sent by each peer. fn split_by_peer_id(self) -> HashMap<(ValidatorIndex, PeerId), Vec> { @@ -556,6 +609,8 @@ impl PeerMessagesGenerator { (!self.options.stop_when_approved && message.tranche <= self.options.send_till_tranche) { + message.record_vote(block_info); + for (peer, messages) in message.split_by_peer_id() { for message in messages { let latency = message.get_latency(); @@ -1364,6 +1419,49 @@ pub async fn bench_approvals(env: &mut TestEnvironment, state: ApprovalTestState ); tokio::time::sleep(Duration::from_secs(6)).await; } + tokio::time::sleep(Duration::from_secs(6)).await; + tokio::time::sleep(Duration::from_secs(6)).await; + + for info in &state.per_slot_heads { + for (index, candidates) in info.candidates.iter().enumerate() { + match candidates { + CandidateEvent::CandidateBacked(_, _, _, _) => todo!(), + CandidateEvent::CandidateIncluded(receipt_fetch, head, _, _) => { + let (tx, rx) = oneshot::channel(); + + let msg = ApprovalVotingMessage::GetApprovalSignaturesForCandidate( + receipt_fetch.hash(), + tx, + ); + env.send_message(AllMessages::ApprovalVoting(msg)).await; + let result = rx.await.unwrap(); + + for (validator, votes) in result.iter() { + let result = info + .votes + .get(validator.0 as usize) + .unwrap() + .get(index) + .unwrap() + .store(false, std::sync::atomic::Ordering::SeqCst); + } + }, + + CandidateEvent::CandidateTimedOut(_, _, _) => todo!(), + }; + } + } + + for state in &state.per_slot_heads { + for (validator, votes) in state.votes.as_ref().iter().enumerate() { + for (index, candidate) in votes.iter().enumerate() { + assert_eq!( + (validator, index, candidate.load(std::sync::atomic::Ordering::SeqCst)), + (validator, index, false) + ); + } + } + } env.stop().await; From 83a7325f2ee8d26dfd31810fe0b90f33fe89a3d7 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 11 Dec 2023 16:19:56 +0200 Subject: [PATCH 140/192] Fix GetApprovalSignatures ... discovered during benchmarking Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 08a5e57a9fb1..d520febaef51 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -599,17 +599,29 @@ impl BlockEntry { &self, candidate_index: CandidateIndex, ) -> Vec { - let result: Option> = - self.candidates.get(candidate_index as usize).map(|candidate_entry| { - candidate_entry - .assignments - .iter() - .filter_map(|(validator, assignment_bitfield)| { - self.approval_entries.get(&(*validator, assignment_bitfield.clone())) - }) - .flat_map(|approval_entry| approval_entry.approvals.clone().into_iter()) - .collect() - }); + let result: Option< + HashMap<(ValidatorIndex, CandidateBitfield), IndirectSignedApprovalVoteV2>, + > = self.candidates.get(candidate_index as usize).map(|candidate_entry| { + candidate_entry + .assignments + .iter() + .filter_map(|(validator, assignment_bitfield)| { + self.approval_entries.get(&(*validator, assignment_bitfield.clone())) + }) + .flat_map(|approval_entry| { + approval_entry + .approvals + .clone() + .into_iter() + .filter(|(approved_candidates, _)| { + approved_candidates.bit_at(candidate_index.as_bit_index()) + }) + .map(|(approved_candidates, vote)| { + ((approval_entry.validator_index, approved_candidates), vote) + }) + }) + .collect() + }); result.map(|result| result.into_values().collect_vec()).unwrap_or_default() } From 4d33c9d632fe537a257c1458df4c7c5c73357760 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 11 Dec 2023 16:35:09 +0200 Subject: [PATCH 141/192] Fix formatting issues Signed-off-by: Alexandru Gheorghe --- polkadot/cli/Cargo.toml | 2 +- polkadot/node/service/Cargo.toml | 2 +- polkadot/roadmap/implementers-guide/src/protocol-approval.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/polkadot/cli/Cargo.toml b/polkadot/cli/Cargo.toml index 3f877923923b..5f20d99c2f64 100644 --- a/polkadot/cli/Cargo.toml +++ b/polkadot/cli/Cargo.toml @@ -74,4 +74,4 @@ malus = ["full-node", "service/malus"] runtime-metrics = [ "polkadot-node-metrics/runtime-metrics", "service/runtime-metrics", -] \ No newline at end of file +] diff --git a/polkadot/node/service/Cargo.toml b/polkadot/node/service/Cargo.toml index 23a7bace30eb..81eff49ee305 100644 --- a/polkadot/node/service/Cargo.toml +++ b/polkadot/node/service/Cargo.toml @@ -224,4 +224,4 @@ runtime-metrics = [ "polkadot-runtime-parachains/runtime-metrics", "rococo-runtime?/runtime-metrics", "westend-runtime?/runtime-metrics", -] \ No newline at end of file +] diff --git a/polkadot/roadmap/implementers-guide/src/protocol-approval.md b/polkadot/roadmap/implementers-guide/src/protocol-approval.md index ebe4894f0b3e..b6aa16646ad2 100644 --- a/polkadot/roadmap/implementers-guide/src/protocol-approval.md +++ b/polkadot/roadmap/implementers-guide/src/protocol-approval.md @@ -301,9 +301,9 @@ To reduce the necessary network bandwidth and cpu time when a validator has more doing our best effort to send a single message that approves all available candidates with a single signature. The implemented heuristic, is that each time we are ready to create a signature and send a vote for a candidate we delay sending it until one of three things happen: -- We gathered a maximum of `MAX_APPROVAL_COALESCE_COUNT` candidates that we have already checked and we are +- We gathered a maximum of `MAX_APPROVAL_COALESCE_COUNT` candidates that we have already checked and we are ready to sign approval for. -- `MAX_APPROVAL_COALESCE_WAIT_TICKS` have passed since checking oldest candidate and we were ready to sign +- `MAX_APPROVAL_COALESCE_WAIT_TICKS` have passed since checking oldest candidate and we were ready to sign and send the approval message. - We are already in the last third of the no-show period in order to avoid creating accidental no-shows, which in turn might trigger other assignments. From 30950d2abda45d8486adec5d68d24c5ac3e11fc9 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 11 Dec 2023 16:40:37 +0200 Subject: [PATCH 142/192] Fixup cargo fmt Signed-off-by: Alexandru Gheorghe --- .../relay-chain-rpc-interface/src/rpc_client.rs | 3 +-- .../approval-voting/src/approval_db/v2/tests.rs | 2 +- .../approval-voting/src/approval_db/v3/tests.rs | 2 +- polkadot/node/subsystem-types/src/messages.rs | 17 +++++++++-------- polkadot/runtime/parachains/src/disputes.rs | 6 ++++-- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs b/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs index 205a5717f7a3..c64fff77a29f 100644 --- a/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs +++ b/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs @@ -32,8 +32,7 @@ use cumulus_primitives_core::{ relay_chain::{ async_backing::{AsyncBackingParams, BackingState}, slashing, - vstaging::ApprovalVotingParams, - vstaging::NodeFeatures, + vstaging::{ApprovalVotingParams, NodeFeatures}, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash as RelayHash, Header as RelayHeader, InboundHrmpMessage, OccupiedCoreAssumption, diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs index 3dbda3639d28..6021b44c2765 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs @@ -18,7 +18,7 @@ use crate::{ approval_db::{ - common::{DbBackend, StoredBlockRange, *, migration_helpers::make_bitvec}, + common::{migration_helpers::make_bitvec, DbBackend, StoredBlockRange, *}, v2::*, v3::{load_block_entry_v2, load_candidate_entry_v2}, }, diff --git a/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs b/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs index 29b3373e3f25..08c65461bca8 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs @@ -18,7 +18,7 @@ use crate::{ approval_db::{ - common::{DbBackend, StoredBlockRange, *, migration_helpers::make_bitvec}, + common::{migration_helpers::make_bitvec, DbBackend, StoredBlockRange, *}, v3::*, }, backend::{Backend, OverlayedBackend}, diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index 8d5d3f055a03..c7675c84b91c 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -42,14 +42,15 @@ use polkadot_node_primitives::{ ValidationResult, }; use polkadot_primitives::{ - async_backing, slashing, vstaging::{NodeFeatures, ApprovalVotingParams}, AuthorityDiscoveryId, BackedCandidate, - BlockNumber, CandidateEvent, CandidateHash, CandidateIndex, CandidateReceipt, CollatorId, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupIndex, - GroupRotationInfo, Hash, Header as BlockHeader, Id as ParaId, InboundDownwardMessage, - InboundHrmpMessage, MultiDisputeStatementSet, OccupiedCoreAssumption, PersistedValidationData, - PvfCheckStatement, PvfExecKind, SessionIndex, SessionInfo, SignedAvailabilityBitfield, - SignedAvailabilityBitfields, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, - ValidatorSignature, + async_backing, slashing, + vstaging::{ApprovalVotingParams, NodeFeatures}, + AuthorityDiscoveryId, BackedCandidate, BlockNumber, CandidateEvent, CandidateHash, + CandidateIndex, CandidateReceipt, CollatorId, CommittedCandidateReceipt, CoreState, + DisputeState, ExecutorParams, GroupIndex, GroupRotationInfo, Hash, Header as BlockHeader, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, MultiDisputeStatementSet, + OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, PvfExecKind, SessionIndex, + SessionInfo, SignedAvailabilityBitfield, SignedAvailabilityBitfields, ValidationCode, + ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; use polkadot_statement_table::v2::Misbehavior; use std::{ diff --git a/polkadot/runtime/parachains/src/disputes.rs b/polkadot/runtime/parachains/src/disputes.rs index ca275cf6442c..c2383dad3053 100644 --- a/polkadot/runtime/parachains/src/disputes.rs +++ b/polkadot/runtime/parachains/src/disputes.rs @@ -1017,8 +1017,10 @@ impl Pallet { set.session, statement, signature, - // This is here to prevent malicious nodes of generating `ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates` - // before that is enabled, via setting `max_approval_coalesce_count` in the parachain host config. + // This is here to prevent malicious nodes of generating + // `ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates` before that + // is enabled, via setting `max_approval_coalesce_count` in the parachain host + // config. config.approval_voting_params.max_approval_coalesce_count > 1, ) { log::warn!("Failed to check dispute signature"); From 593b82ab003288e57c8b8c83bf6d57a72c8fd50c Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 11 Dec 2023 17:13:01 +0200 Subject: [PATCH 143/192] Fix some logging messed during rebase Signed-off-by: Alexandru Gheorghe --- .../runtime/parachains/src/configuration/migration/v11.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/polkadot/runtime/parachains/src/configuration/migration/v11.rs b/polkadot/runtime/parachains/src/configuration/migration/v11.rs index 053386be6e46..b7dec7070f93 100644 --- a/polkadot/runtime/parachains/src/configuration/migration/v11.rs +++ b/polkadot/runtime/parachains/src/configuration/migration/v11.rs @@ -71,22 +71,22 @@ pub struct UncheckedMigrateToV11(sp_std::marker::PhantomData); impl OnRuntimeUpgrade for UncheckedMigrateToV11 { #[cfg(feature = "try-runtime")] fn pre_upgrade() -> Result, sp_runtime::TryRuntimeError> { - log::trace!(target: crate::configuration::LOG_TARGET, "Running pre_upgrade() for HostConfiguration MigrateToV10"); + log::trace!(target: crate::configuration::LOG_TARGET, "Running pre_upgrade() for HostConfiguration MigrateToV11"); Ok(Vec::new()) } fn on_runtime_upgrade() -> Weight { - log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV10 started"); + log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV11 started"); let weight_consumed = migrate_to_v11::(); - log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV10 executed successfully"); + log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV11 executed successfully"); weight_consumed } #[cfg(feature = "try-runtime")] fn post_upgrade(_state: Vec) -> Result<(), sp_runtime::TryRuntimeError> { - log::trace!(target: crate::configuration::LOG_TARGET, "Running post_upgrade() for HostConfiguration MigrateToV10"); + log::trace!(target: crate::configuration::LOG_TARGET, "Running post_upgrade() for HostConfiguration MigrateToV11"); ensure!( StorageVersion::get::>() >= 11, "Storage version should be >= 11 after the migration" From f8f03e572a589156d9ac8a7500aacac3e61bcd04 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 11 Dec 2023 17:39:06 +0200 Subject: [PATCH 144/192] Add prdoc Signed-off-by: Alexandru Gheorghe --- prdoc/pr_1191.prdoc | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 prdoc/pr_1191.prdoc diff --git a/prdoc/pr_1191.prdoc b/prdoc/pr_1191.prdoc new file mode 100644 index 000000000000..26626731be46 --- /dev/null +++ b/prdoc/pr_1191.prdoc @@ -0,0 +1,21 @@ +title: Approve multiple candidates with a single signature + +doc: + - audience: Node Operator + description: | + Changed approval-voting, approval-distribution to approve multiple candidate with a single message, it adds: + * A new parachains_db version. + * A new validation protocol to support the new message types. + The new logic will be disabled and will be enabled at a later date after all validators have upgraded. + +migrations: + db: + - name: Parachains database change from v4 to v5. + description: | + Approval-voting column format has been updated with several new fields. All existing data will be automatically + be migrated to the new values. + +crates: + - name: "polkadot" + +host_functions: [] From 8099c16e7b2c9a24be929f51083c6a31d770528a Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 12 Dec 2023 13:51:03 +0200 Subject: [PATCH 145/192] Fixup 0002-upgrade-node failures V2 was not put into the list of fallbacks for the validation protocol, so the test wrongly fall-backed on v1. Signed-off-by: Alexandru Gheorghe --- .../node/network/protocol/src/peer_set.rs | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/polkadot/node/network/protocol/src/peer_set.rs b/polkadot/node/network/protocol/src/peer_set.rs index 3a3c3edbbd0b..cb329607ad61 100644 --- a/polkadot/node/network/protocol/src/peer_set.rs +++ b/polkadot/node/network/protocol/src/peer_set.rs @@ -73,7 +73,11 @@ impl PeerSet { // Networking layer relies on `get_main_name()` being the main name of the protocol // for peersets and connection management. let protocol = peerset_protocol_names.get_main_name(self); - let fallback_names = PeerSetProtocolNames::get_fallback_names(self); + let fallback_names = PeerSetProtocolNames::get_fallback_names( + self, + &peerset_protocol_names.genesis_hash, + peerset_protocol_names.fork_id.as_deref(), + ); let max_notification_size = self.get_max_notification_size(is_authority); match self { @@ -293,6 +297,8 @@ impl From for ProtocolVersion { pub struct PeerSetProtocolNames { protocols: HashMap, names: HashMap<(PeerSet, ProtocolVersion), ProtocolName>, + genesis_hash: Hash, + fork_id: Option, } impl PeerSetProtocolNames { @@ -327,7 +333,7 @@ impl PeerSetProtocolNames { } Self::register_legacy_protocol(&mut protocols, protocol); } - Self { protocols, names } + Self { protocols, names, genesis_hash, fork_id: fork_id.map(|fork_id| fork_id.into()) } } /// Helper function to register main protocol. @@ -431,9 +437,30 @@ impl PeerSetProtocolNames { } /// Get the protocol fallback names. Currently only holds the legacy name - /// for `LEGACY_PROTOCOL_VERSION` = 1. - fn get_fallback_names(protocol: PeerSet) -> Vec { - std::iter::once(Self::get_legacy_name(protocol)).collect() + /// for `LEGACY_PROTOCOL_VERSION` = 1 and v2 for validation. + fn get_fallback_names( + protocol: PeerSet, + genesis_hash: &Hash, + fork_id: Option<&str>, + ) -> Vec { + let mut fallbacks = vec![Self::get_legacy_name(protocol)]; + match protocol { + PeerSet::Validation => { + // Fallbacks are tried one by one, till one matches so push v2 at the top, so + // that it is used ahead of the legacy one(v1). + fallbacks.insert( + 0, + Self::generate_name( + genesis_hash, + fork_id, + protocol, + ValidationVersion::V2.into(), + ), + ) + }, + PeerSet::Collation => {}, + }; + fallbacks } } From 0bc7e5fcec8c9811d0a6f68a121dfb43bed4f28e Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 12 Dec 2023 14:08:17 +0200 Subject: [PATCH 146/192] WIP Signed-off-by: Andrei Sandu --- .../src/core/mock/network_bridge.rs | 2 +- .../node/subsystem-bench/src/core/network.rs | 492 +++++++++++++----- 2 files changed, 349 insertions(+), 145 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index 22abd0625c51..0436eea148b3 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -191,7 +191,7 @@ impl MockNetworkBridgeTx { // If peer is disconnected return an error if !self.network.is_peer_connected(&authority_discovery_id) { - let future = async move { + let future: Pin + Send>> = async move { let _ = outgoing_request .pending_response .send(Err(RequestFailure::NotConnected)); diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index e340dba8ed92..f1f0689ba3ea 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -13,15 +13,24 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +//! +//! Implements network emulation and interfaces to control and specialize +//! network peer behaviour. + use super::{ configuration::{TestAuthorities, TestConfiguration}, environment::TestEnvironmentDependencies, *, }; +use parity_scale_codec::Encode; + use colored::Colorize; +use futures::channel::mpsc; +use net_protocol::VersionedValidationProtocol; use polkadot_primitives::AuthorityDiscoveryId; use prometheus_endpoint::U64; use rand::{seq::SliceRandom, thread_rng}; +use sc_network::request_responses::IncomingRequest; use sc_service::SpawnTaskHandle; use std::{ collections::HashMap, @@ -29,10 +38,22 @@ use std::{ atomic::{AtomicU64, Ordering}, Arc, }, - time::{Duration, Instant}, + time::{Duration, Instant}, ops::DerefMut, +}; + +use polkadot_node_network_protocol::{ + self as net_protocol, + + peer_set::{ProtocolVersion, ValidationVersion}, + v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, OurView, PeerId, + UnifiedReputationChange as Rep, Versioned, View, }; -use tokio::sync::mpsc::UnboundedSender; +use futures::channel::mpsc::{UnboundedSender, UnboundedReceiver}; + +use futures::{ + Future, FutureExt, Stream, StreamExt, +}; // An emulated node egress traffic rate_limiter. #[derive(Debug)] pub struct RateLimit { @@ -99,112 +120,251 @@ impl RateLimit { } } -#[cfg(test)] -mod tests { - use std::time::Instant; +// A wrapper for both gossip and request/response protocols along with the destination peer(`AuthorityDiscoveryId``). +pub enum PeerMessage { + Message(AuthorityDiscoveryId, VersionedValidationProtocol), + Request(AuthorityDiscoveryId, IncomingRequest) +} - use super::RateLimit; +impl PeerMessage { + /// Returns the size of the encoded message or request + pub fn size(&self) -> usize { + match &self { + PeerMessage::Message(_peer_id, Versioned::V1(message)) => { + message.encoded_size() + }, + PeerMessage::Request(_peer_id, incoming) => { + incoming.payload.encoded_size() + } + } + } +} - #[tokio::test] - async fn test_expected_rate() { - let tick_rate = 200; - let budget = 1_000_000; - // rate must not exceeed 100 credits per second - let mut rate_limiter = RateLimit::new(tick_rate, budget); - let mut total_sent = 0usize; - let start = Instant::now(); +/// A network interface of the node under test. +pub struct NetworkInterface { + // The network we are connected to. + network: Option, + // Used to receive traffic from the network and implement `rx_limiter` limits. + // from_network: Receiver, + // `tx_limiter` enforces the configured bandwidth. + // Sender for network peers. + network_to_interface_sender: UnboundedSender, + // Used to receive traffic from the `network-bridge-tx` subsystem. + // The network action is forwarded via `network` to the relevant peer(s). + // from_netowork_bridge: Receiver, + // Sender for subsystems. + bridge_to_interface_sender: UnboundedSender, + // A sender to forward actions to the network bridge subsystem. + interface_to_bridge_sender: UnboundedSender, +} - let mut reap_amount = 0; - while rate_limiter.total_ticks < tick_rate { - reap_amount += 1; - reap_amount = reap_amount % 100; +// Wraps the receiving side of a interface to bridge channel. It is a required +// parameter of the `network-bridge` mock. +pub struct NetworkInterfaceReceiver(pub UnboundedReceiver); - rate_limiter.reap(reap_amount).await; - total_sent += reap_amount; +impl NetworkInterface { + /// Create a new `NetworkInterface` + pub fn new( + spawn_task_handle: SpawnTaskHandle, + mut network: NetworkEmulator, + bandiwdth_bps: usize, + ) -> (NetworkInterface, NetworkInterfaceReceiver) { + let mut rx_limiter: RateLimit = RateLimit::new(10, bandiwdth_bps); + let mut tx_limiter: RateLimit = RateLimit::new(10, bandiwdth_bps); + + // A sender (`ingress_tx`) clone will be owned by each `PeerEmulator`, such that it can send + // messages to the node. THe receiver will be polled in the network interface task. + let (network_to_interface_sender, network_to_interface_receiver) = mpsc::unbounded::(); + // The sender (`egress_tx`) is + let (bridge_to_interface_sender, bridge_to_interface_receiver) = mpsc::unbounded::(); + + // Channel for forwarding actions to the bridge. + let (interface_to_bridge_sender, interface_to_bridge_receiver) = mpsc::unbounded::(); + + // Spawn the network interface task. + let task = async move { + let mut network_to_interface_receiver = network_to_interface_receiver.fuse(); + let mut bridge_to_interface_receiver = bridge_to_interface_receiver.fuse(); + + loop { + // TODO (maybe): split into separate rx/tx tasks. + futures::select! { + maybe_peer_message = network_to_interface_receiver.next() => { + if let Some(peer_message) = maybe_peer_message { + rx_limiter.reap(peer_message.size()).await; + + // Forward the message to the bridge. + interface_to_bridge_sender.unbounded_send(peer_message).expect("network bridge subsystem is alive"); + } else { + gum::info!(target: LOG_TARGET, "Uplink channel closed, network interface task exiting"); + break + } + }, + maybe_peer_message = bridge_to_interface_receiver.next() => { + if let Some(peer_message) = maybe_action_from_subsystem { + tx_limiter.reap(peer_message.size()).await; + + // Forward action to the destination network peer. + network.submit_peer_action(peer, action); + } else { + gum::info!(target: LOG_TARGET, "Downlink channel closed, network interface task exiting"); + break + } + } + + } + } } + .boxed(); + + ( + Self { + network: None, + network_to_interface_sender, + bridge_to_interface_sender, + interface_to_bridge_sender, + }, + NetworkInterfaceReceiver(interface_to_bridge_receiver), + ) + } - let end = Instant::now(); + /// Connect the interface to a network. + pub fn connect(&mut self, network: NetworkEmulator) { + self.network = Some(network); + } - println!("duration: {}", (end - start).as_millis()); + /// Get a sender that can be used by a subsystem to send network actions to the network. + pub fn subsystem_sender(&self) -> UnboundedSender { + self.bridge_to_interface_sender.clone() + } - // Allow up to `budget/max_refill` error tolerance - let lower_bound = budget as u128 * ((end - start).as_millis() / 1000u128); - let upper_bound = budget as u128 * - ((end - start).as_millis() / 1000u128 + rate_limiter.max_refill as u128); - assert!(total_sent as u128 >= lower_bound); - assert!(total_sent as u128 <= upper_bound); + /// Get a sender that can be used by the Network to send network actions to the network. + pub fn network_sender(&self) -> UnboundedSender { + self.network_to_interface_sender.clone() } } +/// An emulated peer network interface. +#[derive(Debug)] +pub struct PeerNetworkInterface { + /// Receive rate limiter. + rx_limiter: RateLimit, + /// Transmit rate limiter + tx_limiter: RateLimit, + /// Network receive queue. + /// This is paired to a `Sender` in `NetworkEmulator`. + /// The network interface task will forward messages/requests from the node over + /// this channel. + rx_queue: UnboundedReceiver, + /// Network send queue. + /// Paired to the `rx_queue` receiver on the `NetworkInterface`. + tx_queue: UnboundedSender, +} + +// // A network peer emulator. It spawns a task that accepts `NetworkActions` and // executes them with a configurable delay and bandwidth constraints. Tipically // these actions wrap a future that performs a channel send to the subsystem(s) under test. #[derive(Clone)] -struct PeerEmulator { - // The queue of requests waiting to be served by the emulator +struct EmulatedPeerHandle { + // Send messages to the peer emulator task + messages_tx: UnboundedSender, + // Send actions to the peer emulator task actions_tx: UnboundedSender, } -impl PeerEmulator { - pub fn new( - bandwidth: usize, - spawn_task_handle: SpawnTaskHandle, - stats: Arc, - ) -> Self { - let (actions_tx, mut actions_rx) = tokio::sync::mpsc::unbounded_channel(); - - spawn_task_handle - .clone() - .spawn("peer-emulator", "test-environment", async move { - // Rate limit peer send. - let mut rate_limiter = RateLimit::new(10, bandwidth); - loop { - let stats_clone = stats.clone(); - let maybe_action: Option = actions_rx.recv().await; - if let Some(action) = maybe_action { - let size = action.size(); - rate_limiter.reap(size).await; - if let Some(latency) = action.latency { - spawn_task_handle.spawn( - "peer-emulator-latency", - "test-environment", - async move { - tokio::time::sleep(latency).await; - action.run().await; - stats_clone.inc_sent(size); - }, - ) +/// Interceptor pattern for handling messages. +pub trait HandlePeerMessage { + // Returns `None` if the message was handled, or the `message` + // otherwise. + fn handle(&self, message: PeerMessage, node_sender: &mut UnboundedSender) -> Option; +} + +impl HandlePeerMessage for Arc +where T: HandlePeerMessage { + fn handle(&self, message: PeerMessage, node_sender: &mut UnboundedSender) -> Option { + self.as_ref().handle(message, node_sender) + } +} + +pub fn new_peer( + bandwidth: usize, + spawn_task_handle: SpawnTaskHandle, + handlers: Vec>, + stats: Arc, + network_interface: &NetworkInterface, +) -> Self { + let (messages_tx, mut messages_rx) = mpsc::unbounded::(); + let (actions_tx, mut actions_rx) = mpsc::unbounded::(); + + // We'll use this to send messages from this peer to the node under test (peer 0) + let to_node = network_interface.network_sender(); + + spawn_task_handle + .clone() + .spawn("peer-emulator", "test-environment", async move { + let mut rx_limiter = RateLimit::new(10, bandwidth); + let mut tx_limiter = RateLimit::new(10, bandwidth); + let mut messages_rx = messages_rx.fuse(); + let mut actions_rx = actions_rx.fuse(); + + assert!(message.is_none(), "A peer will process all received messages"); + + // TODO: implement latency and error. + loop { + futures::select! { + maybe_peer_message = messages_rx.next() => { + if let Some(peer_message) = maybe_peer_message { + let size = peer_message.size(); + rx_limiter.reap(size).await; + stats.inc_received(size); + + let message = Some(message); + for handler in handlers.iter() { + message = handler.handle(message, &mut to_node); + if message.is_none() { + break + } + } } else { - action.run().await; - stats_clone.inc_sent(size); + gum::trace!(target: LOG_TARGET, "Downlink channel closed, peer task exiting"); + break } - } else { - break - } + }, + maybe_action = actions_rx.next() => { + if let Some(action) = maybe_action { + match action.kind { + NetworkActionKind::SendMessage(message) => { + let size = message.size(); + tx_limiter.reap(size).await; + stats.inc_sent(size); + to_node.unbounded_send(message); + } + } + } else { + gum::trace!(target: LOG_TARGET, "Action channel closed, peer task exiting"); + break + } + }, } - }); - - Self { actions_tx } - } + } + }); - // Queue a send request from the emulated peer. - pub fn send(&mut self, action: NetworkAction) { - self.actions_tx.send(action).expect("peer emulator task lives"); - } + EmulatedPeerHandle { messages_tx, actions_tx } } -pub type ActionFuture = std::pin::Pin + std::marker::Send>>; +/// Types of actions that an emulated peer can run. +pub enum NetworkActionKind { + /// Send a message to node under test (peer 0) + SendMessage(PeerMessage) +} -/// A network action to be completed by the emulator task. +/// A network action to be completed by an emulator task. pub struct NetworkAction { - // The function that performs the action - run: ActionFuture, - // The payload size that we simulate sending/receiving from a peer - size: usize, + /// The action type + pub kind: NetworkActionKind, // Peer which should run the action. - peer: AuthorityDiscoveryId, - // The amount of time to delay the polling `run` - latency: Option, + pub peer: AuthorityDiscoveryId, } unsafe impl Send for NetworkAction {} @@ -227,31 +387,25 @@ impl PeerEmulatorStats { pub fn inc_received(&self, bytes: usize) { self.metrics.on_peer_received(self.peer_index, bytes); } -} - -impl NetworkAction { - pub fn new( - peer: AuthorityDiscoveryId, - run: ActionFuture, - size: usize, - latency: Option, - ) -> Self { - Self { run, size, peer, latency } - } - - pub fn size(&self) -> usize { - self.size - } - pub async fn run(self) { - self.run.await; + pub fn sent(&self) -> usize { + self.metrics + .peer_total_sent + .get_metric_with_label_values(&[&format!("node{}", self.peer_index)]) + .expect("Metric exists") + .get() as usize } - pub fn peer(&self) -> AuthorityDiscoveryId { - self.peer.clone() + pub fn received(&self) -> usize { + self.metrics + .peer_total_received + .get_metric_with_label_values(&[&format!("node{}", self.peer_index)]) + .expect("Metric exists") + .get() as usize } } + /// The state of a peer on the emulated network. #[derive(Clone)] enum Peer { @@ -284,10 +438,9 @@ impl Peer { } } -/// Mocks the network bridge and an arbitrary number of connected peer nodes. -/// Implements network latency, bandwidth and connection errors. +/// An emulated network implementation. Can be cloned #[derive(Clone)] -pub struct NetworkEmulator { +pub struct NetworkEmulatorHandle { // Per peer network emulation. peers: Vec, /// Per peer stats. @@ -296,55 +449,69 @@ pub struct NetworkEmulator { validator_authority_ids: HashMap, } -impl NetworkEmulator { - pub fn new( - config: &TestConfiguration, - dependencies: &TestEnvironmentDependencies, - authorities: &TestAuthorities, - ) -> Self { - let n_peers = config.n_validators; - gum::info!(target: LOG_TARGET, "{}",format!("Initializing emulation for a {} peer network.", n_peers).bright_blue()); - gum::info!(target: LOG_TARGET, "{}",format!("connectivity {}%, error {}%", config.connectivity, config.error).bright_black()); - - let metrics = - Metrics::new(&dependencies.registry).expect("Metrics always register succesfully"); - let mut validator_authority_id_mapping = HashMap::new(); - - // Create a `PeerEmulator` for each peer. - let (stats, mut peers): (_, Vec<_>) = (0..n_peers) - .zip(authorities.validator_authority_id.clone().into_iter()) - .map(|(peer_index, authority_id)| { - validator_authority_id_mapping.insert(authority_id, peer_index); - let stats = Arc::new(PeerEmulatorStats::new(peer_index, metrics.clone())); - ( - stats.clone(), - Peer::Connected(PeerEmulator::new( - config.peer_bandwidth, - dependencies.task_manager.spawn_handle(), - stats, - )), - ) - }) - .unzip(); - - let connected_count = config.n_validators as f64 / (100.0 / config.connectivity as f64); - - let (_connected, to_disconnect) = - peers.partial_shuffle(&mut thread_rng(), connected_count as usize); - - for peer in to_disconnect { - peer.disconnect(); - } +pub fn new_network( + config: &TestConfiguration, + dependencies: &TestEnvironmentDependencies, + authorities: &TestAuthorities, + handlers: Vec>, + network_interface: &NetworkInterface, +) -> NetworkEmulatorHandle { + let n_peers = config.n_validators; + gum::info!(target: LOG_TARGET, "{}",format!("Initializing emulation for a {} peer network.", n_peers).bright_blue()); + gum::info!(target: LOG_TARGET, "{}",format!("connectivity {}%, error {}%", config.connectivity, config.error).bright_black()); + + let metrics = + Metrics::new(&dependencies.registry).expect("Metrics always register succesfully"); + let mut validator_authority_id_mapping = HashMap::new(); + + // Create a `PeerEmulator` for each peer. + let (stats, mut peers): (_, Vec<_>) = (0..n_peers) + .zip(authorities.validator_authority_id.clone().into_iter()) + .map(|(peer_index, authority_id)| { + validator_authority_id_mapping.insert(authority_id, peer_index); + let stats = Arc::new(PeerEmulatorStats::new(peer_index, metrics.clone())); + ( + stats.clone(), + Peer::Connected(new_peer( + config.peer_bandwidth, + dependencies.task_manager.spawn_handle(), + handlers, + stats, + network_interface, + )), + ) + }) + .unzip(); - gum::info!(target: LOG_TARGET, "{}",format!("Network created, connected validator count {}", connected_count).bright_black()); + let connected_count = config.n_validators as f64 / (100.0 / config.connectivity as f64); - Self { peers, stats, validator_authority_ids: validator_authority_id_mapping } + let (_connected, to_disconnect) = + peers.partial_shuffle(&mut thread_rng(), connected_count as usize); + + for peer in to_disconnect { + peer.disconnect(); } + gum::info!(target: LOG_TARGET, "{}",format!("Network created, connected validator count {}", connected_count).bright_black()); + + NetworkEmulatorHandle { peers, stats, validator_authority_ids: validator_authority_id_mapping } +} + + +impl NetworkEmulatorHandle { pub fn is_peer_connected(&self, peer: &AuthorityDiscoveryId) -> bool { self.peer(peer).is_connected() } + /// Forward `message`` to an emulated `peer``. + /// Panics if peer is not connected. + pub fn forward_message(&self, peer: &AuthorityDiscoveryId, message: PeerMessage) { + assert(!self.peer(peer).is_connected(), "forward message only for connected peers."); + + + } + + /// Run some code in the context of an emulated pub fn submit_peer_action(&mut self, peer: AuthorityDiscoveryId, action: NetworkAction) { let index = self .validator_authority_ids @@ -453,3 +620,40 @@ impl Metrics { .inc_by(bytes as u64); } } + +#[cfg(test)] +mod tests { + use std::time::Instant; + + use super::RateLimit; + + #[tokio::test] + async fn test_expected_rate() { + let tick_rate = 200; + let budget = 1_000_000; + // rate must not exceeed 100 credits per second + let mut rate_limiter = RateLimit::new(tick_rate, budget); + let mut total_sent = 0usize; + let start = Instant::now(); + + let mut reap_amount = 0; + while rate_limiter.total_ticks < tick_rate { + reap_amount += 1; + reap_amount = reap_amount % 100; + + rate_limiter.reap(reap_amount).await; + total_sent += reap_amount; + } + + let end = Instant::now(); + + println!("duration: {}", (end - start).as_millis()); + + // Allow up to `budget/max_refill` error tolerance + let lower_bound = budget as u128 * ((end - start).as_millis() / 1000u128); + let upper_bound = budget as u128 * + ((end - start).as_millis() / 1000u128 + rate_limiter.max_refill as u128); + assert!(total_sent as u128 >= lower_bound); + assert!(total_sent as u128 <= upper_bound); + } +} From dc4b2f96d06440833ea27a021b57800216810a20 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 13 Dec 2023 10:44:44 +0200 Subject: [PATCH 147/192] Fix something here and there Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 6 ++ polkadot/node/overseer/src/lib.rs | 2 +- .../src/approval/mock_runtime_api.rs | 8 ++- .../node/subsystem-bench/src/approval/mod.rs | 61 ++++++++++++++----- .../subsystem-bench/src/core/environment.rs | 3 +- 5 files changed, 61 insertions(+), 19 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index d386146ecfb0..8072198b7c75 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -351,6 +351,7 @@ struct State { /// Aggregated reputation change reputation: ReputationAggregator, + total_num_messages: u64, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -714,7 +715,12 @@ impl State { }); }, NetworkBridgeEvent::PeerMessage(peer_id, message) => { + self.total_num_messages += 1; self.process_incoming_peer_message(ctx, metrics, peer_id, message, rng).await; + + if self.total_num_messages % 100000 == 0 { + gum::info!(target: LOG_TARGET, total_num_messages = self.total_num_messages, "Processed 100k"); + } }, NetworkBridgeEvent::UpdatedAuthorityIds { .. } => { // The approval-distribution subsystem doesn't deal with `AuthorityDiscoveryId`s. diff --git a/polkadot/node/overseer/src/lib.rs b/polkadot/node/overseer/src/lib.rs index f4eddf1f41ce..f687cb4f89e0 100644 --- a/polkadot/node/overseer/src/lib.rs +++ b/polkadot/node/overseer/src/lib.rs @@ -577,7 +577,7 @@ pub struct Overseer { ])] collator_protocol: CollatorProtocol, - #[subsystem(blocking, message_capacity: 64000, ApprovalDistributionMessage, sends: [ + #[subsystem(blocking, message_capacity: 6400000, ApprovalDistributionMessage, sends: [ NetworkBridgeTxMessage, ApprovalVotingMessage, ])] diff --git a/polkadot/node/subsystem-bench/src/approval/mock_runtime_api.rs b/polkadot/node/subsystem-bench/src/approval/mock_runtime_api.rs index 7021aec68c37..b590ab9e2a13 100644 --- a/polkadot/node/subsystem-bench/src/approval/mock_runtime_api.rs +++ b/polkadot/node/subsystem-bench/src/approval/mock_runtime_api.rs @@ -18,7 +18,7 @@ use super::ApprovalTestState; use futures::FutureExt; use polkadot_node_subsystem::{overseer, SpawnedSubsystem, SubsystemError}; use polkadot_node_subsystem_types::messages::{RuntimeApiMessage, RuntimeApiRequest}; -use polkadot_primitives::ExecutorParams; +use polkadot_primitives::{vstaging::NodeFeatures, ExecutorParams}; /// Mock RuntimeApi subsystem used to answer request made by the approval-voting subsystem, /// during benchmark. All the necessary information to answer the requests is stored in the `state` @@ -63,6 +63,12 @@ impl MockRuntimeApi { ) => { let _ = sender.send(Ok(Some(self.state.session_info.clone()))); }, + RuntimeApiMessage::Request( + _request, + RuntimeApiRequest::NodeFeatures(_session_index, sender), + ) => { + let _ = sender.send(Ok(NodeFeatures::EMPTY)); + }, RuntimeApiMessage::Request( _request, RuntimeApiRequest::SessionExecutorParams(_session_index, sender), diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index a72ebfc75ebe..019d30524172 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -122,7 +122,7 @@ pub const NODE_UNDER_TEST: u32 = 0; /// Start generating messages for a slot into the future, so that the /// generation nevers falls behind the current slot. -const BUFFER_FOR_GENERATION_MILLIS: u64 = 6_000; +const BUFFER_FOR_GENERATION_MILLIS: u64 = 30_000; /// Parameters specific to the approvals benchmark #[derive(Debug, Clone, Serialize, Deserialize, clap::Parser)] @@ -140,6 +140,8 @@ pub struct ApprovalsOptions { #[clap(short, long, default_value_t = 1)] /// Max candidate to be signed in a single approval. pub max_coalesce: u32, + /// The maximum tranche diff between approvals coalesced toghther. + pub coalesce_tranche_diff: u32, #[clap(short, long, default_value_t = false)] /// Enable assignments v2. pub enable_assignments_v2: bool, @@ -554,6 +556,13 @@ impl PeerMessagesGenerator { self.options.last_considered_tranche, ); + let bytes = self.validator_index.0.to_be_bytes(); + let seed = [ + bytes[0], bytes[1], bytes[2], bytes[3], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + + let mut rand_chacha = ChaCha20Rng::from_seed(seed); let approvals = issue_approvals( &assignments, block_info.hash, @@ -566,6 +575,7 @@ impl PeerMessagesGenerator { .collect_vec(), block_info.candidates.clone(), &self.options, + &mut rand_chacha, ); let generated_assignments = assignments.into_iter().peekable(); @@ -770,11 +780,13 @@ fn session_info_for_peers( /// Helper function to randomly determine how many approvals we coalesce together in a single /// message. -fn coalesce_approvals_len(min_coalesce: u32, max_coalesce: u32) -> usize { - let seed = [7u8; 32]; - let mut rand_chacha = ChaCha20Rng::from_seed(seed); +fn coalesce_approvals_len( + min_coalesce: u32, + max_coalesce: u32, + rand_chacha: &mut ChaCha20Rng, +) -> usize { let mut sampling: Vec = (min_coalesce as usize..max_coalesce as usize + 1).collect_vec(); - *(sampling.partial_shuffle(&mut rand_chacha, 1).0.first().unwrap()) + *(sampling.partial_shuffle(rand_chacha, 1).0.first().unwrap()) } /// Helper function to create approvals signatures for all assignments passed as arguments. @@ -785,9 +797,11 @@ fn issue_approvals( keyrings: Vec<(Keyring, PeerId)>, candidates: Vec, options: &ApprovalsOptions, + rand_chacha: &mut ChaCha20Rng, ) -> Vec { let mut to_sign: Vec = Vec::new(); - + let mut num_coalesce = + coalesce_approvals_len(options.min_coalesce, options.max_coalesce, rand_chacha); let result = assignments .iter() .enumerate() @@ -806,10 +820,18 @@ fn issue_approvals( let assignment = assignments.first().unwrap(); - if to_sign.len() >= - coalesce_approvals_len(options.min_coalesce, options.max_coalesce) as usize || - (!to_sign.is_empty() && current_validator_index != assignment.0.validator) + let earliest_tranche = + to_sign.first().map(|val| val.tranche).unwrap_or(message.tranche); + + if to_sign.len() >= num_coalesce as usize || + (!to_sign.is_empty() && current_validator_index != assignment.0.validator) || + message.tranche - earliest_tranche >= options.coalesce_tranche_diff { + num_coalesce = coalesce_approvals_len( + options.min_coalesce, + options.max_coalesce, + rand_chacha, + ); approvals_to_create.push(sign_candidates(&mut to_sign, &keyrings, block_hash)) } @@ -824,11 +846,17 @@ fn issue_approvals( sent_by: message.sent_by.clone(), tranche: message.tranche, }); + let earliest_tranche = + to_sign.first().map(|val| val.tranche).unwrap_or(message.tranche); - if to_sign.len() >= - coalesce_approvals_len(options.min_coalesce, options.max_coalesce) - as usize + if to_sign.len() >= num_coalesce || + message.tranche - earliest_tranche >= options.coalesce_tranche_diff { + num_coalesce = coalesce_approvals_len( + options.min_coalesce, + options.max_coalesce, + rand_chacha, + ); approvals_to_create.push(sign_candidates( &mut to_sign, &keyrings, @@ -1279,7 +1307,7 @@ fn build_overseer( config: &TestConfiguration, dependencies: &TestEnvironmentDependencies, ) -> (Overseer, AlwaysSupportsParachains>, OverseerHandleReal) { - let overseer_connector = OverseerConnector::with_event_capacity(640000); + let overseer_connector = OverseerConnector::with_event_capacity(6400000); let spawn_task_handle = dependencies.task_manager.spawn_handle(); @@ -1414,13 +1442,14 @@ pub async fn bench_approvals(env: &mut TestEnvironment, state: ApprovalTestState env.config().num_blocks as u32 { gum::info!( - "Waiting for all blocks to be approved current approved {:}", - state.last_approved_block.load(std::sync::atomic::Ordering::SeqCst) + "Waiting for all blocks to be approved current approved {:} num_sent {:}", + state.last_approved_block.load(std::sync::atomic::Ordering::SeqCst), + state.total_sent_messages.load(std::sync::atomic::Ordering::SeqCst) ); tokio::time::sleep(Duration::from_secs(6)).await; } tokio::time::sleep(Duration::from_secs(6)).await; - tokio::time::sleep(Duration::from_secs(6)).await; + tokio::time::sleep(Duration::from_secs(60)).await; for info in &state.per_slot_heads { for (index, candidates) in info.candidates.iter().enumerate() { diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index e4fcb3922cc8..16ab0d49d869 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -121,7 +121,8 @@ fn new_runtime() -> tokio::runtime::Runtime { tokio::runtime::Builder::new_multi_thread() .thread_name("subsystem-bench") .enable_all() - .thread_stack_size(3 * 1024 * 1024) + .thread_stack_size(128 * 1024 * 1024) + .max_blocking_threads(4096) .build() .unwrap() } From d6b0edae55b0ba07d112712ec1a4eace9d062583 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 13 Dec 2023 15:35:31 +0200 Subject: [PATCH 148/192] Add test message Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 7 +- .../node/core/approval-voting/src/time.rs | 16 +- .../node/subsystem-bench/src/approval/mod.rs | 234 +++++++++++++++--- .../subsystem-bench/src/subsystem-bench.rs | 6 +- 4 files changed, 225 insertions(+), 38 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 30456bd8c0ed..a36800236074 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -159,6 +159,7 @@ pub struct ApprovalVotingSubsystem { db: Arc, mode: Mode, metrics: Metrics, + clock: Box, } #[derive(Clone)] @@ -451,6 +452,7 @@ impl ApprovalVotingSubsystem { keystore: Arc, sync_oracle: Box, metrics: Metrics, + clock: Box, ) -> Self { ApprovalVotingSubsystem { keystore, @@ -459,6 +461,7 @@ impl ApprovalVotingSubsystem { db_config: DatabaseConfig { col_approval_data: config.col_approval_data }, mode: Mode::Syncing(sync_oracle), metrics, + clock, } } @@ -916,7 +919,7 @@ enum Action { async fn run( mut ctx: Context, mut subsystem: ApprovalVotingSubsystem, - clock: Box, + _clock: Box, assignment_criteria: Box, mut backend: B, ) -> SubsystemResult<()> @@ -930,7 +933,7 @@ where let mut state = State { keystore: subsystem.keystore, slot_duration_millis: subsystem.slot_duration_millis, - clock, + clock: subsystem.clock, assignment_criteria, spans: HashMap::new(), }; diff --git a/polkadot/node/core/approval-voting/src/time.rs b/polkadot/node/core/approval-voting/src/time.rs index 6048b9a0238d..15da993c518d 100644 --- a/polkadot/node/core/approval-voting/src/time.rs +++ b/polkadot/node/core/approval-voting/src/time.rs @@ -36,11 +36,11 @@ use polkadot_primitives::{Hash, ValidatorIndex}; const TICK_DURATION_MILLIS: u64 = 500; /// A base unit of time, starting from the Unix epoch, split into half-second intervals. -pub(crate) type Tick = u64; +pub type Tick = u64; /// A clock which allows querying of the current tick as well as /// waiting for a tick to be reached. -pub(crate) trait Clock { +pub trait Clock { /// Yields the current tick. fn tick_now(&self) -> Tick; @@ -61,6 +61,7 @@ impl ClockExt for C { } /// A clock which uses the actual underlying system clock. +#[derive(Clone)] pub struct SystemClock; impl Clock for SystemClock { @@ -93,11 +94,20 @@ fn tick_to_time(tick: Tick) -> SystemTime { } /// assumes `slot_duration_millis` evenly divided by tick duration. -pub(crate) fn slot_number_to_tick(slot_duration_millis: u64, slot: Slot) -> Tick { +pub fn slot_number_to_tick(slot_duration_millis: u64, slot: Slot) -> Tick { let ticks_per_slot = slot_duration_millis / TICK_DURATION_MILLIS; u64::from(slot) * ticks_per_slot } +pub fn tick_to_slot_number(slot_duration_millis: u64, tick: Tick) -> Slot { + let ticks_per_slot = slot_duration_millis / TICK_DURATION_MILLIS; + (tick / ticks_per_slot).into() +} + +pub fn tranche_to_tick(slot_duration_millis: u64, slot: Slot, tranche: u32) -> Tick { + slot_number_to_tick(slot_duration_millis, slot) + tranche as u64 +} + /// A list of delayed futures that gets triggered when the waiting time has expired and it is /// time to sign the candidate. /// We have a timer per relay-chain block. diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 019d30524172..897eff99acb1 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -14,9 +14,13 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use parity_scale_codec::{Decode, Encode}; use serde::{Deserialize, Serialize}; use std::{ + alloc::System, collections::{BTreeMap, HashMap, HashSet}, + fs, + io::Write, sync::{ atomic::{AtomicBool, AtomicU32, AtomicU64}, Arc, @@ -25,16 +29,19 @@ use std::{ }; use colored::Colorize; -use futures::{channel::oneshot, lock::Mutex, FutureExt}; +use futures::{channel::oneshot, lock::Mutex, Future, FutureExt, StreamExt}; use itertools::Itertools; use orchestra::TimeoutExt; -use overseer::{metrics::Metrics as OverseerMetrics, MetricsTrait}; +use overseer::{messages, metrics::Metrics as OverseerMetrics, MetricsTrait}; use polkadot_approval_distribution::{ metrics::Metrics as ApprovalDistributionMetrics, ApprovalDistribution, }; use polkadot_node_core_approval_voting::{ criteria::{compute_assignments, Config}, - time::{ClockExt, SystemClock}, + time::{ + slot_number_to_tick, tick_to_slot_number, tranche_to_tick, Clock, ClockExt, SystemClock, + Tick, + }, ApprovalVotingSubsystem, Metrics as ApprovalVotingMetrics, }; use polkadot_node_primitives::approval::{ @@ -105,6 +112,7 @@ use crate::{ use tokio::time::sleep; +mod message_generator; mod mock_chain_api; mod mock_chain_selection; mod mock_runtime_api; @@ -231,7 +239,7 @@ impl ApprovalTestState { let delta_to_first_slot_under_test = Timestamp::new(BUFFER_FOR_GENERATION_MILLIS); let initial_slot = Slot::from_timestamp( - Timestamp::current() + delta_to_first_slot_under_test, + (*Timestamp::current() - *delta_to_first_slot_under_test).into(), SlotDuration::from_millis(SLOT_DURATION_MILLIS), ); @@ -340,6 +348,74 @@ impl ApprovalTestState { peer_message_source.generate_messages(&spawn_task_handle); } } + + /// Starts the generation of messages(Assignments & Approvals) needed for approving blocks. + fn generate_messages( + &mut self, + network_emulator: &NetworkEmulator, + overseer_handle: OverseerHandleReal, + spawn_task_handle: &SpawnTaskHandle, + ) { + gum::info!(target: LOG_TARGET, "Generate messages"); + + let topology = generate_topology(&self.test_authorities); + + let topology_node_under_test = + topology.compute_grid_neighbors_for(ValidatorIndex(NODE_UNDER_TEST)).unwrap(); + let (tx, mut rx) = futures::channel::mpsc::unbounded(); + for current_validator_index in 1..self.test_authorities.keyrings.len() { + let peer_message_source = message_generator::PeerMessagesGenerator { + topology_node_under_test: topology_node_under_test.clone(), + topology: topology.clone(), + validator_index: ValidatorIndex(current_validator_index as u32), + network: network_emulator.clone(), + overseer_handle: overseer_handle.clone(), + state: self.clone(), + options: self.options.clone(), + tx_messages: tx.clone(), + }; + + peer_message_source.generate_messages(&spawn_task_handle); + } + std::mem::drop(tx); + + let mut file = fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(format!("/tmp/all_messages").as_str()) + .unwrap(); + + let mut all_messages: BTreeMap> = BTreeMap::new(); + + loop { + match rx.try_next() { + Ok(Some((block_hash, messages))) => + for message in messages { + let block_info = self.get_info_by_hash(block_hash); + let tick_to_send = + tranche_to_tick(SLOT_DURATION_MILLIS, block_info.slot, message.tranche); + if all_messages.contains_key(&tick_to_send) { + all_messages.get_mut(&tick_to_send).unwrap().push(message); + } else { + all_messages.insert(tick_to_send, vec![message]); + } + }, + Ok(None) => break, + Err(_) => { + std::thread::sleep(Duration::from_millis(50)); + }, + } + } + let mut count = 0; + for message in all_messages { + for message in message.1 { + count += 1; + file.write_all(&message.serialize()); + } + } + gum::info!("Took something like {:}", count); + } } impl ApprovalTestState { @@ -368,7 +444,8 @@ impl ApprovalTestState { } /// Type of generated messages. -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, Encode, Decode, PartialEq, Eq)] + enum MessageType { Approval, Assignment, @@ -391,6 +468,22 @@ struct TestMessage { typ: MessageType, } +#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +struct TestMessage2 { + /// The actual message + msg: protocol_v3::ApprovalDistributionMessage, + /// The list of peers that would sends this message in a real topology. + /// It includes both the peers that would send the message because of the topology + /// or because of randomly chosing so. + sent_by: Vec, + /// The tranche at which this message should be sent. + tranche: u32, + /// The block hash this message refers to. + block_hash: Hash, + /// The type of the message. + typ: MessageType, +} + impl TestMessage { /// Returns the lantency based on the message type. fn get_latency(&self) -> Option { @@ -468,6 +561,29 @@ impl TestMessage { } result } + + fn serialize(&self) -> Vec { + match &self.msg { + ApprovalDistributionMessage::NewBlocks(_) => todo!(), + ApprovalDistributionMessage::DistributeAssignment(_, _) => todo!(), + ApprovalDistributionMessage::DistributeApproval(_) => todo!(), + ApprovalDistributionMessage::NetworkBridgeUpdate(msg) => match msg { + NetworkBridgeEvent::PeerConnected(_, _, _, _) => todo!(), + NetworkBridgeEvent::PeerDisconnected(_) => todo!(), + NetworkBridgeEvent::NewGossipTopology(_) => todo!(), + NetworkBridgeEvent::PeerMessage(peer, msg) => match msg { + Versioned::V1(_) => todo!(), + Versioned::V2(_) => todo!(), + Versioned::V3(msg) => msg.encode(), + }, + NetworkBridgeEvent::PeerViewChange(_, _) => todo!(), + NetworkBridgeEvent::OurViewChange(_) => todo!(), + NetworkBridgeEvent::UpdatedAuthorityIds(_, _) => todo!(), + }, + ApprovalDistributionMessage::GetApprovalSignatures(_, _) => todo!(), + ApprovalDistributionMessage::ApprovalCheckingLagUpdate(_) => todo!(), + } + } } /// A generator of messages coming from a given Peer/Validator @@ -496,15 +612,19 @@ impl PeerMessagesGenerator { spawn_task_handle.spawn_blocking("generate-messages", "generate-messages", async move { let mut messages_to_send = Vec::new(); let mut already_generated = HashSet::new(); - let system_clock = SystemClock {}; + let real_system_clock = SystemClock {}; + + let system_clock = FakeSystemClock::new( + SystemClock {}, + real_system_clock.tick_now() - + slot_number_to_tick(SLOT_DURATION_MILLIS, self.state.initial_slot), + ); loop { sleep(Duration::from_millis(50)).await; - let current_slot = Slot::from_timestamp( - Timestamp::current(), - SlotDuration::from_millis(SLOT_DURATION_MILLIS), - ); + let current_slot = + tick_to_slot_number(SLOT_DURATION_MILLIS, system_clock.tick_now()); let block_info = self.state.get_info_by_slot(current_slot).map(|block| block.clone()); @@ -577,7 +697,16 @@ impl PeerMessagesGenerator { &self.options, &mut rand_chacha, ); - + let mut file = fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(format!("/tmp/validator{}", self.validator_index.0).as_str()) + .unwrap(); + for assignment in assignments.iter().chain(approvals.iter()) { + let bytes = assignment.serialize(); + file.write_all(&bytes); + } let generated_assignments = assignments.into_iter().peekable(); let approvals = approvals.into_iter().peekable(); @@ -591,10 +720,8 @@ impl PeerMessagesGenerator { // Messages are sorted per block and per tranches, so earlier blocks would be // at the front of messages_to_send, so we always prefer to send all messages // we can send for older blocks. - let current_slot = Slot::from_timestamp( - Timestamp::current(), - SlotDuration::from_millis(SLOT_DURATION_MILLIS), - ); + let current_slot = + tick_to_slot_number(SLOT_DURATION_MILLIS, system_clock.tick_now()); for message_to_send in messages_to_send.iter_mut() { if message_to_send @@ -1315,12 +1442,19 @@ fn build_overseer( let db: polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter = polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[]); let keystore = LocalKeystore::in_memory(); + let real_system_clock = SystemClock {}; + let system_clock = FakeSystemClock::new( + SystemClock {}, + real_system_clock.tick_now() - + slot_number_to_tick(SLOT_DURATION_MILLIS, state.initial_slot), + ); let approval_voting = ApprovalVotingSubsystem::with_config( TEST_CONFIG, Arc::new(db), Arc::new(keystore), Box::new(TestSyncOracle {}), state.approval_voting_metrics.clone(), + Box::new(system_clock), ); let approval_distribution = ApprovalDistribution::new( @@ -1370,12 +1504,17 @@ fn prepare_test_inner( let (overseer, overseer_handle) = build_overseer(&state, &network, &config, &dependencies); - state.start_message_generation( + // state.start_message_generation( + // &network, + // overseer_handle.clone(), + // &dependencies.task_manager.spawn_handle(), + // ); + + state.generate_messages( &network, overseer_handle.clone(), &dependencies.task_manager.spawn_handle(), ); - (TestEnvironment::new(dependencies, config, network, overseer, overseer_handle), state) } @@ -1402,19 +1541,21 @@ pub async fn bench_approvals(env: &mut TestEnvironment, state: ApprovalTestState let start_marker = Instant::now(); + let real_system_clock = SystemClock {}; + let system_clock = FakeSystemClock::new( + SystemClock {}, + real_system_clock.tick_now() - + slot_number_to_tick(SLOT_DURATION_MILLIS, state.initial_slot), + ); + for block_num in 0..env.config().num_blocks { - let mut current_slot = Slot::from_timestamp( - Timestamp::current(), - SlotDuration::from_millis(SLOT_DURATION_MILLIS), - ); + gum::info!("TICK NOW === {:?}", system_clock.tick_now()); + let mut current_slot = tick_to_slot_number(SLOT_DURATION_MILLIS, system_clock.tick_now()); // Wait untill the time arrieves at the first slot under test. while current_slot < state.initial_slot { sleep(Duration::from_millis(5)).await; - current_slot = Slot::from_timestamp( - Timestamp::current(), - SlotDuration::from_millis(SLOT_DURATION_MILLIS), - ); + current_slot = tick_to_slot_number(SLOT_DURATION_MILLIS, system_clock.tick_now()); } gum::info!(target: LOG_TARGET, "Current block {}/{}", block_num + 1, env.config().num_blocks); @@ -1426,14 +1567,22 @@ pub async fn bench_approvals(env: &mut TestEnvironment, state: ApprovalTestState .await; } - let block_time_delta = Duration::from_millis( - (*current_slot + 1) * SLOT_DURATION_MILLIS - Timestamp::current().as_millis(), - ); + // let block_time_delta = Duration::from_millis( + // (*current_slot + 1) * SLOT_DURATION_MILLIS - Timestamp::current().as_millis(), + // ); let block_time = Instant::now().sub(block_start_ts).as_millis() as u64; env.metrics().set_block_time(block_time); gum::info!("Block time {}", format!("{:?}ms", block_time).cyan()); - gum::info!(target: LOG_TARGET,"{}", format!("Sleeping till end of block ({}ms)", block_time_delta.as_millis()).bright_black()); - tokio::time::sleep(block_time_delta).await; + // gum::info!(target: LOG_TARGET,"{}", format!("Sleeping till end of block ({}ms)", + // block_time_delta.as_millis()).bright_black()); tokio::time::sleep(block_time_delta). + // await; + gum::info!( + "Next slot tick {:?}", + slot_number_to_tick(SLOT_DURATION_MILLIS, current_slot + 1) + ); + system_clock + .wait(slot_number_to_tick(SLOT_DURATION_MILLIS, current_slot + 1)) + .await; } // Wait for all blocks to be approved before exiting. @@ -1503,3 +1652,28 @@ pub async fn bench_approvals(env: &mut TestEnvironment, state: ApprovalTestState gum::info!("{}", &env); } + +struct FakeSystemClock { + real_system_clock: SystemClock, + delta_ticks: Tick, +} + +impl FakeSystemClock { + fn new(real_system_clock: SystemClock, delta_ticks: Tick) -> Self { + gum::info!("Delta tick is {:}", delta_ticks); + FakeSystemClock { real_system_clock, delta_ticks } + } +} + +impl Clock for FakeSystemClock { + fn tick_now(&self) -> Tick { + self.real_system_clock.tick_now() - self.delta_ticks + } + + fn wait( + &self, + tick: Tick, + ) -> std::pin::Pin + Send + 'static>> { + self.real_system_clock.wait(tick + self.delta_ticks) + } +} diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 07b2c1bd1f4d..7a8fefe60ea6 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -115,9 +115,9 @@ impl BenchCli { let (mut env, state) = approval::prepare_test(test_config.clone(), options.clone()); - env.runtime().block_on(async { - bench_approvals(&mut env, state).await; - }); + // env.runtime().block_on(async { + // bench_approvals(&mut env, state).await; + // }); }, TestObjective::TestSequence(_) => todo!(), } From 1ed76206da5e529537725313c9f7200b0b464225 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 14 Dec 2023 11:00:52 +0200 Subject: [PATCH 149/192] Approval coalescing serialization Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 2 +- .../node/core/approval-voting/src/time.rs | 2 +- .../src/approval/mock_chain_selection.rs | 12 +- .../node/subsystem-bench/src/approval/mod.rs | 631 ++++++++---------- .../subsystem-bench/src/core/environment.rs | 8 + .../subsystem-bench/src/subsystem-bench.rs | 6 +- 6 files changed, 297 insertions(+), 364 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index a36800236074..dcc575ffaf0d 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -114,7 +114,7 @@ const APPROVAL_CHECKING_TIMEOUT: Duration = Duration::from_secs(120); /// /// Value rather arbitrarily: Should not be hit in practice, it exists to more easily diagnose dead /// lock issues for example. -const WAIT_FOR_SIGS_TIMEOUT: Duration = Duration::from_millis(500); +const WAIT_FOR_SIGS_TIMEOUT: Duration = Duration::from_millis(2000); const APPROVAL_CACHE_SIZE: u32 = 1024; const TICK_TOO_FAR_IN_FUTURE: Tick = 20; // 10 seconds. diff --git a/polkadot/node/core/approval-voting/src/time.rs b/polkadot/node/core/approval-voting/src/time.rs index 15da993c518d..bb17c6bfeb47 100644 --- a/polkadot/node/core/approval-voting/src/time.rs +++ b/polkadot/node/core/approval-voting/src/time.rs @@ -33,7 +33,7 @@ use std::{ }; use polkadot_primitives::{Hash, ValidatorIndex}; -const TICK_DURATION_MILLIS: u64 = 500; +pub const TICK_DURATION_MILLIS: u64 = 500; /// A base unit of time, starting from the Unix epoch, split into half-second intervals. pub type Tick = u64; diff --git a/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs b/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs index 986bcf5db78f..c57102620fd8 100644 --- a/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs +++ b/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs @@ -16,8 +16,9 @@ use crate::approval::{LOG_TARGET, SLOT_DURATION_MILLIS}; -use super::ApprovalTestState; +use super::{ApprovalTestState, FakeSystemClock}; use futures::FutureExt; +use polkadot_node_core_approval_voting::time::{slot_number_to_tick, Clock, TICK_DURATION_MILLIS}; use polkadot_node_subsystem::{overseer, SpawnedSubsystem, SubsystemError}; use polkadot_node_subsystem_types::messages::ChainSelectionMessage; use sp_timestamp::Timestamp; @@ -26,6 +27,7 @@ use sp_timestamp::Timestamp; /// during benchmark. All the necessary information to answer the requests is stored in the `state` pub struct MockChainSelection { pub state: ApprovalTestState, + pub clock: FakeSystemClock, } #[overseer::subsystem(ChainSelection, error=SubsystemError, prefix=self::overseer)] impl MockChainSelection { @@ -52,9 +54,11 @@ impl MockChainSelection { self.state .last_approved_block .store(approved_number, std::sync::atomic::Ordering::SeqCst); - let passed_since_slot_start = Timestamp::current().as_millis() - - *block_info.slot * SLOT_DURATION_MILLIS; - gum::info!(target: LOG_TARGET, ?hash, "Chain selection approved after {:} ms", passed_since_slot_start); + + let approved_in_tick = self.clock.tick_now() - + slot_number_to_tick(SLOT_DURATION_MILLIS, block_info.slot); + + gum::info!(target: LOG_TARGET, ?hash, "Chain selection approved after {:} ms", approved_in_tick * TICK_DURATION_MILLIS); }, _ => {}, }, diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 897eff99acb1..856740564b73 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -20,7 +20,7 @@ use std::{ alloc::System, collections::{BTreeMap, HashMap, HashSet}, fs, - io::Write, + io::{Read, Write}, sync::{ atomic::{AtomicBool, AtomicU32, AtomicU64}, Arc, @@ -78,7 +78,7 @@ use polkadot_primitives::{ }; use polkadot_primitives_test_helpers::dummy_candidate_receipt_bad_sig; use sc_keystore::LocalKeystore; -use sc_network::PeerId; +use sc_network::{network_state::Peer, PeerId}; use sc_service::SpawnTaskHandle; use sp_consensus_babe::{ digests::{CompatibleDigestItem, PreDigest, SecondaryVRFPreDigest}, @@ -126,6 +126,8 @@ pub(crate) const TEST_CONFIG: ApprovalVotingConfig = ApprovalVotingConfig { slot_duration_millis: SLOT_DURATION_MILLIS, }; +const MESSAGES_PATH: &str = "/home/alexggh/messages_tmp/all_messages_shuffled_500"; + pub const NODE_UNDER_TEST: u32 = 0; /// Start generating messages for a slot into the future, so that the @@ -159,6 +161,9 @@ pub struct ApprovalsOptions { #[clap(short, long, default_value_t = true)] /// Sends messages only till block is approved. pub stop_when_approved: bool, + #[clap(short, long, default_value_t = true)] + /// Only generate the messages + pub generate_only: bool, } /// Information about a block. It is part of test state and it is used by the mock @@ -204,13 +209,8 @@ pub struct ApprovalTestState { babe_epoch: BabeEpoch, /// The session info used during testing. session_info: SessionInfo, - /// An array of pre-generated random samplings, that is used to determine, which nodes would - /// send a given assignment, to the node under test because of the random samplings. - /// As an optimization we generate this sampling at the begining of the test and just pick - /// one randomly, because always taking the samples would be too expensive for benchamrk. - random_samplings: Vec>, /// The slot at which this benchamrk begins. - initial_slot: Slot, + generated_state: GeneratedState, /// The test authorities test_authorities: TestAuthorities, /// Last approved block number. @@ -230,29 +230,45 @@ impl ApprovalTestState { dependencies: &TestEnvironmentDependencies, ) -> Self { let test_authorities = configuration.generate_authorities(); + let start = Instant::now(); - let random_samplings = random_samplings_to_node_patterns( - ValidatorIndex(NODE_UNDER_TEST), - test_authorities.keyrings.len(), - test_authorities.keyrings.len() as usize * 2, - ); + let generated_state = if !options.generate_only { + let mut file = fs::OpenOptions::new().read(true).open(MESSAGES_PATH).unwrap(); + let mut content = Vec::::with_capacity(2000000); - let delta_to_first_slot_under_test = Timestamp::new(BUFFER_FOR_GENERATION_MILLIS); - let initial_slot = Slot::from_timestamp( - (*Timestamp::current() - *delta_to_first_slot_under_test).into(), - SlotDuration::from_millis(SLOT_DURATION_MILLIS), - ); + file.read_to_end(&mut content); + let mut content = content.as_slice(); + let generated_state: GeneratedState = + Decode::decode(&mut content).expect("Could not decode messages"); - let babe_epoch = generate_babe_epoch(initial_slot, test_authorities.clone()); + gum::info!( + "It took {:?} ms count {:?}", + start.elapsed().as_millis(), + generated_state.all_messages.as_ref().map(|val| val.len()).unwrap_or_default() + ); + generated_state + } else { + let delta_to_first_slot_under_test = Timestamp::new(BUFFER_FOR_GENERATION_MILLIS); + + GeneratedState { + initial_slot: Slot::from_timestamp( + (*Timestamp::current() - *delta_to_first_slot_under_test).into(), + SlotDuration::from_millis(SLOT_DURATION_MILLIS), + ), + all_messages: Default::default(), + } + }; + + let babe_epoch = + generate_babe_epoch(generated_state.initial_slot, test_authorities.clone()); let session_info = session_info_for_peers(configuration, test_authorities.clone()); let mut state = ApprovalTestState { per_slot_heads: Default::default(), babe_epoch: babe_epoch.clone(), session_info: session_info.clone(), - random_samplings, configuration: configuration.clone(), - initial_slot, + generated_state, test_authorities, last_approved_block: Arc::new(AtomicU32::new(0)), total_sent_messages: Arc::new(AtomicU64::new(0)), @@ -277,7 +293,7 @@ impl ApprovalTestState { .last() .map(|val| val.hash) .unwrap_or(Hash::repeat_byte(0xde)); - let slot_for_block = self.initial_slot + (block_number as u64 - 1); + let slot_for_block = self.generated_state.initial_slot + (block_number as u64 - 1); let header = make_header(parent_hash, slot_for_block, block_number as u32); @@ -321,37 +337,38 @@ impl ApprovalTestState { } /// Starts the generation of messages(Assignments & Approvals) needed for approving blocks. - fn start_message_generation( + async fn start_message_production( &mut self, network_emulator: &NetworkEmulator, overseer_handle: OverseerHandleReal, spawn_task_handle: &SpawnTaskHandle, - ) { - gum::info!(target: LOG_TARGET, "Start assignments/approvals generation"); + ) -> oneshot::Receiver<()> { + gum::info!(target: LOG_TARGET, "Start assignments/approvals production"); let topology = generate_topology(&self.test_authorities); let topology_node_under_test = topology.compute_grid_neighbors_for(ValidatorIndex(NODE_UNDER_TEST)).unwrap(); - for current_validator_index in 1..self.test_authorities.keyrings.len() { - let peer_message_source = PeerMessagesGenerator { - topology_node_under_test: topology_node_under_test.clone(), - topology: topology.clone(), - validator_index: ValidatorIndex(current_validator_index as u32), - network: network_emulator.clone(), - overseer_handle: overseer_handle.clone(), - state: self.clone(), - options: self.options.clone(), - }; + let (producer_tx, producer_rx) = oneshot::channel(); + let peer_message_source = PeerMessageProducer { + network: network_emulator.clone(), + overseer_handle: overseer_handle.clone(), + state: self.clone(), + options: self.options.clone(), + notify_done: producer_tx, + }; - peer_message_source.generate_messages(&spawn_task_handle); - } + peer_message_source.produce_messages( + &spawn_task_handle, + self.generated_state.all_messages.take().unwrap(), + ); + producer_rx } /// Starts the generation of messages(Assignments & Approvals) needed for approving blocks. fn generate_messages( - &mut self, + &self, network_emulator: &NetworkEmulator, overseer_handle: OverseerHandleReal, spawn_task_handle: &SpawnTaskHandle, @@ -360,6 +377,12 @@ impl ApprovalTestState { let topology = generate_topology(&self.test_authorities); + let random_samplings = random_samplings_to_node_patterns( + ValidatorIndex(NODE_UNDER_TEST), + self.test_authorities.keyrings.len(), + self.test_authorities.keyrings.len() as usize * 2, + ); + let topology_node_under_test = topology.compute_grid_neighbors_for(ValidatorIndex(NODE_UNDER_TEST)).unwrap(); let (tx, mut rx) = futures::channel::mpsc::unbounded(); @@ -373,20 +396,23 @@ impl ApprovalTestState { state: self.clone(), options: self.options.clone(), tx_messages: tx.clone(), + random_samplings: random_samplings.clone(), }; peer_message_source.generate_messages(&spawn_task_handle); } std::mem::drop(tx); - + let seed = [0x32; 32]; + let mut rand_chacha = ChaCha20Rng::from_seed(seed); let mut file = fs::OpenOptions::new() .write(true) .create(true) .truncate(true) - .open(format!("/tmp/all_messages").as_str()) + .open(MESSAGES_PATH) .unwrap(); - let mut all_messages: BTreeMap> = BTreeMap::new(); + let mut all_messages: BTreeMap, Vec)> = + BTreeMap::new(); loop { match rx.try_next() { @@ -395,10 +421,15 @@ impl ApprovalTestState { let block_info = self.get_info_by_hash(block_hash); let tick_to_send = tranche_to_tick(SLOT_DURATION_MILLIS, block_info.slot, message.tranche); - if all_messages.contains_key(&tick_to_send) { - all_messages.get_mut(&tick_to_send).unwrap().push(message); - } else { - all_messages.insert(tick_to_send, vec![message]); + if !all_messages.contains_key(&tick_to_send) { + all_messages.insert(tick_to_send, (Vec::new(), Vec::new())); + } + + let to_add = all_messages.get_mut(&tick_to_send).unwrap(); + match message.typ { + MessageType::Approval => to_add.1.push(message), + MessageType::Assignment => to_add.0.push(message), + MessageType::Other => todo!(), } }, Ok(None) => break, @@ -407,16 +438,30 @@ impl ApprovalTestState { }, } } - let mut count = 0; - for message in all_messages { - for message in message.1 { - count += 1; - file.write_all(&message.serialize()); - } - } - gum::info!("Took something like {:}", count); + let all_messages = all_messages + .into_iter() + .map(|(_, (mut assignments, mut approvals))| { + assignments.shuffle(&mut rand_chacha); + approvals.shuffle(&mut rand_chacha); + assignments.into_iter().chain(approvals.into_iter()) + }) + .flatten() + .collect_vec(); + gum::info!("Took something like {:}", all_messages.len()); + + let generated_state = GeneratedState { + all_messages: Some(all_messages), + initial_slot: self.generated_state.initial_slot, + }; + + file.write_all(&generated_state.encode()); } } +#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +struct GeneratedState { + all_messages: Option>, + initial_slot: Slot, +} impl ApprovalTestState { /// Returns test data for the given hash @@ -452,24 +497,8 @@ enum MessageType { Other, } -/// A test message generated by the `PeerMessagesGenerator` -struct TestMessage { - /// The actual message - msg: ApprovalDistributionMessage, - /// The list of peers that would sends this message in a real topology. - /// It includes both the peers that would send the message because of the topology - /// or because of randomly chosing so. - sent_by: HashSet<(ValidatorIndex, PeerId)>, - /// The tranche at which this message should be sent. - tranche: u32, - /// The block hash this message refers to. - block_hash: Hash, - /// The type of the message. - typ: MessageType, -} - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -struct TestMessage2 { +struct TestMessageInfo { /// The actual message msg: protocol_v3::ApprovalDistributionMessage, /// The list of peers that would sends this message in a real topology. @@ -484,8 +513,14 @@ struct TestMessage2 { typ: MessageType, } -impl TestMessage { +impl TestMessageInfo { /// Returns the lantency based on the message type. + fn to_all_messages_from_peer(self, peer: PeerId) -> AllMessages { + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::PeerMessage(peer, Versioned::V3(self.msg)), + )) + } + fn get_latency(&self) -> Option { match &self.typ { // We want assignments to always arrive before approval, so @@ -499,142 +534,91 @@ impl TestMessage { fn record_vote(&self, state: &BlockTestData) { if let MessageType::Approval = self.typ { match &self.msg { - ApprovalDistributionMessage::NewBlocks(_) => todo!(), - ApprovalDistributionMessage::DistributeAssignment(_, _) => todo!(), - ApprovalDistributionMessage::DistributeApproval(_) => todo!(), - ApprovalDistributionMessage::NetworkBridgeUpdate(msg) => match msg { - NetworkBridgeEvent::PeerConnected(_, _, _, _) => todo!(), - NetworkBridgeEvent::PeerDisconnected(_) => todo!(), - NetworkBridgeEvent::NewGossipTopology(_) => todo!(), - NetworkBridgeEvent::PeerMessage(peer, msg) => match msg { - Versioned::V1(_) => todo!(), - Versioned::V2(_) => todo!(), - Versioned::V3(msg) => match msg { - protocol_v3::ApprovalDistributionMessage::Assignments(_) => todo!(), - protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => - for approval in approvals { - for candidate_index in approval.candidate_indices.iter_ones() { - state - .votes - .get(approval.validator.0 as usize) - .unwrap() - .get(candidate_index) - .unwrap() - .store(true, std::sync::atomic::Ordering::SeqCst); - } - }, - }, + protocol_v3::ApprovalDistributionMessage::Assignments(_) => todo!(), + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => + for approval in approvals { + for candidate_index in approval.candidate_indices.iter_ones() { + state + .votes + .get(approval.validator.0 as usize) + .unwrap() + .get(candidate_index) + .unwrap() + .store(true, std::sync::atomic::Ordering::SeqCst); + } }, - NetworkBridgeEvent::PeerViewChange(_, _) => todo!(), - NetworkBridgeEvent::OurViewChange(_) => todo!(), - NetworkBridgeEvent::UpdatedAuthorityIds(_, _) => todo!(), - }, - ApprovalDistributionMessage::GetApprovalSignatures(_, _) => todo!(), - ApprovalDistributionMessage::ApprovalCheckingLagUpdate(_) => todo!(), } } } /// Splits a message into multiple messages based on what peers should send this message. /// It build a HashMap of messages that should be sent by each peer. - fn split_by_peer_id(self) -> HashMap<(ValidatorIndex, PeerId), Vec> { - let mut result: HashMap<(ValidatorIndex, PeerId), Vec> = HashMap::new(); - - for peer in &self.sent_by { - match &self.msg { - ApprovalDistributionMessage::NetworkBridgeUpdate(msg) => { - result.entry(*peer).or_default().push(TestMessage { - msg: ApprovalDistributionMessage::NetworkBridgeUpdate(match msg { - NetworkBridgeEvent::PeerMessage(_, msg) => - NetworkBridgeEvent::PeerMessage(peer.1, msg.clone()), - NetworkBridgeEvent::OurViewChange(_) => todo!(), - _ => todo!(), - }), - sent_by: Default::default(), - tranche: self.tranche, - block_hash: self.block_hash, - typ: self.typ, - }); - }, - _ => {}, - } + fn split_by_peer_id( + self, + authorities: &TestAuthorities, + ) -> HashMap<(ValidatorIndex, PeerId), Vec> { + let mut result: HashMap<(ValidatorIndex, PeerId), Vec> = HashMap::new(); + + for validator_index in &self.sent_by { + let peer = authorities.peer_ids.get(validator_index.0 as usize).unwrap(); + result.entry((*validator_index, *peer)).or_default().push(TestMessageInfo { + msg: self.msg.clone(), + sent_by: Default::default(), + tranche: self.tranche, + block_hash: self.block_hash, + typ: self.typ, + }); } result } - - fn serialize(&self) -> Vec { - match &self.msg { - ApprovalDistributionMessage::NewBlocks(_) => todo!(), - ApprovalDistributionMessage::DistributeAssignment(_, _) => todo!(), - ApprovalDistributionMessage::DistributeApproval(_) => todo!(), - ApprovalDistributionMessage::NetworkBridgeUpdate(msg) => match msg { - NetworkBridgeEvent::PeerConnected(_, _, _, _) => todo!(), - NetworkBridgeEvent::PeerDisconnected(_) => todo!(), - NetworkBridgeEvent::NewGossipTopology(_) => todo!(), - NetworkBridgeEvent::PeerMessage(peer, msg) => match msg { - Versioned::V1(_) => todo!(), - Versioned::V2(_) => todo!(), - Versioned::V3(msg) => msg.encode(), - }, - NetworkBridgeEvent::PeerViewChange(_, _) => todo!(), - NetworkBridgeEvent::OurViewChange(_) => todo!(), - NetworkBridgeEvent::UpdatedAuthorityIds(_, _) => todo!(), - }, - ApprovalDistributionMessage::GetApprovalSignatures(_, _) => todo!(), - ApprovalDistributionMessage::ApprovalCheckingLagUpdate(_) => todo!(), - } - } } /// A generator of messages coming from a given Peer/Validator -struct PeerMessagesGenerator { +struct PeerMessageProducer { /// The state state used to know what messages to generate. state: ApprovalTestState, /// Configuration options, passed at the beginning of the test. options: ApprovalsOptions, - /// The grid neighbors of the node under test. - topology_node_under_test: GridNeighbors, - /// The topology of the network for the epoch under test. - topology: SessionGridTopology, - /// The validator index for this object generates the messages. - validator_index: ValidatorIndex, /// A reference to the network emulator network: NetworkEmulator, /// A handle to the overseer, used for sending messages to the node /// under test. overseer_handle: OverseerHandleReal, + notify_done: oneshot::Sender<()>, } -impl PeerMessagesGenerator { +impl PeerMessageProducer { /// Generates messages by spawning a blocking task in the background which begins creating /// the assignments/approvals and peer view changes at the begining of each block. - fn generate_messages(mut self, spawn_task_handle: &SpawnTaskHandle) { - spawn_task_handle.spawn_blocking("generate-messages", "generate-messages", async move { - let mut messages_to_send = Vec::new(); + fn produce_messages( + mut self, + spawn_task_handle: &SpawnTaskHandle, + all_messages: Vec, + ) { + spawn_task_handle.spawn_blocking("produce-messages", "produce-messages", async move { let mut already_generated = HashSet::new(); let real_system_clock = SystemClock {}; - let system_clock = FakeSystemClock::new( SystemClock {}, real_system_clock.tick_now() - - slot_number_to_tick(SLOT_DURATION_MILLIS, self.state.initial_slot), + slot_number_to_tick( + SLOT_DURATION_MILLIS, + self.state.generated_state.initial_slot, + ), ); - - loop { - sleep(Duration::from_millis(50)).await; - + let mut all_messages = all_messages.into_iter().peekable(); + while all_messages.peek().is_some() { let current_slot = tick_to_slot_number(SLOT_DURATION_MILLIS, system_clock.tick_now()); - let block_info = self.state.get_info_by_slot(current_slot).map(|block| block.clone()); if let Some(block_info) = block_info { let candidates = self.state.approval_voting_metrics.candidates_imported(); - if candidates >= block_info.prev_candidates + block_info.candidates.len() as u64 && already_generated.insert(block_info.hash) { + gum::info!("Setup {:?}", block_info.hash); let (tx, rx) = oneshot::channel(); self.overseer_handle.wait_for_activation(block_info.hash, tx).await; @@ -642,139 +626,88 @@ impl PeerMessagesGenerator { .expect("We should not fail waiting for block to be activated") .expect("We should not fail waiting for block to be activated"); - let peer_id = self - .state - .test_authorities - .peer_ids - .get(self.validator_index.0 as usize) - .unwrap(); - - let view_update = generate_peer_view_change_for( - block_info.hash, - *peer_id, - self.validator_index, - ); + for validator in 1..self.state.test_authorities.keyrings.len() as u32 { + let peer_id = self + .state + .test_authorities + .peer_ids + .get(validator as usize) + .unwrap(); + let validator = ValidatorIndex(validator); - self.send_message(view_update, self.validator_index, None); + let view_update = + generate_peer_view_change_for(block_info.hash, *peer_id, validator); - let assignments = generate_assignments( - &block_info, - self.state - .test_authorities - .keyrings - .clone() - .into_iter() - .zip(self.state.test_authorities.peer_ids.clone().into_iter()) - .collect_vec(), - &self.state.session_info, - self.options.enable_assignments_v2, - &self.state.random_samplings, - self.validator_index.0, - &block_info.relay_vrf_story, - &self.topology_node_under_test, - &self.topology, - self.options.last_considered_tranche, - ); - - let bytes = self.validator_index.0.to_be_bytes(); - let seed = [ - bytes[0], bytes[1], bytes[2], bytes[3], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]; - - let mut rand_chacha = ChaCha20Rng::from_seed(seed); - let approvals = issue_approvals( - &assignments, - block_info.hash, - self.state - .test_authorities - .keyrings - .clone() - .into_iter() - .zip(self.state.test_authorities.peer_ids.clone().into_iter()) - .collect_vec(), - block_info.candidates.clone(), - &self.options, - &mut rand_chacha, - ); - let mut file = fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(format!("/tmp/validator{}", self.validator_index.0).as_str()) - .unwrap(); - for assignment in assignments.iter().chain(approvals.iter()) { - let bytes = assignment.serialize(); - file.write_all(&bytes); + self.send_message(view_update, validator, None); } - let generated_assignments = assignments.into_iter().peekable(); - let approvals = approvals.into_iter().peekable(); - - messages_to_send.push(generated_assignments); - messages_to_send.push(approvals); } } - loop { - let mut at_least_one_sent = false; - // Messages are sorted per block and per tranches, so earlier blocks would be - // at the front of messages_to_send, so we always prefer to send all messages - // we can send for older blocks. - let current_slot = - tick_to_slot_number(SLOT_DURATION_MILLIS, system_clock.tick_now()); - - for message_to_send in messages_to_send.iter_mut() { - if message_to_send - .peek() - .map(|val| { - let block_info = self.state.get_info_by_hash(val.block_hash); - let tranche_now = - system_clock.tranche_now(SLOT_DURATION_MILLIS, block_info.slot); - let tranche_to_send = match val.typ { - MessageType::Approval => val.tranche + 1, - MessageType::Assignment => val.tranche, - MessageType::Other => val.tranche, - }; - tranche_to_send <= tranche_now && current_slot >= block_info.slot - }) - .unwrap_or_default() + while all_messages + .peek() + .map(|val| { + let block_info = self.state.get_info_by_hash(val.block_hash); + let tranche_now = + system_clock.tranche_now(SLOT_DURATION_MILLIS, block_info.slot); + let tranche_to_send = match val.typ { + MessageType::Approval => val.tranche + 1, + MessageType::Assignment => val.tranche, + MessageType::Other => val.tranche, + }; + tranche_to_send <= tranche_now && + current_slot >= block_info.slot && + already_generated.contains(&block_info.hash) + }) + .unwrap_or_default() + { + let message = all_messages.next().unwrap(); + + let block_info = self.state.get_info_by_hash(message.block_hash); + if !block_info.approved.load(std::sync::atomic::Ordering::SeqCst) || + (!self.options.stop_when_approved && + message.tranche <= self.options.send_till_tranche) + { + message.record_vote(block_info); + for (peer, messages) in + message.split_by_peer_id(&self.state.test_authorities) { - let message = message_to_send.next().unwrap(); - - let block_info = self.state.get_info_by_hash(message.block_hash); - if !block_info.approved.load(std::sync::atomic::Ordering::SeqCst) || - (!self.options.stop_when_approved && - message.tranche <= self.options.send_till_tranche) - { - message.record_vote(block_info); - - for (peer, messages) in message.split_by_peer_id() { - for message in messages { - let latency = message.get_latency(); - self.state - .total_sent_messages - .as_ref() - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - self.send_message(message, peer.0, latency) - } - } + for message in messages { + let latency = message.get_latency(); + self.state + .total_sent_messages + .as_ref() + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.send_message( + message.to_all_messages_from_peer(peer.1), + peer.0, + latency, + ) } - at_least_one_sent = true; - break } } - if !at_least_one_sent { - break - } } } + + gum::info!("All messages sent "); + let (tx, rx) = oneshot::channel(); + let msg = ApprovalDistributionMessage::GetApprovalSignatures(HashSet::new(), tx); + self.overseer_handle + .send_msg(AllMessages::ApprovalDistribution(msg), LOG_TARGET) + .timeout(MAX_TIME_OF_FLIGHT) + .await + .unwrap_or_else(|| { + panic!("{} ms maximum time of flight breached", MAX_TIME_OF_FLIGHT.as_millis()) + }); + rx.await; + self.notify_done.send(()); + gum::info!("All messages processed "); }); } /// Queues a message to be sent by the peer identified by the `sent_by` value. fn send_message( &mut self, - message: TestMessage, + message: AllMessages, sent_by: ValidatorIndex, latency: Option, ) { @@ -791,7 +724,7 @@ impl PeerMessagesGenerator { peer.clone(), async move { overseer_handle - .send_msg(AllMessages::ApprovalDistribution(message.msg), LOG_TARGET) + .send_msg(message, LOG_TARGET) .timeout(MAX_TIME_OF_FLIGHT) .await .unwrap_or_else(|| { @@ -919,13 +852,13 @@ fn coalesce_approvals_len( /// Helper function to create approvals signatures for all assignments passed as arguments. /// Returns a list of Approvals messages that need to be sent. fn issue_approvals( - assignments: &Vec, + assignments: &Vec, block_hash: Hash, keyrings: Vec<(Keyring, PeerId)>, candidates: Vec, options: &ApprovalsOptions, rand_chacha: &mut ChaCha20Rng, -) -> Vec { +) -> Vec { let mut to_sign: Vec = Vec::new(); let mut num_coalesce = coalesce_approvals_len(options.min_coalesce, options.max_coalesce, rand_chacha); @@ -933,10 +866,7 @@ fn issue_approvals( .iter() .enumerate() .map(|(_index, message)| match &message.msg { - ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage( - _, - Versioned::V3(protocol_v3::ApprovalDistributionMessage::Assignments(assignments)), - )) => { + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) => { let mut approvals_to_create = Vec::new(); let current_validator_index = @@ -970,7 +900,7 @@ fn issue_approvals( candidate_hash: candidate.hash(), candidate_index: candidate_index as CandidateIndex, validator_index: assignment.0.validator, - sent_by: message.sent_by.clone(), + sent_by: message.sent_by.clone().into_iter().collect(), tranche: message.tranche, }); let earliest_tranche = @@ -1016,7 +946,7 @@ struct TestSignInfo { candidate_hash: CandidateHash, candidate_index: CandidateIndex, validator_index: ValidatorIndex, - sent_by: HashSet<(ValidatorIndex, PeerId)>, + sent_by: HashSet, tranche: u32, } @@ -1026,7 +956,7 @@ fn sign_candidates( to_sign: &mut Vec, keyrings: &Vec<(Keyring, PeerId)>, block_hash: Hash, -) -> TestMessage { +) -> TestMessageInfo { let current_validator_index = to_sign.first().map(|val| val.validator_index).unwrap(); let tranche_trigger_timestamp = to_sign.iter().map(|val| val.tranche).max().unwrap(); let keyring = keyrings.get(current_validator_index.0 as usize).unwrap().clone(); @@ -1043,7 +973,7 @@ fn sign_candidates( .map(|val| val.sent_by.iter()) .flatten() .map(|peer| *peer) - .collect::>(); + .collect::>(); let payload = ApprovalVoteMultipleCandidates(&hashes).signing_payload(1); @@ -1056,12 +986,9 @@ fn sign_candidates( signature, }; let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![indirect]); - TestMessage { - msg: ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage( - keyring.1, - Versioned::V3(msg), - )), - sent_by, + TestMessageInfo { + msg, + sent_by: sent_by.into_iter().collect_vec(), tranche: tranche_trigger_timestamp, block_hash, typ: MessageType::Approval, @@ -1127,7 +1054,7 @@ fn generate_assignments( topology_node_under_test: &GridNeighbors, topology: &SessionGridTopology, last_considered_tranche: u32, -) -> Vec { +) -> Vec { let config = Config::from(session_info); let leaving_cores = block_info @@ -1253,14 +1180,13 @@ fn generate_assignments( let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( indirect.0, indirect.1, )]); - TestMessage { - msg: ApprovalDistributionMessage::NetworkBridgeUpdate( - NetworkBridgeEvent::PeerMessage( - keyrings[validator_index as usize].1, - Versioned::V3(msg), - ), - ), - sent_by: indirect.2, + TestMessageInfo { + msg, + sent_by: indirect + .2 + .into_iter() + .map(|(validator_index, peer_id)| validator_index) + .collect_vec(), tranche: indirect.3, block_hash: block_info.hash, typ: MessageType::Assignment, @@ -1318,24 +1244,18 @@ fn generate_peer_view_change_for( block_hash: Hash, peer_id: PeerId, validator_index: ValidatorIndex, -) -> TestMessage { +) -> AllMessages { let network = NetworkBridgeEvent::PeerViewChange(peer_id, View::new([block_hash].into_iter(), 0)); - TestMessage { - msg: ApprovalDistributionMessage::NetworkBridgeUpdate(network), - sent_by: [(validator_index, peer_id)].into_iter().collect(), - tranche: 0, - block_hash, - typ: MessageType::Other, - } + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate(network)) } /// Generates peer_connected messages for all peers in `test_authorities` fn generate_peer_connected( test_authorities: &TestAuthorities, block_hash: Hash, -) -> Vec { +) -> Vec { let keyrings = test_authorities .keyrings .clone() @@ -1351,13 +1271,9 @@ fn generate_peer_connected( ProtocolVersion::from(ValidationVersion::V3), None, ); - TestMessage { - msg: ApprovalDistributionMessage::NetworkBridgeUpdate(network), - sent_by: Default::default(), - tranche: 0, - block_hash, - typ: MessageType::Other, - } + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate( + network, + )) }) .collect_vec() } @@ -1390,7 +1306,7 @@ fn generate_topology(test_authorities: &TestAuthorities) -> SessionGridTopology fn generate_new_session_topology( test_authorities: &TestAuthorities, block_hash: Hash, -) -> Vec { +) -> Vec { let topology = generate_topology(test_authorities); let event = NetworkBridgeEvent::NewGossipTopology(NewGossipTopology { @@ -1398,13 +1314,7 @@ fn generate_new_session_topology( topology, local_index: Some(ValidatorIndex(NODE_UNDER_TEST)), // TODO }); - vec![TestMessage { - msg: ApprovalDistributionMessage::NetworkBridgeUpdate(event), - sent_by: Default::default(), - tranche: 0, - block_hash, - typ: MessageType::Other, - }] + vec![AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate(event))] } /// Helper function to generate a babe epoch for this benchmark. @@ -1446,7 +1356,7 @@ fn build_overseer( let system_clock = FakeSystemClock::new( SystemClock {}, real_system_clock.tick_now() - - slot_number_to_tick(SLOT_DURATION_MILLIS, state.initial_slot), + slot_number_to_tick(SLOT_DURATION_MILLIS, state.generated_state.initial_slot), ); let approval_voting = ApprovalVotingSubsystem::with_config( TEST_CONFIG, @@ -1454,14 +1364,14 @@ fn build_overseer( Arc::new(keystore), Box::new(TestSyncOracle {}), state.approval_voting_metrics.clone(), - Box::new(system_clock), + Box::new(system_clock.clone()), ); let approval_distribution = ApprovalDistribution::new( ApprovalDistributionMetrics::try_register(&dependencies.registry).unwrap(), ); let mock_chain_api = MockChainApi { state: state.clone() }; - let mock_chain_selection = MockChainSelection { state: state.clone() }; + let mock_chain_selection = MockChainSelection { state: state.clone(), clock: system_clock }; let mock_runtime_api = MockRuntimeApi { state: state.clone() }; let mock_tx_bridge = MockNetworkBridgeTx::new(config.clone(), state.clone(), network.clone()); let overseer_metrics = OverseerMetrics::try_register(&dependencies.registry).unwrap(); @@ -1504,22 +1414,30 @@ fn prepare_test_inner( let (overseer, overseer_handle) = build_overseer(&state, &network, &config, &dependencies); - // state.start_message_generation( - // &network, - // overseer_handle.clone(), - // &dependencies.task_manager.spawn_handle(), - // ); - - state.generate_messages( - &network, - overseer_handle.clone(), - &dependencies.task_manager.spawn_handle(), - ); (TestEnvironment::new(dependencies, config, network, overseer, overseer_handle), state) } +pub async fn bench_approvals(env: &mut TestEnvironment, mut state: ApprovalTestState) { + if state.options.generate_only { + state.generate_messages(env.network(), env.overseer_handle().clone(), &env.spawn_handle()); + } else { + let producer_rx = state + .start_message_production( + env.network(), + env.overseer_handle().clone(), + &env.spawn_handle(), + ) + .await; + bench_approvals_run(env, state, producer_rx).await + } +} + /// Runs the approval benchmark. -pub async fn bench_approvals(env: &mut TestEnvironment, state: ApprovalTestState) { +pub async fn bench_approvals_run( + env: &mut TestEnvironment, + state: ApprovalTestState, + producer_rx: oneshot::Receiver<()>, +) { let config = env.config().clone(); env.metrics().set_n_validators(config.n_validators); @@ -1536,7 +1454,7 @@ pub async fn bench_approvals(env: &mut TestEnvironment, state: ApprovalTestState state.per_slot_heads.first().unwrap().hash, )); for message in initialization_messages { - env.send_message(AllMessages::ApprovalDistribution(message.msg)).await; + env.send_message(message).await; } let start_marker = Instant::now(); @@ -1545,15 +1463,14 @@ pub async fn bench_approvals(env: &mut TestEnvironment, state: ApprovalTestState let system_clock = FakeSystemClock::new( SystemClock {}, real_system_clock.tick_now() - - slot_number_to_tick(SLOT_DURATION_MILLIS, state.initial_slot), + slot_number_to_tick(SLOT_DURATION_MILLIS, state.generated_state.initial_slot), ); for block_num in 0..env.config().num_blocks { - gum::info!("TICK NOW === {:?}", system_clock.tick_now()); let mut current_slot = tick_to_slot_number(SLOT_DURATION_MILLIS, system_clock.tick_now()); // Wait untill the time arrieves at the first slot under test. - while current_slot < state.initial_slot { + while current_slot < state.generated_state.initial_slot { sleep(Duration::from_millis(5)).await; current_slot = tick_to_slot_number(SLOT_DURATION_MILLIS, system_clock.tick_now()); } @@ -1576,10 +1493,6 @@ pub async fn bench_approvals(env: &mut TestEnvironment, state: ApprovalTestState // gum::info!(target: LOG_TARGET,"{}", format!("Sleeping till end of block ({}ms)", // block_time_delta.as_millis()).bright_black()); tokio::time::sleep(block_time_delta). // await; - gum::info!( - "Next slot tick {:?}", - slot_number_to_tick(SLOT_DURATION_MILLIS, current_slot + 1) - ); system_clock .wait(slot_number_to_tick(SLOT_DURATION_MILLIS, current_slot + 1)) .await; @@ -1597,8 +1510,11 @@ pub async fn bench_approvals(env: &mut TestEnvironment, state: ApprovalTestState ); tokio::time::sleep(Duration::from_secs(6)).await; } - tokio::time::sleep(Duration::from_secs(6)).await; - tokio::time::sleep(Duration::from_secs(60)).await; + + gum::info!("Awaiting producer to signal done"); + + producer_rx.await; + gum::info!("Requesting approval votes ms"); for info in &state.per_slot_heads { for (index, candidates) in info.candidates.iter().enumerate() { @@ -1612,6 +1528,9 @@ pub async fn bench_approvals(env: &mut TestEnvironment, state: ApprovalTestState tx, ); env.send_message(AllMessages::ApprovalVoting(msg)).await; + + let start: Instant = Instant::now(); + let result = rx.await.unwrap(); for (validator, votes) in result.iter() { @@ -1653,6 +1572,8 @@ pub async fn bench_approvals(env: &mut TestEnvironment, state: ApprovalTestState gum::info!("{}", &env); } +#[derive(Clone)] + struct FakeSystemClock { real_system_clock: SystemClock, delta_ticks: Tick, diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 16ab0d49d869..a6257dd265b0 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -242,6 +242,14 @@ impl TestEnvironment { &self.network } + pub fn overseer_handle(&self) -> &OverseerHandle { + &self.overseer_handle + } + + pub fn spawn_handle(&self) -> SpawnTaskHandle { + self.dependencies.task_manager.spawn_handle() + } + pub fn registry(&self) -> &Registry { &self.dependencies.registry } diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 7a8fefe60ea6..07b2c1bd1f4d 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -115,9 +115,9 @@ impl BenchCli { let (mut env, state) = approval::prepare_test(test_config.clone(), options.clone()); - // env.runtime().block_on(async { - // bench_approvals(&mut env, state).await; - // }); + env.runtime().block_on(async { + bench_approvals(&mut env, state).await; + }); }, TestObjective::TestSequence(_) => todo!(), } From c77f428ca8f41b2b6121877f3b388899c4f7140c Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 14 Dec 2023 14:39:25 +0200 Subject: [PATCH 150/192] Approval-voting with serialization of messages Signed-off-by: Alexandru Gheorghe --- polkadot/node/subsystem-bench/Cargo.toml | 3 + .../node/subsystem-bench/src/approval/mod.rs | 275 ++++++++++-------- polkadot/node/subsystem-bench/src/cli.rs | 1 + .../subsystem-bench/src/subsystem-bench.rs | 2 + 4 files changed, 161 insertions(+), 120 deletions(-) diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index 977427c68ea0..dbe887e6ed51 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -35,6 +35,9 @@ sp-core = { path = "../../../substrate/primitives/core" } clap = { version = "4.4.6", features = ["derive"] } futures = "0.3.21" futures-timer = "3.0.2" +bincode = "1.3.3" +sha1 = "0.10.6" +hex = "0.4.3" gum = { package = "tracing-gum", path = "../gum" } polkadot-erasure-coding = { package = "polkadot-erasure-coding", path = "../../erasure-coding" } log = "0.4.17" diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 856740564b73..31047cfba347 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -21,6 +21,7 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, fs, io::{Read, Write}, + path::Path, sync::{ atomic::{AtomicBool, AtomicU32, AtomicU64}, Arc, @@ -50,6 +51,8 @@ use polkadot_node_primitives::approval::{ v2::{CoreBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2}, }; +use sha1::Digest as Sha1Digest; + use polkadot_node_network_protocol::{ grid_topology::{ GridNeighbors, RandomRouting, RequiredRouting, SessionGridTopology, TopologyPeerInfo, @@ -99,7 +102,7 @@ use crate::{ mock_runtime_api::MockRuntimeApi, }, core::{ - configuration::{TestAuthorities, TestConfiguration}, + configuration::{TestAuthorities, TestConfiguration, TestObjective}, environment::{TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT}, keyring::Keyring, mock::{ @@ -126,7 +129,7 @@ pub(crate) const TEST_CONFIG: ApprovalVotingConfig = ApprovalVotingConfig { slot_duration_millis: SLOT_DURATION_MILLIS, }; -const MESSAGES_PATH: &str = "/home/alexggh/messages_tmp/all_messages_shuffled_500"; +const MESSAGES_PATH: &str = "/home/alexggh/messages_tmp/"; pub const NODE_UNDER_TEST: u32 = 0; @@ -161,9 +164,18 @@ pub struct ApprovalsOptions { #[clap(short, long, default_value_t = true)] /// Sends messages only till block is approved. pub stop_when_approved: bool, - #[clap(short, long, default_value_t = true)] - /// Only generate the messages - pub generate_only: bool, +} + +impl ApprovalsOptions { + fn gen_key(&self) -> Vec { + let mut bytes = Vec::new(); + bytes.extend(self.last_considered_tranche.to_be_bytes()); + bytes.extend(self.min_coalesce.to_be_bytes()); + bytes.extend(self.max_coalesce.to_be_bytes()); + bytes.extend(self.coalesce_tranche_diff.to_be_bytes()); + bytes.extend((self.enable_assignments_v2 as i32).to_be_bytes()); + bytes + } } /// Information about a block. It is part of test state and it is used by the mock @@ -219,6 +231,7 @@ pub struct ApprovalTestState { total_sent_messages: Arc, /// Approval voting metrics. approval_voting_metrics: ApprovalVotingMetrics, + delta_tick_from_generated: Arc, } impl ApprovalTestState { @@ -232,39 +245,67 @@ impl ApprovalTestState { let test_authorities = configuration.generate_authorities(); let start = Instant::now(); - let generated_state = if !options.generate_only { - let mut file = fs::OpenOptions::new().read(true).open(MESSAGES_PATH).unwrap(); - let mut content = Vec::::with_capacity(2000000); - - file.read_to_end(&mut content); - let mut content = content.as_slice(); - let generated_state: GeneratedState = - Decode::decode(&mut content).expect("Could not decode messages"); + let mut key = options.gen_key(); + let mut exclude_objective = configuration.clone(); + exclude_objective.objective = TestObjective::Unimplemented; + let configuration_bytes = bincode::serialize(&exclude_objective).unwrap(); + key.extend(configuration_bytes); + let mut sha1 = sha1::Sha1::new(); + sha1.update(key); + let result = sha1.finalize(); + let path_name = format!("{}/{}", MESSAGES_PATH, hex::encode(result)); + let delta_to_first_slot_under_test = Timestamp::new(BUFFER_FOR_GENERATION_MILLIS); + + let messages = Path::new(&path_name); + if !messages.exists() { + gum::info!("Generate message because filed does not exist"); + let delta_to_first_slot_under_test = Timestamp::new(BUFFER_FOR_GENERATION_MILLIS); + let initial_slot = Slot::from_timestamp( + (*Timestamp::current() - *delta_to_first_slot_under_test).into(), + SlotDuration::from_millis(SLOT_DURATION_MILLIS), + ); - gum::info!( - "It took {:?} ms count {:?}", - start.elapsed().as_millis(), - generated_state.all_messages.as_ref().map(|val| val.len()).unwrap_or_default() + let babe_epoch = generate_babe_epoch(initial_slot, test_authorities.clone()); + let session_info = session_info_for_peers(configuration, test_authorities.clone()); + let blocks = + Self::generate_blocks_information(configuration, &babe_epoch, initial_slot); + + let generate = ApprovalTestState::generate_messages( + initial_slot, + &test_authorities, + &blocks, + &session_info, + &options, + &dependencies.task_manager.spawn_handle(), + &messages, ); - generated_state - } else { - let delta_to_first_slot_under_test = Timestamp::new(BUFFER_FOR_GENERATION_MILLIS); + } - GeneratedState { - initial_slot: Slot::from_timestamp( - (*Timestamp::current() - *delta_to_first_slot_under_test).into(), - SlotDuration::from_millis(SLOT_DURATION_MILLIS), - ), - all_messages: Default::default(), - } - }; + let mut file = fs::OpenOptions::new().read(true).open(messages).unwrap(); + let mut content = Vec::::with_capacity(2000000); + + file.read_to_end(&mut content); + let mut content = content.as_slice(); + let generated_state: GeneratedState = + Decode::decode(&mut content).expect("Could not decode messages"); + + gum::info!( + "It took {:?} ms count {:?}", + start.elapsed().as_millis(), + generated_state.all_messages.as_ref().map(|val| val.len()).unwrap_or_default() + ); let babe_epoch = generate_babe_epoch(generated_state.initial_slot, test_authorities.clone()); let session_info = session_info_for_peers(configuration, test_authorities.clone()); + let blocks = Self::generate_blocks_information( + configuration, + &babe_epoch, + generated_state.initial_slot, + ); let mut state = ApprovalTestState { - per_slot_heads: Default::default(), + per_slot_heads: blocks, babe_epoch: babe_epoch.clone(), session_info: session_info.clone(), configuration: configuration.clone(), @@ -275,8 +316,8 @@ impl ApprovalTestState { options, approval_voting_metrics: ApprovalVotingMetrics::try_register(&dependencies.registry) .unwrap(), + delta_tick_from_generated: Arc::new(AtomicU64::new(630720000)), }; - state.generate_blocks_information(); gum::info!("Built testing state"); state @@ -284,16 +325,18 @@ impl ApprovalTestState { /// Generates the blocks and the information about the blocks that will be used /// to drive this test. - fn generate_blocks_information(&mut self) { + fn generate_blocks_information( + configuration: &TestConfiguration, + babe_epoch: &BabeEpoch, + initial_slot: Slot, + ) -> Vec { + let mut per_block_heads: Vec = Vec::new(); let mut prev_candidates = 0; - for block_number in 1..self.configuration.num_blocks + 1 { + for block_number in 1..configuration.num_blocks + 1 { let block_hash = Hash::repeat_byte(block_number as u8); - let parent_hash = self - .per_slot_heads - .last() - .map(|val| val.hash) - .unwrap_or(Hash::repeat_byte(0xde)); - let slot_for_block = self.generated_state.initial_slot + (block_number as u64 - 1); + let parent_hash = + per_block_heads.last().map(|val| val.hash).unwrap_or(Hash::repeat_byte(0xde)); + let slot_for_block = initial_slot + (block_number as u64 - 1); let header = make_header(parent_hash, slot_for_block, block_number as u32); @@ -301,9 +344,9 @@ impl ApprovalTestState { .expect("Can not continue without vrf generator"); let relay_vrf_story = unsafe_vrf .compute_randomness( - &self.babe_epoch.authorities, - &self.babe_epoch.randomness, - self.babe_epoch.epoch_index, + &babe_epoch.authorities, + &babe_epoch.randomness, + babe_epoch.epoch_index, ) .expect("Can not continue without vrf story"); @@ -315,25 +358,24 @@ impl ApprovalTestState { candidates: make_candidates( block_hash, block_number as BlockNumber, - self.configuration.n_cores as u32, - self.configuration.n_included_candidates as u32, + configuration.n_cores as u32, + configuration.n_included_candidates as u32, ), relay_vrf_story, approved: Arc::new(AtomicBool::new(false)), prev_candidates, votes: Arc::new( - (0..self.configuration.n_validators) + (0..configuration.n_validators) .map(|_| { - (0..self.configuration.n_cores) - .map(|_| AtomicBool::new(false)) - .collect_vec() + (0..configuration.n_cores).map(|_| AtomicBool::new(false)).collect_vec() }) .collect_vec(), ), }; prev_candidates += block_info.candidates.len() as u64; - self.per_slot_heads.push(block_info) + per_block_heads.push(block_info) } + per_block_heads } /// Starts the generation of messages(Assignments & Approvals) needed for approving blocks. @@ -368,47 +410,56 @@ impl ApprovalTestState { /// Starts the generation of messages(Assignments & Approvals) needed for approving blocks. fn generate_messages( - &self, - network_emulator: &NetworkEmulator, - overseer_handle: OverseerHandleReal, + initial_slot: Slot, + test_authorities: &TestAuthorities, + blocks: &Vec, + session_info: &SessionInfo, + options: &ApprovalsOptions, spawn_task_handle: &SpawnTaskHandle, + path: &Path, ) { gum::info!(target: LOG_TARGET, "Generate messages"); - - let topology = generate_topology(&self.test_authorities); + let topology = generate_topology(&test_authorities); let random_samplings = random_samplings_to_node_patterns( ValidatorIndex(NODE_UNDER_TEST), - self.test_authorities.keyrings.len(), - self.test_authorities.keyrings.len() as usize * 2, + test_authorities.keyrings.len(), + test_authorities.keyrings.len() as usize * 2, ); let topology_node_under_test = topology.compute_grid_neighbors_for(ValidatorIndex(NODE_UNDER_TEST)).unwrap(); + let (tx, mut rx) = futures::channel::mpsc::unbounded(); - for current_validator_index in 1..self.test_authorities.keyrings.len() { + for current_validator_index in 1..test_authorities.keyrings.len() { let peer_message_source = message_generator::PeerMessagesGenerator { topology_node_under_test: topology_node_under_test.clone(), topology: topology.clone(), validator_index: ValidatorIndex(current_validator_index as u32), - network: network_emulator.clone(), - overseer_handle: overseer_handle.clone(), - state: self.clone(), - options: self.options.clone(), + test_authorities: test_authorities.clone(), + session_info: session_info.clone(), + blocks: blocks.clone(), tx_messages: tx.clone(), random_samplings: random_samplings.clone(), + enable_assignments_v2: options.enable_assignments_v2, + last_considered_tranche: options.last_considered_tranche, + min_coalesce: options.min_coalesce, + max_coalesce: options.max_coalesce, + coalesce_tranche_diff: options.coalesce_tranche_diff, }; peer_message_source.generate_messages(&spawn_task_handle); } + std::mem::drop(tx); + let seed = [0x32; 32]; let mut rand_chacha = ChaCha20Rng::from_seed(seed); let mut file = fs::OpenOptions::new() .write(true) .create(true) .truncate(true) - .open(MESSAGES_PATH) + .open(path) .unwrap(); let mut all_messages: BTreeMap, Vec)> = @@ -418,7 +469,10 @@ impl ApprovalTestState { match rx.try_next() { Ok(Some((block_hash, messages))) => for message in messages { - let block_info = self.get_info_by_hash(block_hash); + let block_info = blocks + .iter() + .find(|val| val.hash == block_hash) + .expect("Should find blocks"); let tick_to_send = tranche_to_tick(SLOT_DURATION_MILLIS, block_info.slot, message.tranche); if !all_messages.contains_key(&tick_to_send) { @@ -449,10 +503,7 @@ impl ApprovalTestState { .collect_vec(); gum::info!("Took something like {:}", all_messages.len()); - let generated_state = GeneratedState { - all_messages: Some(all_messages), - initial_slot: self.generated_state.initial_slot, - }; + let generated_state = GeneratedState { all_messages: Some(all_messages), initial_slot }; file.write_all(&generated_state.encode()); } @@ -597,15 +648,8 @@ impl PeerMessageProducer { ) { spawn_task_handle.spawn_blocking("produce-messages", "produce-messages", async move { let mut already_generated = HashSet::new(); - let real_system_clock = SystemClock {}; - let system_clock = FakeSystemClock::new( - SystemClock {}, - real_system_clock.tick_now() - - slot_number_to_tick( - SLOT_DURATION_MILLIS, - self.state.generated_state.initial_slot, - ), - ); + let system_clock = + FakeSystemClock::new(SystemClock {}, self.state.delta_tick_from_generated.clone()); let mut all_messages = all_messages.into_iter().peekable(); while all_messages.peek().is_some() { let current_slot = @@ -654,16 +698,22 @@ impl PeerMessageProducer { MessageType::Assignment => val.tranche, MessageType::Other => val.tranche, }; - tranche_to_send <= tranche_now && + (tranche_to_send <= tranche_now && current_slot >= block_info.slot && - already_generated.contains(&block_info.hash) + already_generated.contains(&block_info.hash)) || + ((val.tranche > self.options.send_till_tranche || + self.options.stop_when_approved) && block_info + .approved + .load(std::sync::atomic::Ordering::SeqCst)) }) .unwrap_or_default() { let message = all_messages.next().unwrap(); let block_info = self.state.get_info_by_hash(message.block_hash); - if !block_info.approved.load(std::sync::atomic::Ordering::SeqCst) || + let block_approved = + block_info.approved.load(std::sync::atomic::Ordering::SeqCst); + if !block_approved || (!self.options.stop_when_approved && message.tranche <= self.options.send_till_tranche) { @@ -856,12 +906,13 @@ fn issue_approvals( block_hash: Hash, keyrings: Vec<(Keyring, PeerId)>, candidates: Vec, - options: &ApprovalsOptions, + min_coalesce: u32, + max_coalesce: u32, + coalesce_tranche_diff: u32, rand_chacha: &mut ChaCha20Rng, ) -> Vec { let mut to_sign: Vec = Vec::new(); - let mut num_coalesce = - coalesce_approvals_len(options.min_coalesce, options.max_coalesce, rand_chacha); + let mut num_coalesce = coalesce_approvals_len(min_coalesce, max_coalesce, rand_chacha); let result = assignments .iter() .enumerate() @@ -882,13 +933,9 @@ fn issue_approvals( if to_sign.len() >= num_coalesce as usize || (!to_sign.is_empty() && current_validator_index != assignment.0.validator) || - message.tranche - earliest_tranche >= options.coalesce_tranche_diff + message.tranche - earliest_tranche >= coalesce_tranche_diff { - num_coalesce = coalesce_approvals_len( - options.min_coalesce, - options.max_coalesce, - rand_chacha, - ); + num_coalesce = coalesce_approvals_len(min_coalesce, max_coalesce, rand_chacha); approvals_to_create.push(sign_candidates(&mut to_sign, &keyrings, block_hash)) } @@ -907,13 +954,10 @@ fn issue_approvals( to_sign.first().map(|val| val.tranche).unwrap_or(message.tranche); if to_sign.len() >= num_coalesce || - message.tranche - earliest_tranche >= options.coalesce_tranche_diff + message.tranche - earliest_tranche >= coalesce_tranche_diff { - num_coalesce = coalesce_approvals_len( - options.min_coalesce, - options.max_coalesce, - rand_chacha, - ); + num_coalesce = + coalesce_approvals_len(min_coalesce, max_coalesce, rand_chacha); approvals_to_create.push(sign_candidates( &mut to_sign, &keyrings, @@ -1353,11 +1397,8 @@ fn build_overseer( polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[]); let keystore = LocalKeystore::in_memory(); let real_system_clock = SystemClock {}; - let system_clock = FakeSystemClock::new( - SystemClock {}, - real_system_clock.tick_now() - - slot_number_to_tick(SLOT_DURATION_MILLIS, state.generated_state.initial_slot), - ); + let system_clock = + FakeSystemClock::new(SystemClock {}, state.delta_tick_from_generated.clone()); let approval_voting = ApprovalVotingSubsystem::with_config( TEST_CONFIG, Arc::new(db), @@ -1410,6 +1451,7 @@ fn prepare_test_inner( gum::info!("Build network emulator"); let network = NetworkEmulator::new(&config, &dependencies, &state.test_authorities); + gum::info!("Build overseer"); let (overseer, overseer_handle) = build_overseer(&state, &network, &config, &dependencies); @@ -1418,18 +1460,10 @@ fn prepare_test_inner( } pub async fn bench_approvals(env: &mut TestEnvironment, mut state: ApprovalTestState) { - if state.options.generate_only { - state.generate_messages(env.network(), env.overseer_handle().clone(), &env.spawn_handle()); - } else { - let producer_rx = state - .start_message_production( - env.network(), - env.overseer_handle().clone(), - &env.spawn_handle(), - ) - .await; - bench_approvals_run(env, state, producer_rx).await - } + let producer_rx = state + .start_message_production(env.network(), env.overseer_handle().clone(), &env.spawn_handle()) + .await; + bench_approvals_run(env, state, producer_rx).await } /// Runs the approval benchmark. @@ -1458,13 +1492,13 @@ pub async fn bench_approvals_run( } let start_marker = Instant::now(); - - let real_system_clock = SystemClock {}; - let system_clock = FakeSystemClock::new( - SystemClock {}, - real_system_clock.tick_now() - + let real_clock = SystemClock {}; + state.delta_tick_from_generated.store( + real_clock.tick_now() - slot_number_to_tick(SLOT_DURATION_MILLIS, state.generated_state.initial_slot), + std::sync::atomic::Ordering::SeqCst, ); + let system_clock = FakeSystemClock::new(real_clock, state.delta_tick_from_generated.clone()); for block_num in 0..env.config().num_blocks { let mut current_slot = tick_to_slot_number(SLOT_DURATION_MILLIS, system_clock.tick_now()); @@ -1576,25 +1610,26 @@ pub async fn bench_approvals_run( struct FakeSystemClock { real_system_clock: SystemClock, - delta_ticks: Tick, + delta_ticks: Arc, } impl FakeSystemClock { - fn new(real_system_clock: SystemClock, delta_ticks: Tick) -> Self { - gum::info!("Delta tick is {:}", delta_ticks); + fn new(real_system_clock: SystemClock, delta_ticks: Arc) -> Self { FakeSystemClock { real_system_clock, delta_ticks } } } impl Clock for FakeSystemClock { fn tick_now(&self) -> Tick { - self.real_system_clock.tick_now() - self.delta_ticks + self.real_system_clock.tick_now() - + self.delta_ticks.load(std::sync::atomic::Ordering::SeqCst) } fn wait( &self, tick: Tick, ) -> std::pin::Pin + Send + 'static>> { - self.real_system_clock.wait(tick + self.delta_ticks) + self.real_system_clock + .wait(tick + self.delta_ticks.load(std::sync::atomic::Ordering::SeqCst)) } } diff --git a/polkadot/node/subsystem-bench/src/cli.rs b/polkadot/node/subsystem-bench/src/cli.rs index a5d6fa7fec09..ed387a1bd7e3 100644 --- a/polkadot/node/subsystem-bench/src/cli.rs +++ b/polkadot/node/subsystem-bench/src/cli.rs @@ -36,6 +36,7 @@ pub enum TestObjective { TestSequence(TestSequenceOptions), /// ApprovalsTest(ApprovalsOptions), + Unimplemented, } #[derive(Debug, clap::Parser)] diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 07b2c1bd1f4d..24b82927016a 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -120,6 +120,7 @@ impl BenchCli { }); }, TestObjective::TestSequence(_) => todo!(), + TestObjective::Unimplemented => todo!(), } } return Ok(()) @@ -153,6 +154,7 @@ impl BenchCli { TestObjective::ApprovalsTest(ref options) => { todo!("Not implemented"); }, + TestObjective::Unimplemented => todo!(), }; let mut latency_config = test_config.latency.clone().unwrap_or_default(); From 27f636837ea26c510ff9c8e7c7f006930cc70be1 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 15 Dec 2023 08:41:23 +0200 Subject: [PATCH 151/192] Same tranche Signed-off-by: Alexandru Gheorghe --- polkadot/node/subsystem-bench/src/approval/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 31047cfba347..c813d814be9a 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -694,7 +694,7 @@ impl PeerMessageProducer { let tranche_now = system_clock.tranche_now(SLOT_DURATION_MILLIS, block_info.slot); let tranche_to_send = match val.typ { - MessageType::Approval => val.tranche + 1, + MessageType::Approval => val.tranche, MessageType::Assignment => val.tranche, MessageType::Other => val.tranche, }; From b223f6aa4ff87d52455dadbffd7fb9343c3e6261 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 19 Dec 2023 12:27:05 +0200 Subject: [PATCH 152/192] Latest rafector Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 3 + .../node/subsystem-bench/src/approval/mod.rs | 595 ++++++++++++++---- .../subsystem-bench/src/subsystem-bench.rs | 1 + 3 files changed, 477 insertions(+), 122 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 8072198b7c75..41045ed53202 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -352,6 +352,7 @@ struct State { /// Aggregated reputation change reputation: ReputationAggregator, total_num_messages: u64, + got_votes: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -1578,6 +1579,7 @@ impl State { source: MessageSource, vote: IndirectSignedApprovalVoteV2, ) { + assert!(self.got_votes.is_none()); let _span = self .spans .get(&vote.block_hash) @@ -2422,6 +2424,7 @@ impl ApprovalDistribution { .await; }, ApprovalDistributionMessage::GetApprovalSignatures(indices, tx) => { + state.got_votes = Some(true); let sigs = state.get_approval_signatures(indices); if let Err(_) = tx.send(sigs) { gum::debug!( diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index c813d814be9a..e9063fbf97fd 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -18,6 +18,7 @@ use parity_scale_codec::{Decode, Encode}; use serde::{Deserialize, Serialize}; use std::{ alloc::System, + cmp::max, collections::{BTreeMap, HashMap, HashSet}, fs, io::{Read, Write}, @@ -129,8 +130,6 @@ pub(crate) const TEST_CONFIG: ApprovalVotingConfig = ApprovalVotingConfig { slot_duration_millis: SLOT_DURATION_MILLIS, }; -const MESSAGES_PATH: &str = "/home/alexggh/messages_tmp/"; - pub const NODE_UNDER_TEST: u32 = 0; /// Start generating messages for a slot into the future, so that the @@ -164,6 +163,11 @@ pub struct ApprovalsOptions { #[clap(short, long, default_value_t = true)] /// Sends messages only till block is approved. pub stop_when_approved: bool, + #[clap(short, long)] + /// Sends messages only till block is approved. + pub workdir_prefix: String, + #[clap(short, long, default_value_t = 0)] + pub num_no_shows_per_block: u32, } impl ApprovalsOptions { @@ -202,6 +206,45 @@ struct BlockTestData { approved: Arc, prev_candidates: u64, votes: Arc>>, + per_candidate_votes: Arc>, + per_candidate_min_tranches: Arc>, +} + +#[derive(Debug)] +struct CandidateTestData { + max_no_shows: u32, + last_tranche_with_no_show: u32, + sent_assignment: u32, + num_no_shows: u32, + max_tranche: u32, +} + +impl CandidateTestData { + fn should_send_tranche(&self, tranche: u32) -> bool { + self.sent_assignment <= 30 || tranche <= self.max_tranche + self.num_no_shows + } + + fn set_max_tranche(&mut self, tranche: u32) { + self.max_tranche = max(tranche, self.max_tranche); + } + + fn record_no_show(&mut self, tranche: u32) { + self.num_no_shows += 1; + self.last_tranche_with_no_show = max(tranche, self.last_tranche_with_no_show); + } + + fn sent_assignment(&mut self, tranche: u32) { + if self.sent_assignment < 30 { + self.set_max_tranche(tranche); + } + + self.sent_assignment += 1; + } + + fn should_no_show(&self, tranche: u32) -> bool { + (self.num_no_shows < self.max_no_shows && self.last_tranche_with_no_show < tranche) || + (tranche == 0 && self.num_no_shows == 0 && self.max_no_shows > 0) + } } /// Approval test state used by all mock subsystems to be able to answer messages emitted @@ -229,6 +272,8 @@ pub struct ApprovalTestState { last_approved_block: Arc, /// Total sent messages. total_sent_messages: Arc, + /// Total unique sent messages. + total_unique_messages: Arc, /// Approval voting metrics. approval_voting_metrics: ApprovalVotingMetrics, delta_tick_from_generated: Arc, @@ -253,7 +298,7 @@ impl ApprovalTestState { let mut sha1 = sha1::Sha1::new(); sha1.update(key); let result = sha1.finalize(); - let path_name = format!("{}/{}", MESSAGES_PATH, hex::encode(result)); + let path_name = format!("{}/{}", options.workdir_prefix, hex::encode(result)); let delta_to_first_slot_under_test = Timestamp::new(BUFFER_FOR_GENERATION_MILLIS); let messages = Path::new(&path_name); @@ -313,6 +358,7 @@ impl ApprovalTestState { test_authorities, last_approved_block: Arc::new(AtomicU32::new(0)), total_sent_messages: Arc::new(AtomicU64::new(0)), + total_unique_messages: Arc::new(AtomicU64::new(0)), options, approval_voting_metrics: ApprovalVotingMetrics::try_register(&dependencies.registry) .unwrap(), @@ -371,6 +417,12 @@ impl ApprovalTestState { }) .collect_vec(), ), + per_candidate_votes: Arc::new( + (0..configuration.n_cores).map(|_| AtomicU32::new(0)).collect_vec(), + ), + per_candidate_min_tranches: Arc::new( + (0..configuration.n_cores).map(|_| AtomicU32::new(0)).collect_vec(), + ), }; prev_candidates += block_info.candidates.len() as u64; per_block_heads.push(block_info) @@ -462,8 +514,7 @@ impl ApprovalTestState { .open(path) .unwrap(); - let mut all_messages: BTreeMap, Vec)> = - BTreeMap::new(); + let mut all_messages: BTreeMap> = BTreeMap::new(); loop { match rx.try_next() { @@ -473,18 +524,17 @@ impl ApprovalTestState { .iter() .find(|val| val.hash == block_hash) .expect("Should find blocks"); - let tick_to_send = - tranche_to_tick(SLOT_DURATION_MILLIS, block_info.slot, message.tranche); + let tick_to_send = tranche_to_tick( + SLOT_DURATION_MILLIS, + block_info.slot, + message.tranche_to_send(), + ); if !all_messages.contains_key(&tick_to_send) { - all_messages.insert(tick_to_send, (Vec::new(), Vec::new())); + all_messages.insert(tick_to_send, Vec::new()); } let to_add = all_messages.get_mut(&tick_to_send).unwrap(); - match message.typ { - MessageType::Approval => to_add.1.push(message), - MessageType::Assignment => to_add.0.push(message), - MessageType::Other => todo!(), - } + to_add.push(message); }, Ok(None) => break, Err(_) => { @@ -494,10 +544,9 @@ impl ApprovalTestState { } let all_messages = all_messages .into_iter() - .map(|(_, (mut assignments, mut approvals))| { - assignments.shuffle(&mut rand_chacha); - approvals.shuffle(&mut rand_chacha); - assignments.into_iter().chain(approvals.into_iter()) + .map(|(_, mut messages)| { + messages.shuffle(&mut rand_chacha); + messages }) .flatten() .collect_vec(); @@ -510,7 +559,7 @@ impl ApprovalTestState { } #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] struct GeneratedState { - all_messages: Option>, + all_messages: Option>, initial_slot: Slot, } @@ -540,7 +589,7 @@ impl ApprovalTestState { } /// Type of generated messages. -#[derive(Debug, Copy, Clone, Encode, Decode, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, Encode, Decode, PartialEq, Eq, Hash)] enum MessageType { Approval, @@ -564,6 +613,77 @@ struct TestMessageInfo { typ: MessageType, } +impl std::hash::Hash for TestMessageInfo { + fn hash(&self, state: &mut H) { + match &self.msg { + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) => { + for (assignment, candidates) in assignments { + (assignment.block_hash, assignment.validator).hash(state); + candidates.hash(state); + } + }, + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => { + for approval in approvals { + (approval.block_hash, approval.validator).hash(state); + approval.candidate_indices.hash(state); + } + }, + }; + self.typ.hash(state); + } +} + +#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +struct DependentMessageInfo { + assignments: Vec, + approvals: Vec, +} + +impl DependentMessageInfo { + fn tranche_to_send(&self) -> u32 { + self.assignments + .iter() + .chain(self.approvals.iter()) + .max_by(|a, b| a.tranche.cmp(&b.tranche)) + .unwrap() + .tranche + } + + fn min_tranche(&self) -> u32 { + self.assignments + .iter() + .chain(self.approvals.iter()) + .min_by(|a, b| a.tranche.cmp(&b.tranche)) + .unwrap() + .tranche + } + + fn bundle_needed( + &self, + candidates_test_data: &HashMap<(Hash, CandidateIndex), CandidateTestData>, + options: &ApprovalsOptions, + ) -> bool { + self.should_send(candidates_test_data) || + (!options.stop_when_approved && self.min_tranche() <= options.send_till_tranche) + } + + fn should_send( + &self, + candidates_test_data: &HashMap<(Hash, CandidateIndex), CandidateTestData>, + ) -> bool { + self.assignments.iter().any(|message| message.should_send(candidates_test_data)) + } + + fn record_sent_assignment( + &self, + candidates_test_data: &mut HashMap<(Hash, CandidateIndex), CandidateTestData>, + ) { + self.assignments + .iter() + .for_each(|assignment| assignment.record_sent_assignment(candidates_test_data)); + } +} + impl TestMessageInfo { /// Returns the lantency based on the message type. fn to_all_messages_from_peer(self, peer: PeerId) -> AllMessages { @@ -602,6 +722,130 @@ impl TestMessageInfo { } } + fn record_sent_assignment( + &self, + candidates_test_data: &mut HashMap<(Hash, CandidateIndex), CandidateTestData>, + ) { + match &self.msg { + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) => { + for (assignment, candidate_indices) in assignments { + for candidate_index in candidate_indices.iter_ones() { + let candidate_test_data = candidates_test_data + .get_mut(&(assignment.block_hash, candidate_index as CandidateIndex)) + .unwrap(); + candidate_test_data.sent_assignment(self.tranche) + } + } + }, + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => todo!(), + } + } + + fn candidate_indices(&self) -> HashSet { + let mut unique_candidate_indicies = HashSet::new(); + match &self.msg { + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) => + for (assignment, candidate_indices) in assignments { + for candidate_index in candidate_indices.iter_ones() { + unique_candidate_indicies.insert(candidate_index); + } + }, + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => + for approval in approvals { + for candidate_index in approval.candidate_indices.iter_ones() { + unique_candidate_indicies.insert(candidate_index); + } + }, + } + unique_candidate_indicies + } + + fn no_show_if_required( + &self, + assignments: &Vec, + candidates_test_data: &mut HashMap<(Hash, CandidateIndex), CandidateTestData>, + ) -> bool { + let mut should_no_show = false; + if let MessageType::Approval = self.typ { + let covered_candidates = assignments + .iter() + .map(|assignment| (assignment, assignment.candidate_indices())) + .collect_vec(); + match &self.msg { + protocol_v3::ApprovalDistributionMessage::Assignments(_) => todo!(), + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => { + assert_eq!(approvals.len(), 1); + + for approval in approvals { + should_no_show = should_no_show || + approval.candidate_indices.iter_ones().all(|candidate_index| { + let candidate_test_data = candidates_test_data + .get_mut(&( + approval.block_hash, + candidate_index as CandidateIndex, + )) + .unwrap(); + let assignment = covered_candidates + .iter() + .filter(|(assignment, candidates)| { + candidates.contains(&candidate_index) + }) + .next() + .unwrap(); + candidate_test_data.should_no_show(assignment.0.tranche) + }); + + if should_no_show { + for candidate_index in approval.candidate_indices.iter_ones() { + let candidate_test_data = candidates_test_data + .get_mut(&( + approval.block_hash, + candidate_index as CandidateIndex, + )) + .unwrap(); + let assignment = covered_candidates + .iter() + .filter(|(assignment, candidates)| { + candidates.contains(&candidate_index) + }) + .next() + .unwrap(); + candidate_test_data.record_no_show(assignment.0.tranche) + } + } + } + }, + } + } + should_no_show + } + + fn should_send( + &self, + candidates_test_data: &HashMap<(Hash, CandidateIndex), CandidateTestData>, + ) -> bool { + match &self.msg { + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) => + assignments.iter().any(|(assignment, candidate_indices)| { + candidate_indices.iter_ones().any(|candidate_index| { + candidates_test_data + .get(&(assignment.block_hash, candidate_index as CandidateIndex)) + .map(|data| data.should_send_tranche(self.tranche)) + .unwrap_or_default() + }) + }), + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => + approvals.iter().any(|approval| { + approval.candidate_indices.iter_ones().any(|candidate_index| { + candidates_test_data + .get(&(approval.block_hash, candidate_index as CandidateIndex)) + .map(|data| data.should_send_tranche(self.tranche)) + .unwrap_or_default() + }) + }), + } + } + /// Splits a message into multiple messages based on what peers should send this message. /// It build a HashMap of messages that should be sent by each peer. fn split_by_peer_id( @@ -644,13 +888,35 @@ impl PeerMessageProducer { fn produce_messages( mut self, spawn_task_handle: &SpawnTaskHandle, - all_messages: Vec, + all_messages: Vec, ) { spawn_task_handle.spawn_blocking("produce-messages", "produce-messages", async move { let mut already_generated = HashSet::new(); let system_clock = FakeSystemClock::new(SystemClock {}, self.state.delta_tick_from_generated.clone()); let mut all_messages = all_messages.into_iter().peekable(); + let mut no_shows: HashMap<(Hash, usize), HashSet> = HashMap::new(); + let mut per_candidate_data: HashMap<(Hash, CandidateIndex), CandidateTestData> = + HashMap::new(); + + for block_info in self.state.per_slot_heads.iter() { + for (candidate_index, _) in block_info.candidates.iter().enumerate() { + per_candidate_data.insert( + (block_info.hash, candidate_index as CandidateIndex), + CandidateTestData { + max_no_shows: self.options.num_no_shows_per_block, + last_tranche_with_no_show: 0, + sent_assignment: 0, + num_no_shows: 0, + max_tranche: 0, + }, + ); + } + } + + let mut skipped_messages: Vec = Vec::new(); + let mut re_process_skipped = false; + while all_messages.peek().is_some() { let current_slot = tick_to_slot_number(SLOT_DURATION_MILLIS, system_clock.tick_now()); @@ -687,73 +953,140 @@ impl PeerMessageProducer { } } - while all_messages + let mut maybe_need_skip = skipped_messages.clone().into_iter().peekable(); + + let progressing_iterator = if !re_process_skipped { + &mut all_messages + } else { + re_process_skipped = false; + &mut maybe_need_skip + }; + + while progressing_iterator .peek() - .map(|val| { - let block_info = self.state.get_info_by_hash(val.block_hash); - let tranche_now = - system_clock.tranche_now(SLOT_DURATION_MILLIS, block_info.slot); - let tranche_to_send = match val.typ { - MessageType::Approval => val.tranche, - MessageType::Assignment => val.tranche, - MessageType::Other => val.tranche, - }; - (tranche_to_send <= tranche_now && - current_slot >= block_info.slot && - already_generated.contains(&block_info.hash)) || - ((val.tranche > self.options.send_till_tranche || - self.options.stop_when_approved) && block_info - .approved - .load(std::sync::atomic::Ordering::SeqCst)) + .map(|bundle| { + self.time_to_process_message( + bundle, + current_slot, + &already_generated, + &system_clock, + &per_candidate_data, + ) }) .unwrap_or_default() { - let message = all_messages.next().unwrap(); - - let block_info = self.state.get_info_by_hash(message.block_hash); - let block_approved = - block_info.approved.load(std::sync::atomic::Ordering::SeqCst); - if !block_approved || - (!self.options.stop_when_approved && - message.tranche <= self.options.send_till_tranche) - { - message.record_vote(block_info); - for (peer, messages) in - message.split_by_peer_id(&self.state.test_authorities) - { - for message in messages { - let latency = message.get_latency(); - self.state - .total_sent_messages - .as_ref() - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - self.send_message( - message.to_all_messages_from_peer(peer.1), - peer.0, - latency, - ) - } - } - } + let bundle = progressing_iterator.next().unwrap(); + re_process_skipped = re_process_skipped || + self.process_message( + bundle, + &mut per_candidate_data, + &already_generated, + &mut skipped_messages, + ); } } - gum::info!("All messages sent "); + gum::info!( + "All messages sent max_tranche {:?} last_tranche_with_no_show {:?}", + per_candidate_data.values().map(|data| data.max_tranche).max(), + per_candidate_data.values().map(|data| data.last_tranche_with_no_show).max() + ); let (tx, rx) = oneshot::channel(); let msg = ApprovalDistributionMessage::GetApprovalSignatures(HashSet::new(), tx); - self.overseer_handle - .send_msg(AllMessages::ApprovalDistribution(msg), LOG_TARGET) - .timeout(MAX_TIME_OF_FLIGHT) - .await - .unwrap_or_else(|| { - panic!("{} ms maximum time of flight breached", MAX_TIME_OF_FLIGHT.as_millis()) - }); + self.send_message(AllMessages::ApprovalDistribution(msg), ValidatorIndex(0), None); rx.await; self.notify_done.send(()); gum::info!("All messages processed "); }); } + pub fn process_message( + &mut self, + bundle: DependentMessageInfo, + per_candidate_data: &mut HashMap<(Hash, CandidateIndex), CandidateTestData>, + initialized_blocks: &HashSet, + skipped_messages: &mut Vec, + ) -> bool { + let mut reprocess_skipped = false; + let block_info = self + .state + .get_info_by_hash(bundle.assignments.first().unwrap().block_hash) + .clone(); + + if bundle.bundle_needed(&per_candidate_data, &self.options) { + // if bundle.min_tranche() >= 85 { + // gum::info!("============= {:?} =====", bundle); + // } + + bundle.record_sent_assignment(per_candidate_data); + + let assignments = bundle.assignments.clone(); + + for message in bundle.assignments.into_iter().chain(bundle.approvals.into_iter()) { + if message.no_show_if_required(&assignments, per_candidate_data) { + reprocess_skipped = true; + continue; + } else { + message.record_vote(&block_info); + } + self.state + .total_unique_messages + .as_ref() + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + for (peer, messages) in message.split_by_peer_id(&self.state.test_authorities) { + for message in messages { + let latency = message.get_latency(); + self.state + .total_sent_messages + .as_ref() + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.send_message( + message.to_all_messages_from_peer(peer.1), + peer.0, + latency, + ) + } + } + } + } else if !block_info.approved.load(std::sync::atomic::Ordering::SeqCst) { + skipped_messages.push(bundle); + } + reprocess_skipped + } + + pub fn time_to_process_message( + &self, + bundle: &DependentMessageInfo, + current_slot: Slot, + initialized_blocks: &HashSet, + system_clock: &FakeSystemClock, + per_candidate_data: &HashMap<(Hash, CandidateIndex), CandidateTestData>, + ) -> bool { + let block_info = + self.state.get_info_by_hash(bundle.assignments.first().unwrap().block_hash); + let tranche_now = system_clock.tranche_now(SLOT_DURATION_MILLIS, block_info.slot); + + Self::time_to_send( + bundle, + tranche_now, + current_slot, + block_info, + initialized_blocks.contains(&block_info.hash), + ) || !bundle.bundle_needed(&per_candidate_data, &self.options) + } + + pub fn time_to_send( + bundle: &DependentMessageInfo, + tranche_now: u32, + current_slot: Slot, + block_info: &BlockTestData, + block_initialized: bool, + ) -> bool { + bundle.tranche_to_send() <= tranche_now && + current_slot >= block_info.slot && + block_initialized + } + /// Queues a message to be sent by the peer identified by the `sent_by` value. fn send_message( &mut self, @@ -902,7 +1235,7 @@ fn coalesce_approvals_len( /// Helper function to create approvals signatures for all assignments passed as arguments. /// Returns a list of Approvals messages that need to be sent. fn issue_approvals( - assignments: &Vec, + assignments: Vec, block_hash: Hash, keyrings: Vec<(Keyring, PeerId)>, candidates: Vec, @@ -910,7 +1243,7 @@ fn issue_approvals( max_coalesce: u32, coalesce_tranche_diff: u32, rand_chacha: &mut ChaCha20Rng, -) -> Vec { +) -> Vec { let mut to_sign: Vec = Vec::new(); let mut num_coalesce = coalesce_approvals_len(min_coalesce, max_coalesce, rand_chacha); let result = assignments @@ -936,7 +1269,12 @@ fn issue_approvals( message.tranche - earliest_tranche >= coalesce_tranche_diff { num_coalesce = coalesce_approvals_len(min_coalesce, max_coalesce, rand_chacha); - approvals_to_create.push(sign_candidates(&mut to_sign, &keyrings, block_hash)) + approvals_to_create.push(sign_candidates( + &mut to_sign, + &keyrings, + block_hash, + num_coalesce, + )) } // If more that one candidate was in the assignment queue all of them. @@ -949,21 +1287,8 @@ fn issue_approvals( validator_index: assignment.0.validator, sent_by: message.sent_by.clone().into_iter().collect(), tranche: message.tranche, + assignment: message.clone(), }); - let earliest_tranche = - to_sign.first().map(|val| val.tranche).unwrap_or(message.tranche); - - if to_sign.len() >= num_coalesce || - message.tranche - earliest_tranche >= coalesce_tranche_diff - { - num_coalesce = - coalesce_approvals_len(min_coalesce, max_coalesce, rand_chacha); - approvals_to_create.push(sign_candidates( - &mut to_sign, - &keyrings, - block_hash, - )) - } } else { todo!("Other enum variants are not used in this benchmark"); } @@ -979,7 +1304,7 @@ fn issue_approvals( let mut result = result.into_iter().flatten().collect_vec(); if !to_sign.is_empty() { - result.push(sign_candidates(&mut to_sign, &keyrings, block_hash)); + result.push(sign_candidates(&mut to_sign, &keyrings, block_hash, num_coalesce)); } result } @@ -992,6 +1317,7 @@ struct TestSignInfo { validator_index: ValidatorIndex, sent_by: HashSet, tranche: u32, + assignment: TestMessageInfo, } /// Helper function to create a signture for all candidates in `to_sign` parameter. @@ -1000,43 +1326,59 @@ fn sign_candidates( to_sign: &mut Vec, keyrings: &Vec<(Keyring, PeerId)>, block_hash: Hash, -) -> TestMessageInfo { + num_coalesce: usize, +) -> DependentMessageInfo { let current_validator_index = to_sign.first().map(|val| val.validator_index).unwrap(); let tranche_trigger_timestamp = to_sign.iter().map(|val| val.tranche).max().unwrap(); let keyring = keyrings.get(current_validator_index.0 as usize).unwrap().clone(); - let to_sign = to_sign + let mut unique_assignments: HashSet = + to_sign.iter().map(|info| info.assignment.clone()).collect(); + + let mut to_sign = to_sign .drain(..) .sorted_by(|val1, val2| val1.candidate_index.cmp(&val2.candidate_index)) - .collect_vec(); + .peekable(); - let hashes = to_sign.iter().map(|val| val.candidate_hash).collect_vec(); - let candidate_indices = to_sign.iter().map(|val| val.candidate_index).collect_vec(); - let sent_by = to_sign - .iter() - .map(|val| val.sent_by.iter()) - .flatten() - .map(|peer| *peer) - .collect::>(); - - let payload = ApprovalVoteMultipleCandidates(&hashes).signing_payload(1); - - let validator_key: ValidatorPair = keyring.0.pair().into(); - let signature = validator_key.sign(&payload[..]); - let indirect = IndirectSignedApprovalVoteV2 { - block_hash, - candidate_indices: candidate_indices.try_into().unwrap(), - validator: current_validator_index, - signature, + let mut bundle = DependentMessageInfo { + assignments: unique_assignments.into_iter().collect_vec(), + approvals: Vec::new(), }; - let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![indirect]); - TestMessageInfo { - msg, - sent_by: sent_by.into_iter().collect_vec(), - tranche: tranche_trigger_timestamp, - block_hash, - typ: MessageType::Approval, + let mut unique_assignments: HashSet = HashSet::new(); + while to_sign.peek().is_some() { + let to_sign = to_sign.by_ref().take(num_coalesce).collect_vec(); + + let hashes = to_sign.iter().map(|val| val.candidate_hash).collect_vec(); + let candidate_indices = to_sign.iter().map(|val| val.candidate_index).collect_vec(); + + let sent_by = to_sign + .iter() + .map(|val| val.sent_by.iter()) + .flatten() + .map(|peer| *peer) + .collect::>(); + + let payload = ApprovalVoteMultipleCandidates(&hashes).signing_payload(1); + + let validator_key: ValidatorPair = keyring.0.clone().pair().into(); + let signature = validator_key.sign(&payload[..]); + let indirect = IndirectSignedApprovalVoteV2 { + block_hash, + candidate_indices: candidate_indices.try_into().unwrap(), + validator: current_validator_index, + signature, + }; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![indirect]); + + bundle.approvals.push(TestMessageInfo { + msg, + sent_by: sent_by.into_iter().collect_vec(), + tranche: tranche_trigger_timestamp, + block_hash, + typ: MessageType::Approval, + }); } + bundle } fn neighbours_that_would_sent_message( @@ -1396,6 +1738,13 @@ fn build_overseer( let db: polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter = polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[]); let keystore = LocalKeystore::in_memory(); + // let _public = keystore + // .sr25519_generate_new( + // ASSIGNMENT_KEY_TYPE_ID, + // Some(state.test_authorities.keyrings[NODE_UNDER_TEST as usize].seed().as_str()), + // ) + // .expect("should not fail"); + let real_system_clock = SystemClock {}; let system_clock = FakeSystemClock::new(SystemClock {}, state.delta_tick_from_generated.clone()); @@ -1538,9 +1887,10 @@ pub async fn bench_approvals_run( env.config().num_blocks as u32 { gum::info!( - "Waiting for all blocks to be approved current approved {:} num_sent {:}", + "Waiting for all blocks to be approved current approved {:} num_sent {:} num_unique {:}", state.last_approved_block.load(std::sync::atomic::Ordering::SeqCst), - state.total_sent_messages.load(std::sync::atomic::Ordering::SeqCst) + state.total_sent_messages.load(std::sync::atomic::Ordering::SeqCst), + state.total_unique_messages.load(std::sync::atomic::Ordering::SeqCst) ); tokio::time::sleep(Duration::from_secs(6)).await; } @@ -1598,9 +1948,10 @@ pub async fn bench_approvals_run( let duration: u128 = start_marker.elapsed().as_millis(); gum::info!( - "All blocks processed in {} num_messages {}", + "All blocks processed in {} num_messages {} num_unique_messages {}", format!("{:?}ms", duration).cyan(), - state.total_sent_messages.load(std::sync::atomic::Ordering::SeqCst) + state.total_sent_messages.load(std::sync::atomic::Ordering::SeqCst), + state.total_unique_messages.load(std::sync::atomic::Ordering::SeqCst) ); gum::info!("{}", &env); diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 24b82927016a..cb9b81937f91 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -199,6 +199,7 @@ fn main() -> eyre::Result<()> { .filter(Some("hyper"), log::LevelFilter::Info) // Avoid `Terminating due to subsystem exit subsystem` warnings .filter(Some("polkadot_overseer"), log::LevelFilter::Error) + .filter(Some("parachain"), log::LevelFilter::Info) .filter(None, log::LevelFilter::Info) .format_timestamp(Some(env_logger::TimestampPrecision::Millis)) .try_init() From 40ef9d5d64f015de458d2ea8ea94fcbecf8b6daa Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 20 Dec 2023 16:47:32 +0200 Subject: [PATCH 153/192] More optimizations Signed-off-by: Alexandru Gheorghe --- polkadot/node/subsystem-bench/src/approval/mod.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index e9063fbf97fd..7a3c6e1e9825 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -898,7 +898,6 @@ impl PeerMessageProducer { let mut no_shows: HashMap<(Hash, usize), HashSet> = HashMap::new(); let mut per_candidate_data: HashMap<(Hash, CandidateIndex), CandidateTestData> = HashMap::new(); - for block_info in self.state.per_slot_heads.iter() { for (candidate_index, _) in block_info.candidates.iter().enumerate() { per_candidate_data.insert( @@ -918,6 +917,8 @@ impl PeerMessageProducer { let mut re_process_skipped = false; while all_messages.peek().is_some() { + sleep(Duration::from_millis(50)).await; + let current_slot = tick_to_slot_number(SLOT_DURATION_MILLIS, system_clock.tick_now()); let block_info = @@ -1048,7 +1049,9 @@ impl PeerMessageProducer { } } } - } else if !block_info.approved.load(std::sync::atomic::Ordering::SeqCst) { + } else if !block_info.approved.load(std::sync::atomic::Ordering::SeqCst) && + self.options.num_no_shows_per_block > 0 + { skipped_messages.push(bundle); } reprocess_skipped From b4afb81c04eb6acca0c063e9f145639b307d5d7a Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 22 Dec 2023 09:33:59 +0200 Subject: [PATCH 154/192] Refactoring Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 2 - polkadot/node/overseer/src/metrics.rs | 4 +- .../src/approval/mock_chain_selection.rs | 5 +- .../node/subsystem-bench/src/approval/mod.rs | 1258 ++--------------- .../subsystem-bench/src/subsystem-bench.rs | 2 +- 5 files changed, 93 insertions(+), 1178 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 41045ed53202..626a7c6be3f5 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -21,8 +21,6 @@ //! //! [approval-distribution-page]: https://paritytech.github.io/polkadot-sdk/book/node/approval/approval-distribution.html -#![warn(missing_docs)] - use self::metrics::Metrics; use futures::{channel::oneshot, select, FutureExt as _}; use itertools::Itertools; diff --git a/polkadot/node/overseer/src/metrics.rs b/polkadot/node/overseer/src/metrics.rs index 9b6053ccf769..18080468dc14 100644 --- a/polkadot/node/overseer/src/metrics.rs +++ b/polkadot/node/overseer/src/metrics.rs @@ -168,7 +168,7 @@ impl MetricsTrait for Metrics { ) .buckets(vec![ 0.0001, 0.0004, 0.0016, 0.0064, 0.0256, 0.1024, 0.4096, 1.6384, 3.2768, - 4.9152, 6.5536, + 4.9152, 6.1, 7.1, 8.0, 13.12, 26.25, ]), &["subsystem_name"], )?, @@ -212,7 +212,7 @@ impl MetricsTrait for Metrics { ) .buckets(vec![ 0.0001, 0.0004, 0.0016, 0.0064, 0.0256, 0.1024, 0.4096, 1.6384, 3.2768, - 4.9152, 6.5536, + 4.9152, 6.1, 7.1, 8.0, 13.12, 26.25, ]), &["subsystem_name"], )?, diff --git a/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs b/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs index c57102620fd8..6892f32209f3 100644 --- a/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs +++ b/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs @@ -16,18 +16,17 @@ use crate::approval::{LOG_TARGET, SLOT_DURATION_MILLIS}; -use super::{ApprovalTestState, FakeSystemClock}; +use super::{ApprovalTestState, PastSystemClock}; use futures::FutureExt; use polkadot_node_core_approval_voting::time::{slot_number_to_tick, Clock, TICK_DURATION_MILLIS}; use polkadot_node_subsystem::{overseer, SpawnedSubsystem, SubsystemError}; use polkadot_node_subsystem_types::messages::ChainSelectionMessage; -use sp_timestamp::Timestamp; /// Mock ChainSelection subsystem used to answer request made by the approval-voting subsystem, /// during benchmark. All the necessary information to answer the requests is stored in the `state` pub struct MockChainSelection { pub state: ApprovalTestState, - pub clock: FakeSystemClock, + pub clock: PastSystemClock, } #[overseer::subsystem(ChainSelection, error=SubsystemError, prefix=self::overseer)] impl MockChainSelection { diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index d0cad8302aae..273a0ae18c01 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -17,12 +17,10 @@ use parity_scale_codec::{Decode, Encode}; use serde::{Deserialize, Serialize}; use std::{ - alloc::System, cmp::max, - collections::{BTreeMap, HashMap, HashSet}, + collections::{HashMap, HashSet}, fs, - io::{Read, Write}, - path::Path, + io::Read, sync::{ atomic::{AtomicBool, AtomicU32, AtomicU64}, Arc, @@ -31,95 +29,61 @@ use std::{ }; use colored::Colorize; -use futures::{channel::oneshot, lock::Mutex, Future, FutureExt, StreamExt}; +use futures::{channel::oneshot, FutureExt}; use itertools::Itertools; use orchestra::TimeoutExt; -use overseer::{messages, metrics::Metrics as OverseerMetrics, MetricsTrait}; +use overseer::{metrics::Metrics as OverseerMetrics, MetricsTrait}; use polkadot_approval_distribution::{ metrics::Metrics as ApprovalDistributionMetrics, ApprovalDistribution, }; use polkadot_node_core_approval_voting::{ - criteria::{compute_assignments, Config}, - time::{ - slot_number_to_tick, tick_to_slot_number, tranche_to_tick, Clock, ClockExt, SystemClock, - Tick, - }, + time::{slot_number_to_tick, tick_to_slot_number, Clock, ClockExt, SystemClock}, ApprovalVotingSubsystem, Metrics as ApprovalVotingMetrics, }; -use polkadot_node_primitives::approval::{ - self, - v1::RelayVRFStory, - v2::{CoreBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2}, -}; - -use sha1::Digest as Sha1Digest; - -use polkadot_node_network_protocol::{ - grid_topology::{ - GridNeighbors, RandomRouting, RequiredRouting, SessionGridTopology, TopologyPeerInfo, - }, - peer_set::{ProtocolVersion, ValidationVersion}, - v3 as protocol_v3, ObservedRole, Versioned, View, -}; +use polkadot_node_primitives::approval::{self, v1::RelayVRFStory}; use polkadot_node_subsystem::{overseer, AllMessages, Overseer, OverseerConnector, SpawnGlue}; use polkadot_node_subsystem_test_helpers::mock::new_block_import_info; use polkadot_overseer::Handle as OverseerHandleReal; -use polkadot_node_core_approval_voting::Config as ApprovalVotingConfig; -use polkadot_node_subsystem_types::messages::{ - network_bridge_event::NewGossipTopology, ApprovalDistributionMessage, ApprovalVotingMessage, - NetworkBridgeEvent, +use self::{ + helpers::{make_candidates, make_header}, + test_message::MessagesBundle, }; - -use rand::{seq::SliceRandom, RngCore, SeedableRng}; -use rand_chacha::ChaCha20Rng; - -use polkadot_primitives::{ - vstaging::ApprovalVoteMultipleCandidates, ApprovalVote, BlockNumber, CandidateEvent, - CandidateHash, CandidateIndex, CandidateReceipt, CoreIndex, GroupIndex, Hash, Header, - Id as ParaId, IndexedVec, SessionInfo, Slot, ValidatorIndex, ValidatorPair, - ASSIGNMENT_KEY_TYPE_ID, -}; -use polkadot_primitives_test_helpers::dummy_candidate_receipt_bad_sig; -use sc_keystore::LocalKeystore; -use sc_network::{network_state::Peer, PeerId}; -use sc_service::SpawnTaskHandle; -use sp_consensus_babe::{ - digests::{CompatibleDigestItem, PreDigest, SecondaryVRFPreDigest}, - AllowedSlots, BabeEpochConfiguration, Epoch as BabeEpoch, SlotDuration, VrfSignature, - VrfTranscript, -}; -use sp_core::{crypto::VrfSecret, Pair}; -use sp_keystore::Keystore; -use sp_runtime::{Digest, DigestItem}; -use std::ops::Sub; - -use sp_keyring::sr25519::Keyring as Sr25519Keyring; -use sp_timestamp::Timestamp; - use crate::{ approval::{ - mock_chain_api::MockChainApi, mock_chain_selection::MockChainSelection, + helpers::{ + generate_babe_epoch, generate_new_session_topology, generate_peer_connected, + generate_peer_view_change_for, session_info_for_peers, PastSystemClock, + }, + message_generator::PeerMessagesGenerator, + mock_chain_api::MockChainApi, + mock_chain_selection::MockChainSelection, mock_runtime_api::MockRuntimeApi, }, core::{ - configuration::{TestAuthorities, TestConfiguration, TestObjective}, + configuration::{TestAuthorities, TestConfiguration}, environment::{TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT}, - keyring::Keyring, - mock::{ - dummy_builder, AlwaysSupportsParachains, MockNetworkBridgeTx, RespondToRequestsInfo, - TestSyncOracle, - }, + mock::{dummy_builder, AlwaysSupportsParachains, MockNetworkBridgeTx, TestSyncOracle}, network::{NetworkAction, NetworkEmulator}, }, }; - +use polkadot_node_core_approval_voting::Config as ApprovalVotingConfig; +use polkadot_node_subsystem_types::messages::{ApprovalDistributionMessage, ApprovalVotingMessage}; +use polkadot_primitives::{ + BlockNumber, CandidateEvent, CandidateIndex, Hash, Header, SessionInfo, Slot, ValidatorIndex, +}; +use sc_keystore::LocalKeystore; +use sc_service::SpawnTaskHandle; +use sp_consensus_babe::Epoch as BabeEpoch; +use std::ops::Sub; use tokio::time::sleep; +mod helpers; mod message_generator; mod mock_chain_api; mod mock_chain_selection; mod mock_runtime_api; +mod test_message; pub const LOG_TARGET: &str = "bench::approval"; const DATA_COL: u32 = 0; @@ -131,6 +95,7 @@ pub(crate) const TEST_CONFIG: ApprovalVotingConfig = ApprovalVotingConfig { }; pub const NODE_UNDER_TEST: u32 = 0; +pub const NEEDED_APPROVALS: u32 = 30; /// Start generating messages for a slot into the future, so that the /// generation nevers falls behind the current slot. @@ -171,7 +136,7 @@ pub struct ApprovalsOptions { } impl ApprovalsOptions { - fn gen_key(&self) -> Vec { + fn fingerprint(&self) -> Vec { let mut bytes = Vec::new(); bytes.extend(self.last_considered_tranche.to_be_bytes()); bytes.extend(self.min_coalesce.to_be_bytes()); @@ -204,43 +169,57 @@ struct BlockTestData { /// This set on `true` when ChainSelectionMessage::Approved is received inside the chain /// selection mock subsystem. approved: Arc, - prev_candidates: u64, + /// The total number of candidates before this block. + total_candidates_before: u64, + /// The votes we sent. + /// votes[validator_index][candidate_index] tells if validator sent vote for candidate. + /// We use this to mark the test as succesfull if GetApprovalSignatures returns all the votes + /// from here. votes: Arc>>, - per_candidate_votes: Arc>, - per_candidate_min_tranches: Arc>, } +/// Candidate information used during the test to decide if more messages are needed. #[derive(Debug)] struct CandidateTestData { + /// The configured maximum number of no-shows for this candidate. max_no_shows: u32, + /// The last tranche where we had a no-show. last_tranche_with_no_show: u32, + /// The number of sent assignments. sent_assignment: u32, + /// The number of no-shows. num_no_shows: u32, + /// The maximum tranche were we covered the needed approvals max_tranche: u32, } impl CandidateTestData { + /// If message in this tranche needs to be sent. fn should_send_tranche(&self, tranche: u32) -> bool { - self.sent_assignment <= 30 || tranche <= self.max_tranche + self.num_no_shows + self.sent_assignment <= NEEDED_APPROVALS || tranche <= self.max_tranche + self.num_no_shows } + /// Sets max tranche fn set_max_tranche(&mut self, tranche: u32) { self.max_tranche = max(tranche, self.max_tranche); } + /// Records no-show for candidate. fn record_no_show(&mut self, tranche: u32) { self.num_no_shows += 1; self.last_tranche_with_no_show = max(tranche, self.last_tranche_with_no_show); } + /// Marks an assignment sent. fn sent_assignment(&mut self, tranche: u32) { - if self.sent_assignment < 30 { + if self.sent_assignment < NEEDED_APPROVALS { self.set_max_tranche(tranche); } self.sent_assignment += 1; } + /// Tells if a message in this tranche should be a no-show. fn should_no_show(&self, tranche: u32) -> bool { (self.num_no_shows < self.max_no_shows && self.last_tranche_with_no_show < tranche) || (tranche == 0 && self.num_no_shows == 0 && self.max_no_shows > 0) @@ -254,8 +233,6 @@ impl CandidateTestData { /// updated between subsystems, they would have to be wrapped in Arc's. #[derive(Clone)] pub struct ApprovalTestState { - /// The generic test configuration passed when starting the benchmark. - configuration: TestConfiguration, /// The specific test configurations passed when starting the benchmark. options: ApprovalsOptions, /// The list of blocks used for testing. @@ -276,6 +253,8 @@ pub struct ApprovalTestState { total_unique_messages: Arc, /// Approval voting metrics. approval_voting_metrics: ApprovalVotingMetrics, + /// The delta ticks from the tick the messages were generated to the the time we start this + /// message. delta_tick_from_generated: Arc, } @@ -290,52 +269,23 @@ impl ApprovalTestState { let test_authorities = configuration.generate_authorities(); let start = Instant::now(); - let mut key = options.gen_key(); - let mut exclude_objective = configuration.clone(); - exclude_objective.objective = TestObjective::Unimplemented; - let configuration_bytes = bincode::serialize(&exclude_objective).unwrap(); - key.extend(configuration_bytes); - let mut sha1 = sha1::Sha1::new(); - sha1.update(key); - let result = sha1.finalize(); - let path_name = format!("{}/{}", options.workdir_prefix, hex::encode(result)); - let delta_to_first_slot_under_test = Timestamp::new(BUFFER_FOR_GENERATION_MILLIS); - - let messages = Path::new(&path_name); - if !messages.exists() { - gum::info!("Generate message because filed does not exist"); - let delta_to_first_slot_under_test = Timestamp::new(BUFFER_FOR_GENERATION_MILLIS); - let initial_slot = Slot::from_timestamp( - (*Timestamp::current() - *delta_to_first_slot_under_test).into(), - SlotDuration::from_millis(SLOT_DURATION_MILLIS), - ); - - let babe_epoch = generate_babe_epoch(initial_slot, test_authorities.clone()); - let session_info = session_info_for_peers(configuration, test_authorities.clone()); - let blocks = - Self::generate_blocks_information(configuration, &babe_epoch, initial_slot); - - let generate = ApprovalTestState::generate_messages( - initial_slot, - &test_authorities, - &blocks, - &session_info, - &options, - &dependencies.task_manager.spawn_handle(), - &messages, - ); - } + let messages_path = PeerMessagesGenerator::generate_all_messages( + configuration, + &test_authorities, + &options, + &dependencies.task_manager.spawn_handle(), + ); - let mut file = fs::OpenOptions::new().read(true).open(messages).unwrap(); + let mut file = fs::OpenOptions::new().read(true).open(messages_path.as_path()).unwrap(); let mut content = Vec::::with_capacity(2000000); - file.read_to_end(&mut content); + file.read_to_end(&mut content).expect("Could not initialize list of messages"); let mut content = content.as_slice(); let generated_state: GeneratedState = Decode::decode(&mut content).expect("Could not decode messages"); gum::info!( - "It took {:?} ms count {:?}", + "It took {:?} ms to load {:?} unique messages", start.elapsed().as_millis(), generated_state.all_messages.as_ref().map(|val| val.len()).unwrap_or_default() ); @@ -349,11 +299,10 @@ impl ApprovalTestState { generated_state.initial_slot, ); - let mut state = ApprovalTestState { + let state = ApprovalTestState { per_slot_heads: blocks, babe_epoch: babe_epoch.clone(), session_info: session_info.clone(), - configuration: configuration.clone(), generated_state, test_authorities, last_approved_block: Arc::new(AtomicU32::new(0)), @@ -409,7 +358,7 @@ impl ApprovalTestState { ), relay_vrf_story, approved: Arc::new(AtomicBool::new(false)), - prev_candidates, + total_candidates_before: prev_candidates, votes: Arc::new( (0..configuration.n_validators) .map(|_| { @@ -417,12 +366,6 @@ impl ApprovalTestState { }) .collect_vec(), ), - per_candidate_votes: Arc::new( - (0..configuration.n_cores).map(|_| AtomicU32::new(0)).collect_vec(), - ), - per_candidate_min_tranches: Arc::new( - (0..configuration.n_cores).map(|_| AtomicU32::new(0)).collect_vec(), - ), }; prev_candidates += block_info.candidates.len() as u64; per_block_heads.push(block_info) @@ -439,11 +382,6 @@ impl ApprovalTestState { ) -> oneshot::Receiver<()> { gum::info!(target: LOG_TARGET, "Start assignments/approvals production"); - let topology = generate_topology(&self.test_authorities); - - let topology_node_under_test = - topology.compute_grid_neighbors_for(ValidatorIndex(NODE_UNDER_TEST)).unwrap(); - let (producer_tx, producer_rx) = oneshot::channel(); let peer_message_source = PeerMessageProducer { network: network_emulator.clone(), @@ -459,107 +397,11 @@ impl ApprovalTestState { ); producer_rx } - - /// Starts the generation of messages(Assignments & Approvals) needed for approving blocks. - fn generate_messages( - initial_slot: Slot, - test_authorities: &TestAuthorities, - blocks: &Vec, - session_info: &SessionInfo, - options: &ApprovalsOptions, - spawn_task_handle: &SpawnTaskHandle, - path: &Path, - ) { - gum::info!(target: LOG_TARGET, "Generate messages"); - let topology = generate_topology(&test_authorities); - - let random_samplings = random_samplings_to_node_patterns( - ValidatorIndex(NODE_UNDER_TEST), - test_authorities.keyrings.len(), - test_authorities.keyrings.len() as usize * 2, - ); - - let topology_node_under_test = - topology.compute_grid_neighbors_for(ValidatorIndex(NODE_UNDER_TEST)).unwrap(); - - let (tx, mut rx) = futures::channel::mpsc::unbounded(); - for current_validator_index in 1..test_authorities.keyrings.len() { - let peer_message_source = message_generator::PeerMessagesGenerator { - topology_node_under_test: topology_node_under_test.clone(), - topology: topology.clone(), - validator_index: ValidatorIndex(current_validator_index as u32), - test_authorities: test_authorities.clone(), - session_info: session_info.clone(), - blocks: blocks.clone(), - tx_messages: tx.clone(), - random_samplings: random_samplings.clone(), - enable_assignments_v2: options.enable_assignments_v2, - last_considered_tranche: options.last_considered_tranche, - min_coalesce: options.min_coalesce, - max_coalesce: options.max_coalesce, - coalesce_tranche_diff: options.coalesce_tranche_diff, - }; - - peer_message_source.generate_messages(&spawn_task_handle); - } - - std::mem::drop(tx); - - let seed = [0x32; 32]; - let mut rand_chacha = ChaCha20Rng::from_seed(seed); - let mut file = fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(path) - .unwrap(); - - let mut all_messages: BTreeMap> = BTreeMap::new(); - - loop { - match rx.try_next() { - Ok(Some((block_hash, messages))) => - for message in messages { - let block_info = blocks - .iter() - .find(|val| val.hash == block_hash) - .expect("Should find blocks"); - let tick_to_send = tranche_to_tick( - SLOT_DURATION_MILLIS, - block_info.slot, - message.tranche_to_send(), - ); - if !all_messages.contains_key(&tick_to_send) { - all_messages.insert(tick_to_send, Vec::new()); - } - - let to_add = all_messages.get_mut(&tick_to_send).unwrap(); - to_add.push(message); - }, - Ok(None) => break, - Err(_) => { - std::thread::sleep(Duration::from_millis(50)); - }, - } - } - let all_messages = all_messages - .into_iter() - .map(|(_, mut messages)| { - messages.shuffle(&mut rand_chacha); - messages - }) - .flatten() - .collect_vec(); - gum::info!("Took something like {:}", all_messages.len()); - - let generated_state = GeneratedState { all_messages: Some(all_messages), initial_slot }; - - file.write_all(&generated_state.encode()); - } } + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] struct GeneratedState { - all_messages: Option>, + all_messages: Option>, initial_slot: Slot, } @@ -588,286 +430,6 @@ impl ApprovalTestState { } } -/// Type of generated messages. -#[derive(Debug, Copy, Clone, Encode, Decode, PartialEq, Eq, Hash)] - -enum MessageType { - Approval, - Assignment, - Other, -} - -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -struct TestMessageInfo { - /// The actual message - msg: protocol_v3::ApprovalDistributionMessage, - /// The list of peers that would sends this message in a real topology. - /// It includes both the peers that would send the message because of the topology - /// or because of randomly chosing so. - sent_by: Vec, - /// The tranche at which this message should be sent. - tranche: u32, - /// The block hash this message refers to. - block_hash: Hash, - /// The type of the message. - typ: MessageType, -} - -impl std::hash::Hash for TestMessageInfo { - fn hash(&self, state: &mut H) { - match &self.msg { - protocol_v3::ApprovalDistributionMessage::Assignments(assignments) => { - for (assignment, candidates) in assignments { - (assignment.block_hash, assignment.validator).hash(state); - candidates.hash(state); - } - }, - protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => { - for approval in approvals { - (approval.block_hash, approval.validator).hash(state); - approval.candidate_indices.hash(state); - } - }, - }; - self.typ.hash(state); - } -} - -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -struct DependentMessageInfo { - assignments: Vec, - approvals: Vec, -} - -impl DependentMessageInfo { - fn tranche_to_send(&self) -> u32 { - self.assignments - .iter() - .chain(self.approvals.iter()) - .max_by(|a, b| a.tranche.cmp(&b.tranche)) - .unwrap() - .tranche - } - - fn min_tranche(&self) -> u32 { - self.assignments - .iter() - .chain(self.approvals.iter()) - .min_by(|a, b| a.tranche.cmp(&b.tranche)) - .unwrap() - .tranche - } - - fn bundle_needed( - &self, - candidates_test_data: &HashMap<(Hash, CandidateIndex), CandidateTestData>, - options: &ApprovalsOptions, - ) -> bool { - self.should_send(candidates_test_data) || - (!options.stop_when_approved && self.min_tranche() <= options.send_till_tranche) - } - - fn should_send( - &self, - candidates_test_data: &HashMap<(Hash, CandidateIndex), CandidateTestData>, - ) -> bool { - self.assignments.iter().any(|message| message.should_send(candidates_test_data)) - } - - fn record_sent_assignment( - &self, - candidates_test_data: &mut HashMap<(Hash, CandidateIndex), CandidateTestData>, - ) { - self.assignments - .iter() - .for_each(|assignment| assignment.record_sent_assignment(candidates_test_data)); - } -} - -impl TestMessageInfo { - /// Returns the lantency based on the message type. - fn to_all_messages_from_peer(self, peer: PeerId) -> AllMessages { - AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate( - NetworkBridgeEvent::PeerMessage(peer, Versioned::V3(self.msg)), - )) - } - - fn get_latency(&self) -> Option { - match &self.typ { - // We want assignments to always arrive before approval, so - // we don't send them with a latency. - MessageType::Approval => None, - MessageType::Assignment => None, - MessageType::Other => None, - } - } - - fn record_vote(&self, state: &BlockTestData) { - if let MessageType::Approval = self.typ { - match &self.msg { - protocol_v3::ApprovalDistributionMessage::Assignments(_) => todo!(), - protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => - for approval in approvals { - for candidate_index in approval.candidate_indices.iter_ones() { - state - .votes - .get(approval.validator.0 as usize) - .unwrap() - .get(candidate_index) - .unwrap() - .store(true, std::sync::atomic::Ordering::SeqCst); - } - }, - } - } - } - - fn record_sent_assignment( - &self, - candidates_test_data: &mut HashMap<(Hash, CandidateIndex), CandidateTestData>, - ) { - match &self.msg { - protocol_v3::ApprovalDistributionMessage::Assignments(assignments) => { - for (assignment, candidate_indices) in assignments { - for candidate_index in candidate_indices.iter_ones() { - let candidate_test_data = candidates_test_data - .get_mut(&(assignment.block_hash, candidate_index as CandidateIndex)) - .unwrap(); - candidate_test_data.sent_assignment(self.tranche) - } - } - }, - protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => todo!(), - } - } - - fn candidate_indices(&self) -> HashSet { - let mut unique_candidate_indicies = HashSet::new(); - match &self.msg { - protocol_v3::ApprovalDistributionMessage::Assignments(assignments) => - for (assignment, candidate_indices) in assignments { - for candidate_index in candidate_indices.iter_ones() { - unique_candidate_indicies.insert(candidate_index); - } - }, - protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => - for approval in approvals { - for candidate_index in approval.candidate_indices.iter_ones() { - unique_candidate_indicies.insert(candidate_index); - } - }, - } - unique_candidate_indicies - } - - fn no_show_if_required( - &self, - assignments: &Vec, - candidates_test_data: &mut HashMap<(Hash, CandidateIndex), CandidateTestData>, - ) -> bool { - let mut should_no_show = false; - if let MessageType::Approval = self.typ { - let covered_candidates = assignments - .iter() - .map(|assignment| (assignment, assignment.candidate_indices())) - .collect_vec(); - match &self.msg { - protocol_v3::ApprovalDistributionMessage::Assignments(_) => todo!(), - protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => { - assert_eq!(approvals.len(), 1); - - for approval in approvals { - should_no_show = should_no_show || - approval.candidate_indices.iter_ones().all(|candidate_index| { - let candidate_test_data = candidates_test_data - .get_mut(&( - approval.block_hash, - candidate_index as CandidateIndex, - )) - .unwrap(); - let assignment = covered_candidates - .iter() - .filter(|(assignment, candidates)| { - candidates.contains(&candidate_index) - }) - .next() - .unwrap(); - candidate_test_data.should_no_show(assignment.0.tranche) - }); - - if should_no_show { - for candidate_index in approval.candidate_indices.iter_ones() { - let candidate_test_data = candidates_test_data - .get_mut(&( - approval.block_hash, - candidate_index as CandidateIndex, - )) - .unwrap(); - let assignment = covered_candidates - .iter() - .filter(|(assignment, candidates)| { - candidates.contains(&candidate_index) - }) - .next() - .unwrap(); - candidate_test_data.record_no_show(assignment.0.tranche) - } - } - } - }, - } - } - should_no_show - } - - fn should_send( - &self, - candidates_test_data: &HashMap<(Hash, CandidateIndex), CandidateTestData>, - ) -> bool { - match &self.msg { - protocol_v3::ApprovalDistributionMessage::Assignments(assignments) => - assignments.iter().any(|(assignment, candidate_indices)| { - candidate_indices.iter_ones().any(|candidate_index| { - candidates_test_data - .get(&(assignment.block_hash, candidate_index as CandidateIndex)) - .map(|data| data.should_send_tranche(self.tranche)) - .unwrap_or_default() - }) - }), - protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => - approvals.iter().any(|approval| { - approval.candidate_indices.iter_ones().any(|candidate_index| { - candidates_test_data - .get(&(approval.block_hash, candidate_index as CandidateIndex)) - .map(|data| data.should_send_tranche(self.tranche)) - .unwrap_or_default() - }) - }), - } - } - - /// Splits a message into multiple messages based on what peers should send this message. - /// It build a HashMap of messages that should be sent by each peer. - fn split_by_peer_id( - self, - authorities: &TestAuthorities, - ) -> HashMap<(ValidatorIndex, PeerId), Vec> { - let mut result: HashMap<(ValidatorIndex, PeerId), Vec> = HashMap::new(); - - for validator_index in &self.sent_by { - let peer = authorities.peer_ids.get(validator_index.0 as usize).unwrap(); - result.entry((*validator_index, *peer)).or_default().push(TestMessageInfo { - msg: self.msg.clone(), - sent_by: Default::default(), - tranche: self.tranche, - block_hash: self.block_hash, - typ: self.typ, - }); - } - result - } -} - /// A generator of messages coming from a given Peer/Validator struct PeerMessageProducer { /// The state state used to know what messages to generate. @@ -888,14 +450,13 @@ impl PeerMessageProducer { fn produce_messages( mut self, spawn_task_handle: &SpawnTaskHandle, - all_messages: Vec, + all_messages: Vec, ) { spawn_task_handle.spawn_blocking("produce-messages", "produce-messages", async move { let mut already_generated = HashSet::new(); let system_clock = - FakeSystemClock::new(SystemClock {}, self.state.delta_tick_from_generated.clone()); + PastSystemClock::new(SystemClock {}, self.state.delta_tick_from_generated.clone()); let mut all_messages = all_messages.into_iter().peekable(); - let mut no_shows: HashMap<(Hash, usize), HashSet> = HashMap::new(); let mut per_candidate_data: HashMap<(Hash, CandidateIndex), CandidateTestData> = HashMap::new(); for block_info in self.state.per_slot_heads.iter() { @@ -913,7 +474,7 @@ impl PeerMessageProducer { } } - let mut skipped_messages: Vec = Vec::new(); + let mut skipped_messages: Vec = Vec::new(); let mut re_process_skipped = false; while all_messages.peek().is_some() { @@ -926,7 +487,8 @@ impl PeerMessageProducer { if let Some(block_info) = block_info { let candidates = self.state.approval_voting_metrics.candidates_imported(); - if candidates >= block_info.prev_candidates + block_info.candidates.len() as u64 && + if candidates >= + block_info.total_candidates_before + block_info.candidates.len() as u64 && already_generated.insert(block_info.hash) { gum::info!("Setup {:?}", block_info.hash); @@ -947,7 +509,7 @@ impl PeerMessageProducer { let validator = ValidatorIndex(validator); let view_update = - generate_peer_view_change_for(block_info.hash, *peer_id, validator); + generate_peer_view_change_for(block_info.hash, *peer_id); self.send_message(view_update, validator, None); } @@ -981,7 +543,6 @@ impl PeerMessageProducer { self.process_message( bundle, &mut per_candidate_data, - &already_generated, &mut skipped_messages, ); } @@ -996,18 +557,17 @@ impl PeerMessageProducer { let (tx, rx) = oneshot::channel(); let msg = ApprovalDistributionMessage::GetApprovalSignatures(HashSet::new(), tx); self.send_message(AllMessages::ApprovalDistribution(msg), ValidatorIndex(0), None); - rx.await; - self.notify_done.send(()); + rx.await.expect("Failed to get signatures"); + self.notify_done.send(()).expect("Failed to notify main loop"); gum::info!("All messages processed "); }); } pub fn process_message( &mut self, - bundle: DependentMessageInfo, + bundle: MessagesBundle, per_candidate_data: &mut HashMap<(Hash, CandidateIndex), CandidateTestData>, - initialized_blocks: &HashSet, - skipped_messages: &mut Vec, + skipped_messages: &mut Vec, ) -> bool { let mut reprocess_skipped = false; let block_info = self @@ -1016,10 +576,6 @@ impl PeerMessageProducer { .clone(); if bundle.bundle_needed(&per_candidate_data, &self.options) { - // if bundle.min_tranche() >= 85 { - // gum::info!("============= {:?} =====", bundle); - // } - bundle.record_sent_assignment(per_candidate_data); let assignments = bundle.assignments.clone(); @@ -1060,10 +616,10 @@ impl PeerMessageProducer { pub fn time_to_process_message( &self, - bundle: &DependentMessageInfo, + bundle: &MessagesBundle, current_slot: Slot, initialized_blocks: &HashSet, - system_clock: &FakeSystemClock, + system_clock: &PastSystemClock, per_candidate_data: &HashMap<(Hash, CandidateIndex), CandidateTestData>, ) -> bool { let block_info = @@ -1080,7 +636,7 @@ impl PeerMessageProducer { } pub fn time_to_send( - bundle: &DependentMessageInfo, + bundle: &MessagesBundle, tranche_now: u32, current_slot: Slot, block_info: &BlockTestData, @@ -1129,603 +685,6 @@ impl PeerMessageProducer { } } -/// Helper function to create a a signature for the block header. -fn garbage_vrf_signature() -> VrfSignature { - let transcript = VrfTranscript::new(b"test-garbage", &[]); - Sr25519Keyring::Alice.pair().vrf_sign(&transcript.into()) -} - -/// Helper function to create a block header. -fn make_header(parent_hash: Hash, slot: Slot, number: u32) -> Header { - let digest = - { - let mut digest = Digest::default(); - let vrf_signature = garbage_vrf_signature(); - digest.push(DigestItem::babe_pre_digest(PreDigest::SecondaryVRF( - SecondaryVRFPreDigest { authority_index: 0, slot, vrf_signature }, - ))); - digest - }; - - Header { - digest, - extrinsics_root: Default::default(), - number, - state_root: Default::default(), - parent_hash, - } -} - -/// Helper function to create a candidate receipt. -fn make_candidate(para_id: ParaId, hash: &Hash) -> CandidateReceipt { - let mut r = dummy_candidate_receipt_bad_sig(*hash, Some(Default::default())); - r.descriptor.para_id = para_id; - r -} - -/// Helper function to create a list of candidates that are included in the block -fn make_candidates( - block_hash: Hash, - block_number: BlockNumber, - num_cores: u32, - num_candidates: u32, -) -> Vec { - let seed = [block_number as u8; 32]; - let mut rand_chacha = ChaCha20Rng::from_seed(seed); - let mut candidates = (0..num_cores) - .map(|core| { - CandidateEvent::CandidateIncluded( - make_candidate(ParaId::from(core), &block_hash), - Vec::new().into(), - CoreIndex(core), - GroupIndex(core), - ) - }) - .collect_vec(); - let (candidates, _) = candidates.partial_shuffle(&mut rand_chacha, num_candidates as usize); - candidates - .into_iter() - .map(|val| val.clone()) - .sorted_by(|a, b| match (a, b) { - ( - CandidateEvent::CandidateIncluded(_, _, core_a, _), - CandidateEvent::CandidateIncluded(_, _, core_b, _), - ) => core_a.0.cmp(&core_b.0), - (_, _) => todo!("Should not happen"), - }) - .collect_vec() -} - -/// Generates a test session info with all passed authorities as consensus validators. -fn session_info_for_peers( - configuration: &TestConfiguration, - authorities: TestAuthorities, -) -> SessionInfo { - let keys = authorities.keyrings.iter().zip(authorities.peer_ids.iter()); - SessionInfo { - validators: keys.clone().map(|v| v.0.clone().public().into()).collect(), - discovery_keys: keys.clone().map(|v| v.0.clone().public().into()).collect(), - assignment_keys: keys.clone().map(|v| v.0.clone().public().into()).collect(), - validator_groups: IndexedVec::>::from( - (0..authorities.keyrings.len()) - .map(|index| vec![ValidatorIndex(index as u32)]) - .collect_vec(), - ), - n_cores: configuration.n_cores as u32, - needed_approvals: 30, - zeroth_delay_tranche_width: 0, - relay_vrf_modulo_samples: 6, - n_delay_tranches: 89, - no_show_slots: 3, - active_validator_indices: (0..authorities.keyrings.len()) - .map(|index| ValidatorIndex(index as u32)) - .collect_vec(), - dispute_period: 6, - random_seed: [0u8; 32], - } -} - -/// Helper function to randomly determine how many approvals we coalesce together in a single -/// message. -fn coalesce_approvals_len( - min_coalesce: u32, - max_coalesce: u32, - rand_chacha: &mut ChaCha20Rng, -) -> usize { - let mut sampling: Vec = (min_coalesce as usize..max_coalesce as usize + 1).collect_vec(); - *(sampling.partial_shuffle(rand_chacha, 1).0.first().unwrap()) -} - -/// Helper function to create approvals signatures for all assignments passed as arguments. -/// Returns a list of Approvals messages that need to be sent. -fn issue_approvals( - assignments: Vec, - block_hash: Hash, - keyrings: Vec<(Keyring, PeerId)>, - candidates: Vec, - min_coalesce: u32, - max_coalesce: u32, - coalesce_tranche_diff: u32, - rand_chacha: &mut ChaCha20Rng, -) -> Vec { - let mut to_sign: Vec = Vec::new(); - let mut num_coalesce = coalesce_approvals_len(min_coalesce, max_coalesce, rand_chacha); - let result = assignments - .iter() - .enumerate() - .map(|(_index, message)| match &message.msg { - protocol_v3::ApprovalDistributionMessage::Assignments(assignments) => { - let mut approvals_to_create = Vec::new(); - - let current_validator_index = - to_sign.first().map(|msg| msg.validator_index).unwrap_or(ValidatorIndex(999)); - - // Invariant for this benchmark. - assert_eq!(assignments.len(), 1); - - let assignment = assignments.first().unwrap(); - - let earliest_tranche = - to_sign.first().map(|val| val.tranche).unwrap_or(message.tranche); - - if to_sign.len() >= num_coalesce as usize || - (!to_sign.is_empty() && current_validator_index != assignment.0.validator) || - message.tranche - earliest_tranche >= coalesce_tranche_diff - { - num_coalesce = coalesce_approvals_len(min_coalesce, max_coalesce, rand_chacha); - approvals_to_create.push(sign_candidates( - &mut to_sign, - &keyrings, - block_hash, - num_coalesce, - )) - } - - // If more that one candidate was in the assignment queue all of them. - for candidate_index in assignment.1.iter_ones() { - let candidate = candidates.get(candidate_index).unwrap(); - if let CandidateEvent::CandidateIncluded(candidate, _, _, _) = candidate { - to_sign.push(TestSignInfo { - candidate_hash: candidate.hash(), - candidate_index: candidate_index as CandidateIndex, - validator_index: assignment.0.validator, - sent_by: message.sent_by.clone().into_iter().collect(), - tranche: message.tranche, - assignment: message.clone(), - }); - } else { - todo!("Other enum variants are not used in this benchmark"); - } - } - approvals_to_create - }, - _ => { - todo!("Other enum variants are not used in this benchmark"); - }, - }) - .collect_vec(); - - let mut result = result.into_iter().flatten().collect_vec(); - - if !to_sign.is_empty() { - result.push(sign_candidates(&mut to_sign, &keyrings, block_hash, num_coalesce)); - } - result -} - -/// Helper struct to gather information about more than one candidate an sign it in a single -/// approval message. -struct TestSignInfo { - candidate_hash: CandidateHash, - candidate_index: CandidateIndex, - validator_index: ValidatorIndex, - sent_by: HashSet, - tranche: u32, - assignment: TestMessageInfo, -} - -/// Helper function to create a signture for all candidates in `to_sign` parameter. -/// Returns a TestMessage -fn sign_candidates( - to_sign: &mut Vec, - keyrings: &Vec<(Keyring, PeerId)>, - block_hash: Hash, - num_coalesce: usize, -) -> DependentMessageInfo { - let current_validator_index = to_sign.first().map(|val| val.validator_index).unwrap(); - let tranche_trigger_timestamp = to_sign.iter().map(|val| val.tranche).max().unwrap(); - let keyring = keyrings.get(current_validator_index.0 as usize).unwrap().clone(); - - let mut unique_assignments: HashSet = - to_sign.iter().map(|info| info.assignment.clone()).collect(); - - let mut to_sign = to_sign - .drain(..) - .sorted_by(|val1, val2| val1.candidate_index.cmp(&val2.candidate_index)) - .peekable(); - - let mut bundle = DependentMessageInfo { - assignments: unique_assignments.into_iter().collect_vec(), - approvals: Vec::new(), - }; - let mut unique_assignments: HashSet = HashSet::new(); - while to_sign.peek().is_some() { - let to_sign = to_sign.by_ref().take(num_coalesce).collect_vec(); - - let hashes = to_sign.iter().map(|val| val.candidate_hash).collect_vec(); - let candidate_indices = to_sign.iter().map(|val| val.candidate_index).collect_vec(); - - let sent_by = to_sign - .iter() - .map(|val| val.sent_by.iter()) - .flatten() - .map(|peer| *peer) - .collect::>(); - - let payload = ApprovalVoteMultipleCandidates(&hashes).signing_payload(1); - - let validator_key: ValidatorPair = keyring.0.clone().pair().into(); - let signature = validator_key.sign(&payload[..]); - let indirect = IndirectSignedApprovalVoteV2 { - block_hash, - candidate_indices: candidate_indices.try_into().unwrap(), - validator: current_validator_index, - signature, - }; - let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![indirect]); - - bundle.approvals.push(TestMessageInfo { - msg, - sent_by: sent_by.into_iter().collect_vec(), - tranche: tranche_trigger_timestamp, - block_hash, - typ: MessageType::Approval, - }); - } - bundle -} - -fn neighbours_that_would_sent_message( - keyrings: &Vec<(Keyring, PeerId)>, - current_validator_index: u32, - topology_node_under_test: &GridNeighbors, - topology: &SessionGridTopology, -) -> Vec<(ValidatorIndex, PeerId)> { - let topology_originator = topology - .compute_grid_neighbors_for(ValidatorIndex(current_validator_index as u32)) - .unwrap(); - - let originator_y = topology_originator - .validator_indices_y - .iter() - .filter(|validator| { - topology_node_under_test.required_routing_by_index(**validator, false) == - RequiredRouting::GridY - }) - .next(); - let originator_x = topology_originator - .validator_indices_x - .iter() - .filter(|validator| { - topology_node_under_test.required_routing_by_index(**validator, false) == - RequiredRouting::GridX - }) - .next(); - - let is_neighbour = topology_originator - .validator_indices_x - .contains(&ValidatorIndex(NODE_UNDER_TEST)) || - topology_originator - .validator_indices_y - .contains(&ValidatorIndex(NODE_UNDER_TEST)); - - let mut to_be_sent_by = originator_y - .into_iter() - .chain(originator_x) - .map(|val| (*val, keyrings[val.0 as usize].1)) - .collect_vec(); - - if is_neighbour { - to_be_sent_by.push((ValidatorIndex(NODE_UNDER_TEST), keyrings[0].1)); - } - to_be_sent_by -} - -/// Generates assignments for the given `current_validator_index` -/// Returns a list of assignments to be sent sorted by tranche. -fn generate_assignments( - block_info: &BlockTestData, - keyrings: Vec<(Keyring, PeerId)>, - session_info: &SessionInfo, - generate_v2_assignments: bool, - random_samplings: &Vec>, - current_validator_index: u32, - relay_vrf_story: &RelayVRFStory, - topology_node_under_test: &GridNeighbors, - topology: &SessionGridTopology, - last_considered_tranche: u32, -) -> Vec { - let config = Config::from(session_info); - - let leaving_cores = block_info - .candidates - .clone() - .into_iter() - .map(|candidate_event| { - if let CandidateEvent::CandidateIncluded(candidate, _, core_index, group_index) = - candidate_event - { - (candidate.hash(), core_index, group_index) - } else { - todo!("Variant is never created in this benchmark") - } - }) - .collect_vec(); - - let mut assignments_by_tranche = BTreeMap::new(); - - let bytes = current_validator_index.to_be_bytes(); - let seed = [ - bytes[0], bytes[1], bytes[2], bytes[3], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]; - let mut rand_chacha = ChaCha20Rng::from_seed(seed); - - let to_be_sent_by = neighbours_that_would_sent_message( - &keyrings, - current_validator_index, - topology_node_under_test, - topology, - ); - - let store = LocalKeystore::in_memory(); - let _public = store - .sr25519_generate_new( - ASSIGNMENT_KEY_TYPE_ID, - Some(keyrings[current_validator_index as usize].0.seed().as_str()), - ) - .expect("should not fail"); - - let leaving_cores = leaving_cores - .clone() - .into_iter() - .filter(|(_, core_index, _group_index)| core_index.0 != current_validator_index) - .collect_vec(); - - let assignments = compute_assignments( - &store, - relay_vrf_story.clone(), - &config, - leaving_cores.clone(), - generate_v2_assignments, - ); - - let random_sending_nodes = random_samplings - .get(rand_chacha.next_u32() as usize % random_samplings.len()) - .unwrap(); - let random_sending_peer_ids = random_sending_nodes - .into_iter() - .map(|validator| (*validator, keyrings[validator.0 as usize].1)) - .collect_vec(); - - let mut no_duplicates = HashSet::new(); - for (core_index, assignment) in assignments { - let assigned_cores = match &assignment.cert().kind { - approval::v2::AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => - core_bitfield.iter_ones().map(|val| CoreIndex::from(val as u32)).collect_vec(), - approval::v2::AssignmentCertKindV2::RelayVRFDelay { core_index } => vec![*core_index], - approval::v2::AssignmentCertKindV2::RelayVRFModulo { sample: _ } => vec![core_index], - }; - if assignment.tranche() > last_considered_tranche { - continue - } - - let bitfiled: CoreBitfield = assigned_cores.clone().try_into().unwrap(); - - // For the cases where tranch0 assignments are in a single certificate we need to make - // sure we create a single message. - if no_duplicates.insert(bitfiled) { - if !assignments_by_tranche.contains_key(&assignment.tranche()) { - assignments_by_tranche.insert(assignment.tranche(), Vec::new()); - } - let this_tranche_assignments = - assignments_by_tranche.get_mut(&assignment.tranche()).unwrap(); - this_tranche_assignments.push(( - IndirectAssignmentCertV2 { - block_hash: block_info.hash, - validator: ValidatorIndex(current_validator_index), - cert: assignment.cert().clone(), - }, - block_info - .candidates - .iter() - .enumerate() - .filter(|(_index, candidate)| { - if let CandidateEvent::CandidateIncluded(_, _, core, _) = candidate { - assigned_cores.contains(core) - } else { - panic!("Should not happen"); - } - }) - .map(|(index, _)| index as u32) - .collect_vec() - .try_into() - .unwrap(), - to_be_sent_by - .iter() - .chain(random_sending_peer_ids.iter()) - .map(|peer| *peer) - .collect::>(), - assignment.tranche(), - )); - } - } - - let res = assignments_by_tranche - .into_values() - .map(|assignments| assignments.into_iter()) - .flatten() - .map(|indirect| { - let validator_index = indirect.0.validator.0; - let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( - indirect.0, indirect.1, - )]); - TestMessageInfo { - msg, - sent_by: indirect - .2 - .into_iter() - .map(|(validator_index, peer_id)| validator_index) - .collect_vec(), - tranche: indirect.3, - block_hash: block_info.hash, - typ: MessageType::Assignment, - } - }) - .collect_vec(); - - res -} - -/// A list of random samplings that we use to determine which nodes should send a given message to -/// the node under test. -/// We can not sample every time for all the messages because that would be too expensive to -/// perform, so pre-generate a list of samples for a given network size. -fn random_samplings_to_node_patterns( - node_under_test: ValidatorIndex, - num_validators: usize, - num_patterns: usize, -) -> Vec> { - let seed = [7u8; 32]; - let mut rand_chacha = ChaCha20Rng::from_seed(seed); - - (0..num_patterns) - .map(|_| { - (0..num_validators) - .map(|sending_validator_index| { - let mut validators = (0..num_validators).map(|val| val).collect_vec(); - validators.shuffle(&mut rand_chacha); - - let mut random_routing = RandomRouting::default(); - validators - .into_iter() - .flat_map(|validator_to_send| { - if random_routing.sample(num_validators, &mut rand_chacha) { - random_routing.inc_sent(); - if validator_to_send == node_under_test.0 as usize { - Some(ValidatorIndex(sending_validator_index as u32)) - } else { - None - } - } else { - None - } - }) - .collect_vec() - }) - .flatten() - .collect_vec() - }) - .collect_vec() -} - -/// Generates a peer view change for the passed `block_hash` -fn generate_peer_view_change_for( - block_hash: Hash, - peer_id: PeerId, - validator_index: ValidatorIndex, -) -> AllMessages { - let network = - NetworkBridgeEvent::PeerViewChange(peer_id, View::new([block_hash].into_iter(), 0)); - - AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate(network)) -} - -/// Generates peer_connected messages for all peers in `test_authorities` -fn generate_peer_connected( - test_authorities: &TestAuthorities, - block_hash: Hash, -) -> Vec { - let keyrings = test_authorities - .keyrings - .clone() - .into_iter() - .zip(test_authorities.peer_ids.clone().into_iter()) - .collect_vec(); - keyrings - .into_iter() - .map(|(_, peer_id)| { - let network = NetworkBridgeEvent::PeerConnected( - peer_id, - ObservedRole::Full, - ProtocolVersion::from(ValidationVersion::V3), - None, - ); - AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate( - network, - )) - }) - .collect_vec() -} - -/// Generates a topology to be used for this benchmark. -fn generate_topology(test_authorities: &TestAuthorities) -> SessionGridTopology { - let keyrings = test_authorities - .keyrings - .clone() - .into_iter() - .zip(test_authorities.peer_ids.clone().into_iter()) - .collect_vec(); - - let topology = keyrings - .clone() - .into_iter() - .enumerate() - .map(|(index, (keyring, peer_id))| TopologyPeerInfo { - peer_ids: vec![peer_id], - validator_index: ValidatorIndex(index as u32), - discovery_id: keyring.public().into(), - }) - .collect_vec(); - let shuffled = (0..keyrings.len()).collect_vec(); - - SessionGridTopology::new(shuffled, topology) -} - -/// Generates new session topology message. -fn generate_new_session_topology( - test_authorities: &TestAuthorities, - block_hash: Hash, -) -> Vec { - let topology = generate_topology(test_authorities); - - let event = NetworkBridgeEvent::NewGossipTopology(NewGossipTopology { - session: 1, - topology, - local_index: Some(ValidatorIndex(NODE_UNDER_TEST)), // TODO - }); - vec![AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate(event))] -} - -/// Helper function to generate a babe epoch for this benchmark. -/// It does not change for the duration of the test. -fn generate_babe_epoch(current_slot: Slot, authorities: TestAuthorities) -> BabeEpoch { - let authorities = authorities - .keyrings - .into_iter() - .enumerate() - .map(|(index, keyring)| (keyring.public().into(), index as u64)) - .collect_vec(); - BabeEpoch { - epoch_index: 1, - start_slot: current_slot.saturating_sub(1u64), - duration: 200, - authorities, - randomness: [0xde; 32], - config: BabeEpochConfiguration { c: (1, 4), allowed_slots: AllowedSlots::PrimarySlots }, - } -} - /// Helper function to build an overseer with the real implementation for `ApprovalDistribution` and /// `ApprovalVoting` subystems and mock subsytems for all others. fn build_overseer( @@ -1742,16 +701,9 @@ fn build_overseer( let db: polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter = polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[]); let keystore = LocalKeystore::in_memory(); - // let _public = keystore - // .sr25519_generate_new( - // ASSIGNMENT_KEY_TYPE_ID, - // Some(state.test_authorities.keyrings[NODE_UNDER_TEST as usize].seed().as_str()), - // ) - // .expect("should not fail"); - - let real_system_clock = SystemClock {}; + let system_clock = - FakeSystemClock::new(SystemClock {}, state.delta_tick_from_generated.clone()); + PastSystemClock::new(SystemClock {}, state.delta_tick_from_generated.clone()); let approval_voting = ApprovalVotingSubsystem::with_config( TEST_CONFIG, Arc::new(db), @@ -1799,7 +751,7 @@ fn prepare_test_inner( options: ApprovalsOptions, ) -> (TestEnvironment, ApprovalTestState) { gum::info!("Prepare test state"); - let mut state = ApprovalTestState::new(&config, options, &dependencies); + let state = ApprovalTestState::new(&config, options, &dependencies); gum::info!("Build network emulator"); @@ -1832,13 +784,10 @@ pub async fn bench_approvals_run( // First create the initialization messages that make sure that then node under // tests receives notifications about the topology used and the connected peers. - let mut initialization_messages = generate_peer_connected( - &state.test_authorities, - state.per_slot_heads.first().unwrap().hash, - ); + let mut initialization_messages = generate_peer_connected(&state.test_authorities); initialization_messages.extend(generate_new_session_topology( &state.test_authorities, - state.per_slot_heads.first().unwrap().hash, + ValidatorIndex(NODE_UNDER_TEST), )); for message in initialization_messages { env.send_message(message).await; @@ -1851,7 +800,7 @@ pub async fn bench_approvals_run( slot_number_to_tick(SLOT_DURATION_MILLIS, state.generated_state.initial_slot), std::sync::atomic::Ordering::SeqCst, ); - let system_clock = FakeSystemClock::new(real_clock, state.delta_tick_from_generated.clone()); + let system_clock = PastSystemClock::new(real_clock, state.delta_tick_from_generated.clone()); for block_num in 0..env.config().num_blocks { let mut current_slot = tick_to_slot_number(SLOT_DURATION_MILLIS, system_clock.tick_now()); @@ -1901,14 +850,14 @@ pub async fn bench_approvals_run( gum::info!("Awaiting producer to signal done"); - producer_rx.await; + producer_rx.await.expect("Failed to receive done from message producer"); gum::info!("Requesting approval votes ms"); for info in &state.per_slot_heads { for (index, candidates) in info.candidates.iter().enumerate() { match candidates { CandidateEvent::CandidateBacked(_, _, _, _) => todo!(), - CandidateEvent::CandidateIncluded(receipt_fetch, head, _, _) => { + CandidateEvent::CandidateIncluded(receipt_fetch, _head, _, _) => { let (tx, rx) = oneshot::channel(); let msg = ApprovalVotingMessage::GetApprovalSignaturesForCandidate( @@ -1917,13 +866,10 @@ pub async fn bench_approvals_run( ); env.send_message(AllMessages::ApprovalVoting(msg)).await; - let start: Instant = Instant::now(); - let result = rx.await.unwrap(); - for (validator, votes) in result.iter() { - let result = info - .votes + for (validator, _) in result.iter() { + info.votes .get(validator.0 as usize) .unwrap() .get(index) @@ -1960,31 +906,3 @@ pub async fn bench_approvals_run( gum::info!("{}", &env); } - -#[derive(Clone)] - -struct FakeSystemClock { - real_system_clock: SystemClock, - delta_ticks: Arc, -} - -impl FakeSystemClock { - fn new(real_system_clock: SystemClock, delta_ticks: Arc) -> Self { - FakeSystemClock { real_system_clock, delta_ticks } - } -} - -impl Clock for FakeSystemClock { - fn tick_now(&self) -> Tick { - self.real_system_clock.tick_now() - - self.delta_ticks.load(std::sync::atomic::Ordering::SeqCst) - } - - fn wait( - &self, - tick: Tick, - ) -> std::pin::Pin + Send + 'static>> { - self.real_system_clock - .wait(tick + self.delta_ticks.load(std::sync::atomic::Ordering::SeqCst)) - } -} diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 8ad14e8aa261..e53069c25cd4 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -174,7 +174,7 @@ impl BenchCli { configuration.max_pov_size, ), }, - TestObjective::ApprovalsTest(ref options) => { + TestObjective::ApprovalsTest(ref _options) => { todo!("Not implemented"); }, TestObjective::Unimplemented => todo!(), From 28e19f8fa9d2e06152ace1e93943da1ff104358a Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 22 Dec 2023 09:34:40 +0200 Subject: [PATCH 155/192] Add new files Signed-off-by: Alexandru Gheorghe --- .../subsystem-bench/src/approval/helpers.rs | 265 +++++++ .../src/approval/message_generator.rs | 709 ++++++++++++++++++ .../src/approval/test_message.rs | 324 ++++++++ 3 files changed, 1298 insertions(+) create mode 100644 polkadot/node/subsystem-bench/src/approval/helpers.rs create mode 100644 polkadot/node/subsystem-bench/src/approval/message_generator.rs create mode 100644 polkadot/node/subsystem-bench/src/approval/test_message.rs diff --git a/polkadot/node/subsystem-bench/src/approval/helpers.rs b/polkadot/node/subsystem-bench/src/approval/helpers.rs new file mode 100644 index 000000000000..dc811f660bcc --- /dev/null +++ b/polkadot/node/subsystem-bench/src/approval/helpers.rs @@ -0,0 +1,265 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +use crate::core::configuration::{TestAuthorities, TestConfiguration}; +use itertools::Itertools; +use polkadot_node_core_approval_voting::time::{Clock, SystemClock, Tick}; +use polkadot_node_network_protocol::{ + grid_topology::{SessionGridTopology, TopologyPeerInfo}, + peer_set::{ProtocolVersion, ValidationVersion}, + ObservedRole, View, +}; +use polkadot_node_subsystem_types::messages::{ + network_bridge_event::NewGossipTopology, ApprovalDistributionMessage, NetworkBridgeEvent, +}; +use polkadot_overseer::AllMessages; +use polkadot_primitives::{ + BlockNumber, CandidateEvent, CandidateReceipt, CoreIndex, GroupIndex, Hash, Header, + Id as ParaId, IndexedVec, SessionInfo, Slot, ValidatorIndex, +}; +use polkadot_primitives_test_helpers::dummy_candidate_receipt_bad_sig; +use rand::{seq::SliceRandom, SeedableRng}; +use rand_chacha::ChaCha20Rng; +use sc_network::PeerId; +use sp_consensus_babe::{ + digests::{CompatibleDigestItem, PreDigest, SecondaryVRFPreDigest}, + AllowedSlots, BabeEpochConfiguration, Epoch as BabeEpoch, VrfSignature, VrfTranscript, +}; +use sp_core::crypto::VrfSecret; +use sp_keyring::sr25519::Keyring as Sr25519Keyring; +use sp_runtime::{Digest, DigestItem}; +use std::sync::{atomic::AtomicU64, Arc}; + +use super::NEEDED_APPROVALS; + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +/// A fake system clock used for driving the approval voting and make +/// it process blocks, assignments and approvals from the past. +#[derive(Clone)] + +pub struct PastSystemClock { + /// The real system clock + real_system_clock: SystemClock, + /// The difference in ticks between the real system clock and the current clock. + delta_ticks: Arc, +} + +impl PastSystemClock { + /// Creates a new fake system clock with `delta_ticks` between the real time and the fake one. + pub fn new(real_system_clock: SystemClock, delta_ticks: Arc) -> Self { + PastSystemClock { real_system_clock, delta_ticks } + } +} + +impl Clock for PastSystemClock { + fn tick_now(&self) -> Tick { + self.real_system_clock.tick_now() - + self.delta_ticks.load(std::sync::atomic::Ordering::SeqCst) + } + + fn wait( + &self, + tick: Tick, + ) -> std::pin::Pin + Send + 'static>> { + self.real_system_clock + .wait(tick + self.delta_ticks.load(std::sync::atomic::Ordering::SeqCst)) + } +} + +/// Helper function to generate a babe epoch for this benchmark. +/// It does not change for the duration of the test. +pub fn generate_babe_epoch(current_slot: Slot, authorities: TestAuthorities) -> BabeEpoch { + let authorities = authorities + .keyrings + .into_iter() + .enumerate() + .map(|(index, keyring)| (keyring.public().into(), index as u64)) + .collect_vec(); + BabeEpoch { + epoch_index: 1, + start_slot: current_slot.saturating_sub(1u64), + duration: 200, + authorities, + randomness: [0xde; 32], + config: BabeEpochConfiguration { c: (1, 4), allowed_slots: AllowedSlots::PrimarySlots }, + } +} + +/// Generates a topology to be used for this benchmark. +pub fn generate_topology(test_authorities: &TestAuthorities) -> SessionGridTopology { + let keyrings = test_authorities + .keyrings + .clone() + .into_iter() + .zip(test_authorities.peer_ids.clone().into_iter()) + .collect_vec(); + + let topology = keyrings + .clone() + .into_iter() + .enumerate() + .map(|(index, (keyring, peer_id))| TopologyPeerInfo { + peer_ids: vec![peer_id], + validator_index: ValidatorIndex(index as u32), + discovery_id: keyring.public().into(), + }) + .collect_vec(); + let shuffled = (0..keyrings.len()).collect_vec(); + + SessionGridTopology::new(shuffled, topology) +} + +/// Generates new session topology message. +pub fn generate_new_session_topology( + test_authorities: &TestAuthorities, + test_node: ValidatorIndex, +) -> Vec { + let topology = generate_topology(test_authorities); + + let event = NetworkBridgeEvent::NewGossipTopology(NewGossipTopology { + session: 1, + topology, + local_index: Some(test_node), + }); + vec![AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate(event))] +} + +/// Generates peer_connected messages for all peers in `test_authorities` +pub fn generate_peer_connected(test_authorities: &TestAuthorities) -> Vec { + let keyrings = test_authorities + .keyrings + .clone() + .into_iter() + .zip(test_authorities.peer_ids.clone().into_iter()) + .collect_vec(); + keyrings + .into_iter() + .map(|(_, peer_id)| { + let network = NetworkBridgeEvent::PeerConnected( + peer_id, + ObservedRole::Full, + ProtocolVersion::from(ValidationVersion::V3), + None, + ); + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate( + network, + )) + }) + .collect_vec() +} + +/// Generates a peer view change for the passed `block_hash` +pub fn generate_peer_view_change_for(block_hash: Hash, peer_id: PeerId) -> AllMessages { + let network = + NetworkBridgeEvent::PeerViewChange(peer_id, View::new([block_hash].into_iter(), 0)); + + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate(network)) +} + +/// Generates a test session info with all passed authorities as consensus validators. +pub fn session_info_for_peers( + configuration: &TestConfiguration, + authorities: TestAuthorities, +) -> SessionInfo { + let keys = authorities.keyrings.iter().zip(authorities.peer_ids.iter()); + SessionInfo { + validators: keys.clone().map(|v| v.0.clone().public().into()).collect(), + discovery_keys: keys.clone().map(|v| v.0.clone().public().into()).collect(), + assignment_keys: keys.clone().map(|v| v.0.clone().public().into()).collect(), + validator_groups: IndexedVec::>::from( + (0..authorities.keyrings.len()) + .map(|index| vec![ValidatorIndex(index as u32)]) + .collect_vec(), + ), + n_cores: configuration.n_cores as u32, + needed_approvals: NEEDED_APPROVALS, + zeroth_delay_tranche_width: 0, + relay_vrf_modulo_samples: 6, + n_delay_tranches: 89, + no_show_slots: 3, + active_validator_indices: (0..authorities.keyrings.len()) + .map(|index| ValidatorIndex(index as u32)) + .collect_vec(), + dispute_period: 6, + random_seed: [0u8; 32], + } +} + +/// Helper function to create a a signature for the block header. +fn garbage_vrf_signature() -> VrfSignature { + let transcript = VrfTranscript::new(b"test-garbage", &[]); + Sr25519Keyring::Alice.pair().vrf_sign(&transcript.into()) +} + +/// Helper function to create a block header. +pub fn make_header(parent_hash: Hash, slot: Slot, number: u32) -> Header { + let digest = + { + let mut digest = Digest::default(); + let vrf_signature = garbage_vrf_signature(); + digest.push(DigestItem::babe_pre_digest(PreDigest::SecondaryVRF( + SecondaryVRFPreDigest { authority_index: 0, slot, vrf_signature }, + ))); + digest + }; + + Header { + digest, + extrinsics_root: Default::default(), + number, + state_root: Default::default(), + parent_hash, + } +} + +/// Helper function to create a candidate receipt. +fn make_candidate(para_id: ParaId, hash: &Hash) -> CandidateReceipt { + let mut r = dummy_candidate_receipt_bad_sig(*hash, Some(Default::default())); + r.descriptor.para_id = para_id; + r +} + +/// Helper function to create a list of candidates that are included in the block +pub fn make_candidates( + block_hash: Hash, + block_number: BlockNumber, + num_cores: u32, + num_candidates: u32, +) -> Vec { + let seed = [block_number as u8; 32]; + let mut rand_chacha = ChaCha20Rng::from_seed(seed); + let mut candidates = (0..num_cores) + .map(|core| { + CandidateEvent::CandidateIncluded( + make_candidate(ParaId::from(core), &block_hash), + Vec::new().into(), + CoreIndex(core), + GroupIndex(core), + ) + }) + .collect_vec(); + let (candidates, _) = candidates.partial_shuffle(&mut rand_chacha, num_candidates as usize); + candidates + .into_iter() + .map(|val| val.clone()) + .sorted_by(|a, b| match (a, b) { + ( + CandidateEvent::CandidateIncluded(_, _, core_a, _), + CandidateEvent::CandidateIncluded(_, _, core_b, _), + ) => core_a.0.cmp(&core_b.0), + (_, _) => todo!("Should not happen"), + }) + .collect_vec() +} diff --git a/polkadot/node/subsystem-bench/src/approval/message_generator.rs b/polkadot/node/subsystem-bench/src/approval/message_generator.rs new file mode 100644 index 000000000000..2341a6b8d2d5 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/approval/message_generator.rs @@ -0,0 +1,709 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use std::{ + collections::{BTreeMap, HashSet}, + fs, + io::Write, + path::{Path, PathBuf}, + time::Duration, +}; + +use futures::SinkExt; +use itertools::Itertools; +use parity_scale_codec::Encode; +use polkadot_node_core_approval_voting::{ + criteria::{compute_assignments, Config}, + time::tranche_to_tick, +}; +use polkadot_node_network_protocol::grid_topology::{ + GridNeighbors, RandomRouting, RequiredRouting, SessionGridTopology, +}; +use polkadot_node_primitives::approval::{ + self, + v1::RelayVRFStory, + v2::{CoreBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2}, +}; +use polkadot_primitives::{ + vstaging::ApprovalVoteMultipleCandidates, CandidateEvent, CandidateHash, CandidateIndex, + CoreIndex, SessionInfo, Slot, ValidatorIndex, ValidatorPair, ASSIGNMENT_KEY_TYPE_ID, +}; +use rand::{seq::SliceRandom, RngCore, SeedableRng}; +use rand_chacha::ChaCha20Rng; +use sc_keystore::LocalKeystore; +use sc_network::PeerId; +use sha1::Digest; +use sp_consensus_babe::SlotDuration; +use sp_core::Pair; +use sp_keystore::Keystore; +use sp_timestamp::Timestamp; + +use super::{ + test_message::{MessagesBundle, TestMessageInfo}, + ApprovalTestState, ApprovalsOptions, BlockTestData, +}; +use crate::{ + approval::{ + helpers::{generate_babe_epoch, generate_topology, session_info_for_peers}, + GeneratedState, BUFFER_FOR_GENERATION_MILLIS, LOG_TARGET, NODE_UNDER_TEST, + SLOT_DURATION_MILLIS, + }, + core::{ + configuration::{TestAuthorities, TestConfiguration, TestObjective}, + keyring::Keyring, + }, +}; +use polkadot_node_network_protocol::v3 as protocol_v3; +use polkadot_primitives::Hash; +use sc_service::SpawnTaskHandle; +/// A generator of messages coming from a given Peer/Validator +pub struct PeerMessagesGenerator { + /// The grid neighbors of the node under test. + pub topology_node_under_test: GridNeighbors, + /// The topology of the network for the epoch under test. + pub topology: SessionGridTopology, + /// The validator index for this object generates the messages. + pub validator_index: ValidatorIndex, + /// An array of pre-generated random samplings, that is used to determine, which nodes would + /// send a given assignment, to the node under test because of the random samplings. + /// As an optimization we generate this sampling at the begining of the test and just pick + /// one randomly, because always taking the samples would be too expensive for benchamrk. + pub random_samplings: Vec>, + /// Channel for sending the generated messages to the aggregator + pub tx_messages: futures::channel::mpsc::UnboundedSender<(Hash, Vec)>, + /// The list of test authorities + pub test_authorities: TestAuthorities, + //// The session info used for the test. + pub session_info: SessionInfo, + /// The blocks used for testing + pub blocks: Vec, + /// If v2_assignments is enabled + pub enable_assignments_v2: bool, + /// Last considered tranche for generating assignments and approvals + pub last_considered_tranche: u32, + /// Minimum of approvals coalesced together. + pub min_coalesce: u32, + /// Maximum of approvals coalesced together. + pub max_coalesce: u32, + /// Maximum tranche diffs between two coalesced approvals. + pub coalesce_tranche_diff: u32, +} + +impl PeerMessagesGenerator { + /// Generates messages by spawning a blocking task in the background which begins creating + /// the assignments/approvals and peer view changes at the begining of each block. + pub fn generate_messages(mut self, spawn_task_handle: &SpawnTaskHandle) { + spawn_task_handle.spawn_blocking("generate-messages", "generate-messages", async move { + for block_info in self.blocks { + let assignments = generate_assignments( + &block_info, + self.test_authorities + .keyrings + .clone() + .into_iter() + .zip(self.test_authorities.peer_ids.clone().into_iter()) + .collect_vec(), + &self.session_info, + self.enable_assignments_v2, + &self.random_samplings, + self.validator_index.0, + &block_info.relay_vrf_story, + &self.topology_node_under_test, + &self.topology, + self.last_considered_tranche, + ); + + let bytes = self.validator_index.0.to_be_bytes(); + let seed = [ + bytes[0], bytes[1], bytes[2], bytes[3], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + + let mut rand_chacha = ChaCha20Rng::from_seed(seed); + let approvals = issue_approvals( + assignments, + block_info.hash, + self.test_authorities + .keyrings + .clone() + .into_iter() + .zip(self.test_authorities.peer_ids.clone().into_iter()) + .collect_vec(), + block_info.candidates.clone(), + self.min_coalesce, + self.max_coalesce, + self.coalesce_tranche_diff, + &mut rand_chacha, + ); + + self.tx_messages + .send((block_info.hash, approvals)) + .await + .expect("Should not fail"); + } + }) + } + + // Builds the messages finger print corresponding to this configuration. + // When the finger print exists already on disk the messages are not re-generated. + fn messages_fingerprint( + configuration: &TestConfiguration, + options: &ApprovalsOptions, + ) -> String { + let mut fingerprint = options.fingerprint(); + let mut exclude_objective = configuration.clone(); + // The objective contains the full content of `ApprovalOptions`, we don't want to put all of + // that in fingerprint, so execlute it because we add it manually see above. + exclude_objective.objective = TestObjective::Unimplemented; + let configuration_bytes = bincode::serialize(&exclude_objective).unwrap(); + fingerprint.extend(configuration_bytes); + let mut sha1 = sha1::Sha1::new(); + sha1.update(fingerprint); + let result = sha1.finalize(); + hex::encode(result) + } + + /// Generate all messages(Assignments & Approvals) needed for approving `blocks``. + pub fn generate_all_messages( + configuration: &TestConfiguration, + test_authorities: &TestAuthorities, + options: &ApprovalsOptions, + spawn_task_handle: &SpawnTaskHandle, + ) -> PathBuf { + let path_name = format!( + "{}/{}", + options.workdir_prefix, + Self::messages_fingerprint(configuration, options) + ); + + let path = Path::new(&path_name); + if path.exists() { + return path.to_path_buf(); + } + + gum::info!("Generate message because filed does not exist"); + let delta_to_first_slot_under_test = Timestamp::new(BUFFER_FOR_GENERATION_MILLIS); + let initial_slot = Slot::from_timestamp( + (*Timestamp::current() - *delta_to_first_slot_under_test).into(), + SlotDuration::from_millis(SLOT_DURATION_MILLIS), + ); + + let babe_epoch = generate_babe_epoch(initial_slot, test_authorities.clone()); + let session_info = session_info_for_peers(configuration, test_authorities.clone()); + let blocks = ApprovalTestState::generate_blocks_information( + configuration, + &babe_epoch, + initial_slot, + ); + + gum::info!(target: LOG_TARGET, "Generate messages"); + let topology = generate_topology(&test_authorities); + + let random_samplings = random_samplings_to_node_patterns( + ValidatorIndex(NODE_UNDER_TEST), + test_authorities.keyrings.len(), + test_authorities.keyrings.len() as usize * 2, + ); + + let topology_node_under_test = + topology.compute_grid_neighbors_for(ValidatorIndex(NODE_UNDER_TEST)).unwrap(); + + let (tx, mut rx) = futures::channel::mpsc::unbounded(); + + // Spawn a thread to generate the messages for each validator, so that we speed up the + // generation a bit. + for current_validator_index in 1..test_authorities.keyrings.len() { + let peer_message_source = PeerMessagesGenerator { + topology_node_under_test: topology_node_under_test.clone(), + topology: topology.clone(), + validator_index: ValidatorIndex(current_validator_index as u32), + test_authorities: test_authorities.clone(), + session_info: session_info.clone(), + blocks: blocks.clone(), + tx_messages: tx.clone(), + random_samplings: random_samplings.clone(), + enable_assignments_v2: options.enable_assignments_v2, + last_considered_tranche: options.last_considered_tranche, + min_coalesce: options.min_coalesce, + max_coalesce: options.max_coalesce, + coalesce_tranche_diff: options.coalesce_tranche_diff, + }; + + peer_message_source.generate_messages(&spawn_task_handle); + } + + std::mem::drop(tx); + + let seed = [0x32; 32]; + let mut rand_chacha = ChaCha20Rng::from_seed(seed); + + let mut all_messages: BTreeMap> = BTreeMap::new(); + // Receive all messages and sort them by Tick they have to be sent. + loop { + match rx.try_next() { + Ok(Some((block_hash, messages))) => + for message in messages { + let block_info = blocks + .iter() + .find(|val| val.hash == block_hash) + .expect("Should find blocks"); + let tick_to_send = tranche_to_tick( + SLOT_DURATION_MILLIS, + block_info.slot, + message.tranche_to_send(), + ); + if !all_messages.contains_key(&tick_to_send) { + all_messages.insert(tick_to_send, Vec::new()); + } + + let to_add = all_messages.get_mut(&tick_to_send).unwrap(); + to_add.push(message); + }, + Ok(None) => break, + Err(_) => { + std::thread::sleep(Duration::from_millis(50)); + }, + } + } + let all_messages = all_messages + .into_iter() + .map(|(_, mut messages)| { + // Shuffle the messages inside the same tick, so that we don't priorites messages + // for older nodes. we try to simulate the same behaviour as in real world. + messages.shuffle(&mut rand_chacha); + messages + }) + .flatten() + .collect_vec(); + + gum::info!("Generated a number of {:} unique messages", all_messages.len()); + + let generated_state = GeneratedState { all_messages: Some(all_messages), initial_slot }; + + let mut messages_file = fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(path) + .unwrap(); + + messages_file + .write_all(&generated_state.encode()) + .expect("Could not update message file"); + path.to_path_buf() + } +} + +/// A list of random samplings that we use to determine which nodes should send a given message to +/// the node under test. +/// We can not sample every time for all the messages because that would be too expensive to +/// perform, so pre-generate a list of samples for a given network size. +fn random_samplings_to_node_patterns( + node_under_test: ValidatorIndex, + num_validators: usize, + num_patterns: usize, +) -> Vec> { + let seed = [7u8; 32]; + let mut rand_chacha = ChaCha20Rng::from_seed(seed); + + (0..num_patterns) + .map(|_| { + (0..num_validators) + .map(|sending_validator_index| { + let mut validators = (0..num_validators).map(|val| val).collect_vec(); + validators.shuffle(&mut rand_chacha); + + let mut random_routing = RandomRouting::default(); + validators + .into_iter() + .flat_map(|validator_to_send| { + if random_routing.sample(num_validators, &mut rand_chacha) { + random_routing.inc_sent(); + if validator_to_send == node_under_test.0 as usize { + Some(ValidatorIndex(sending_validator_index as u32)) + } else { + None + } + } else { + None + } + }) + .collect_vec() + }) + .flatten() + .collect_vec() + }) + .collect_vec() +} + +/// Helper function to randomly determine how many approvals we coalesce together in a single +/// message. +fn coalesce_approvals_len( + min_coalesce: u32, + max_coalesce: u32, + rand_chacha: &mut ChaCha20Rng, +) -> usize { + let mut sampling: Vec = (min_coalesce as usize..max_coalesce as usize + 1).collect_vec(); + *(sampling.partial_shuffle(rand_chacha, 1).0.first().unwrap()) +} + +/// Helper function to create approvals signatures for all assignments passed as arguments. +/// Returns a list of Approvals messages that need to be sent. +fn issue_approvals( + assignments: Vec, + block_hash: Hash, + keyrings: Vec<(Keyring, PeerId)>, + candidates: Vec, + min_coalesce: u32, + max_coalesce: u32, + coalesce_tranche_diff: u32, + rand_chacha: &mut ChaCha20Rng, +) -> Vec { + let mut to_sign: Vec = Vec::new(); + let mut num_coalesce = coalesce_approvals_len(min_coalesce, max_coalesce, rand_chacha); + let result = assignments + .iter() + .enumerate() + .map(|(_index, message)| match &message.msg { + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) => { + let mut approvals_to_create = Vec::new(); + + let current_validator_index = + to_sign.first().map(|msg| msg.validator_index).unwrap_or(ValidatorIndex(999)); + + // Invariant for this benchmark. + assert_eq!(assignments.len(), 1); + + let assignment = assignments.first().unwrap(); + + let earliest_tranche = + to_sign.first().map(|val| val.tranche).unwrap_or(message.tranche); + + if to_sign.len() >= num_coalesce as usize || + (!to_sign.is_empty() && current_validator_index != assignment.0.validator) || + message.tranche - earliest_tranche >= coalesce_tranche_diff + { + num_coalesce = coalesce_approvals_len(min_coalesce, max_coalesce, rand_chacha); + approvals_to_create.push(sign_candidates( + &mut to_sign, + &keyrings, + block_hash, + num_coalesce, + )) + } + + // If more that one candidate was in the assignment queue all of them. + for candidate_index in assignment.1.iter_ones() { + let candidate = candidates.get(candidate_index).unwrap(); + if let CandidateEvent::CandidateIncluded(candidate, _, _, _) = candidate { + to_sign.push(TestSignInfo { + candidate_hash: candidate.hash(), + candidate_index: candidate_index as CandidateIndex, + validator_index: assignment.0.validator, + sent_by: message.sent_by.clone().into_iter().collect(), + tranche: message.tranche, + assignment: message.clone(), + }); + } else { + todo!("Other enum variants are not used in this benchmark"); + } + } + approvals_to_create + }, + _ => { + todo!("Other enum variants are not used in this benchmark"); + }, + }) + .collect_vec(); + + let mut result = result.into_iter().flatten().collect_vec(); + + if !to_sign.is_empty() { + result.push(sign_candidates(&mut to_sign, &keyrings, block_hash, num_coalesce)); + } + result +} + +/// Helper struct to gather information about more than one candidate an sign it in a single +/// approval message. +struct TestSignInfo { + candidate_hash: CandidateHash, + candidate_index: CandidateIndex, + validator_index: ValidatorIndex, + sent_by: HashSet, + tranche: u32, + assignment: TestMessageInfo, +} + +/// Helper function to create a signture for all candidates in `to_sign` parameter. +/// Returns a TestMessage +fn sign_candidates( + to_sign: &mut Vec, + keyrings: &Vec<(Keyring, PeerId)>, + block_hash: Hash, + num_coalesce: usize, +) -> MessagesBundle { + let current_validator_index = to_sign.first().map(|val| val.validator_index).unwrap(); + let tranche_trigger_timestamp = to_sign.iter().map(|val| val.tranche).max().unwrap(); + let keyring = keyrings.get(current_validator_index.0 as usize).unwrap().clone(); + + let unique_assignments: HashSet = + to_sign.iter().map(|info| info.assignment.clone()).collect(); + + let mut to_sign = to_sign + .drain(..) + .sorted_by(|val1, val2| val1.candidate_index.cmp(&val2.candidate_index)) + .peekable(); + + let mut bundle = MessagesBundle { + assignments: unique_assignments.into_iter().collect_vec(), + approvals: Vec::new(), + }; + + while to_sign.peek().is_some() { + let to_sign = to_sign.by_ref().take(num_coalesce).collect_vec(); + + let hashes = to_sign.iter().map(|val| val.candidate_hash).collect_vec(); + let candidate_indices = to_sign.iter().map(|val| val.candidate_index).collect_vec(); + + let sent_by = to_sign + .iter() + .map(|val| val.sent_by.iter()) + .flatten() + .map(|peer| *peer) + .collect::>(); + + let payload = ApprovalVoteMultipleCandidates(&hashes).signing_payload(1); + + let validator_key: ValidatorPair = keyring.0.clone().pair().into(); + let signature = validator_key.sign(&payload[..]); + let indirect = IndirectSignedApprovalVoteV2 { + block_hash, + candidate_indices: candidate_indices.try_into().unwrap(), + validator: current_validator_index, + signature, + }; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![indirect]); + + bundle.approvals.push(TestMessageInfo { + msg, + sent_by: sent_by.into_iter().collect_vec(), + tranche: tranche_trigger_timestamp, + block_hash, + }); + } + bundle +} + +fn neighbours_that_would_sent_message( + keyrings: &Vec<(Keyring, PeerId)>, + current_validator_index: u32, + topology_node_under_test: &GridNeighbors, + topology: &SessionGridTopology, +) -> Vec<(ValidatorIndex, PeerId)> { + let topology_originator = topology + .compute_grid_neighbors_for(ValidatorIndex(current_validator_index as u32)) + .unwrap(); + + let originator_y = topology_originator + .validator_indices_y + .iter() + .filter(|validator| { + topology_node_under_test.required_routing_by_index(**validator, false) == + RequiredRouting::GridY + }) + .next(); + let originator_x = topology_originator + .validator_indices_x + .iter() + .filter(|validator| { + topology_node_under_test.required_routing_by_index(**validator, false) == + RequiredRouting::GridX + }) + .next(); + + let is_neighbour = topology_originator + .validator_indices_x + .contains(&ValidatorIndex(NODE_UNDER_TEST)) || + topology_originator + .validator_indices_y + .contains(&ValidatorIndex(NODE_UNDER_TEST)); + + let mut to_be_sent_by = originator_y + .into_iter() + .chain(originator_x) + .map(|val| (*val, keyrings[val.0 as usize].1)) + .collect_vec(); + + if is_neighbour { + to_be_sent_by.push((ValidatorIndex(NODE_UNDER_TEST), keyrings[0].1)); + } + to_be_sent_by +} + +/// Generates assignments for the given `current_validator_index` +/// Returns a list of assignments to be sent sorted by tranche. +fn generate_assignments( + block_info: &BlockTestData, + keyrings: Vec<(Keyring, PeerId)>, + session_info: &SessionInfo, + generate_v2_assignments: bool, + random_samplings: &Vec>, + current_validator_index: u32, + relay_vrf_story: &RelayVRFStory, + topology_node_under_test: &GridNeighbors, + topology: &SessionGridTopology, + last_considered_tranche: u32, +) -> Vec { + let config = Config::from(session_info); + + let leaving_cores = block_info + .candidates + .clone() + .into_iter() + .map(|candidate_event| { + if let CandidateEvent::CandidateIncluded(candidate, _, core_index, group_index) = + candidate_event + { + (candidate.hash(), core_index, group_index) + } else { + todo!("Variant is never created in this benchmark") + } + }) + .collect_vec(); + + let mut assignments_by_tranche = BTreeMap::new(); + + let bytes = current_validator_index.to_be_bytes(); + let seed = [ + bytes[0], bytes[1], bytes[2], bytes[3], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + let mut rand_chacha = ChaCha20Rng::from_seed(seed); + + let to_be_sent_by = neighbours_that_would_sent_message( + &keyrings, + current_validator_index, + topology_node_under_test, + topology, + ); + + let store = LocalKeystore::in_memory(); + let _public = store + .sr25519_generate_new( + ASSIGNMENT_KEY_TYPE_ID, + Some(keyrings[current_validator_index as usize].0.seed().as_str()), + ) + .expect("should not fail"); + + let leaving_cores = leaving_cores + .clone() + .into_iter() + .filter(|(_, core_index, _group_index)| core_index.0 != current_validator_index) + .collect_vec(); + + let assignments = compute_assignments( + &store, + relay_vrf_story.clone(), + &config, + leaving_cores.clone(), + generate_v2_assignments, + ); + + let random_sending_nodes = random_samplings + .get(rand_chacha.next_u32() as usize % random_samplings.len()) + .unwrap(); + let random_sending_peer_ids = random_sending_nodes + .into_iter() + .map(|validator| (*validator, keyrings[validator.0 as usize].1)) + .collect_vec(); + + let mut no_duplicates = HashSet::new(); + for (core_index, assignment) in assignments { + let assigned_cores = match &assignment.cert().kind { + approval::v2::AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => + core_bitfield.iter_ones().map(|val| CoreIndex::from(val as u32)).collect_vec(), + approval::v2::AssignmentCertKindV2::RelayVRFDelay { core_index } => vec![*core_index], + approval::v2::AssignmentCertKindV2::RelayVRFModulo { sample: _ } => vec![core_index], + }; + if assignment.tranche() > last_considered_tranche { + continue + } + + let bitfiled: CoreBitfield = assigned_cores.clone().try_into().unwrap(); + + // For the cases where tranch0 assignments are in a single certificate we need to make + // sure we create a single message. + if no_duplicates.insert(bitfiled) { + if !assignments_by_tranche.contains_key(&assignment.tranche()) { + assignments_by_tranche.insert(assignment.tranche(), Vec::new()); + } + let this_tranche_assignments = + assignments_by_tranche.get_mut(&assignment.tranche()).unwrap(); + this_tranche_assignments.push(( + IndirectAssignmentCertV2 { + block_hash: block_info.hash, + validator: ValidatorIndex(current_validator_index), + cert: assignment.cert().clone(), + }, + block_info + .candidates + .iter() + .enumerate() + .filter(|(_index, candidate)| { + if let CandidateEvent::CandidateIncluded(_, _, core, _) = candidate { + assigned_cores.contains(core) + } else { + panic!("Should not happen"); + } + }) + .map(|(index, _)| index as u32) + .collect_vec() + .try_into() + .unwrap(), + to_be_sent_by + .iter() + .chain(random_sending_peer_ids.iter()) + .map(|peer| *peer) + .collect::>(), + assignment.tranche(), + )); + } + } + + let res = assignments_by_tranche + .into_values() + .map(|assignments| assignments.into_iter()) + .flatten() + .map(|indirect| { + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( + indirect.0, indirect.1, + )]); + TestMessageInfo { + msg, + sent_by: indirect + .2 + .into_iter() + .map(|(validator_index, _)| validator_index) + .collect_vec(), + tranche: indirect.3, + block_hash: block_info.hash, + } + }) + .collect_vec(); + + res +} diff --git a/polkadot/node/subsystem-bench/src/approval/test_message.rs b/polkadot/node/subsystem-bench/src/approval/test_message.rs new file mode 100644 index 000000000000..491154151803 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/approval/test_message.rs @@ -0,0 +1,324 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use super::{ApprovalsOptions, BlockTestData, CandidateTestData}; +use crate::core::configuration::TestAuthorities; +use itertools::Itertools; +use parity_scale_codec::{Decode, Encode}; +use polkadot_node_network_protocol::{v3 as protocol_v3, Versioned}; +use polkadot_node_subsystem_types::messages::{ApprovalDistributionMessage, NetworkBridgeEvent}; +use polkadot_overseer::AllMessages; +use polkadot_primitives::{CandidateIndex, Hash, ValidatorIndex}; +use sc_network::PeerId; +use std::{ + collections::{HashMap, HashSet}, + time::Duration, +}; + +#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +pub struct TestMessageInfo { + /// The actual message + pub msg: protocol_v3::ApprovalDistributionMessage, + /// The list of peers that would sends this message in a real topology. + /// It includes both the peers that would send the message because of the topology + /// or because of randomly chosing so. + pub sent_by: Vec, + /// The tranche at which this message should be sent. + pub tranche: u32, + /// The block hash this message refers to. + pub block_hash: Hash, +} + +impl std::hash::Hash for TestMessageInfo { + fn hash(&self, state: &mut H) { + match &self.msg { + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) => { + for (assignment, candidates) in assignments { + (assignment.block_hash, assignment.validator).hash(state); + candidates.hash(state); + } + }, + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => { + for approval in approvals { + (approval.block_hash, approval.validator).hash(state); + approval.candidate_indices.hash(state); + } + }, + }; + } +} + +#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +/// A list of messages that depend of each-other, approvals cover one of the assignments and +/// vice-versa. +pub struct MessagesBundle { + pub assignments: Vec, + pub approvals: Vec, +} + +impl MessagesBundle { + /// The tranche when this bundle can be sent correctly, so no assignments or approvals will be + /// from the future. + pub fn tranche_to_send(&self) -> u32 { + self.assignments + .iter() + .chain(self.approvals.iter()) + .max_by(|a, b| a.tranche.cmp(&b.tranche)) + .unwrap() + .tranche + } + + /// The min tranche in the bundle. + pub fn min_tranche(&self) -> u32 { + self.assignments + .iter() + .chain(self.approvals.iter()) + .min_by(|a, b| a.tranche.cmp(&b.tranche)) + .unwrap() + .tranche + } + + /// Tells if the bundle is needed for sending. + /// We either send it because we need more assignments and approvals to approve the candidates + /// or because we configured the test to send messages untill a given tranche. + pub fn bundle_needed( + &self, + candidates_test_data: &HashMap<(Hash, CandidateIndex), CandidateTestData>, + options: &ApprovalsOptions, + ) -> bool { + self.needed_for_approval(candidates_test_data) || + (!options.stop_when_approved && self.min_tranche() <= options.send_till_tranche) + } + + /// Tells if the bundle is needed because we need more messages to approve the candidates. + pub fn needed_for_approval( + &self, + candidates_test_data: &HashMap<(Hash, CandidateIndex), CandidateTestData>, + ) -> bool { + self.assignments + .iter() + .any(|message| message.needed_for_approval(candidates_test_data)) + } + + /// Mark the assignments in the bundle as sent. + pub fn record_sent_assignment( + &self, + candidates_test_data: &mut HashMap<(Hash, CandidateIndex), CandidateTestData>, + ) { + self.assignments + .iter() + .for_each(|assignment| assignment.record_sent_assignment(candidates_test_data)); + } +} + +impl TestMessageInfo { + /// Converts message to `AllMessages` to be sent to overseer. + pub fn to_all_messages_from_peer(self, peer: PeerId) -> AllMessages { + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::PeerMessage(peer, Versioned::V3(self.msg)), + )) + } + + /// Gives us the latency for this message. + pub fn get_latency(&self) -> Option { + match &self.msg { + protocol_v3::ApprovalDistributionMessage::Assignments(_) => None, + protocol_v3::ApprovalDistributionMessage::Approvals(_) => None, + } + } + + /// Tells if the message is an approval. + fn is_approval(&self) -> bool { + match self.msg { + protocol_v3::ApprovalDistributionMessage::Assignments(_) => false, + protocol_v3::ApprovalDistributionMessage::Approvals(_) => true, + } + } + + /// Records an approval. + /// We use this to check after all messages have been processed that we didn't loose any + /// message. + pub fn record_vote(&self, state: &BlockTestData) { + if self.is_approval() { + match &self.msg { + protocol_v3::ApprovalDistributionMessage::Assignments(_) => todo!(), + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => + for approval in approvals { + for candidate_index in approval.candidate_indices.iter_ones() { + state + .votes + .get(approval.validator.0 as usize) + .unwrap() + .get(candidate_index) + .unwrap() + .store(true, std::sync::atomic::Ordering::SeqCst); + } + }, + } + } + } + + /// Mark the assignments in the message as sent. + pub fn record_sent_assignment( + &self, + candidates_test_data: &mut HashMap<(Hash, CandidateIndex), CandidateTestData>, + ) { + match &self.msg { + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) => { + for (assignment, candidate_indices) in assignments { + for candidate_index in candidate_indices.iter_ones() { + let candidate_test_data = candidates_test_data + .get_mut(&(assignment.block_hash, candidate_index as CandidateIndex)) + .unwrap(); + candidate_test_data.sent_assignment(self.tranche) + } + } + }, + protocol_v3::ApprovalDistributionMessage::Approvals(_approvals) => todo!(), + } + } + + /// Returns a list of candidates indicies in this message + pub fn candidate_indices(&self) -> HashSet { + let mut unique_candidate_indicies = HashSet::new(); + match &self.msg { + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) => + for (_assignment, candidate_indices) in assignments { + for candidate_index in candidate_indices.iter_ones() { + unique_candidate_indicies.insert(candidate_index); + } + }, + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => + for approval in approvals { + for candidate_index in approval.candidate_indices.iter_ones() { + unique_candidate_indicies.insert(candidate_index); + } + }, + } + unique_candidate_indicies + } + + /// Marks this message as no-shows if the number of configured no-shows is above the registered + /// no-shows. + /// Returns true if the message is a no-show. + pub fn no_show_if_required( + &self, + assignments: &Vec, + candidates_test_data: &mut HashMap<(Hash, CandidateIndex), CandidateTestData>, + ) -> bool { + let mut should_no_show = false; + if self.is_approval() { + let covered_candidates = assignments + .iter() + .map(|assignment| (assignment, assignment.candidate_indices())) + .collect_vec(); + + match &self.msg { + protocol_v3::ApprovalDistributionMessage::Assignments(_) => todo!(), + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => { + assert_eq!(approvals.len(), 1); + + for approval in approvals { + should_no_show = should_no_show || + approval.candidate_indices.iter_ones().all(|candidate_index| { + let candidate_test_data = candidates_test_data + .get_mut(&( + approval.block_hash, + candidate_index as CandidateIndex, + )) + .unwrap(); + let assignment = covered_candidates + .iter() + .filter(|(_assignment, candidates)| { + candidates.contains(&candidate_index) + }) + .next() + .unwrap(); + candidate_test_data.should_no_show(assignment.0.tranche) + }); + + if should_no_show { + for candidate_index in approval.candidate_indices.iter_ones() { + let candidate_test_data = candidates_test_data + .get_mut(&( + approval.block_hash, + candidate_index as CandidateIndex, + )) + .unwrap(); + let assignment = covered_candidates + .iter() + .filter(|(_assignment, candidates)| { + candidates.contains(&candidate_index) + }) + .next() + .unwrap(); + candidate_test_data.record_no_show(assignment.0.tranche) + } + } + } + }, + } + } + should_no_show + } + + /// Tells if a message is needed for approval + pub fn needed_for_approval( + &self, + candidates_test_data: &HashMap<(Hash, CandidateIndex), CandidateTestData>, + ) -> bool { + match &self.msg { + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) => + assignments.iter().any(|(assignment, candidate_indices)| { + candidate_indices.iter_ones().any(|candidate_index| { + candidates_test_data + .get(&(assignment.block_hash, candidate_index as CandidateIndex)) + .map(|data| data.should_send_tranche(self.tranche)) + .unwrap_or_default() + }) + }), + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) => + approvals.iter().any(|approval| { + approval.candidate_indices.iter_ones().any(|candidate_index| { + candidates_test_data + .get(&(approval.block_hash, candidate_index as CandidateIndex)) + .map(|data| data.should_send_tranche(self.tranche)) + .unwrap_or_default() + }) + }), + } + } + + /// Splits a message into multiple messages based on what peers should send this message. + /// It build a HashMap of messages that should be sent by each peer. + pub fn split_by_peer_id( + self, + authorities: &TestAuthorities, + ) -> HashMap<(ValidatorIndex, PeerId), Vec> { + let mut result: HashMap<(ValidatorIndex, PeerId), Vec> = HashMap::new(); + + for validator_index in &self.sent_by { + let peer = authorities.peer_ids.get(validator_index.0 as usize).unwrap(); + result.entry((*validator_index, *peer)).or_default().push(TestMessageInfo { + msg: self.msg.clone(), + sent_by: Default::default(), + tranche: self.tranche, + block_hash: self.block_hash, + }); + } + result + } +} From 0bfc13e8c9a812a52c5a73a9f6584ca3caa4e6a0 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 22 Dec 2023 13:30:20 +0200 Subject: [PATCH 156/192] Fixup some other stuff Signed-off-by: Alexandru Gheorghe --- .../src/approval/message_generator.rs | 182 ++++++++-------- .../node/subsystem-bench/src/approval/mod.rs | 197 ++++++++++++------ .../node/subsystem-bench/src/core/display.rs | 69 +++++- .../subsystem-bench/src/core/environment.rs | 29 +++ 4 files changed, 323 insertions(+), 154 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/approval/message_generator.rs b/polkadot/node/subsystem-bench/src/approval/message_generator.rs index 2341a6b8d2d5..3a28f0c8996d 100644 --- a/polkadot/node/subsystem-bench/src/approval/message_generator.rs +++ b/polkadot/node/subsystem-bench/src/approval/message_generator.rs @@ -177,7 +177,7 @@ impl PeerMessagesGenerator { } /// Generate all messages(Assignments & Approvals) needed for approving `blocks``. - pub fn generate_all_messages( + pub fn generate_messages_if_needed( configuration: &TestConfiguration, test_authorities: &TestAuthorities, options: &ApprovalsOptions, @@ -194,7 +194,7 @@ impl PeerMessagesGenerator { return path.to_path_buf(); } - gum::info!("Generate message because filed does not exist"); + gum::info!("Generate message because file does not exist"); let delta_to_first_slot_under_test = Timestamp::new(BUFFER_FOR_GENERATION_MILLIS); let initial_slot = Slot::from_timestamp( (*Timestamp::current() - *delta_to_first_slot_under_test).into(), @@ -372,7 +372,7 @@ fn issue_approvals( coalesce_tranche_diff: u32, rand_chacha: &mut ChaCha20Rng, ) -> Vec { - let mut to_sign: Vec = Vec::new(); + let mut queued_to_sign: Vec = Vec::new(); let mut num_coalesce = coalesce_approvals_len(min_coalesce, max_coalesce, rand_chacha); let result = assignments .iter() @@ -381,40 +381,44 @@ fn issue_approvals( protocol_v3::ApprovalDistributionMessage::Assignments(assignments) => { let mut approvals_to_create = Vec::new(); - let current_validator_index = - to_sign.first().map(|msg| msg.validator_index).unwrap_or(ValidatorIndex(999)); + let current_validator_index = queued_to_sign + .first() + .map(|msg| msg.validator_index) + .unwrap_or(ValidatorIndex(99999)); // Invariant for this benchmark. assert_eq!(assignments.len(), 1); let assignment = assignments.first().unwrap(); - let earliest_tranche = - to_sign.first().map(|val| val.tranche).unwrap_or(message.tranche); + let earliest_tranche = queued_to_sign + .first() + .map(|val| val.assignment.tranche) + .unwrap_or(message.tranche); - if to_sign.len() >= num_coalesce as usize || - (!to_sign.is_empty() && current_validator_index != assignment.0.validator) || + if queued_to_sign.len() >= num_coalesce as usize || + (!queued_to_sign.is_empty() && + current_validator_index != assignment.0.validator) || message.tranche - earliest_tranche >= coalesce_tranche_diff { - num_coalesce = coalesce_approvals_len(min_coalesce, max_coalesce, rand_chacha); - approvals_to_create.push(sign_candidates( - &mut to_sign, + approvals_to_create.push(TestSignInfo::sign_candidates( + &mut queued_to_sign, &keyrings, block_hash, num_coalesce, - )) + )); + num_coalesce = coalesce_approvals_len(min_coalesce, max_coalesce, rand_chacha); } - // If more that one candidate was in the assignment queue all of them. + // If more that one candidate was in the assignment queue all of them for issuing + // approvals for candidate_index in assignment.1.iter_ones() { let candidate = candidates.get(candidate_index).unwrap(); if let CandidateEvent::CandidateIncluded(candidate, _, _, _) = candidate { - to_sign.push(TestSignInfo { + queued_to_sign.push(TestSignInfo { candidate_hash: candidate.hash(), candidate_index: candidate_index as CandidateIndex, validator_index: assignment.0.validator, - sent_by: message.sent_by.clone().into_iter().collect(), - tranche: message.tranche, assignment: message.clone(), }); } else { @@ -429,85 +433,96 @@ fn issue_approvals( }) .collect_vec(); - let mut result = result.into_iter().flatten().collect_vec(); + let mut messages = result.into_iter().flatten().collect_vec(); - if !to_sign.is_empty() { - result.push(sign_candidates(&mut to_sign, &keyrings, block_hash, num_coalesce)); + if !queued_to_sign.is_empty() { + messages.push(TestSignInfo::sign_candidates( + &mut queued_to_sign, + &keyrings, + block_hash, + num_coalesce, + )); } - result + messages } /// Helper struct to gather information about more than one candidate an sign it in a single /// approval message. struct TestSignInfo { + /// The candidate hash candidate_hash: CandidateHash, + /// The candidate index candidate_index: CandidateIndex, + /// The validator sending the assignments validator_index: ValidatorIndex, - sent_by: HashSet, - tranche: u32, + /// The assignments convering this candidate assignment: TestMessageInfo, } -/// Helper function to create a signture for all candidates in `to_sign` parameter. -/// Returns a TestMessage -fn sign_candidates( - to_sign: &mut Vec, - keyrings: &Vec<(Keyring, PeerId)>, - block_hash: Hash, - num_coalesce: usize, -) -> MessagesBundle { - let current_validator_index = to_sign.first().map(|val| val.validator_index).unwrap(); - let tranche_trigger_timestamp = to_sign.iter().map(|val| val.tranche).max().unwrap(); - let keyring = keyrings.get(current_validator_index.0 as usize).unwrap().clone(); - - let unique_assignments: HashSet = - to_sign.iter().map(|info| info.assignment.clone()).collect(); - - let mut to_sign = to_sign - .drain(..) - .sorted_by(|val1, val2| val1.candidate_index.cmp(&val2.candidate_index)) - .peekable(); - - let mut bundle = MessagesBundle { - assignments: unique_assignments.into_iter().collect_vec(), - approvals: Vec::new(), - }; - - while to_sign.peek().is_some() { - let to_sign = to_sign.by_ref().take(num_coalesce).collect_vec(); - - let hashes = to_sign.iter().map(|val| val.candidate_hash).collect_vec(); - let candidate_indices = to_sign.iter().map(|val| val.candidate_index).collect_vec(); - - let sent_by = to_sign - .iter() - .map(|val| val.sent_by.iter()) - .flatten() - .map(|peer| *peer) - .collect::>(); +impl TestSignInfo { + /// Helper function to create a signture for all candidates in `to_sign` parameter. + /// Returns a TestMessage + fn sign_candidates( + to_sign: &mut Vec, + keyrings: &Vec<(Keyring, PeerId)>, + block_hash: Hash, + num_coalesce: usize, + ) -> MessagesBundle { + let current_validator_index = to_sign.first().map(|val| val.validator_index).unwrap(); + let tranche_approval_can_be_sent = + to_sign.iter().map(|val| val.assignment.tranche).max().unwrap(); + let keyring = keyrings.get(current_validator_index.0 as usize).unwrap().clone(); + + let unique_assignments: HashSet = + to_sign.iter().map(|info| info.assignment.clone()).collect(); + + let mut to_sign = to_sign + .drain(..) + .sorted_by(|val1, val2| val1.candidate_index.cmp(&val2.candidate_index)) + .peekable(); + + let mut bundle = MessagesBundle { + assignments: unique_assignments.into_iter().collect_vec(), + approvals: Vec::new(), + }; - let payload = ApprovalVoteMultipleCandidates(&hashes).signing_payload(1); + while to_sign.peek().is_some() { + let to_sign = to_sign.by_ref().take(num_coalesce).collect_vec(); - let validator_key: ValidatorPair = keyring.0.clone().pair().into(); - let signature = validator_key.sign(&payload[..]); - let indirect = IndirectSignedApprovalVoteV2 { - block_hash, - candidate_indices: candidate_indices.try_into().unwrap(), - validator: current_validator_index, - signature, - }; - let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![indirect]); + let hashes = to_sign.iter().map(|val| val.candidate_hash).collect_vec(); + let candidate_indices = to_sign.iter().map(|val| val.candidate_index).collect_vec(); - bundle.approvals.push(TestMessageInfo { - msg, - sent_by: sent_by.into_iter().collect_vec(), - tranche: tranche_trigger_timestamp, - block_hash, - }); + let sent_by = to_sign + .iter() + .map(|val| val.assignment.sent_by.iter()) + .flatten() + .map(|peer| *peer) + .collect::>(); + + let payload = ApprovalVoteMultipleCandidates(&hashes).signing_payload(1); + + let validator_key: ValidatorPair = keyring.0.clone().pair().into(); + let signature = validator_key.sign(&payload[..]); + let indirect = IndirectSignedApprovalVoteV2 { + block_hash, + candidate_indices: candidate_indices.try_into().unwrap(), + validator: current_validator_index, + signature, + }; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![indirect]); + + bundle.approvals.push(TestMessageInfo { + msg, + sent_by: sent_by.into_iter().collect_vec(), + tranche: tranche_approval_can_be_sent, + block_hash, + }); + } + bundle } - bundle } +/// Determine what neighbours would send a given message to the node under test. fn neighbours_that_would_sent_message( keyrings: &Vec<(Keyring, PeerId)>, current_validator_index: u32, @@ -631,7 +646,7 @@ fn generate_assignments( .map(|validator| (*validator, keyrings[validator.0 as usize].1)) .collect_vec(); - let mut no_duplicates = HashSet::new(); + let mut unique_assignments = HashSet::new(); for (core_index, assignment) in assignments { let assigned_cores = match &assignment.cert().kind { approval::v2::AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => @@ -647,7 +662,7 @@ fn generate_assignments( // For the cases where tranch0 assignments are in a single certificate we need to make // sure we create a single message. - if no_duplicates.insert(bitfiled) { + if unique_assignments.insert(bitfiled) { if !assignments_by_tranche.contains_key(&assignment.tranche()) { assignments_by_tranche.insert(assignment.tranche(), Vec::new()); } @@ -688,18 +703,19 @@ fn generate_assignments( .into_values() .map(|assignments| assignments.into_iter()) .flatten() - .map(|indirect| { + .map(|assignment| { let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( - indirect.0, indirect.1, + assignment.0, + assignment.1, )]); TestMessageInfo { msg, - sent_by: indirect + sent_by: assignment .2 .into_iter() .map(|(validator_index, _)| validator_index) .collect_vec(), - tranche: indirect.3, + tranche: assignment.3, block_hash: block_info.hash, } }) diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 273a0ae18c01..41bcce0a7373 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -15,6 +15,7 @@ // along with Polkadot. If not, see . use parity_scale_codec::{Decode, Encode}; +use prometheus::Registry; use serde::{Deserialize, Serialize}; use std::{ cmp::max, @@ -129,13 +130,24 @@ pub struct ApprovalsOptions { /// Sends messages only till block is approved. pub stop_when_approved: bool, #[clap(short, long)] - /// Sends messages only till block is approved. + /// Work directory. pub workdir_prefix: String, + /// The number of no shows per candidate #[clap(short, long, default_value_t = 0)] - pub num_no_shows_per_block: u32, + pub num_no_shows_per_candidate: u32, + /// Max expected time of flight for approval-distribution. + #[clap(short, long, default_value_t = 6.0)] + pub approval_distribution_expected_tof: f64, + /// Max expected cpu usage by approval-distribution. + #[clap(short, long, default_value_t = 6.0)] + pub approval_distribution_cpu_ms: f64, + /// Max expected cpu usage by approval-voting. + #[clap(short, long, default_value_t = 6.0)] + pub approval_voting_cpu_ms: f64, } impl ApprovalsOptions { + // Generates a fingerprint use to determine if messages need to be re-generated. fn fingerprint(&self) -> Vec { let mut bytes = Vec::new(); bytes.extend(self.last_considered_tranche.to_be_bytes()); @@ -226,6 +238,16 @@ impl CandidateTestData { } } +/// Test state that is pre-generated and loaded from a file that matches the fingerprint +/// of the TestConfiguration. +#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +struct GeneratedState { + /// All assignments and approvals + all_messages: Option>, + /// The first slot in the test. + initial_slot: Slot, +} + /// Approval test state used by all mock subsystems to be able to answer messages emitted /// by the approval-voting and approval-distribution-subystems. /// @@ -269,20 +291,22 @@ impl ApprovalTestState { let test_authorities = configuration.generate_authorities(); let start = Instant::now(); - let messages_path = PeerMessagesGenerator::generate_all_messages( + let messages_path = PeerMessagesGenerator::generate_messages_if_needed( configuration, &test_authorities, &options, &dependencies.task_manager.spawn_handle(), ); - let mut file = fs::OpenOptions::new().read(true).open(messages_path.as_path()).unwrap(); - let mut content = Vec::::with_capacity(2000000); + let mut messages_file = + fs::OpenOptions::new().read(true).open(messages_path.as_path()).unwrap(); + let mut messages_bytes = Vec::::with_capacity(2000000); - file.read_to_end(&mut content).expect("Could not initialize list of messages"); - let mut content = content.as_slice(); + messages_file + .read_to_end(&mut messages_bytes) + .expect("Could not initialize list of messages"); let generated_state: GeneratedState = - Decode::decode(&mut content).expect("Could not decode messages"); + Decode::decode(&mut messages_bytes.as_slice()).expect("Could not decode messages"); gum::info!( "It took {:?} ms to load {:?} unique messages", @@ -313,6 +337,7 @@ impl ApprovalTestState { .unwrap(), delta_tick_from_generated: Arc::new(AtomicU64::new(630720000)), }; + gum::info!("Built testing state"); state @@ -379,6 +404,7 @@ impl ApprovalTestState { network_emulator: &NetworkEmulator, overseer_handle: OverseerHandleReal, spawn_task_handle: &SpawnTaskHandle, + registry: Registry, ) -> oneshot::Receiver<()> { gum::info!(target: LOG_TARGET, "Start assignments/approvals production"); @@ -389,6 +415,7 @@ impl ApprovalTestState { state: self.clone(), options: self.options.clone(), notify_done: producer_tx, + registry, }; peer_message_source.produce_messages( @@ -399,12 +426,6 @@ impl ApprovalTestState { } } -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -struct GeneratedState { - all_messages: Option>, - initial_slot: Slot, -} - impl ApprovalTestState { /// Returns test data for the given hash fn get_info_by_hash(&self, requested_hash: Hash) -> &BlockTestData { @@ -442,6 +463,7 @@ struct PeerMessageProducer { /// under test. overseer_handle: OverseerHandleReal, notify_done: oneshot::Sender<()>, + registry: Registry, } impl PeerMessageProducer { @@ -453,75 +475,47 @@ impl PeerMessageProducer { all_messages: Vec, ) { spawn_task_handle.spawn_blocking("produce-messages", "produce-messages", async move { - let mut already_generated = HashSet::new(); - let system_clock = - PastSystemClock::new(SystemClock {}, self.state.delta_tick_from_generated.clone()); - let mut all_messages = all_messages.into_iter().peekable(); + let mut initialized_blocks = HashSet::new(); let mut per_candidate_data: HashMap<(Hash, CandidateIndex), CandidateTestData> = - HashMap::new(); - for block_info in self.state.per_slot_heads.iter() { - for (candidate_index, _) in block_info.candidates.iter().enumerate() { - per_candidate_data.insert( - (block_info.hash, candidate_index as CandidateIndex), - CandidateTestData { - max_no_shows: self.options.num_no_shows_per_block, - last_tranche_with_no_show: 0, - sent_assignment: 0, - num_no_shows: 0, - max_tranche: 0, - }, - ); - } - } - + self.initialize_candidates_test_data(); let mut skipped_messages: Vec = Vec::new(); let mut re_process_skipped = false; - while all_messages.peek().is_some() { - sleep(Duration::from_millis(50)).await; + let system_clock = + PastSystemClock::new(SystemClock {}, self.state.delta_tick_from_generated.clone()); + let mut all_messages = all_messages.into_iter().peekable(); + while all_messages.peek().is_some() { let current_slot = tick_to_slot_number(SLOT_DURATION_MILLIS, system_clock.tick_now()); let block_info = self.state.get_info_by_slot(current_slot).map(|block| block.clone()); if let Some(block_info) = block_info { - let candidates = self.state.approval_voting_metrics.candidates_imported(); - if candidates >= - block_info.total_candidates_before + block_info.candidates.len() as u64 && - already_generated.insert(block_info.hash) - { - gum::info!("Setup {:?}", block_info.hash); - let (tx, rx) = oneshot::channel(); - self.overseer_handle.wait_for_activation(block_info.hash, tx).await; - - rx.await - .expect("We should not fail waiting for block to be activated") - .expect("We should not fail waiting for block to be activated"); - - for validator in 1..self.state.test_authorities.keyrings.len() as u32 { - let peer_id = self - .state - .test_authorities - .peer_ids - .get(validator as usize) - .unwrap(); - let validator = ValidatorIndex(validator); - - let view_update = - generate_peer_view_change_for(block_info.hash, *peer_id); - - self.send_message(view_update, validator, None); - } + if !initialized_blocks.contains(&block_info.hash) && + !TestEnvironment::metric_lower_than( + &self.registry, + "polkadot_parachain_imported_candidates_total", + (block_info.total_candidates_before + + block_info.candidates.len() as u64 - + 1) as f64, + ) { + initialized_blocks.insert(block_info.hash); + self.initialize_block(&block_info).await; } } - let mut maybe_need_skip = skipped_messages.clone().into_iter().peekable(); + let mut maybe_need_skip = if re_process_skipped { + skipped_messages.clone().into_iter().peekable() + } else { + vec![].into_iter().peekable() + }; let progressing_iterator = if !re_process_skipped { &mut all_messages } else { re_process_skipped = false; + skipped_messages.clear(); &mut maybe_need_skip }; @@ -531,7 +525,7 @@ impl PeerMessageProducer { self.time_to_process_message( bundle, current_slot, - &already_generated, + &initialized_blocks, &system_clock, &per_candidate_data, ) @@ -546,6 +540,7 @@ impl PeerMessageProducer { &mut skipped_messages, ); } + sleep(Duration::from_millis(50)).await; } gum::info!( @@ -607,7 +602,7 @@ impl PeerMessageProducer { } } } else if !block_info.approved.load(std::sync::atomic::Ordering::SeqCst) && - self.options.num_no_shows_per_block > 0 + self.options.num_no_shows_per_candidate > 0 { skipped_messages.push(bundle); } @@ -683,6 +678,47 @@ impl PeerMessageProducer { ); self.network.submit_peer_action(peer, network_action); } + + async fn initialize_block(&mut self, block_info: &BlockTestData) { + gum::info!("Initialize block {:?}", block_info.hash); + let (tx, rx) = oneshot::channel(); + self.overseer_handle.wait_for_activation(block_info.hash, tx).await; + + rx.await + .expect("We should not fail waiting for block to be activated") + .expect("We should not fail waiting for block to be activated"); + + for validator in 1..self.state.test_authorities.keyrings.len() as u32 { + let peer_id = self.state.test_authorities.peer_ids.get(validator as usize).unwrap(); + let validator = ValidatorIndex(validator); + + let view_update = generate_peer_view_change_for(block_info.hash, *peer_id); + + self.send_message(view_update, validator, None); + } + } + + fn initialize_candidates_test_data( + &self, + ) -> HashMap<(Hash, CandidateIndex), CandidateTestData> { + let mut per_candidate_data: HashMap<(Hash, CandidateIndex), CandidateTestData> = + HashMap::new(); + for block_info in self.state.per_slot_heads.iter() { + for (candidate_index, _) in block_info.candidates.iter().enumerate() { + per_candidate_data.insert( + (block_info.hash, candidate_index as CandidateIndex), + CandidateTestData { + max_no_shows: self.options.num_no_shows_per_candidate, + last_tranche_with_no_show: 0, + sent_assignment: 0, + num_no_shows: 0, + max_tranche: 0, + }, + ); + } + } + per_candidate_data + } } /// Helper function to build an overseer with the real implementation for `ApprovalDistribution` and @@ -766,7 +802,12 @@ fn prepare_test_inner( pub async fn bench_approvals(env: &mut TestEnvironment, mut state: ApprovalTestState) { let producer_rx = state - .start_message_production(env.network(), env.overseer_handle().clone(), &env.spawn_handle()) + .start_message_production( + env.network(), + env.overseer_handle().clone(), + &env.spawn_handle(), + env.registry().clone(), + ) .await; bench_approvals_run(env, state, producer_rx).await } @@ -903,6 +944,26 @@ pub async fn bench_approvals_run( state.total_sent_messages.load(std::sync::atomic::Ordering::SeqCst), state.total_unique_messages.load(std::sync::atomic::Ordering::SeqCst) ); - gum::info!("{}", &env); + + assert!(env.bucket_metric_lower_than( + "polkadot_parachain_subsystem_bounded_tof_bucket", + "subsystem_name", + "approval-distribution-subsystem", + state.options.approval_distribution_expected_tof + )); + + assert!(env.metric_with_label_lower_than( + "substrate_tasks_polling_duration_sum", + "task_group", + "approval-voting", + state.options.approval_voting_cpu_ms + )); + + assert!(env.metric_with_label_lower_than( + "substrate_tasks_polling_duration_sum", + "task_group", + "approval-distribution", + state.options.approval_distribution_cpu_ms + )); } diff --git a/polkadot/node/subsystem-bench/src/core/display.rs b/polkadot/node/subsystem-bench/src/core/display.rs index d600cc484c14..345f1720fa10 100644 --- a/polkadot/node/subsystem-bench/src/core/display.rs +++ b/polkadot/node/subsystem-bench/src/core/display.rs @@ -20,13 +20,14 @@ //! Currently histogram buckets are skipped. use super::{configuration::TestConfiguration, LOG_TARGET}; use colored::Colorize; +use itertools::Itertools; use prometheus::{ - proto::{MetricFamily, MetricType}, + proto::{Bucket, MetricFamily, MetricType}, Registry, }; use std::fmt::Display; -#[derive(Default)] +#[derive(Default, Debug)] pub struct MetricCollection(Vec); impl From> for MetricCollection { @@ -49,6 +50,53 @@ impl MetricCollection { .sum() } + /// Filters metrics by name + pub fn subset_with_name(&self, name: &str) -> MetricCollection { + self.all() + .iter() + .filter(|metric| metric.name == name) + .map(|metric| metric.clone()) + .collect::>() + .into() + } + + /// Tells if entries in bucket metric is lower than `value` + pub fn metric_lower_than(&self, metric_name: &str, value: f64) -> bool { + self.sum_by(metric_name) < value + } + + /// Tells if entries in bucket metric is lower than `value` + pub fn bucket_metric_lower_than( + &self, + metric_name: &str, + label_name: &str, + label_value: &str, + value: f64, + ) -> bool { + let metric = self + .subset_with_name(metric_name) + .subset_with_label_value(label_name, label_value); + + metric.all().iter().all(|metric| { + metric + .bucket + .as_ref() + .and_then(|buckets| { + let mut prev_value = 0; + for bucket in buckets { + if value < bucket.get_upper_bound() && + prev_value != bucket.get_cumulative_count() + { + return Some(false); + } + prev_value = bucket.get_cumulative_count(); + } + Some(true) + }) + .unwrap_or_default() + }) + } + pub fn subset_with_label_value(&self, label_name: &str, label_value: &str) -> MetricCollection { self.0 .iter() @@ -85,6 +133,7 @@ pub struct TestMetric { label_names: Vec, label_values: Vec, value: f64, + bucket: Option>, } impl Display for TestMetric { @@ -138,6 +187,7 @@ pub fn parse_metrics(registry: &Registry) -> MetricCollection { label_names, label_values, value: m.get_counter().get_value(), + bucket: None, }); }, MetricType::GAUGE => { @@ -146,6 +196,7 @@ pub fn parse_metrics(registry: &Registry) -> MetricCollection { label_names, label_values, value: m.get_gauge().get_value(), + bucket: None, }); }, MetricType::HISTOGRAM => { @@ -156,14 +207,26 @@ pub fn parse_metrics(registry: &Registry) -> MetricCollection { label_names: label_names.clone(), label_values: label_values.clone(), value: h.get_sample_sum(), + bucket: None, }); let h_name = name.clone() + "_count"; + test_metrics.push(TestMetric { + name: h_name, + label_names: label_names.clone(), + label_values: label_values.clone(), + value: h.get_sample_count() as f64, + bucket: None, + }); + let h_name = name.clone() + "_bucket"; test_metrics.push(TestMetric { name: h_name, label_names, label_values, - value: h.get_sample_sum(), + value: 0.0, + bucket: Some( + h.get_bucket().iter().map(|bucket| bucket.clone()).collect_vec(), + ), }); }, MetricType::SUMMARY => { diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index a6257dd265b0..ad97a3a1594b 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -283,7 +283,36 @@ impl TestEnvironment { panic!("{}ms maximum time of flight breached", MAX_TIME_OF_FLIGHT.as_millis()) }); } + /// Tells if entries in bucket metric is lower than `value` + pub fn metric_with_label_lower_than( + &self, + metric_name: &str, + label_name: &str, + label_value: &str, + value: f64, + ) -> bool { + let test_metrics = super::display::parse_metrics(self.registry()) + .subset_with_label_value(label_name, label_value); + test_metrics.metric_lower_than(metric_name, value) + } + + /// Tells if entries in bucket metric is lower than `value` + pub fn metric_lower_than(registry: &Registry, metric_name: &str, value: f64) -> bool { + let test_metrics = super::display::parse_metrics(registry); + test_metrics.metric_lower_than(metric_name, value) + } + pub fn bucket_metric_lower_than( + &self, + metric_name: &str, + label_name: &str, + label_value: &str, + value: f64, + ) -> bool { + let test_metrics = super::display::parse_metrics(self.registry()); + + test_metrics.bucket_metric_lower_than(metric_name, label_name, label_value, value) + } // Stop overseer and subsystems. pub async fn stop(&mut self) { self.overseer_handle.stop().await; From 8d8de390dfaeed291ea85cd457609fd5a4560846 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 22 Dec 2023 13:33:13 +0200 Subject: [PATCH 157/192] Address some other fixups Signed-off-by: Alexandru Gheorghe --- .../src/approval/mock_chain_api.rs | 4 ++-- .../node/subsystem-bench/src/approval/mod.rs | 20 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/approval/mock_chain_api.rs b/polkadot/node/subsystem-bench/src/approval/mock_chain_api.rs index 8c6c3ee780b3..4e490e60f34c 100644 --- a/polkadot/node/subsystem-bench/src/approval/mock_chain_api.rs +++ b/polkadot/node/subsystem-bench/src/approval/mock_chain_api.rs @@ -63,11 +63,11 @@ impl MockChainApi { ChainApiMessage::Ancestors { hash, k: _, response_channel } => { let position = self .state - .per_slot_heads + .blocks .iter() .find_position(|block_info| block_info.hash == hash) .unwrap(); - let (ancestors, _) = self.state.per_slot_heads.split_at(position.0); + let (ancestors, _) = self.state.blocks.split_at(position.0); let ancestors = ancestors.iter().rev().map(|val| val.hash).collect_vec(); response_channel.send(Ok(ancestors)).unwrap(); diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 41bcce0a7373..258f789dc42d 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -258,7 +258,7 @@ pub struct ApprovalTestState { /// The specific test configurations passed when starting the benchmark. options: ApprovalsOptions, /// The list of blocks used for testing. - per_slot_heads: Vec, + blocks: Vec, /// The babe epoch used during testing. babe_epoch: BabeEpoch, /// The session info used during testing. @@ -324,7 +324,7 @@ impl ApprovalTestState { ); let state = ApprovalTestState { - per_slot_heads: blocks, + blocks, babe_epoch: babe_epoch.clone(), session_info: session_info.clone(), generated_state, @@ -429,7 +429,7 @@ impl ApprovalTestState { impl ApprovalTestState { /// Returns test data for the given hash fn get_info_by_hash(&self, requested_hash: Hash) -> &BlockTestData { - self.per_slot_heads + self.blocks .iter() .filter(|block| block.hash == requested_hash) .next() @@ -438,7 +438,7 @@ impl ApprovalTestState { /// Returns test data for the given block number fn get_info_by_number(&self, requested_number: u32) -> &BlockTestData { - self.per_slot_heads + self.blocks .iter() .filter(|block| block.block_number == requested_number) .next() @@ -447,7 +447,7 @@ impl ApprovalTestState { /// Returns test data for the given slot fn get_info_by_slot(&self, slot: Slot) -> Option<&BlockTestData> { - self.per_slot_heads.iter().filter(|block| block.slot == slot).next() + self.blocks.iter().filter(|block| block.slot == slot).next() } } @@ -703,7 +703,7 @@ impl PeerMessageProducer { ) -> HashMap<(Hash, CandidateIndex), CandidateTestData> { let mut per_candidate_data: HashMap<(Hash, CandidateIndex), CandidateTestData> = HashMap::new(); - for block_info in self.state.per_slot_heads.iter() { + for block_info in self.state.blocks.iter() { for (candidate_index, _) in block_info.candidates.iter().enumerate() { per_candidate_data.insert( (block_info.hash, candidate_index as CandidateIndex), @@ -894,7 +894,7 @@ pub async fn bench_approvals_run( producer_rx.await.expect("Failed to receive done from message producer"); gum::info!("Requesting approval votes ms"); - for info in &state.per_slot_heads { + for info in &state.blocks { for (index, candidates) in info.candidates.iter().enumerate() { match candidates { CandidateEvent::CandidateBacked(_, _, _, _) => todo!(), @@ -924,7 +924,7 @@ pub async fn bench_approvals_run( } } - for state in &state.per_slot_heads { + for state in &state.blocks { for (validator, votes) in state.votes.as_ref().iter().enumerate() { for (index, candidate) in votes.iter().enumerate() { assert_eq!( @@ -957,13 +957,13 @@ pub async fn bench_approvals_run( "substrate_tasks_polling_duration_sum", "task_group", "approval-voting", - state.options.approval_voting_cpu_ms + state.options.approval_voting_cpu_ms * state.blocks.len() as f64 )); assert!(env.metric_with_label_lower_than( "substrate_tasks_polling_duration_sum", "task_group", "approval-distribution", - state.options.approval_distribution_cpu_ms + state.options.approval_distribution_cpu_ms * state.blocks.len() as f64 )); } From 366e26747ac013644184ad2e7e735142a4346729 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 22 Dec 2023 13:39:35 +0200 Subject: [PATCH 158/192] Fixup stuff Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 258 ++------------------- README.md | 280 ++++------------------- polkadot/node/subsystem-bench/Cargo.toml | 2 +- 3 files changed, 61 insertions(+), 479 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 43d24da7bc95..aff3af9193d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1226,11 +1226,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -1247,11 +1243,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -1433,11 +1425,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -2711,11 +2699,7 @@ dependencies = [ "heck", "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -3925,11 +3909,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -4440,11 +4420,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -4484,11 +4460,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -4505,11 +4477,7 @@ checksum = "50c49547d73ba8dcfd4ad7325d64c6d5391ff4224d498fc39a6f3f49825a530d" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -4717,11 +4685,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -4782,11 +4746,7 @@ dependencies = [ "proc-macro2", "quote", "regex", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master "termcolor", "toml 0.7.8", "walkdir", @@ -5014,11 +4974,7 @@ checksum = "5e9a1f9f7d83e59740248a6e14ecf93929ade55027844dfcea78beafccc15745" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -5029,11 +4985,7 @@ checksum = "c2ad8cef1d801a4686bfd8919f0b30eac4c8e48968c437a6405ded4fb5272d2b" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -5061,9 +5013,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece" dependencies = [ "humantime", "is-terminal", @@ -5201,11 +5153,7 @@ dependencies = [ "fs-err", "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -5320,7 +5268,7 @@ dependencies = [ [[package]] name = "fflonk" version = "0.1.0" -source = "git+https://github.com/w3f/fflonk#1beb0585e1c8488956fac7f05da061f9b41e8948" +source = "git+https://github.com/w3f/fflonk#1e854f35e9a65d08b11a86291405cdc95baa0a35" dependencies = [ "ark-ec", "ark-ff 0.4.2", @@ -5342,7 +5290,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84f2e425d9790201ba4af4630191feac6dcc98765b118d4d18e91d23c2353866" dependencies = [ - "env_logger 0.10.0", + "env_logger 0.10.1", "log", ] @@ -5593,11 +5541,7 @@ dependencies = [ "quote", "scale-info", "sp-arithmetic", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master "trybuild", ] @@ -5750,11 +5694,7 @@ dependencies = [ "quote", "regex", "sp-core-hashing", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -5765,11 +5705,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -5778,11 +5714,7 @@ version = "3.0.0" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -6015,11 +5947,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -7956,11 +7884,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -7974,11 +7898,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -7989,11 +7909,7 @@ checksum = "9ea73aa640dc01d62a590d48c0c3521ed739d53b27f919b25c3551e233481654" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -8004,11 +7920,7 @@ checksum = "ef9d79ae96aaba821963320eb2b6e34d17df1e5a83d8a1985c29cc5be59577b3" dependencies = [ "macro_magic_core", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -9747,11 +9659,7 @@ version = "4.0.0-dev" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -10911,11 +10819,7 @@ dependencies = [ "proc-macro2", "quote", "sp-runtime", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -11528,9 +11432,9 @@ dependencies = [ [[package]] name = "parity-scale-codec" -version = "3.6.9" +version = "3.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "881331e34fa842a2fb61cc2db9643a8fedc615e47cfcc52597d1af0db9a7e8fe" +checksum = "0dec8a8073036902368c2cdc0387e85ff9a37054d7e7c98e592145e0c92cd4fb" dependencies = [ "arrayvec 0.7.4", "bitvec", @@ -11543,11 +11447,11 @@ dependencies = [ [[package]] name = "parity-scale-codec-derive" -version = "3.6.9" +version = "3.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be30eaf4b0a9fba5336683b38de57bb86d179a35862ba6bfcf57625d006bde5b" +checksum = "312270ee71e1cd70289dacf597cab7b207aa107d2f28191c2ae45b2ece18a260" dependencies = [ - "proc-macro-crate 2.0.1", + "proc-macro-crate 1.3.1", "proc-macro2", "quote", "syn 1.0.109", @@ -11808,11 +11712,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -11853,11 +11753,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -13419,6 +13315,7 @@ version = "1.0.0" dependencies = [ "assert_matches", "async-trait", + "bincode", "clap 4.4.11", "clap-num", "color-eyre", @@ -13426,24 +13323,17 @@ dependencies = [ "env_logger 0.9.3", "futures", "futures-timer", + "hex", "itertools 0.11.0", -<<<<<<< HEAD "kvdb-memorydb", -======= ->>>>>>> origin/master "log", "orchestra", "parity-scale-codec", "paste", -<<<<<<< HEAD "polkadot-approval-distribution", "polkadot-availability-recovery", "polkadot-erasure-coding", "polkadot-node-core-approval-voting", -======= - "polkadot-availability-recovery", - "polkadot-erasure-coding", ->>>>>>> origin/master "polkadot-node-metrics", "polkadot-node-network-protocol", "polkadot-node-primitives", @@ -13455,16 +13345,18 @@ dependencies = [ "polkadot-primitives", "polkadot-primitives-test-helpers", "prometheus", -<<<<<<< HEAD + "pyroscope", + "pyroscope_pprofrs", "rand 0.8.5", "rand_chacha 0.3.1", - "rand_core 0.5.1", + "rand_core 0.6.4", "sc-keystore", "sc-network", "sc-service", "schnorrkel 0.9.1", "serde", "serde_yaml", + "sha1", "sp-application-crypto", "sp-consensus", "sp-consensus-babe", @@ -13473,20 +13365,6 @@ dependencies = [ "sp-keystore", "sp-runtime", "sp-timestamp", -======= - "pyroscope", - "pyroscope_pprofrs", - "rand 0.8.5", - "sc-keystore", - "sc-network", - "sc-service", - "serde", - "serde_yaml", - "sp-application-crypto", - "sp-core", - "sp-keyring", - "sp-keystore", ->>>>>>> origin/master "substrate-prometheus-endpoint", "tokio", "tracing-gum", @@ -13852,11 +13730,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" dependencies = [ "proc-macro2", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -13948,18 +13822,14 @@ checksum = "9b698b0b09d40e9b7c1a47b132d66a8b54bcd20583d9b6d06e4535e383b4405c" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] name = "proc-macro2" -version = "1.0.70" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] @@ -14024,11 +13894,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -14446,11 +14312,7 @@ checksum = "7f7473c2cfcf90008193dd0e3e16599455cb601a9fce322b5bb55de799664925" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -15284,11 +15146,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -16544,11 +16402,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -16966,11 +16820,7 @@ checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -17060,11 +16910,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -17093,9 +16939,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", @@ -17854,11 +17700,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -18193,11 +18035,7 @@ version = "9.0.0" dependencies = [ "quote", "sp-core-hashing", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -18255,11 +18093,7 @@ version = "8.0.0" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -18269,11 +18103,7 @@ source = "git+https://github.com/paritytech/polkadot-sdk#82912acb33a9030c0ef3bf5 dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -18546,11 +18376,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -18562,11 +18388,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -18837,11 +18659,7 @@ dependencies = [ "proc-macro2", "quote", "sp-version", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -19675,15 +19493,9 @@ dependencies = [ [[package]] name = "syn" -<<<<<<< HEAD -version = "2.0.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13fa70a4ee923979ffb522cacce59d34421ebdea5625e1073c4326ef9d2dd42e" -======= version = "2.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" ->>>>>>> origin/master dependencies = [ "proc-macro2", "quote", @@ -19950,11 +19762,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -20115,11 +19923,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -20325,11 +20129,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -20371,11 +20171,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] @@ -20905,11 +20701,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master "wasm-bindgen-shared", ] @@ -20943,11 +20735,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -21914,11 +21702,7 @@ dependencies = [ "proc-macro2", "quote", "staging-xcm", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master "trybuild", ] @@ -22038,11 +21822,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", -<<<<<<< HEAD - "syn 2.0.40", -======= "syn 2.0.41", ->>>>>>> origin/master ] [[package]] diff --git a/README.md b/README.md index d4eccf238cd0..1f255823b5b6 100644 --- a/README.md +++ b/README.md @@ -1,258 +1,60 @@ -

Pyroscope

+> NOTE: We have recently made significant changes to our repository structure. In order to streamline our development +process and foster better contributions, we have merged three separate repositories Cumulus, Substrate and Polkadot into +this repository. Read more about the changes [ +here](https://polkadot-public.notion.site/Polkadot-SDK-FAQ-fbc4cecc2c46443fb37b9eeec2f0d85f). +# Polkadot SDK -[![ci](https://github.com/grafana/pyroscope/actions/workflows/test.yml/badge.svg)](https://github.com/grafana/pyroscope/actions/workflows/test.yml) -[![JS Tests Status](https://github.com/grafana/pyroscope/workflows/JS%20Tests/badge.svg)](https://github.com/grafana/pyroscope/actions?query=workflow%3AJS%20Tests) -[![Go Report](https://goreportcard.com/badge/github.com/grafana/pyroscope)](https://goreportcard.com/report/github.com/grafana/pyroscope) -[![License: AGPLv3](https://img.shields.io/badge/License-AGPL%20v3-blue.svg)](LICENSE) -[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fgrafana%2Fpyroscope.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fgrafana%2Fpyroscope?ref=badge_shield) -[![Latest release](https://img.shields.io/github/release/grafana/pyroscope.svg)](https://github.com/grafana/pyroscope/releases) -[![DockerHub](https://img.shields.io/docker/pulls/grafana/pyroscope.svg)](https://hub.docker.com/r/grafana/pyroscope) -[![GoDoc](https://godoc.org/github.com/grafana/pyroscope?status.svg)](https://godoc.org/github.com/grafana/pyroscope) +![](https://cms.polkadot.network/content/images/2021/06/1-xPcVR_fkITd0ssKBvJ3GMw.png) -### News +[![StackExchange](https://img.shields.io/badge/StackExchange-Community%20&%20Support-222222?logo=stackexchange)](https://substrate.stackexchange.com/) -On 2023-03-15, [Grafana Labs acquired Pyroscope](https://grafana.com/blog/2023/03/15/pyroscope-grafana-phlare-join-for-oss-continuous-profiling/). +The Polkadot SDK repository provides all the resources needed to start building on the Polkadot network, a multi-chain +blockchain platform that enables different blockchains to interoperate and share information in a secure and scalable +way. The Polkadot SDK comprises three main pieces of software: -The teams and codebases of both [Grafana Phlare](https://github.com/grafana/phlare) and Pyroscope have **merged into Grafana Pyroscope**. +## [Polkadot](./polkadot/) +[![PolkadotForum](https://img.shields.io/badge/Polkadot_Forum-e6007a?logo=polkadot)](https://forum.polkadot.network/) +[![Polkadot-license](https://img.shields.io/badge/License-GPL3-blue)](./polkadot/LICENSE) -### What is Grafana Pyroscope? +Implementation of a node for the https://polkadot.network in Rust, using the Substrate framework. This directory +currently contains runtimes for the Polkadot, Kusama, Westend, and Rococo networks. In the future, these will be +relocated to the [`runtimes`](https://github.com/polkadot-fellows/runtimes/) repository. -Grafana Pyroscope is an open source continuous profiling platform. It will help you: -* Find performance issues and bottlenecks in your code -* Use high-cardinality tags/labels to analyze your application -* Resolve issues with high CPU utilization -* Track down memory leaks -* Understand the call tree of your application -* Auto-instrument your code to link profiling data to traces +## [Substrate](./substrate/) + [![SubstrateRustDocs](https://img.shields.io/badge/Rust_Docs-Substrate-24CC85?logo=rust)](https://paritytech.github.io/substrate/master/substrate/index.html) + [![Substrate-license](https://img.shields.io/badge/License-GPL3%2FApache2.0-blue)](./substrate/README.md#LICENSE) -## 🔥 [Pyroscope Live Demo](https://demo.pyroscope.io/) 🔥 +Substrate is the primary blockchain SDK used by developers to create the parachains that make up the Polkadot network. +Additionally, it allows for the development of self-sovereign blockchains that operate completely independently of +Polkadot. -[![Pyroscope GIF Demo](https://user-images.githubusercontent.com/23323466/143324845-16ff72df-231e-412d-bd0a-38ef2e09cba8.gif)](https://demo.pyroscope.io/) +## [Cumulus](./cumulus/) +[![CumulusRustDocs](https://img.shields.io/badge/Rust_Docs-Cumulus-222222?logo=rust)](https://paritytech.github.io/cumulus/cumulus_client_collator/index.html) +[![Cumulus-license](https://img.shields.io/badge/License-GPL3-blue)](./cumulus/LICENSE) -## Features +Cumulus is a set of tools for writing Substrate-based Polkadot parachains. -* Minimal CPU overhead -* Efficient compression, low disk space requirements -* Can handle high-cardinality tags/labels -* Calculate the performance "diff" between various tags/labels and time periods -* Can store years of profiling data from multiple applications -* Advanced analysis UI +## Upstream Dependencies -## Run Pyroscope Locally +Below are the primary upstream dependencies utilized in this project: -```shell -docker run -it -p 4040:4040 grafana/pyroscope -``` +- [`parity-scale-codec`](https://crates.io/crates/parity-scale-codec) +- [`parity-db`](https://crates.io/crates/parity-db) +- [`parity-common`](https://github.com/paritytech/parity-common) +- [`trie`](https://github.com/paritytech/trie) -For more documentation on how to configure Pyroscope server, see [our server documentation](https://grafana.com/docs/pyroscope/latest/configure-server/). +## Security +The security policy and procedures can be found in [docs/contributor/SECURITY.md](./docs/contributor/SECURITY.md). -## Send data to server via Pyroscope agent (language specific) +## Contributing & Code of Conduct -For more documentation on how to add the Pyroscope agent to your code, see the [agent documentation](https://grafana.com/docs/pyroscope/latest/configure-client/) on our website or find language specific examples and documentation below: - - - - - - - - - - - - - -

- Golang
- Documentation
- Examples -
- Java
- Documentation
- Examples -
- Python
- Documentation
- Examples -
- Ruby
- Documentation
- Examples -

- NodeJS
- Documentation
- Examples -
- Dotnet
- Documentation
- Examples -
- eBPF
- Documentation
- Examples -
- Rust
- Documentation
- Examples -
+Ensure you follow our [contribution guidelines](./docs/contributor/CONTRIBUTING.md). In every interaction and +contribution, this project adheres to the [Contributor Covenant Code of Conduct](./docs/contributor/CODE_OF_CONDUCT.md). -## Deployment Diagram +## Additional Resources -![agent_server_diagram_11-01](https://user-images.githubusercontent.com/23323466/178165230-a94e1ee2-9725-4752-97ff-542158d1b703.svg) - -## Third-Party Integrations - -Pyroscope also supports several third-party integrations, notably: -- [Jaeger UI](https://github.com/pyroscope-io/jaeger-ui) -- [OTel Golang (tracing)](https://github.com/grafana/otel-profiling-go) -- [AWS Lambda Extension](https://github.com/grafana/pyroscope-lambda-extension) - -## Documentation - -For more information on how to use Pyroscope with other programming languages, install it on Linux, or use it in production environment, check out our documentation: - -* [Getting Started](https://grafana.com/docs/pyroscope/latest/get-started/) -* [Deployment Guide](https://grafana.com/docs/pyroscope/latest/deploy-kubernetes/) -* [Pyroscope Architecture](https://grafana.com/docs/pyroscope/latest/reference-pyroscope-architecture/) - - -## Downloads - -You can download the latest version of pyroscope for macOS, linux and Docker from our [Releases page](https://github.com/grafana/pyroscope/releases). - -## Supported Languages - -* [x] Go (via `pprof`) -* [x] Python (via `py-spy`) -* [x] Ruby (via `rbspy`) -* [x] Linux eBPF -* [x] Java (via `async-profiler`) -* [x] Rust (via `pprof-rs`) -* [x] .NET -* [x] PHP (via `phpspy`) -* [x] Node - -Let us know what other integrations you want to see in [our issues](https://github.com/grafana/pyroscope/issues?q=is%3Aissue+is%3Aopen+label%3Anew-profilers) or in [our slack](https://slack.grafana.com). - -## Credits - -Pyroscope is possible thanks to the excellent work of many people, including but not limited to: - -* Brendan Gregg — inventor of Flame Graphs -* Julia Evans — creator of rbspy — sampling profiler for Ruby -* Vladimir Agafonkin — creator of flamebearer — fast flamegraph renderer -* Ben Frederickson — creator of py-spy — sampling profiler for Python -* Adam Saponara — creator of phpspy — sampling profiler for PHP -* Alexei Starovoitov, Brendan Gregg, and many others who made BPF based profiling in Linux kernel possible -* Jamie Wong — creator of speedscope — interactive flamegraph visualizer - -## Contributing - -To start contributing, check out our [Contributing Guide](CONTRIBUTING.md) - - -### Thanks to the contributors of Pyroscope! - -[//]: contributor-faces - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -[//]: contributor-faces +- For monitoring upcoming changes and current proposals related to the technical implementation of the Polkadot network, + visit the [`Requests for Comment (RFC)`](https://github.com/polkadot-fellows/RFCs) repository. While it's maintained + by the Polkadot Fellowship, the RFC process welcomes contributions from everyone. diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index fc9e888da070..9a17cbabc811 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -69,7 +69,7 @@ sp-runtime = { path = "../../../substrate/primitives/runtime", default-features sp-timestamp = { path = "../../../substrate/primitives/timestamp" } schnorrkel = { version = "0.9.1", default-features = false } -rand_core = "0.5.1" # should match schnorrkel +rand_core = "0.6.2" # should match schnorrkel rand_chacha = { version = "0.3.1" } orchestra = { version = "0.3.3", default-features = false, features = ["futures_channel"] } pyroscope = "0.5.7" From 290eef2fe301a311a1a95f43af14a29d97a4d100 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 22 Dec 2023 14:54:01 +0200 Subject: [PATCH 159/192] Clippy Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 41 ++++---- .../node/core/approval-voting/src/tests.rs | 4 +- .../node/core/approval-voting/src/time.rs | 2 + .../network/approval-distribution/src/lib.rs | 23 +---- .../approval-distribution/src/metrics.rs | 1 - .../network/availability-recovery/src/lib.rs | 2 +- polkadot/node/subsystem-bench/Cargo.toml | 2 +- polkadot/node/subsystem-bench/README.md | 98 ------------------- .../examples/approvals_throughput.yaml | 16 ++- .../subsystem-bench/src/approval/helpers.rs | 9 +- .../src/approval/message_generator.rs | 78 ++++++--------- .../src/approval/mock_chain_selection.rs | 6 +- .../node/subsystem-bench/src/approval/mod.rs | 94 ++++++++---------- .../src/approval/test_message.rs | 10 +- polkadot/node/subsystem-bench/src/cli.rs | 2 +- .../node/subsystem-bench/src/core/display.rs | 12 +-- .../subsystem-bench/src/core/environment.rs | 1 + .../src/core/mock/network_bridge.rs | 10 +- .../node/subsystem-bench/src/core/network.rs | 4 +- .../subsystem-bench/src/subsystem-bench.rs | 2 +- 20 files changed, 143 insertions(+), 274 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index dcc575ffaf0d..b7de923b808d 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -114,7 +114,7 @@ const APPROVAL_CHECKING_TIMEOUT: Duration = Duration::from_secs(120); /// /// Value rather arbitrarily: Should not be hit in practice, it exists to more easily diagnose dead /// lock issues for example. -const WAIT_FOR_SIGS_TIMEOUT: Duration = Duration::from_millis(2000); +const WAIT_FOR_SIGS_TIMEOUT: Duration = Duration::from_millis(500); const APPROVAL_CACHE_SIZE: u32 = 1024; const TICK_TOO_FAR_IN_FUTURE: Tick = 20; // 10 seconds. @@ -195,13 +195,6 @@ impl Metrics { } } - pub fn candidates_imported(&self) -> u64 { - self.0 - .as_ref() - .map(|metrics| metrics.imported_candidates_total.get()) - .unwrap_or_default() - } - fn on_assignment_produced(&self, tranche: DelayTranche) { if let Some(metrics) = &self.0 { metrics.assignments_produced.observe(tranche as f64); @@ -452,6 +445,24 @@ impl ApprovalVotingSubsystem { keystore: Arc, sync_oracle: Box, metrics: Metrics, + ) -> Self { + ApprovalVotingSubsystem::with_config_and_clock( + config, + db, + keystore, + sync_oracle, + metrics, + Box::new(SystemClock {}), + ) + } + + /// Create a new approval voting subsystem with the given keystore, config, and database. + pub fn with_config_and_clock( + config: Config, + db: Arc, + keystore: Arc, + sync_oracle: Box, + metrics: Metrics, clock: Box, ) -> Self { ApprovalVotingSubsystem { @@ -503,15 +514,10 @@ fn db_sanity_check(db: Arc, config: DatabaseConfig) -> SubsystemRe impl ApprovalVotingSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let backend = DbBackend::new(self.db.clone(), self.db_config); - let future = run::( - ctx, - self, - Box::new(SystemClock), - Box::new(RealAssignmentCriteria), - backend, - ) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) - .boxed(); + let future = + run::(ctx, self, Box::new(RealAssignmentCriteria), backend) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) + .boxed(); SpawnedSubsystem { name: "approval-voting-subsystem", future } } @@ -919,7 +925,6 @@ enum Action { async fn run( mut ctx: Context, mut subsystem: ApprovalVotingSubsystem, - _clock: Box, assignment_criteria: Box, mut backend: B, ) -> SubsystemResult<()> diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 7a0bde6a55e2..9220e84a2554 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -549,7 +549,7 @@ fn test_harness>( let subsystem = run( context, - ApprovalVotingSubsystem::with_config( + ApprovalVotingSubsystem::with_config_and_clock( Config { col_approval_data: test_constants::TEST_CONFIG.col_approval_data, slot_duration_millis: SLOT_DURATION_MILLIS, @@ -558,8 +558,8 @@ fn test_harness>( Arc::new(keystore), sync_oracle, Metrics::default(), + clock.clone(), ), - clock.clone(), assignment_criteria, backend, ); diff --git a/polkadot/node/core/approval-voting/src/time.rs b/polkadot/node/core/approval-voting/src/time.rs index bb17c6bfeb47..99dfbe07678f 100644 --- a/polkadot/node/core/approval-voting/src/time.rs +++ b/polkadot/node/core/approval-voting/src/time.rs @@ -99,11 +99,13 @@ pub fn slot_number_to_tick(slot_duration_millis: u64, slot: Slot) -> Tick { u64::from(slot) * ticks_per_slot } +/// Converts a tick to the slot number. pub fn tick_to_slot_number(slot_duration_millis: u64, tick: Tick) -> Slot { let ticks_per_slot = slot_duration_millis / TICK_DURATION_MILLIS; (tick / ticks_per_slot).into() } +/// Converts a tranche from a slot to the tick number. pub fn tranche_to_tick(slot_duration_millis: u64, slot: Slot, tranche: u32) -> Tick { slot_number_to_tick(slot_duration_millis, slot) + tranche as u64 } diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 626a7c6be3f5..d520febaef51 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -21,6 +21,8 @@ //! //! [approval-distribution-page]: https://paritytech.github.io/polkadot-sdk/book/node/approval/approval-distribution.html +#![warn(missing_docs)] + use self::metrics::Metrics; use futures::{channel::oneshot, select, FutureExt as _}; use itertools::Itertools; @@ -57,7 +59,7 @@ use std::{ time::Duration, }; -pub mod metrics; +mod metrics; #[cfg(test)] mod tests; @@ -349,8 +351,6 @@ struct State { /// Aggregated reputation change reputation: ReputationAggregator, - total_num_messages: u64, - got_votes: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -714,12 +714,7 @@ impl State { }); }, NetworkBridgeEvent::PeerMessage(peer_id, message) => { - self.total_num_messages += 1; self.process_incoming_peer_message(ctx, metrics, peer_id, message, rng).await; - - if self.total_num_messages % 100000 == 0 { - gum::info!(target: LOG_TARGET, total_num_messages = self.total_num_messages, "Processed 100k"); - } }, NetworkBridgeEvent::UpdatedAuthorityIds { .. } => { // The approval-distribution subsystem doesn't deal with `AuthorityDiscoveryId`s. @@ -1172,7 +1167,7 @@ impl State { Some(entry) => entry, None => { if let Some(peer_id) = source.peer_id() { - gum::info!( + gum::trace!( target: LOG_TARGET, ?peer_id, hash = ?block_hash, @@ -1191,12 +1186,6 @@ impl State { metrics.on_assignment_recent_outdated(); } } - gum::info!( - target: LOG_TARGET, - hash = ?block_hash, - ?validator_index, - "Unexpected assignment invalid", - ); metrics.on_assignment_invalid_block(); return }, @@ -1501,7 +1490,7 @@ impl State { ) -> bool { for message_subject in assignments_knowledge_key { if !entry.knowledge.contains(&message_subject.0, message_subject.1) { - gum::info!( + gum::trace!( target: LOG_TARGET, ?peer_id, ?message_subject, @@ -1577,7 +1566,6 @@ impl State { source: MessageSource, vote: IndirectSignedApprovalVoteV2, ) { - assert!(self.got_votes.is_none()); let _span = self .spans .get(&vote.block_hash) @@ -2422,7 +2410,6 @@ impl ApprovalDistribution { .await; }, ApprovalDistributionMessage::GetApprovalSignatures(indices, tx) => { - state.got_votes = Some(true); let sigs = state.get_approval_signatures(indices); if let Err(_) = tx.send(sigs) { gum::debug!( diff --git a/polkadot/node/network/approval-distribution/src/metrics.rs b/polkadot/node/network/approval-distribution/src/metrics.rs index a4658fd1b838..0642b1b2e0cd 100644 --- a/polkadot/node/network/approval-distribution/src/metrics.rs +++ b/polkadot/node/network/approval-distribution/src/metrics.rs @@ -14,7 +14,6 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -//! [`ApprovalDistribution`] metrics implementation. use polkadot_node_metrics::metrics::{prometheus, Metrics as MetricsTrait}; use polkadot_node_primitives::approval::v2::AssignmentCertKindV2; diff --git a/polkadot/node/network/availability-recovery/src/lib.rs b/polkadot/node/network/availability-recovery/src/lib.rs index d029bce04173..fb8064878f4f 100644 --- a/polkadot/node/network/availability-recovery/src/lib.rs +++ b/polkadot/node/network/availability-recovery/src/lib.rs @@ -667,7 +667,7 @@ impl AvailabilityRecoverySubsystem { } }, None => { - gum::trace!( + gum::debug!( target: LOG_TARGET, "Erasure task channel closed", ); diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index 9a17cbabc811..13226fabdd56 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -22,7 +22,7 @@ polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-primitives = { path = "../primitives" } polkadot-primitives = { path = "../../primitives" } polkadot-node-network-protocol = { path = "../network/protocol" } -polkadot-availability-recovery = { path = "../network/availability-recovery", features=["subsystem-benchmarks"]} +polkadot-availability-recovery = { path = "../network/availability-recovery", features = ["subsystem-benchmarks"] } color-eyre = { version = "0.6.1", default-features = false } polkadot-overseer = { path = "../overseer" } diff --git a/polkadot/node/subsystem-bench/README.md b/polkadot/node/subsystem-bench/README.md index 9d423b01f7e0..b1476db27548 100644 --- a/polkadot/node/subsystem-bench/README.md +++ b/polkadot/node/subsystem-bench/README.md @@ -26,19 +26,6 @@ The output binary will be placed in `target/testnet/subsystem-bench`. ### Test metrics -<<<<<<< HEAD -Subsystem, CPU usage and network metrics are exposed via a prometheus endpoint during the test execution. -A small subset of these collected metrics are displayed in the CLI, but for an in depth analysys of the test results, -a local Grafana/Prometheus stack is needed. - -### Install Prometheus - -Please follow the [official installation guide](https://prometheus.io/docs/prometheus/latest/installation/) for your -platform/OS. - -After succesfully installing and starting up Prometheus, we need to alter it's configuration such that it -will scrape the benchmark prometheus endpoint `127.0.0.1:9999`. Please check the prometheus official documentation -======= Subsystem, CPU usage and network metrics are exposed via a prometheus endpoint during the test execution. A small subset of these collected metrics are displayed in the CLI, but for an in depth analysys of the test results, a local Grafana/Prometheus stack is needed. @@ -59,7 +46,6 @@ platform/OS. After succesfully installing and starting up Prometheus, we need to alter it's configuration such that it will scrape the benchmark prometheus endpoint `127.0.0.1:9999`. Please check the prometheus official documentation ->>>>>>> origin/master regarding the location of `prometheus.yml`. On MacOS for example the full path `/opt/homebrew/etc/prometheus.yml` prometheus.yml: @@ -80,9 +66,6 @@ scrape_configs: To complete this step restart Prometheus server such that it picks up the new configuration. -<<<<<<< HEAD -### Install and setup Grafana -======= ### Install Pyroscope To collect CPU profiling data, you must be running the Pyroscope server. @@ -90,19 +73,10 @@ Follow the [installation guide](https://grafana.com/docs/pyroscope/latest/get-st relevant to your operating system. ### Install Grafana ->>>>>>> origin/master Follow the [installation guide](https://grafana.com/docs/grafana/latest/setup-grafana/installation/) relevant to your operating system. -<<<<<<< HEAD -Once you have the installation up and running, configure the local Prometheus as a data source by following -[this guide](https://grafana.com/docs/grafana/latest/datasources/prometheus/configure-prometheus-data-source/) - -#### Import dashboards - -Follow [this guide](https://grafana.com/docs/grafana/latest/dashboards/manage-dashboards/#export-and-import-dashboards) -======= ### Setup Grafana Once you have the installation up and running, configure the local Prometheus and Pyroscope (if needed) @@ -119,7 +93,6 @@ If you are running the servers in Docker, use the following URLs: #### Import dashboards Follow [this guide](https://grafana.com/docs/grafana/latest/dashboards/manage-dashboards/#export-and-import-dashboards) ->>>>>>> origin/master to import the dashboards from the repository `grafana` folder. ## How to run a test @@ -138,28 +111,6 @@ Commands: ``` Note: `test-sequence` is a special test objective that wraps up an arbitrary number of test objectives. It is tipically -<<<<<<< HEAD - used to run a suite of tests defined in a `yaml` file like in this [example](examples/availability_read.yaml). - -### Standard test options - -``` -Options: - --network The type of network to be emulated [default: ideal] [possible values: - ideal, healthy, degraded] - --n-cores Number of cores to fetch availability for [default: 100] - --n-validators Number of validators to fetch chunks from [default: 500] - --min-pov-size The minimum pov size in KiB [default: 5120] - --max-pov-size The maximum pov size bytes [default: 5120] - -n, --num-blocks The number of blocks the test is going to run [default: 1] - -p, --peer-bandwidth The bandwidth of simulated remote peers in KiB - -b, --bandwidth The bandwidth of our simulated node in KiB - --peer-error Simulated conection error ratio [0-100] - --peer-min-latency Minimum remote peer latency in milliseconds [0-5000] - --peer-max-latency Maximum remote peer latency in milliseconds [0-5000] - -h, --help Print help - -V, --version Print version -======= used to run a suite of tests defined in a `yaml` file like in this [example](examples/availability_read.yaml). ### Standard test options @@ -183,7 +134,6 @@ Options: --pyroscope-sample-rate Pyroscope Sample Rate [default: 113] -h, --help Print help -V, --version Print version ->>>>>>> origin/master ``` These apply to all test objectives, except `test-sequence` which relies on the values being specified in a file. @@ -201,13 +151,8 @@ Benchmark availability recovery strategies Usage: subsystem-bench data-availability-read [OPTIONS] Options: -<<<<<<< HEAD - -f, --fetch-from-backers Turbo boost AD Read by fetching the full availability datafrom backers first. Saves CPU - as we don't need to re-construct from chunks. Tipically this is only faster if nodes -======= -f, --fetch-from-backers Turbo boost AD Read by fetching the full availability datafrom backers first. Saves CPU as we don't need to re-construct from chunks. Tipically this is only faster if nodes ->>>>>>> origin/master have enough bandwidth -h, --help Print help ``` @@ -224,15 +169,9 @@ usage: - for how many blocks the test should run (`num_blocks`) From the perspective of the subsystem under test, this means that it will receive an `ActiveLeavesUpdate` signal -<<<<<<< HEAD -followed by an arbitrary amount of messages. This process repeats itself for `num_blocks`. The messages are generally -test payloads pre-generated before the test run, or constructed on pre-genereated payloads. For example the -`AvailabilityRecoveryMessage::RecoverAvailableData` message includes a `CandidateReceipt` which is generated before -======= followed by an arbitrary amount of messages. This process repeats itself for `num_blocks`. The messages are generally test payloads pre-generated before the test run, or constructed on pre-genereated payloads. For example the `AvailabilityRecoveryMessage::RecoverAvailableData` message includes a `CandidateReceipt` which is generated before ->>>>>>> origin/master the test is started. ### Example run @@ -241,13 +180,8 @@ Let's run an availabilty read test which will recover availability for 10 cores node validator network. ``` -<<<<<<< HEAD - target/testnet/subsystem-bench --n-cores 10 data-availability-read -[2023-11-28T09:01:59Z INFO subsystem_bench::core::display] n_validators = 500, n_cores = 10, pov_size = 5120 - 5120, -======= target/testnet/subsystem-bench --n-cores 10 data-availability-read [2023-11-28T09:01:59Z INFO subsystem_bench::core::display] n_validators = 500, n_cores = 10, pov_size = 5120 - 5120, ->>>>>>> origin/master error = 0, latency = None [2023-11-28T09:01:59Z INFO subsystem-bench::availability] Generating template candidate index=0 pov_size=5242880 [2023-11-28T09:01:59Z INFO subsystem-bench::availability] Created test environment. @@ -261,13 +195,8 @@ node validator network. [2023-11-28T09:02:07Z INFO subsystem_bench::availability] All blocks processed in 6001ms [2023-11-28T09:02:07Z INFO subsystem_bench::availability] Throughput: 51200 KiB/block [2023-11-28T09:02:07Z INFO subsystem_bench::availability] Block time: 6001 ms -<<<<<<< HEAD -[2023-11-28T09:02:07Z INFO subsystem_bench::availability] - -======= [2023-11-28T09:02:07Z INFO subsystem_bench::availability] ->>>>>>> origin/master Total received from network: 66 MiB Total sent to network: 58 KiB Total subsystem CPU usage 4.16s @@ -276,20 +205,12 @@ node validator network. CPU usage per block 0.00s ``` -<<<<<<< HEAD -`Block time` in the context of `data-availability-read` has a different meaning. It measures the amount of time it -======= `Block time` in the context of `data-availability-read` has a different meaning. It measures the amount of time it ->>>>>>> origin/master took the subsystem to finish processing all of the messages sent in the context of the current test block. ### Test logs -<<<<<<< HEAD -You can select log target, subtarget and verbosity just like with Polkadot node CLI, simply setting -======= You can select log target, subtarget and verbosity just like with Polkadot node CLI, simply setting ->>>>>>> origin/master `RUST_LOOG="parachain=debug"` turns on debug logs for all parachain consensus subsystems in the test. ### View test metrics @@ -297,45 +218,26 @@ You can select log target, subtarget and verbosity just like with Polkadot node Assuming the Grafana/Prometheus stack installation steps completed succesfully, you should be able to view the test progress in real time by accessing [this link](http://localhost:3000/goto/SM5B8pNSR?orgId=1). -<<<<<<< HEAD -Now run -`target/testnet/subsystem-bench test-sequence --path polkadot/node/subsystem-bench/examples/availability_read.yaml` -and view the metrics in real time and spot differences between different `n_valiator` values. - -======= Now run `target/testnet/subsystem-bench test-sequence --path polkadot/node/subsystem-bench/examples/availability_read.yaml` and view the metrics in real time and spot differences between different `n_validators` values. ->>>>>>> origin/master ## Create new test objectives This tool is intended to make it easy to write new test objectives that focus individual subsystems, or even multiple subsystems (for example `approval-distribution` and `approval-voting`). A special kind of test objectives are performance regression tests for the CI pipeline. These should be sequences -<<<<<<< HEAD -of tests that check the performance characteristics (such as CPU usage, speed) of the subsystem under test in both -======= of tests that check the performance characteristics (such as CPU usage, speed) of the subsystem under test in both ->>>>>>> origin/master happy and negative scenarios (low bandwidth, network errors and low connectivity). ### Reusable test components -<<<<<<< HEAD -To faster write a new test objective you need to use some higher level wrappers and logic: `TestEnvironment`, -======= To faster write a new test objective you need to use some higher level wrappers and logic: `TestEnvironment`, ->>>>>>> origin/master `TestConfiguration`, `TestAuthorities`, `NetworkEmulator`. To create the `TestEnvironment` you will need to also build an `Overseer`, but that should be easy using the mockups for subsystems in`core::mock`. ### Mocking Ideally we want to have a single mock implementation for subsystems that can be minimally configured to -<<<<<<< HEAD -be used in different tests. A good example is `runtime-api` which currently only responds to session information -======= be used in different tests. A good example is `runtime-api` which currently only responds to session information ->>>>>>> origin/master requests based on static data. It can be easily extended to service other requests. diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml index 18f099656f7a..f086ac5eff57 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml @@ -3,11 +3,19 @@ TestConfiguration: - objective: !ApprovalsTest last_considered_tranche: 89 min_coalesce: 1 - max_coalesce: 1 - enable_assignments_v2: false + max_coalesce: 6 + enable_assignments_v2: true + send_till_tranche: 60 + stop_when_approved: false + coalesce_tranche_diff: 12 + workdir_prefix: "/tmp/" + num_no_shows_per_candidate: 0 + approval_distribution_expected_tof: 6.0 + approval_distribution_cpu_ms: 3.0 + approval_voting_cpu_ms: 4.30 n_validators: 500 n_cores: 100 - n_included_candidates: 70 + n_included_candidates: 100 min_pov_size: 1120 max_pov_size: 5120 peer_bandwidth: 524288000000 @@ -19,5 +27,5 @@ TestConfiguration: max_latency: secs: 0 nanos: 100000000 - error: 3 + error: 0 num_blocks: 10 diff --git a/polkadot/node/subsystem-bench/src/approval/helpers.rs b/polkadot/node/subsystem-bench/src/approval/helpers.rs index dc811f660bcc..ca5b8a0cd27f 100644 --- a/polkadot/node/subsystem-bench/src/approval/helpers.rs +++ b/polkadot/node/subsystem-bench/src/approval/helpers.rs @@ -103,7 +103,7 @@ pub fn generate_topology(test_authorities: &TestAuthorities) -> SessionGridTopol .keyrings .clone() .into_iter() - .zip(test_authorities.peer_ids.clone().into_iter()) + .zip(test_authorities.peer_ids.clone()) .collect_vec(); let topology = keyrings @@ -142,7 +142,7 @@ pub fn generate_peer_connected(test_authorities: &TestAuthorities) -> Vec Vec AllMessages { - let network = - NetworkBridgeEvent::PeerViewChange(peer_id, View::new([block_hash].into_iter(), 0)); + let network = NetworkBridgeEvent::PeerViewChange(peer_id, View::new([block_hash], 0)); AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate(network)) } @@ -252,7 +251,7 @@ pub fn make_candidates( .collect_vec(); let (candidates, _) = candidates.partial_shuffle(&mut rand_chacha, num_candidates as usize); candidates - .into_iter() + .iter_mut() .map(|val| val.clone()) .sorted_by(|a, b| match (a, b) { ( diff --git a/polkadot/node/subsystem-bench/src/approval/message_generator.rs b/polkadot/node/subsystem-bench/src/approval/message_generator.rs index 3a28f0c8996d..d755eb463a97 100644 --- a/polkadot/node/subsystem-bench/src/approval/message_generator.rs +++ b/polkadot/node/subsystem-bench/src/approval/message_generator.rs @@ -210,12 +210,12 @@ impl PeerMessagesGenerator { ); gum::info!(target: LOG_TARGET, "Generate messages"); - let topology = generate_topology(&test_authorities); + let topology = generate_topology(test_authorities); let random_samplings = random_samplings_to_node_patterns( ValidatorIndex(NODE_UNDER_TEST), test_authorities.keyrings.len(), - test_authorities.keyrings.len() as usize * 2, + test_authorities.keyrings.len() * 2, ); let topology_node_under_test = @@ -242,7 +242,7 @@ impl PeerMessagesGenerator { coalesce_tranche_diff: options.coalesce_tranche_diff, }; - peer_message_source.generate_messages(&spawn_task_handle); + peer_message_source.generate_messages(spawn_task_handle); } std::mem::drop(tx); @@ -265,11 +265,7 @@ impl PeerMessagesGenerator { block_info.slot, message.tranche_to_send(), ); - if !all_messages.contains_key(&tick_to_send) { - all_messages.insert(tick_to_send, Vec::new()); - } - - let to_add = all_messages.get_mut(&tick_to_send).unwrap(); + let to_add = all_messages.entry(tick_to_send).or_default(); to_add.push(message); }, Ok(None) => break, @@ -280,13 +276,12 @@ impl PeerMessagesGenerator { } let all_messages = all_messages .into_iter() - .map(|(_, mut messages)| { + .flat_map(|(_, mut messages)| { // Shuffle the messages inside the same tick, so that we don't priorites messages // for older nodes. we try to simulate the same behaviour as in real world. messages.shuffle(&mut rand_chacha); messages }) - .flatten() .collect_vec(); gum::info!("Generated a number of {:} unique messages", all_messages.len()); @@ -322,8 +317,8 @@ fn random_samplings_to_node_patterns( (0..num_patterns) .map(|_| { (0..num_validators) - .map(|sending_validator_index| { - let mut validators = (0..num_validators).map(|val| val).collect_vec(); + .flat_map(|sending_validator_index| { + let mut validators = (0..num_validators).collect_vec(); validators.shuffle(&mut rand_chacha); let mut random_routing = RandomRouting::default(); @@ -343,7 +338,6 @@ fn random_samplings_to_node_patterns( }) .collect_vec() }) - .flatten() .collect_vec() }) .collect_vec() @@ -362,6 +356,7 @@ fn coalesce_approvals_len( /// Helper function to create approvals signatures for all assignments passed as arguments. /// Returns a list of Approvals messages that need to be sent. +#[allow(clippy::too_many_arguments)] fn issue_approvals( assignments: Vec, block_hash: Hash, @@ -396,7 +391,7 @@ fn issue_approvals( .map(|val| val.assignment.tranche) .unwrap_or(message.tranche); - if queued_to_sign.len() >= num_coalesce as usize || + if queued_to_sign.len() >= num_coalesce || (!queued_to_sign.is_empty() && current_validator_index != assignment.0.validator) || message.tranche - earliest_tranche >= coalesce_tranche_diff @@ -464,7 +459,7 @@ impl TestSignInfo { /// Returns a TestMessage fn sign_candidates( to_sign: &mut Vec, - keyrings: &Vec<(Keyring, PeerId)>, + keyrings: &[(Keyring, PeerId)], block_hash: Hash, num_coalesce: usize, ) -> MessagesBundle { @@ -494,9 +489,8 @@ impl TestSignInfo { let sent_by = to_sign .iter() - .map(|val| val.assignment.sent_by.iter()) - .flatten() - .map(|peer| *peer) + .flat_map(|val| val.assignment.sent_by.iter()) + .copied() .collect::>(); let payload = ApprovalVoteMultipleCandidates(&hashes).signing_payload(1); @@ -524,31 +518,23 @@ impl TestSignInfo { /// Determine what neighbours would send a given message to the node under test. fn neighbours_that_would_sent_message( - keyrings: &Vec<(Keyring, PeerId)>, + keyrings: &[(Keyring, PeerId)], current_validator_index: u32, topology_node_under_test: &GridNeighbors, topology: &SessionGridTopology, ) -> Vec<(ValidatorIndex, PeerId)> { let topology_originator = topology - .compute_grid_neighbors_for(ValidatorIndex(current_validator_index as u32)) + .compute_grid_neighbors_for(ValidatorIndex(current_validator_index)) .unwrap(); - let originator_y = topology_originator - .validator_indices_y - .iter() - .filter(|validator| { - topology_node_under_test.required_routing_by_index(**validator, false) == - RequiredRouting::GridY - }) - .next(); - let originator_x = topology_originator - .validator_indices_x - .iter() - .filter(|validator| { - topology_node_under_test.required_routing_by_index(**validator, false) == - RequiredRouting::GridX - }) - .next(); + let originator_y = topology_originator.validator_indices_y.iter().find(|validator| { + topology_node_under_test.required_routing_by_index(**validator, false) == + RequiredRouting::GridY + }); + let originator_x = topology_originator.validator_indices_x.iter().find(|validator| { + topology_node_under_test.required_routing_by_index(**validator, false) == + RequiredRouting::GridX + }); let is_neighbour = topology_originator .validator_indices_x @@ -571,6 +557,7 @@ fn neighbours_that_would_sent_message( /// Generates assignments for the given `current_validator_index` /// Returns a list of assignments to be sent sorted by tranche. +#[allow(clippy::too_many_arguments)] fn generate_assignments( block_info: &BlockTestData, keyrings: Vec<(Keyring, PeerId)>, @@ -642,7 +629,7 @@ fn generate_assignments( .get(rand_chacha.next_u32() as usize % random_samplings.len()) .unwrap(); let random_sending_peer_ids = random_sending_nodes - .into_iter() + .iter() .map(|validator| (*validator, keyrings[validator.0 as usize].1)) .collect_vec(); @@ -663,11 +650,9 @@ fn generate_assignments( // For the cases where tranch0 assignments are in a single certificate we need to make // sure we create a single message. if unique_assignments.insert(bitfiled) { - if !assignments_by_tranche.contains_key(&assignment.tranche()) { - assignments_by_tranche.insert(assignment.tranche(), Vec::new()); - } let this_tranche_assignments = - assignments_by_tranche.get_mut(&assignment.tranche()).unwrap(); + assignments_by_tranche.entry(assignment.tranche()).or_insert_with(Vec::new); + this_tranche_assignments.push(( IndirectAssignmentCertV2 { block_hash: block_info.hash, @@ -692,17 +677,16 @@ fn generate_assignments( to_be_sent_by .iter() .chain(random_sending_peer_ids.iter()) - .map(|peer| *peer) + .copied() .collect::>(), assignment.tranche(), )); } } - let res = assignments_by_tranche + assignments_by_tranche .into_values() - .map(|assignments| assignments.into_iter()) - .flatten() + .flat_map(|assignments| assignments.into_iter()) .map(|assignment| { let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( assignment.0, @@ -719,7 +703,5 @@ fn generate_assignments( block_hash: block_info.hash, } }) - .collect_vec(); - - res + .collect_vec() } diff --git a/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs b/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs index 6892f32209f3..b7cff86f3dfe 100644 --- a/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs +++ b/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs @@ -44,8 +44,8 @@ impl MockChainSelection { let msg = ctx.recv().await.expect("Should not fail"); match msg { orchestra::FromOrchestra::Signal(_) => {}, - orchestra::FromOrchestra::Communication { msg } => match msg { - ChainSelectionMessage::Approved(hash) => { + orchestra::FromOrchestra::Communication { msg } => + if let ChainSelectionMessage::Approved(hash) = msg { let block_info = self.state.get_info_by_hash(hash); let approved_number = block_info.block_number; @@ -59,8 +59,6 @@ impl MockChainSelection { gum::info!(target: LOG_TARGET, ?hash, "Chain selection approved after {:} ms", approved_in_tick * TICK_DURATION_MILLIS); }, - _ => {}, - }, } } } diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 258f789dc42d..01982656d145 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -14,38 +14,6 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use parity_scale_codec::{Decode, Encode}; -use prometheus::Registry; -use serde::{Deserialize, Serialize}; -use std::{ - cmp::max, - collections::{HashMap, HashSet}, - fs, - io::Read, - sync::{ - atomic::{AtomicBool, AtomicU32, AtomicU64}, - Arc, - }, - time::{Duration, Instant}, -}; - -use colored::Colorize; -use futures::{channel::oneshot, FutureExt}; -use itertools::Itertools; -use orchestra::TimeoutExt; -use overseer::{metrics::Metrics as OverseerMetrics, MetricsTrait}; -use polkadot_approval_distribution::{ - metrics::Metrics as ApprovalDistributionMetrics, ApprovalDistribution, -}; -use polkadot_node_core_approval_voting::{ - time::{slot_number_to_tick, tick_to_slot_number, Clock, ClockExt, SystemClock}, - ApprovalVotingSubsystem, Metrics as ApprovalVotingMetrics, -}; -use polkadot_node_primitives::approval::{self, v1::RelayVRFStory}; -use polkadot_node_subsystem::{overseer, AllMessages, Overseer, OverseerConnector, SpawnGlue}; -use polkadot_node_subsystem_test_helpers::mock::new_block_import_info; -use polkadot_overseer::Handle as OverseerHandleReal; - use self::{ helpers::{make_candidates, make_header}, test_message::MessagesBundle, @@ -68,15 +36,43 @@ use crate::{ network::{NetworkAction, NetworkEmulator}, }, }; -use polkadot_node_core_approval_voting::Config as ApprovalVotingConfig; +use colored::Colorize; +use futures::{channel::oneshot, FutureExt}; +use itertools::Itertools; +use orchestra::TimeoutExt; +use overseer::{metrics::Metrics as OverseerMetrics, MetricsTrait}; +use parity_scale_codec::{Decode, Encode}; +use polkadot_approval_distribution::ApprovalDistribution; +use polkadot_node_core_approval_voting::{ + time::{slot_number_to_tick, tick_to_slot_number, Clock, ClockExt, SystemClock}, + ApprovalVotingSubsystem, Config as ApprovalVotingConfig, Metrics as ApprovalVotingMetrics, +}; +use polkadot_node_primitives::approval::{self, v1::RelayVRFStory}; +use polkadot_node_subsystem::{overseer, AllMessages, Overseer, OverseerConnector, SpawnGlue}; +use polkadot_node_subsystem_test_helpers::mock::new_block_import_info; use polkadot_node_subsystem_types::messages::{ApprovalDistributionMessage, ApprovalVotingMessage}; +use polkadot_node_subsystem_util::metrics::Metrics; +use polkadot_overseer::Handle as OverseerHandleReal; use polkadot_primitives::{ BlockNumber, CandidateEvent, CandidateIndex, Hash, Header, SessionInfo, Slot, ValidatorIndex, }; +use prometheus::Registry; use sc_keystore::LocalKeystore; use sc_service::SpawnTaskHandle; +use serde::{Deserialize, Serialize}; use sp_consensus_babe::Epoch as BabeEpoch; -use std::ops::Sub; +use std::{ + cmp::max, + collections::{HashMap, HashSet}, + fs, + io::Read, + ops::Sub, + sync::{ + atomic::{AtomicBool, AtomicU32, AtomicU64}, + Arc, + }, + time::{Duration, Instant}, +}; use tokio::time::sleep; mod helpers; @@ -418,10 +414,8 @@ impl ApprovalTestState { registry, }; - peer_message_source.produce_messages( - &spawn_task_handle, - self.generated_state.all_messages.take().unwrap(), - ); + peer_message_source + .produce_messages(spawn_task_handle, self.generated_state.all_messages.take().unwrap()); producer_rx } } @@ -431,8 +425,7 @@ impl ApprovalTestState { fn get_info_by_hash(&self, requested_hash: Hash) -> &BlockTestData { self.blocks .iter() - .filter(|block| block.hash == requested_hash) - .next() + .find(|block| block.hash == requested_hash) .expect("Mocks should not use unknown hashes") } @@ -440,14 +433,13 @@ impl ApprovalTestState { fn get_info_by_number(&self, requested_number: u32) -> &BlockTestData { self.blocks .iter() - .filter(|block| block.block_number == requested_number) - .next() + .find(|block| block.block_number == requested_number) .expect("Mocks should not use unknown numbers") } /// Returns test data for the given slot fn get_info_by_slot(&self, slot: Slot) -> Option<&BlockTestData> { - self.blocks.iter().filter(|block| block.slot == slot).next() + self.blocks.iter().find(|block| block.slot == slot) } } @@ -488,8 +480,7 @@ impl PeerMessageProducer { while all_messages.peek().is_some() { let current_slot = tick_to_slot_number(SLOT_DURATION_MILLIS, system_clock.tick_now()); - let block_info = - self.state.get_info_by_slot(current_slot).map(|block| block.clone()); + let block_info = self.state.get_info_by_slot(current_slot).cloned(); if let Some(block_info) = block_info { if !initialized_blocks.contains(&block_info.hash) && @@ -570,7 +561,7 @@ impl PeerMessageProducer { .get_info_by_hash(bundle.assignments.first().unwrap().block_hash) .clone(); - if bundle.bundle_needed(&per_candidate_data, &self.options) { + if bundle.bundle_needed(per_candidate_data, &self.options) { bundle.record_sent_assignment(per_candidate_data); let assignments = bundle.assignments.clone(); @@ -594,7 +585,7 @@ impl PeerMessageProducer { .as_ref() .fetch_add(1, std::sync::atomic::Ordering::SeqCst); self.send_message( - message.to_all_messages_from_peer(peer.1), + message.into_all_messages_from_peer(peer.1), peer.0, latency, ) @@ -627,7 +618,7 @@ impl PeerMessageProducer { current_slot, block_info, initialized_blocks.contains(&block_info.hash), - ) || !bundle.bundle_needed(&per_candidate_data, &self.options) + ) || !bundle.bundle_needed(per_candidate_data, &self.options) } pub fn time_to_send( @@ -740,7 +731,7 @@ fn build_overseer( let system_clock = PastSystemClock::new(SystemClock {}, state.delta_tick_from_generated.clone()); - let approval_voting = ApprovalVotingSubsystem::with_config( + let approval_voting = ApprovalVotingSubsystem::with_config_and_clock( TEST_CONFIG, Arc::new(db), Arc::new(keystore), @@ -749,9 +740,8 @@ fn build_overseer( Box::new(system_clock.clone()), ); - let approval_distribution = ApprovalDistribution::new( - ApprovalDistributionMetrics::try_register(&dependencies.registry).unwrap(), - ); + let approval_distribution = + ApprovalDistribution::new(Metrics::register(Some(&dependencies.registry)).unwrap()); let mock_chain_api = MockChainApi { state: state.clone() }; let mock_chain_selection = MockChainSelection { state: state.clone(), clock: system_clock }; let mock_runtime_api = MockRuntimeApi { state: state.clone() }; diff --git a/polkadot/node/subsystem-bench/src/approval/test_message.rs b/polkadot/node/subsystem-bench/src/approval/test_message.rs index 491154151803..5fb37a5160f0 100644 --- a/polkadot/node/subsystem-bench/src/approval/test_message.rs +++ b/polkadot/node/subsystem-bench/src/approval/test_message.rs @@ -126,7 +126,7 @@ impl MessagesBundle { impl TestMessageInfo { /// Converts message to `AllMessages` to be sent to overseer. - pub fn to_all_messages_from_peer(self, peer: PeerId) -> AllMessages { + pub fn into_all_messages_from_peer(self, peer: PeerId) -> AllMessages { AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate( NetworkBridgeEvent::PeerMessage(peer, Versioned::V3(self.msg)), )) @@ -216,7 +216,7 @@ impl TestMessageInfo { /// Returns true if the message is a no-show. pub fn no_show_if_required( &self, - assignments: &Vec, + assignments: &[TestMessageInfo], candidates_test_data: &mut HashMap<(Hash, CandidateIndex), CandidateTestData>, ) -> bool { let mut should_no_show = false; @@ -242,10 +242,9 @@ impl TestMessageInfo { .unwrap(); let assignment = covered_candidates .iter() - .filter(|(_assignment, candidates)| { + .find(|(_assignment, candidates)| { candidates.contains(&candidate_index) }) - .next() .unwrap(); candidate_test_data.should_no_show(assignment.0.tranche) }); @@ -260,10 +259,9 @@ impl TestMessageInfo { .unwrap(); let assignment = covered_candidates .iter() - .filter(|(_assignment, candidates)| { + .find(|(_assignment, candidates)| { candidates.contains(&candidate_index) }) - .next() .unwrap(); candidate_test_data.record_no_show(assignment.0.tranche) } diff --git a/polkadot/node/subsystem-bench/src/cli.rs b/polkadot/node/subsystem-bench/src/cli.rs index ed387a1bd7e3..d6046dfa54e9 100644 --- a/polkadot/node/subsystem-bench/src/cli.rs +++ b/polkadot/node/subsystem-bench/src/cli.rs @@ -34,7 +34,7 @@ pub enum TestObjective { DataAvailabilityRead(DataAvailabilityReadOptions), /// Run a test sequence specified in a file TestSequence(TestSequenceOptions), - /// + /// Benchmark the approval-voting and approval-distribution subsystems. ApprovalsTest(ApprovalsOptions), Unimplemented, } diff --git a/polkadot/node/subsystem-bench/src/core/display.rs b/polkadot/node/subsystem-bench/src/core/display.rs index 345f1720fa10..e9f250b23028 100644 --- a/polkadot/node/subsystem-bench/src/core/display.rs +++ b/polkadot/node/subsystem-bench/src/core/display.rs @@ -55,7 +55,7 @@ impl MetricCollection { self.all() .iter() .filter(|metric| metric.name == name) - .map(|metric| metric.clone()) + .cloned() .collect::>() .into() } @@ -81,17 +81,17 @@ impl MetricCollection { metric .bucket .as_ref() - .and_then(|buckets| { + .map(|buckets| { let mut prev_value = 0; for bucket in buckets { if value < bucket.get_upper_bound() && prev_value != bucket.get_cumulative_count() { - return Some(false); + return false; } prev_value = bucket.get_cumulative_count(); } - Some(true) + true }) .unwrap_or_default() }) @@ -224,9 +224,7 @@ pub fn parse_metrics(registry: &Registry) -> MetricCollection { label_names, label_values, value: 0.0, - bucket: Some( - h.get_bucket().iter().map(|bucket| bucket.clone()).collect_vec(), - ), + bucket: Some(h.get_bucket().iter().cloned().collect_vec()), }); }, MetricType::SUMMARY => { diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index ad97a3a1594b..af7973d355f6 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -302,6 +302,7 @@ impl TestEnvironment { test_metrics.metric_lower_than(metric_name, value) } + /// Tells if all entries in a Histogram are lower than value pub fn bucket_metric_lower_than( &self, metric_name: &str, diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index f84a61ab29f4..f6241f489e15 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -171,10 +171,10 @@ impl MockNetworkBridgeTx { .expect("candidate was generated previously; qed"); gum::warn!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); - let chunk: ChunkResponse = - self.state.chunks().get(*candidate_index as usize).unwrap()[validator_index] - .clone() - .into(); + let chunk: ChunkResponse = self.state.chunks().get(*candidate_index).unwrap() + [validator_index] + .clone() + .into(); let mut size = chunk.encoded_size(); let response = if random_error(self.config.error) { @@ -243,7 +243,7 @@ impl MockNetworkBridgeTx { .inc_received(outgoing_request.payload.encoded_size()); let available_data = - self.state.available_data().get(*candidate_index as usize).unwrap().clone(); + self.state.available_data().get(*candidate_index).unwrap().clone(); let size = available_data.encoded_size(); diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index f8e802fb5409..f12c7448b645 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -340,7 +340,7 @@ impl NetworkEmulator { // Create a `PeerEmulator` for each peer. let (stats, mut peers): (_, Vec<_>) = (0..n_peers) - .zip(authorities.validator_authority_id.clone().into_iter()) + .zip(authorities.validator_authority_id.clone()) .map(|(peer_index, authority_id)| { validator_authority_id_mapping.insert(authority_id, peer_index); let stats = Arc::new(PeerEmulatorStats::new(peer_index, metrics.clone())); @@ -386,7 +386,7 @@ impl NetworkEmulator { let our_network = self.clone(); // This task will handle node messages receipt from the simulated network. - let _ = spawn_task_handle.spawn_blocking( + spawn_task_handle.spawn_blocking( "network-receiver-0", "network-receiver", async move { diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index e53069c25cd4..613b1683b1f5 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -223,7 +223,7 @@ impl BenchCli { fn main() -> eyre::Result<()> { color_eyre::install()?; - let _ = env_logger::builder() + env_logger::builder() .filter(Some("hyper"), log::LevelFilter::Info) // Avoid `Terminating due to subsystem exit subsystem` warnings .filter(Some("polkadot_overseer"), log::LevelFilter::Error) From ba461846d74dab986c89b98d8ba8fcaf61334148 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 28 Dec 2023 18:29:40 +0200 Subject: [PATCH 160/192] complete networking refactor Signed-off-by: Andrei Sandu --- .../subsystem-bench/src/availability/mod.rs | 323 ++++++---- .../subsystem-bench/src/core/configuration.rs | 1 + .../subsystem-bench/src/core/environment.rs | 64 +- .../src/core/mock/network_bridge.rs | 381 +++++------- .../node/subsystem-bench/src/core/network.rs | 566 ++++++++++++------ 5 files changed, 773 insertions(+), 562 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 1130a417db71..24be6e647191 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -13,7 +13,6 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use super::core::environment::MAX_TIME_OF_FLIGHT; use crate::{core::mock::ChainApiState, TestEnvironment}; use bitvec::bitvec; use colored::Colorize; @@ -25,6 +24,8 @@ use polkadot_node_subsystem_types::{ messages::{AvailabilityStoreMessage, NetworkBridgeEvent}, Span, }; +use polkadot_primitives::AuthorityDiscoveryId; + use polkadot_overseer::Handle as OverseerHandle; use sc_network::{request_responses::ProtocolConfig, PeerId}; use sp_core::H256; @@ -39,16 +40,23 @@ use polkadot_node_metrics::metrics::Metrics; use polkadot_node_subsystem::TimeoutExt; use polkadot_availability_recovery::AvailabilityRecoverySubsystem; +use polkadot_node_primitives::{AvailableData, ErasureChunk}; use crate::GENESIS_HASH; use futures::FutureExt; use parity_scale_codec::Encode; use polkadot_node_network_protocol::{ request_response::{ - v1::ChunkFetchingRequest, IncomingRequest, OutgoingRequest, ReqProtocolNames, Requests, + v1::{ + AvailableDataFetchingResponse, ChunkFetchingRequest, ChunkFetchingResponse, + ChunkResponse, + }, + IncomingRequest, OutgoingRequest, ReqProtocolNames, Requests, }, BitfieldDistributionMessage, OurView, Versioned, View, }; +use sc_network::request_responses::IncomingRequest as RawIncomingRequest; + use polkadot_node_primitives::{BlockData, PoV, Proof}; use polkadot_node_subsystem::messages::{AllMessages, AvailabilityRecoveryMessage}; @@ -56,7 +64,7 @@ use crate::core::{ environment::TestEnvironmentDependencies, mock::{ av_store, - network_bridge::{self, MockNetworkBridgeTx, NetworkAvailabilityState}, + network_bridge::{self, MockNetworkBridgeRx, MockNetworkBridgeTx}, runtime_api, MockAvailabilityStore, MockChainApi, MockRuntimeApi, }, }; @@ -65,8 +73,6 @@ use super::core::{configuration::TestConfiguration, mock::dummy_builder, network const LOG_TARGET: &str = "subsystem-bench::availability"; -use polkadot_node_primitives::{AvailableData, ErasureChunk}; - use super::{cli::TestObjective, core::mock::AlwaysSupportsParachains}; use polkadot_node_subsystem_test_helpers::{ derive_erasure_chunks_with_proofs_and_root, mock::new_block_import_info, @@ -86,7 +92,7 @@ fn build_overseer_for_availability_read( spawn_task_handle: SpawnTaskHandle, runtime_api: MockRuntimeApi, av_store: MockAvailabilityStore, - network_bridge: MockNetworkBridgeTx, + network_bridge: (MockNetworkBridgeTx, MockNetworkBridgeRx), availability_recovery: AvailabilityRecoverySubsystem, ) -> (Overseer, AlwaysSupportsParachains>, OverseerHandle) { let overseer_connector = OverseerConnector::with_event_capacity(64000); @@ -94,7 +100,8 @@ fn build_overseer_for_availability_read( let builder = dummy .replace_runtime_api(|_| runtime_api) .replace_availability_store(|_| av_store) - .replace_network_bridge_tx(|_| network_bridge) + .replace_network_bridge_tx(|_| network_bridge.0) + .replace_network_bridge_rx(|_| network_bridge.1) .replace_availability_recovery(|_| availability_recovery); let (overseer, raw_handle) = @@ -106,7 +113,7 @@ fn build_overseer_for_availability_read( fn build_overseer_for_availability_write( spawn_task_handle: SpawnTaskHandle, runtime_api: MockRuntimeApi, - network_bridge: MockNetworkBridgeTx, + network_bridge: (MockNetworkBridgeTx, MockNetworkBridgeRx), availability_distribution: AvailabilityDistributionSubsystem, chain_api: MockChainApi, availability_store: AvailabilityStoreSubsystem, @@ -117,7 +124,8 @@ fn build_overseer_for_availability_write( let builder = dummy .replace_runtime_api(|_| runtime_api) .replace_availability_store(|_| availability_store) - .replace_network_bridge_tx(|_| network_bridge) + .replace_network_bridge_tx(|_| network_bridge.0) + .replace_network_bridge_rx(|_| network_bridge.1) .replace_chain_api(|_| chain_api) .replace_bitfield_distribution(|_| bitfield_distribution) // This is needed to test own chunk recovery for `n_cores`. @@ -129,6 +137,80 @@ fn build_overseer_for_availability_write( (overseer, OverseerHandle::new(raw_handle)) } +/// The availability store state of all emulated peers. +/// TODO: Move to common code. +pub struct NetworkAvailabilityState { + pub candidate_hashes: HashMap, + pub available_data: Vec, + pub chunks: Vec>, +} + +impl HandlePeerMessage for NetworkAvailabilityState { + fn handle( + &self, + message: PeerMessage, + node_sender: &mut futures::channel::mpsc::UnboundedSender, + ) -> Option { + match message { + PeerMessage::RequestFromNode(peer, request) => match request { + Requests::ChunkFetchingV1(outgoing_request) => { + gum::debug!(target: LOG_TARGET, request = ?outgoing_request, "Received `RequestFromNode`"); + let validator_index: usize = outgoing_request.payload.index.0 as usize; + let candidate_hash = outgoing_request.payload.candidate_hash; + + let candidate_index = self + .candidate_hashes + .get(&candidate_hash) + .expect("candidate was generated previously; qed"); + gum::warn!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); + + let chunk: ChunkResponse = self.chunks.get(*candidate_index as usize).unwrap() + [validator_index] + .clone() + .into(); + let mut size = chunk.encoded_size(); + + let response = Ok(ChunkFetchingResponse::from(Some(chunk)).encode()); + + if let Err(err) = outgoing_request.pending_response.send(response) { + gum::error!(target: LOG_TARGET, "Failed to send `ChunkFetchingResponse`"); + } + // let _ = outgoing_request + // .pending_response + // .send(response) + // .expect("Response is always sent succesfully"); + + None + }, + Requests::AvailableDataFetchingV1(outgoing_request) => { + let candidate_hash = outgoing_request.payload.candidate_hash; + let candidate_index = self + .candidate_hashes + .get(&candidate_hash) + .expect("candidate was generated previously; qed"); + gum::debug!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); + + let available_data = + self.available_data.get(*candidate_index as usize).unwrap().clone(); + + let size = available_data.encoded_size(); + + let response = + Ok(AvailableDataFetchingResponse::from(Some(available_data)).encode()); + let _ = outgoing_request + .pending_response + .send(response) + .expect("Response is always sent succesfully"); + None + }, + _ => Some(PeerMessage::RequestFromNode(peer, request)), + }, + + message => Some(message), + } + } +} + /// Takes a test configuration and uses it to creates the `TestEnvironment`. pub fn prepare_test( config: TestConfiguration, @@ -148,7 +230,8 @@ fn prepare_test_inner( let mut candidate_hashes: HashMap> = HashMap::new(); // Prepare per block candidates. - for block_num in 0..config.num_blocks { + // Genesis block is always finalized, so we start at 1. + for block_num in 0..=config.num_blocks { for _ in 0..config.n_cores { candidate_hashes .entry(Hash::repeat_byte(block_num as u8)) @@ -179,15 +262,30 @@ fn prepare_test_inner( chunks: state.chunks.clone(), }; - let network = NetworkEmulator::new(&config, &dependencies, &test_authorities); + let mut req_cfgs = Vec::new(); + + let (collation_req_receiver, collation_req_cfg) = + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); + req_cfgs.push(collation_req_cfg); + + let (pov_req_receiver, pov_req_cfg) = + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); + + let (chunk_req_receiver, chunk_req_cfg) = + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); + req_cfgs.push(pov_req_cfg); + + let (network, network_interface, network_receiver) = + new_network(&config, &dependencies, &test_authorities, vec![Arc::new(availability_state)]); let network_bridge_tx = network_bridge::MockNetworkBridgeTx::new( - config.clone(), - availability_state, network.clone(), + network_interface.subsystem_sender(), ); - let mut req_cfgs = Vec::new(); + let network_bridge_rx = + network_bridge::MockNetworkBridgeRx::new(network_receiver, Some(chunk_req_cfg.clone())); + state.set_chunk_request_protocol(chunk_req_cfg); let (overseer, overseer_handle) = match &state.config().objective { TestObjective::DataAvailabilityRead(_options) => { @@ -196,10 +294,6 @@ fn prepare_test_inner( _ => panic!("Unexpected objective"), }; - let (collation_req_receiver, req_cfg) = - IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); - req_cfgs.push(req_cfg); - let subsystem = if use_fast_path { AvailabilityRecoverySubsystem::with_fast_path( collation_req_receiver, @@ -223,26 +317,18 @@ fn prepare_test_inner( dependencies.task_manager.spawn_handle(), runtime_api, av_store, - network_bridge_tx, + (network_bridge_tx, network_bridge_rx), subsystem, ) }, TestObjective::DataAvailabilityWrite => { - let (pov_req_receiver, pov_req_cfg) = - IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); - let (chunk_req_receiver, chunk_req_cfg) = - IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); - req_cfgs.push(pov_req_cfg); - - state.set_chunk_request_protocol(chunk_req_cfg); - let availability_distribution = AvailabilityDistributionSubsystem::new( test_authorities.keyring.keystore(), IncomingRequestReceivers { pov_req_receiver, chunk_req_receiver }, Metrics::try_register(&dependencies.registry).unwrap(), ); - let block_headers = (0..config.num_blocks) + let block_headers = (0..=config.num_blocks) .map(|block_number| { ( Hash::repeat_byte(block_number as u8), @@ -264,7 +350,7 @@ fn prepare_test_inner( build_overseer_for_availability_write( dependencies.task_manager.spawn_handle(), runtime_api, - network_bridge_tx, + (network_bridge_tx, network_bridge_rx), availability_distribution, chain_api, new_av_store(&dependencies), @@ -284,6 +370,7 @@ fn prepare_test_inner( overseer, overseer_handle, test_authorities, + network_interface, ), req_cfgs, ) @@ -312,7 +399,7 @@ pub struct TestState { chunk_request_protocol: Option, // Availability distribution. pov_request_protocol: Option, - // Per relay chain block - our backed candidate + // Per relay chain block - candidate backed by our backing group backed_candidates: Vec, } @@ -450,8 +537,8 @@ pub async fn benchmark_availability_read(env: &mut TestEnvironment, mut state: T env.metrics().set_n_validators(config.n_validators); env.metrics().set_n_cores(config.n_cores); - for block_num in 0..env.config().num_blocks { - gum::info!(target: LOG_TARGET, "Current block {}/{}", block_num + 1, env.config().num_blocks); + for block_num in 1..=env.config().num_blocks { + gum::info!(target: LOG_TARGET, "Current block {}/{}", block_num, env.config().num_blocks); env.metrics().set_current_block(block_num); let block_start_ts = Instant::now(); @@ -535,7 +622,7 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: gum::info!("Done"); - for block_num in 0..env.config().num_blocks { + for block_num in 1..=env.config().num_blocks { gum::info!(target: LOG_TARGET, "Current block #{}", block_num); env.metrics().set_current_block(block_num); @@ -544,7 +631,7 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: env.import_block(new_block_import_info(relay_block_hash, block_num as BlockNumber)) .await; - let mut chunk_request_protocol = + let chunk_request_protocol = state.chunk_request_protocol().expect("No chunk fetching protocol configured"); // Inform bitfield distribution about our view of current test block @@ -553,96 +640,96 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: ); env.send_message(AllMessages::BitfieldDistribution(message)).await; - // Request chunks of backed candidate from all validators + let chunk_fetch_start_ts = Instant::now(); + + // Request chunks of our own backed candidate from all other validators. let mut receivers = Vec::new(); for index in 1..config.n_validators { let (pending_response, pending_response_receiver) = oneshot::channel(); - // Our backed candidate is first in candidate hashes entry for current block. - let payload = ChunkFetchingRequest { - candidate_hash: state.backed_candidates()[block_num].hash(), - index: ValidatorIndex(index as u32), - }; - // We don't really care. - let peer = PeerId::random(); - - // They sent it. - env.network().peer_stats(index).inc_sent(payload.encoded_size()); - // We received it. - env.network().inc_received(payload.encoded_size()); - - // TODO: implement TX rate limiter - if let Some(sender) = chunk_request_protocol.inbound_queue.clone() { - receivers.push(pending_response_receiver); - let _ = sender - .send( - IncomingRequest::new(PeerId::random(), payload, pending_response) - .into_raw(), - ) - .await; - } + let message = PeerMessage::RequestFromPeer(RawIncomingRequest { + peer: PeerId::random(), + payload: ChunkFetchingRequest { + candidate_hash: state.backed_candidates()[block_num].hash(), + index: ValidatorIndex(index as u32), + } + .encode(), + pending_response, + }); + + let peer = env + .authorities() + .validator_authority_id + .get(index) + .expect("all validators have keys") + .clone(); + + env.network_mut().submit_peer_action(NetworkAction { message, peer }); + receivers.push(pending_response_receiver); } gum::info!(target: LOG_TARGET, "Waiting for all emulated peers to receive their chunk from us ..."); - for (index, receiver) in receivers.into_iter().enumerate() { + for receiver in receivers.into_iter() { let response = receiver.await.expect("Chunk is always served succesfully"); + // TODO: check if chunk is the one the peer expects to receive. assert!(response.result.is_ok()); - env.network().peer_stats(index).inc_received(response.result.encoded_size()); - env.network().inc_sent(response.result.encoded_size()); } - gum::info!("All chunks sent"); + let chunk_fetch_duration = Instant::now().sub(chunk_fetch_start_ts).as_millis(); - // This reflects the bitfield sign timer, we expect bitfields to come in from the network - // after it expires. - tokio::time::sleep(std::time::Duration::from_millis(1500)).await; - let signing_context = SigningContext { session_index: 0, parent_hash: relay_block_hash }; + gum::info!("All chunks received in {}ms", chunk_fetch_duration); - // Generate `n_validator` - 1 messages and inject them to the subsystem via overseer. - for index in 1..config.n_validators { - let validator_public = env - .authorities() - .validator_public - .get(index) - .expect("All validator keys are known"); - - // Node has all the chunks in the world. - let payload: AvailabilityBitfield = - AvailabilityBitfield(bitvec![u8, bitvec::order::Lsb0; 1u8; 32]); - let signed_bitfield = Signed::::sign( - &env.authorities().keyring.keystore(), - payload, - &signing_context, - ValidatorIndex(index as u32), - &validator_public.clone().into(), - ) - .ok() - .flatten() - .expect("should be signed"); + let signing_context = SigningContext { session_index: 0, parent_hash: relay_block_hash }; + let mut network = env.network().clone(); + let authorities = env.authorities().clone(); + let n_validators = config.n_validators; - let overseer_handle = env.overseer_handle(); - let (run, size) = - send_peer_bitfield(overseer_handle, relay_block_hash, signed_bitfield); - let network_action = NetworkAction::new( - env.authorities() - .validator_authority_id + // Spawn a task that will generate `n_validator` - 1 signed bitfiends and + // send them from the emulated peers to the subsystem. + env.spawn("send-bitfields", async move { + for index in 1..n_validators { + let validator_public = authorities + .validator_public .get(index) - .cloned() - .expect("All validator keys are known"), - run, - size, - None, - ); - env.network_mut().submit_peer_action(network_action.peer(), network_action); - } + .expect("All validator keys are known"); + + // Node has all the chunks in the world. + let payload: AvailabilityBitfield = + AvailabilityBitfield(bitvec![u8, bitvec::order::Lsb0; 1u8; 32]); + let signed_bitfield = Signed::::sign( + &authorities.keyring.keystore(), + payload, + &signing_context, + ValidatorIndex(index as u32), + &validator_public.clone().into(), + ) + .ok() + .flatten() + .expect("should be signed"); + + let from_peer = authorities.validator_authority_id[index].clone(); + // Send to our node. + let to_peer = authorities.validator_authority_id[0].clone(); + + let action = + send_peer_bitfield(relay_block_hash, signed_bitfield, &from_peer, &to_peer); + + // Send the action to `to_peer`. + network.submit_peer_action(action); + } + + gum::info!("Waiting for {} bitfields to be received and processed", n_validators - 1); + }); // Wait for all bitfields to be processed. env.wait_until_metric_ge( "polkadot_parachain_received_availabilty_bitfields_total", - (config.n_validators - 1) * (block_num + 1), + (config.n_validators - 1) * (block_num), ) .await; + + gum::info!("All bitfields processed"); let block_time = Instant::now().sub(block_start_ts).as_millis() as u64; env.metrics().set_block_time(block_time); @@ -669,32 +756,24 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: } pub fn send_peer_bitfield( - mut overseer_handle: OverseerHandle, relay_hash: H256, signed_bitfield: Signed, -) -> (Pin + std::marker::Send + 'static)>>, usize) { + from_peer: &AuthorityDiscoveryId, + to_peer: &AuthorityDiscoveryId, +) -> NetworkAction { let bitfield = polkadot_node_network_protocol::v2::BitfieldDistributionMessage::Bitfield( relay_hash, signed_bitfield.into(), ); - let payload_size = bitfield.encoded_size(); - let message = - polkadot_node_subsystem_types::messages::BitfieldDistributionMessage::NetworkBridgeUpdate( - NetworkBridgeEvent::PeerMessage(PeerId::random(), Versioned::V2(bitfield)), - ); + let to_peer = to_peer.clone(); - ( - async move { - overseer_handle - .send_msg(AllMessages::BitfieldDistribution(message), LOG_TARGET) - .timeout(MAX_TIME_OF_FLIGHT) - .await - .unwrap_or_else(|| { - panic!("{}ms maximum time of flight breached", MAX_TIME_OF_FLIGHT.as_millis()) - }); - } - .boxed(), - payload_size, - ) + let message = PeerMessage::Message( + to_peer, + Versioned::V2( + polkadot_node_network_protocol::v2::ValidationProtocol::BitfieldDistribution(bitfield), + ), + ); + + NetworkAction { peer: from_peer.clone(), message } } diff --git a/polkadot/node/subsystem-bench/src/core/configuration.rs b/polkadot/node/subsystem-bench/src/core/configuration.rs index fe2e045fb0b2..a5e4110bc595 100644 --- a/polkadot/node/subsystem-bench/src/core/configuration.rs +++ b/polkadot/node/subsystem-bench/src/core/configuration.rs @@ -258,6 +258,7 @@ impl TestConfiguration { } /// Produce a randomized duration between `min` and `max`. +/// TODO: Use normal distribution. pub fn random_latency(maybe_peer_latency: Option<&PeerLatency>) -> Option { if let Some(peer_latency) = maybe_peer_latency { Some( diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 4a965d44d2b9..16883350f6ac 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -15,12 +15,12 @@ // along with Polkadot. If not, see . //! Test environment implementation use crate::{ - core::{mock::AlwaysSupportsParachains, network::NetworkEmulator}, + core::{mock::AlwaysSupportsParachains, network::NetworkEmulatorHandle}, TestConfiguration, }; use colored::Colorize; use core::time::Duration; -use futures::FutureExt; +use futures::{Future, FutureExt}; use polkadot_overseer::{BlockInfo, Handle as OverseerHandle}; use polkadot_node_subsystem::{messages::AllMessages, Overseer, SpawnGlue, TimeoutExt}; @@ -37,7 +37,7 @@ use std::{ use tokio::runtime::Handle; const LOG_TARGET: &str = "subsystem-bench::environment"; -use super::configuration::TestAuthorities; +use super::{configuration::TestAuthorities, network::NetworkInterface}; const MIB: usize = 1024 * 1024; @@ -58,7 +58,7 @@ pub struct TestEnvironmentMetrics { impl TestEnvironmentMetrics { pub fn new(registry: &Registry) -> Result { - let mut buckets = prometheus::exponential_buckets(16384.0, 2.0, 9) + let buckets = prometheus::exponential_buckets(16384.0, 2.0, 9) .expect("arguments are always valid; qed"); Ok(Self { @@ -190,11 +190,13 @@ pub struct TestEnvironment { /// The test configuration. config: TestConfiguration, /// A handle to the network emulator. - network: NetworkEmulator, + network: NetworkEmulatorHandle, /// Configuration/env metrics metrics: TestEnvironmentMetrics, /// Test authorities generated from the configuration. authorities: TestAuthorities, + /// The network interface used by the node. + network_interface: NetworkInterface, } impl TestEnvironment { @@ -202,10 +204,11 @@ impl TestEnvironment { pub fn new( dependencies: TestEnvironmentDependencies, config: TestConfiguration, - network: NetworkEmulator, + network: NetworkEmulatorHandle, overseer: Overseer, AlwaysSupportsParachains>, overseer_handle: OverseerHandle, authorities: TestAuthorities, + network_interface: NetworkInterface, ) -> Self { let metrics = TestEnvironmentMetrics::new(&dependencies.registry) .expect("Metrics need to be registered"); @@ -235,42 +238,54 @@ impl TestEnvironment { network, metrics, authorities, + network_interface, } } + /// Returns the test configuration. pub fn config(&self) -> &TestConfiguration { &self.config } - pub fn network(&self) -> &NetworkEmulator { + /// Returns a reference to the inner network emulator handle. + pub fn network(&self) -> &NetworkEmulatorHandle { &self.network } - pub fn network_mut(&mut self) -> &mut NetworkEmulator { + /// Returns a mutable reference to the inner network emulator handle. + pub fn network_mut(&mut self) -> &mut NetworkEmulatorHandle { &mut self.network } + /// Returns the Prometheus registry. pub fn registry(&self) -> &Registry { &self.dependencies.registry } + /// Spawn a named task in the `test-environment` task group. + pub fn spawn(&self, name: &'static str, task: impl Future + Send + 'static) { + self.dependencies + .task_manager + .spawn_handle() + .spawn(name, "test-environment", task); + } + + /// Returns a reference to the test environment metrics instance pub fn metrics(&self) -> &TestEnvironmentMetrics { &self.metrics } + /// Returns a handle to the tokio runtime. pub fn runtime(&self) -> Handle { self.runtime_handle.clone() } + /// Returns a reference to the authority keys used in the test. pub fn authorities(&self) -> &TestAuthorities { &self.authorities } - pub fn overseer_handle(&self) -> OverseerHandle { - self.overseer_handle.clone() - } - - // Send a message to the subsystem under test environment. + /// Send a message to the subsystem under test environment. pub async fn send_message(&mut self, msg: AllMessages) { self.overseer_handle .send_msg(msg, LOG_TARGET) @@ -281,7 +296,7 @@ impl TestEnvironment { }); } - // Send an `ActiveLeavesUpdate` signal to all subsystems under test. + /// Send an `ActiveLeavesUpdate` signal to all subsystems under test. pub async fn import_block(&mut self, block: BlockInfo) { self.overseer_handle .block_imported(block) @@ -292,12 +307,12 @@ impl TestEnvironment { }); } - // Stop overseer and subsystems. + /// Stop overseer and subsystems. pub async fn stop(&mut self) { self.overseer_handle.stop().await; } - // Blocks until `metric_name` >= `value` + /// Blocks until `metric_name` >= `value` pub async fn wait_until_metric_ge(&self, metric_name: &str, value: usize) { let value = value as f64; loop { @@ -314,29 +329,28 @@ impl TestEnvironment { } } + /// Display network usage stats. pub fn display_network_usage(&self) { - let test_metrics = super::display::parse_metrics(self.registry()); - let stats = self.network().peer_stats(0); - let total_node_received = stats.received() / MIB; - let total_node_sent = stats.sent() / MIB; + let total_node_received = stats.received() / 1024; + let total_node_sent = stats.sent() / 1024; println!( "\nPayload bytes received from peers: {}, {}", - format!("{:.2} MiB total", total_node_received).blue(), - format!("{:.2} MiB/block", total_node_received / self.config().num_blocks) + format!("{:.2} KiB total", total_node_received).blue(), + format!("{:.2} KiB/block", total_node_received / self.config().num_blocks) .bright_blue() ); println!( "Payload bytes sent to peers: {}, {}", - format!("{:.2} MiB total", total_node_sent).blue(), - format!("{:.2} MiB/block", total_node_sent / self.config().num_blocks).bright_blue() + format!("{:.2} KiB total", total_node_sent).blue(), + format!("{:.2} KiB/block", total_node_sent / self.config().num_blocks).bright_blue() ); } - // Print CPU usage stats in the CLI. + /// Print CPU usage stats in the CLI. pub fn display_cpu_usage(&self, subsystems_under_test: &[&str]) { let test_metrics = super::display::parse_metrics(self.registry()); diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index 0436eea148b3..4edc1c9e08c4 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -14,280 +14,131 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . //! -//! A generic av store subsystem mockup suitable to be used in benchmarks. - -use futures::Future; +//! Mocked `network-bridge` subsystems that uses a `NetworkInterface` to access +//! the emulated network. +use futures::{ + channel::{ + mpsc::{UnboundedReceiver, UnboundedSender}, + oneshot, + }, + Future, +}; +use overseer::AllMessages; use parity_scale_codec::Encode; -use polkadot_node_subsystem_types::OverseerSignal; -use std::{collections::HashMap, pin::Pin}; - -use futures::FutureExt; +use polkadot_node_subsystem_types::{ + messages::{BitfieldDistributionMessage, NetworkBridgeEvent}, + OverseerSignal, +}; +use std::{collections::HashMap, f32::consts::E, pin::Pin}; -use polkadot_node_primitives::{AvailableData, ErasureChunk}; +use futures::{FutureExt, Stream, StreamExt}; use polkadot_primitives::CandidateHash; -use sc_network::{OutboundFailure, RequestFailure}; +use sc_network::{ + network_state::Peer, + request_responses::{IncomingRequest, ProtocolConfig}, + OutboundFailure, PeerId, RequestFailure, +}; use polkadot_node_subsystem::{ messages::NetworkBridgeTxMessage, overseer, SpawnedSubsystem, SubsystemError, }; -use polkadot_node_network_protocol::request_response::{ - self as req_res, v1::ChunkResponse, Requests, +use polkadot_node_network_protocol::{ + request_response::{self as req_res, v1::ChunkResponse, Recipient, Requests}, + Versioned, }; use polkadot_primitives::AuthorityDiscoveryId; use crate::core::{ configuration::{random_error, random_latency, TestConfiguration}, - network::{NetworkAction, NetworkEmulator, RateLimit}, + network::{ + NetworkAction, NetworkEmulatorHandle, NetworkInterfaceReceiver, PeerMessage, RateLimit, + }, }; -/// The availability store state of all emulated peers. -/// The network bridge tx mock will respond to requests as if the request is being serviced -/// by a remote peer on the network -pub struct NetworkAvailabilityState { - pub candidate_hashes: HashMap, - pub available_data: Vec, - pub chunks: Vec>, -} - const LOG_TARGET: &str = "subsystem-bench::network-bridge-tx-mock"; /// A mock of the network bridge tx subsystem. pub struct MockNetworkBridgeTx { - /// The test configurationg - config: TestConfiguration, - /// The network availability state - availabilty: NetworkAvailabilityState, - /// A network emulator instance - network: NetworkEmulator, + /// A network emulator handle + network: NetworkEmulatorHandle, + /// A channel to the network interface, + to_network_interface: UnboundedSender, +} + +/// A mock of the network bridge tx subsystem. +pub struct MockNetworkBridgeRx { + /// A network interface receiver + network_receiver: NetworkInterfaceReceiver, + /// Chunk request sender + chunk_request_sender: Option, } impl MockNetworkBridgeTx { pub fn new( - config: TestConfiguration, - availabilty: NetworkAvailabilityState, - network: NetworkEmulator, + network: NetworkEmulatorHandle, + to_network_interface: UnboundedSender, ) -> MockNetworkBridgeTx { - Self { config, availabilty, network } + Self { network, to_network_interface } } +} - fn not_connected_response( - &self, - authority_discovery_id: &AuthorityDiscoveryId, - future: Pin + Send>>, - ) -> NetworkAction { - // The network action will send the error after a random delay expires. - return NetworkAction::new( - authority_discovery_id.clone(), - future, - 0, - // Generate a random latency based on configuration. - random_latency(self.config.latency.as_ref()), - ) +impl MockNetworkBridgeRx { + pub fn new( + network_receiver: NetworkInterfaceReceiver, + chunk_request_sender: Option, + ) -> MockNetworkBridgeRx { + Self { network_receiver, chunk_request_sender } } - /// Returns an `NetworkAction` corresponding to the peer sending the response. If - /// the peer is connected, the error is sent with a randomized latency as defined in - /// configuration. - fn respond_to_send_request( - &mut self, - request: Requests, - ingress_tx: &mut tokio::sync::mpsc::UnboundedSender, - ) -> NetworkAction { - let ingress_tx = ingress_tx.clone(); - - match request { - Requests::ChunkFetchingV1(outgoing_request) => { - let authority_discovery_id = match outgoing_request.peer { - req_res::Recipient::Authority(authority_discovery_id) => authority_discovery_id, - _ => unimplemented!("Peer recipient not supported yet"), - }; - // Account our sent request bytes. - self.network.peer_stats(0).inc_sent(outgoing_request.payload.encoded_size()); +} - // If peer is disconnected return an error - if !self.network.is_peer_connected(&authority_discovery_id) { - // We always send `NotConnected` error and we ignore `IfDisconnected` value in - // the caller. - let future = async move { - let _ = outgoing_request - .pending_response - .send(Err(RequestFailure::NotConnected)); - } - .boxed(); - return self.not_connected_response(&authority_discovery_id, future) - } +#[overseer::subsystem(NetworkBridgeTx, error=SubsystemError, prefix=self::overseer)] +impl MockNetworkBridgeTx { + fn start(self, ctx: Context) -> SpawnedSubsystem { + let future = self.run(ctx).map(|_| Ok(())).boxed(); - // Account for remote received request bytes. - self.network - .peer_stats_by_id(&authority_discovery_id) - .inc_received(outgoing_request.payload.encoded_size()); + SpawnedSubsystem { name: "network-bridge-tx", future } + } +} - let validator_index: usize = outgoing_request.payload.index.0 as usize; - let candidate_hash = outgoing_request.payload.candidate_hash; +#[overseer::subsystem(NetworkBridgeRx, error=SubsystemError, prefix=self::overseer)] +impl MockNetworkBridgeRx { + fn start(self, ctx: Context) -> SpawnedSubsystem { + let future = self.run(ctx).map(|_| Ok(())).boxed(); - let candidate_index = self - .availabilty - .candidate_hashes - .get(&candidate_hash) - .expect("candidate was generated previously; qed"); - gum::warn!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); + SpawnedSubsystem { name: "network-bridge-rx", future } + } +} - let chunk: ChunkResponse = - self.availabilty.chunks.get(*candidate_index as usize).unwrap() - [validator_index] - .clone() - .into(); - let mut size = chunk.encoded_size(); +trait RequestAuthority { + fn authority_id(&self) -> Option<&AuthorityDiscoveryId>; +} - let response = if random_error(self.config.error) { - // Error will not account to any bandwidth used. - size = 0; - Err(RequestFailure::Network(OutboundFailure::ConnectionClosed)) +impl RequestAuthority for Requests { + fn authority_id(&self) -> Option<&AuthorityDiscoveryId> { + match self { + Requests::ChunkFetchingV1(request) => { + if let Recipient::Authority(authority_id) = &request.peer { + Some(authority_id) } else { - Ok(req_res::v1::ChunkFetchingResponse::from(Some(chunk)).encode()) - }; - - let authority_discovery_id_clone = authority_discovery_id.clone(); - - let future = async move { - let _ = outgoing_request.pending_response.send(response); + None } - .boxed(); - - let future_wrapper = async move { - // Forward the response to the ingress channel of our node. - // On receive side we apply our node receiving rate limit. - let action = - NetworkAction::new(authority_discovery_id_clone, future, size, None); - ingress_tx.send(action).unwrap(); - } - .boxed(); - - NetworkAction::new( - authority_discovery_id, - future_wrapper, - size, - // Generate a random latency based on configuration. - random_latency(self.config.latency.as_ref()), - ) }, - Requests::AvailableDataFetchingV1(outgoing_request) => { - let candidate_hash = outgoing_request.payload.candidate_hash; - let candidate_index = self - .availabilty - .candidate_hashes - .get(&candidate_hash) - .expect("candidate was generated previously; qed"); - gum::debug!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); - - let authority_discovery_id = match outgoing_request.peer { - req_res::Recipient::Authority(authority_discovery_id) => authority_discovery_id, - _ => unimplemented!("Peer recipient not supported yet"), - }; - - // Account our sent request bytes. - self.network.peer_stats(0).inc_sent(outgoing_request.payload.encoded_size()); - - // If peer is disconnected return an error - if !self.network.is_peer_connected(&authority_discovery_id) { - let future: Pin + Send>> = async move { - let _ = outgoing_request - .pending_response - .send(Err(RequestFailure::NotConnected)); - } - .boxed(); - return self.not_connected_response(&authority_discovery_id, future) - } - - // Account for remote received request bytes. - self.network - .peer_stats_by_id(&authority_discovery_id) - .inc_received(outgoing_request.payload.encoded_size()); - - let available_data = - self.availabilty.available_data.get(*candidate_index as usize).unwrap().clone(); - - let size = available_data.encoded_size(); - - let response = if random_error(self.config.error) { - Err(RequestFailure::Network(OutboundFailure::ConnectionClosed)) - } else { - Ok(req_res::v1::AvailableDataFetchingResponse::from(Some(available_data)) - .encode()) - }; - - let future = async move { - let _ = outgoing_request.pending_response.send(response); - } - .boxed(); - - let authority_discovery_id_clone = authority_discovery_id.clone(); - - let future_wrapper = async move { - // Forward the response to the ingress channel of our node. - // On receive side we apply our node receiving rate limit. - let action = - NetworkAction::new(authority_discovery_id_clone, future, size, None); - ingress_tx.send(action).unwrap(); - } - .boxed(); - - NetworkAction::new( - authority_discovery_id, - future_wrapper, - size, - // Generate a random latency based on configuration. - random_latency(self.config.latency.as_ref()), - ) + request => { + unimplemented!("RequestAuthority not implemented for {:?}", request) }, - _ => panic!("received an unexpected request"), } } } -#[overseer::subsystem(NetworkBridgeTx, error=SubsystemError, prefix=self::overseer)] -impl MockNetworkBridgeTx { - fn start(self, ctx: Context) -> SpawnedSubsystem { - let future = self.run(ctx).map(|_| Ok(())).boxed(); - - SpawnedSubsystem { name: "test-environment", future } - } -} - #[overseer::contextbounds(NetworkBridgeTx, prefix = self::overseer)] impl MockNetworkBridgeTx { - async fn run(mut self, mut ctx: Context) { - let (mut ingress_tx, mut ingress_rx) = - tokio::sync::mpsc::unbounded_channel::(); - - // Initialize our node bandwidth limits. - let mut rx_limiter = RateLimit::new(10, self.config.bandwidth); - - let our_network = self.network.clone(); - - // This task will handle node messages receipt from the simulated network. - let _ = ctx - .spawn_blocking( - "network-receive", - async move { - while let Some(action) = ingress_rx.recv().await { - let size = action.size(); - // account for our node receiving the data. - our_network.inc_received(size); - rx_limiter.reap(size).await; - action.run().await; - } - } - .boxed(), - ) - .expect("We never fail to spawn tasks"); - + async fn run(self, mut ctx: Context) { // Main subsystem loop. loop { - let msg = ctx.recv().await.expect("Overseer never fails us"); - - match msg { + let subsystem_message = ctx.recv().await.expect("Overseer never fails us"); + match subsystem_message { orchestra::FromOrchestra::Signal(signal) => match signal { OverseerSignal::Conclude => return, _ => {}, @@ -296,14 +147,16 @@ impl MockNetworkBridgeTx { NetworkBridgeTxMessage::SendRequests(requests, _if_disconnected) => { for request in requests { gum::debug!(target: LOG_TARGET, request = ?request, "Processing request"); - self.network.inc_sent(request_size(&request)); - let action = self.respond_to_send_request(request, &mut ingress_tx); - - // Will account for our node sending the request over the emulated - // network. - self.network.submit_peer_action(action.peer(), action); + let peer_id = + request.authority_id().expect("all nodes are authorities").clone(); + let peer_message = + PeerMessage::RequestFromNode(peer_id.clone(), request); + let _ = self.to_network_interface.unbounded_send(peer_message); } }, + NetworkBridgeTxMessage::ReportPeer(_) => { + // ingore rep changes + }, _ => { unimplemented!("Unexpected network bridge message") }, @@ -313,12 +166,58 @@ impl MockNetworkBridgeTx { } } -// A helper to determine the request payload size. -fn request_size(request: &Requests) -> usize { - match request { - Requests::ChunkFetchingV1(outgoing_request) => outgoing_request.payload.encoded_size(), - Requests::AvailableDataFetchingV1(outgoing_request) => - outgoing_request.payload.encoded_size(), - _ => unimplemented!("received an unexpected request"), +#[overseer::contextbounds(NetworkBridgeRx, prefix = self::overseer)] +impl MockNetworkBridgeRx { + async fn run(mut self, mut ctx: Context) { + // Main subsystem loop. + let mut from_network_interface = self.network_receiver.0; + loop { + futures::select! { + maybe_peer_message = from_network_interface.next() => { + if let Some(message) = maybe_peer_message { + match message { + PeerMessage::Message(peer, message) => match message { + Versioned::V2( + polkadot_node_network_protocol::v2::ValidationProtocol::BitfieldDistribution( + bitfield, + ), + ) => { + ctx.send_message( + BitfieldDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage(PeerId::random(), polkadot_node_network_protocol::Versioned::V2(bitfield))) + ).await; + }, + _ => { + unimplemented!("We only talk v2 network protocol") + }, + }, + PeerMessage::RequestFromPeer(request) => { + if let Some(protocol) = self.chunk_request_sender.as_mut() { + if let Some(inbound_queue) = protocol.inbound_queue.as_ref() { + let _ = inbound_queue + .send(request) + .await + .expect("Forwarding requests to subsystem never fails"); + } + } + }, + _ => { + panic!("1PeerMessage::RequestFromNode1 is not expected to be received from a peer") + } + } + } + }, + subsystem_message = ctx.recv().fuse() => { + match subsystem_message.expect("Overseer never fails us") { + orchestra::FromOrchestra::Signal(signal) => match signal { + OverseerSignal::Conclude => return, + _ => {}, + }, + _ => { + unimplemented!("Unexpected network bridge rx message") + }, + } + } + } + } } } diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index f1f0689ba3ea..95a98f010a5b 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -22,38 +22,47 @@ use super::{ environment::TestEnvironmentDependencies, *, }; -use parity_scale_codec::Encode; - use colored::Colorize; -use futures::channel::mpsc; -use net_protocol::VersionedValidationProtocol; +use futures::{ + channel::{mpsc, oneshot}, + future::FusedFuture, + lock::Mutex, + stream::FuturesUnordered, +}; +use net_protocol::{ + request_response::{OutgoingRequest, Requests}, + VersionedValidationProtocol, +}; +use parity_scale_codec::Encode; use polkadot_primitives::AuthorityDiscoveryId; use prometheus_endpoint::U64; use rand::{seq::SliceRandom, thread_rng}; -use sc_network::request_responses::IncomingRequest; +use sc_network::{ + request_responses::{IncomingRequest, OutgoingResponse}, + RequestFailure, +}; use sc_service::SpawnTaskHandle; use std::{ collections::HashMap, + ops::DerefMut, + pin::Pin, sync::{ atomic::{AtomicU64, Ordering}, Arc, }, - time::{Duration, Instant}, ops::DerefMut, + time::{Duration, Instant}, }; use polkadot_node_network_protocol::{ self as net_protocol, - peer_set::{ProtocolVersion, ValidationVersion}, v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, OurView, PeerId, UnifiedReputationChange as Rep, Versioned, View, }; -use futures::channel::mpsc::{UnboundedSender, UnboundedReceiver}; +use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender}; -use futures::{ - Future, FutureExt, Stream, StreamExt, -}; +use futures::{Future, FutureExt, Stream, StreamExt}; // An emulated node egress traffic rate_limiter. #[derive(Debug)] pub struct RateLimit { @@ -120,35 +129,42 @@ impl RateLimit { } } -// A wrapper for both gossip and request/response protocols along with the destination peer(`AuthorityDiscoveryId``). +/// A wrapper for both gossip and request/response protocols along with the destination +/// peer(`AuthorityDiscoveryId``). pub enum PeerMessage { + /// A gossip message Message(AuthorityDiscoveryId, VersionedValidationProtocol), - Request(AuthorityDiscoveryId, IncomingRequest) + /// A request originating from our node + RequestFromNode(AuthorityDiscoveryId, Requests), + /// A request originating from an emultated peer + RequestFromPeer(IncomingRequest), } impl PeerMessage { /// Returns the size of the encoded message or request pub fn size(&self) -> usize { match &self { - PeerMessage::Message(_peer_id, Versioned::V1(message)) => { - message.encoded_size() - }, - PeerMessage::Request(_peer_id, incoming) => { - incoming.payload.encoded_size() - } + PeerMessage::Message(_peer_id, Versioned::V2(message)) => message.encoded_size(), + PeerMessage::Message(_peer_id, Versioned::V1(message)) => message.encoded_size(), + PeerMessage::Message(_peer_id, Versioned::VStaging(message)) => message.encoded_size(), + PeerMessage::RequestFromNode(_peer_id, incoming) => request_size(incoming), + PeerMessage::RequestFromPeer(request) => request.payload.encoded_size(), + } + } + + /// Returns the destination peer from the message + pub fn peer(&self) -> Option<&AuthorityDiscoveryId> { + match &self { + PeerMessage::Message(peer_id, _) | PeerMessage::RequestFromNode(peer_id, _) => + Some(peer_id), + _ => None, } } } /// A network interface of the node under test. +/// TODO(soon): Implement latency and connection errors here, instead of doing it on the peers. pub struct NetworkInterface { - // The network we are connected to. - network: Option, - // Used to receive traffic from the network and implement `rx_limiter` limits. - // from_network: Receiver, - // `tx_limiter` enforces the configured bandwidth. - // Sender for network peers. - network_to_interface_sender: UnboundedSender, // Used to receive traffic from the `network-bridge-tx` subsystem. // The network action is forwarded via `network` to the relevant peer(s). // from_netowork_bridge: Receiver, @@ -162,77 +178,167 @@ pub struct NetworkInterface { // parameter of the `network-bridge` mock. pub struct NetworkInterfaceReceiver(pub UnboundedReceiver); +struct ProxiedRequest { + sender: Option>, + receiver: oneshot::Receiver, +} + +struct ProxiedResponse { + pub sender: oneshot::Sender, + pub result: Result, RequestFailure>, +} + +use std::task::Poll; + +impl Future for ProxiedRequest { + // The sender and result. + type Output = ProxiedResponse; + + fn poll( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll { + match self.receiver.poll_unpin(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(response) => Poll::Ready(ProxiedResponse { + sender: self.sender.take().expect("sender already used"), + result: response + .expect("Response is always succesfully received.") + .result + .map_err(|_| RequestFailure::Refused), + }), + } + } +} + impl NetworkInterface { /// Create a new `NetworkInterface` pub fn new( spawn_task_handle: SpawnTaskHandle, - mut network: NetworkEmulator, + mut network: NetworkEmulatorHandle, bandiwdth_bps: usize, + mut from_network: UnboundedReceiver, ) -> (NetworkInterface, NetworkInterfaceReceiver) { - let mut rx_limiter: RateLimit = RateLimit::new(10, bandiwdth_bps); - let mut tx_limiter: RateLimit = RateLimit::new(10, bandiwdth_bps); + let mut rx_limiter = RateLimit::new(10, bandiwdth_bps); + // We need to share the transimit limiter as we handle incoming request/response on rx + // thread. + let mut tx_limiter = Arc::new(Mutex::new(RateLimit::new(10, bandiwdth_bps))); + let mut proxied_requests = FuturesUnordered::new(); - // A sender (`ingress_tx`) clone will be owned by each `PeerEmulator`, such that it can send - // messages to the node. THe receiver will be polled in the network interface task. - let (network_to_interface_sender, network_to_interface_receiver) = mpsc::unbounded::(); // The sender (`egress_tx`) is - let (bridge_to_interface_sender, bridge_to_interface_receiver) = mpsc::unbounded::(); + let (bridge_to_interface_sender, mut bridge_to_interface_receiver) = + mpsc::unbounded::(); // Channel for forwarding actions to the bridge. - let (interface_to_bridge_sender, interface_to_bridge_receiver) = mpsc::unbounded::(); + let (interface_to_bridge_sender, interface_to_bridge_receiver) = + mpsc::unbounded::(); - // Spawn the network interface task. - let task = async move { - let mut network_to_interface_receiver = network_to_interface_receiver.fuse(); - let mut bridge_to_interface_receiver = bridge_to_interface_receiver.fuse(); + let mut rx_network = network.clone(); + let mut tx_network = network; + + let rx_task_bridge_sender = interface_to_bridge_sender.clone(); + let rx_task_tx_limiter = tx_limiter.clone(); + let tx_task_tx_limiter = tx_limiter; + // Spawn the network interface task. + let rx_task = async move { loop { - // TODO (maybe): split into separate rx/tx tasks. + let mut from_network = from_network.next().fuse(); futures::select! { - maybe_peer_message = network_to_interface_receiver.next() => { + maybe_peer_message = from_network => { if let Some(peer_message) = maybe_peer_message { - rx_limiter.reap(peer_message.size()).await; + let size = peer_message.size(); + // TODO (maybe): Implement limiter as part of something like `RateLimitedChannel`. + rx_limiter.reap(size).await; + rx_network.inc_received(size); + + // We act as an incoming request proxy, so we'll craft a new request and wait for + // the answer from subsystem, and only then, after ratelimiting we send back a response + // to the peer + if let PeerMessage::RequestFromPeer(request) = peer_message { + let (response_sender, response_receiver) = oneshot::channel(); + // Create a new `IncomingRequest` that we forward to the network bridge. + let new_request = IncomingRequest {payload: request.payload, peer: request.peer, pending_response: response_sender}; + + proxied_requests.push(ProxiedRequest {sender: Some(request.pending_response), receiver: response_receiver}); + // Send the new message to network bridge rx + rx_task_bridge_sender + .unbounded_send(PeerMessage::RequestFromPeer(new_request)) + .expect("network bridge subsystem is alive"); + continue + } // Forward the message to the bridge. - interface_to_bridge_sender.unbounded_send(peer_message).expect("network bridge subsystem is alive"); + rx_task_bridge_sender + .unbounded_send(peer_message) + .expect("network bridge subsystem is alive"); } else { gum::info!(target: LOG_TARGET, "Uplink channel closed, network interface task exiting"); break } }, - maybe_peer_message = bridge_to_interface_receiver.next() => { - if let Some(peer_message) = maybe_action_from_subsystem { - tx_limiter.reap(peer_message.size()).await; - - // Forward action to the destination network peer. - network.submit_peer_action(peer, action); + proxied_request = proxied_requests.next() => { + if let Some(proxied_request) = proxied_request { + match proxied_request.result { + Ok(result) => { + let bytes = result.encoded_size(); + gum::trace!(target: LOG_TARGET, size = bytes, "proxied request completed"); + + rx_task_tx_limiter.lock().await.reap(bytes).await; + rx_network.inc_sent(bytes); + + proxied_request.sender.send( + OutgoingResponse { + reputation_changes: Vec::new(), + result: Ok(result), + sent_feedback: None + } + ).expect("network is alive"); + } + Err(e) => { + gum::warn!(target: LOG_TARGET, "Node req/response failure: {:?}", e) + } + } } else { - gum::info!(target: LOG_TARGET, "Downlink channel closed, network interface task exiting"); - break + gum::debug!(target: LOG_TARGET, "No more active proxied requests"); + // break } } + } + } + } + .boxed(); + let tx_task = async move { + loop { + if let Some(peer_message) = bridge_to_interface_receiver.next().await { + let size = peer_message.size(); + tx_task_tx_limiter.lock().await.reap(size).await; + let dst_peer = peer_message + .peer() + .expect( + "Node always needs to specify destination peer when sending a message", + ) + .clone(); + tx_network.submit_peer_message(&dst_peer, peer_message); + tx_network.inc_sent(size); + } else { + gum::info!(target: LOG_TARGET, "Downlink channel closed, network interface task exiting"); + break } } } .boxed(); + spawn_task_handle.spawn("network-interface-rx", "test-environment", rx_task); + spawn_task_handle.spawn("network-interface-tx", "test-environment", tx_task); + ( - Self { - network: None, - network_to_interface_sender, - bridge_to_interface_sender, - interface_to_bridge_sender, - }, + Self { bridge_to_interface_sender, interface_to_bridge_sender }, NetworkInterfaceReceiver(interface_to_bridge_receiver), ) } - /// Connect the interface to a network. - pub fn connect(&mut self, network: NetworkEmulator) { - self.network = Some(network); - } - /// Get a sender that can be used by a subsystem to send network actions to the network. pub fn subsystem_sender(&self) -> UnboundedSender { self.bridge_to_interface_sender.clone() @@ -240,7 +346,7 @@ impl NetworkInterface { /// Get a sender that can be used by the Network to send network actions to the network. pub fn network_sender(&self) -> UnboundedSender { - self.network_to_interface_sender.clone() + self.interface_to_bridge_sender.clone() } } @@ -261,114 +367,204 @@ pub struct PeerNetworkInterface { tx_queue: UnboundedSender, } -// -// A network peer emulator. It spawns a task that accepts `NetworkActions` and -// executes them with a configurable delay and bandwidth constraints. Tipically -// these actions wrap a future that performs a channel send to the subsystem(s) under test. +/// A handle send messages and actions to an emulated peer. #[derive(Clone)] -struct EmulatedPeerHandle { - // Send messages to the peer emulator task +pub struct EmulatedPeerHandle { + /// Send messages to the peer emulator task messages_tx: UnboundedSender, - // Send actions to the peer emulator task + /// Send actions to the peer emulator task actions_tx: UnboundedSender, } +impl EmulatedPeerHandle { + /// Send a message to the peer. + pub fn send_message(&self, message: PeerMessage) { + let _ = self + .messages_tx + .unbounded_send(message) + .expect("Sending message to the peer never fails"); + } + + /// Send a message to the peer. + pub fn send_action(&self, action: NetworkAction) { + let _ = self + .actions_tx + .unbounded_send(action) + .expect("Sending action to the peer never fails"); + } +} + +/// A network peer emulator. Receives `PeerMessages` and `NetworkActions`. Tipically +/// these actions send a message to the node under test. +pub struct EmulatedPeer { + to_node: UnboundedSender, + tx_limiter: RateLimit, + rx_limiter: RateLimit, +} + +impl EmulatedPeer { + pub async fn send_message(&mut self, message: PeerMessage) { + self.tx_limiter.reap(message.size()).await; + let _ = self.to_node.unbounded_send(message).expect("Sending to the node never fails"); + } + + pub fn rx_limiter(&mut self) -> &mut RateLimit { + &mut self.rx_limiter + } + + pub fn tx_limiter(&mut self) -> &mut RateLimit { + &mut self.tx_limiter + } +} + /// Interceptor pattern for handling messages. pub trait HandlePeerMessage { // Returns `None` if the message was handled, or the `message` // otherwise. - fn handle(&self, message: PeerMessage, node_sender: &mut UnboundedSender) -> Option; + fn handle( + &self, + message: PeerMessage, + node_sender: &mut UnboundedSender, + ) -> Option; } -impl HandlePeerMessage for Arc -where T: HandlePeerMessage { - fn handle(&self, message: PeerMessage, node_sender: &mut UnboundedSender) -> Option { +impl HandlePeerMessage for Arc +where + T: HandlePeerMessage, +{ + fn handle( + &self, + message: PeerMessage, + node_sender: &mut UnboundedSender, + ) -> Option { self.as_ref().handle(message, node_sender) } } -pub fn new_peer( - bandwidth: usize, - spawn_task_handle: SpawnTaskHandle, - handlers: Vec>, +async fn emulated_peer_loop( + handlers: Vec>, stats: Arc, - network_interface: &NetworkInterface, -) -> Self { - let (messages_tx, mut messages_rx) = mpsc::unbounded::(); - let (actions_tx, mut actions_rx) = mpsc::unbounded::(); - - // We'll use this to send messages from this peer to the node under test (peer 0) - let to_node = network_interface.network_sender(); - - spawn_task_handle - .clone() - .spawn("peer-emulator", "test-environment", async move { - let mut rx_limiter = RateLimit::new(10, bandwidth); - let mut tx_limiter = RateLimit::new(10, bandwidth); - let mut messages_rx = messages_rx.fuse(); - let mut actions_rx = actions_rx.fuse(); - - assert!(message.is_none(), "A peer will process all received messages"); - - // TODO: implement latency and error. - loop { - futures::select! { - maybe_peer_message = messages_rx.next() => { - if let Some(peer_message) = maybe_peer_message { - let size = peer_message.size(); - rx_limiter.reap(size).await; - stats.inc_received(size); - - let message = Some(message); - for handler in handlers.iter() { - message = handler.handle(message, &mut to_node); - if message.is_none() { - break - } - } - } else { - gum::trace!(target: LOG_TARGET, "Downlink channel closed, peer task exiting"); + mut emulated_peer: EmulatedPeer, + messages_rx: UnboundedReceiver, + actions_rx: UnboundedReceiver, + mut to_network_interface: UnboundedSender, +) { + let mut proxied_requests = FuturesUnordered::new(); + let mut messages_rx = messages_rx.fuse(); + let mut actions_rx = actions_rx.fuse(); + loop { + futures::select! { + maybe_peer_message = messages_rx.next() => { + if let Some(peer_message) = maybe_peer_message { + let size = peer_message.size(); + emulated_peer.rx_limiter().reap(size).await; + stats.inc_received(size); + + let mut message = Some(peer_message); + for handler in handlers.iter() { + // The check below guarantees that message is always `Some`. + message = handler.handle(message.unwrap(), &mut to_network_interface); + if message.is_none() { break } - }, - maybe_action = actions_rx.next() => { - if let Some(action) = maybe_action { - match action.kind { - NetworkActionKind::SendMessage(message) => { - let size = message.size(); - tx_limiter.reap(size).await; - stats.inc_sent(size); - to_node.unbounded_send(message); + } + if let Some(message) = message { + panic!("Emulated message from peer {:?} not handled", message.peer()); + } + } else { + gum::debug!(target: LOG_TARGET, "Downlink channel closed, peer task exiting"); + break + } + }, + maybe_action = actions_rx.next() => { + if let Some(action) = maybe_action { + // We proxy any request being sent to the node to limit bandwidth as we + // do in the `NetworkInterface` task. + if let PeerMessage::RequestFromPeer(request) = action.message { + let (response_sender, response_receiver) = oneshot::channel(); + // Create a new `IncomingRequest` that we forward to the network interface. + let new_request = IncomingRequest {payload: request.payload, peer: request.peer, pending_response: response_sender}; + + proxied_requests.push(ProxiedRequest {sender: Some(request.pending_response), receiver: response_receiver}); + + emulated_peer.send_message(PeerMessage::RequestFromPeer(new_request)).await; + continue + } + + emulated_peer.send_message(action.message).await; + } else { + gum::debug!(target: LOG_TARGET, "Action channel closed, peer task exiting"); + break + } + }, + proxied_request = proxied_requests.next() => { + if let Some(proxied_request) = proxied_request { + match proxied_request.result { + Ok(result) => { + let bytes = result.encoded_size(); + gum::trace!(target: LOG_TARGET, size = bytes, "Peer proxied request completed"); + + emulated_peer.rx_limiter().reap(bytes).await; + stats.inc_received(bytes); + + proxied_request.sender.send( + OutgoingResponse { + reputation_changes: Vec::new(), + result: Ok(result), + sent_feedback: None } - } - } else { - gum::trace!(target: LOG_TARGET, "Action channel closed, peer task exiting"); - break + ).expect("network is alive"); } - }, + Err(e) => { + gum::warn!(target: LOG_TARGET, "Node req/response failure: {:?}", e) + } + } } } - }); - - EmulatedPeerHandle { messages_tx, actions_tx } + } + } } -/// Types of actions that an emulated peer can run. -pub enum NetworkActionKind { - /// Send a message to node under test (peer 0) - SendMessage(PeerMessage) +pub fn new_peer( + bandwidth: usize, + spawn_task_handle: SpawnTaskHandle, + handlers: Vec>, + stats: Arc, + mut to_network_interface: UnboundedSender, +) -> EmulatedPeerHandle { + let (messages_tx, mut messages_rx) = mpsc::unbounded::(); + let (actions_tx, mut actions_rx) = mpsc::unbounded::(); + + let rx_limiter = RateLimit::new(10, bandwidth); + let tx_limiter = RateLimit::new(10, bandwidth); + let mut emulated_peer = + EmulatedPeer { rx_limiter, tx_limiter, to_node: to_network_interface.clone() }; + + spawn_task_handle.clone().spawn( + "peer-emulator", + "test-environment", + emulated_peer_loop( + handlers, + stats, + emulated_peer, + messages_rx, + actions_rx, + to_network_interface, + ) + .boxed(), + ); + + EmulatedPeerHandle { messages_tx, actions_tx } } /// A network action to be completed by an emulator task. pub struct NetworkAction { - /// The action type - pub kind: NetworkActionKind, - // Peer which should run the action. + /// The message to be sent by the peer. + pub message: PeerMessage, + /// Peer which should run the action. pub peer: AuthorityDiscoveryId, } -unsafe impl Send for NetworkAction {} - /// Book keeping of sent and received bytes. pub struct PeerEmulatorStats { metrics: Metrics, @@ -405,12 +601,11 @@ impl PeerEmulatorStats { } } - /// The state of a peer on the emulated network. #[derive(Clone)] enum Peer { - Connected(PeerEmulator), - Disconnected(PeerEmulator), + Connected(EmulatedPeerHandle), + Disconnected(EmulatedPeerHandle), } impl Peer { @@ -423,14 +618,16 @@ impl Peer { } pub fn is_connected(&self) -> bool { - if let Peer::Connected(_) = self { - true - } else { - false - } + matches!(self, Peer::Connected(_)) } - pub fn emulator(&mut self) -> &mut PeerEmulator { + pub fn handle(&self) -> &EmulatedPeerHandle { + match self { + Peer::Connected(ref emulator) => emulator, + Peer::Disconnected(ref emulator) => emulator, + } + } + pub fn handle_mut(&mut self) -> &mut EmulatedPeerHandle { match self { Peer::Connected(ref mut emulator) => emulator, Peer::Disconnected(ref mut emulator) => emulator, @@ -438,7 +635,7 @@ impl Peer { } } -/// An emulated network implementation. Can be cloned +/// A ha emulated network implementation. #[derive(Clone)] pub struct NetworkEmulatorHandle { // Per peer network emulation. @@ -449,13 +646,14 @@ pub struct NetworkEmulatorHandle { validator_authority_ids: HashMap, } +/// Create a new emulated network based on `config`. +/// Each emulated peer will run the specified `handlers` to process incoming messages. pub fn new_network( config: &TestConfiguration, dependencies: &TestEnvironmentDependencies, authorities: &TestAuthorities, - handlers: Vec>, - network_interface: &NetworkInterface, -) -> NetworkEmulatorHandle { + handlers: Vec>, +) -> (NetworkEmulatorHandle, NetworkInterface, NetworkInterfaceReceiver) { let n_peers = config.n_validators; gum::info!(target: LOG_TARGET, "{}",format!("Initializing emulation for a {} peer network.", n_peers).bright_blue()); gum::info!(target: LOG_TARGET, "{}",format!("connectivity {}%, error {}%", config.connectivity, config.error).bright_black()); @@ -464,6 +662,9 @@ pub fn new_network( Metrics::new(&dependencies.registry).expect("Metrics always register succesfully"); let mut validator_authority_id_mapping = HashMap::new(); + // Create the channel from `peer` to `NetworkInterface` . + let (to_network_interface, from_network) = mpsc::unbounded(); + // Create a `PeerEmulator` for each peer. let (stats, mut peers): (_, Vec<_>) = (0..n_peers) .zip(authorities.validator_authority_id.clone().into_iter()) @@ -475,9 +676,9 @@ pub fn new_network( Peer::Connected(new_peer( config.peer_bandwidth, dependencies.task_manager.spawn_handle(), - handlers, + handlers.clone(), stats, - network_interface, + to_network_interface.clone(), )), ) }) @@ -494,40 +695,46 @@ pub fn new_network( gum::info!(target: LOG_TARGET, "{}",format!("Network created, connected validator count {}", connected_count).bright_black()); - NetworkEmulatorHandle { peers, stats, validator_authority_ids: validator_authority_id_mapping } + let handle = NetworkEmulatorHandle { + peers, + stats, + validator_authority_ids: validator_authority_id_mapping, + }; + + // Finally create the `NetworkInterface` with the `from_network` receiver. + let (network_interface, network_interface_receiver) = NetworkInterface::new( + dependencies.task_manager.spawn_handle(), + handle.clone(), + config.bandwidth, + from_network, + ); + + (handle, network_interface, network_interface_receiver) } - impl NetworkEmulatorHandle { pub fn is_peer_connected(&self, peer: &AuthorityDiscoveryId) -> bool { self.peer(peer).is_connected() } - /// Forward `message`` to an emulated `peer``. + /// Forward `message`` to an emulated `peer`. /// Panics if peer is not connected. - pub fn forward_message(&self, peer: &AuthorityDiscoveryId, message: PeerMessage) { - assert(!self.peer(peer).is_connected(), "forward message only for connected peers."); - - + pub fn submit_peer_message(&self, peer: &AuthorityDiscoveryId, message: PeerMessage) { + let peer = self.peer(peer); + assert!(peer.is_connected(), "forward message only for connected peers."); + peer.handle().send_message(message); } - /// Run some code in the context of an emulated - pub fn submit_peer_action(&mut self, peer: AuthorityDiscoveryId, action: NetworkAction) { - let index = self - .validator_authority_ids - .get(&peer) - .expect("all test authorities are valid; qed"); - - let peer = self.peers.get_mut(*index).expect("We just retrieved the index above; qed"); + /// Run a `NetworkAction` in the context of an emulated peer. + pub fn submit_peer_action(&mut self, action: NetworkAction) { + let dst_peer = self.peer(&action.peer); - // Only actions of size 0 are allowed on disconnected peers. - // Typically this are delayed error response sends. - if action.size() > 0 && !peer.is_connected() { - gum::warn!(target: LOG_TARGET, peer_index = index, "Attempted to send data from a disconnected peer, operation ignored"); + if !dst_peer.is_connected() { + gum::warn!(target: LOG_TARGET, "Attempted to send data from a disconnected peer, operation ignored"); return } - peer.emulator().send(action); + dst_peer.handle().send_action(action); } // Returns the sent/received stats for `peer_index`. @@ -547,6 +754,7 @@ impl NetworkEmulatorHandle { fn peer(&self, peer: &AuthorityDiscoveryId) -> &Peer { &self.peers[self.peer_index(peer)] } + // Returns the sent/received stats for `peer`. pub fn peer_stats_by_id(&mut self, peer: &AuthorityDiscoveryId) -> Arc { let peer_index = self.peer_index(peer); @@ -657,3 +865,13 @@ mod tests { assert!(total_sent as u128 <= upper_bound); } } + +/// A helper to determine the request payload size. +pub fn request_size(request: &Requests) -> usize { + match request { + Requests::ChunkFetchingV1(outgoing_request) => outgoing_request.payload.encoded_size(), + Requests::AvailableDataFetchingV1(outgoing_request) => + outgoing_request.payload.encoded_size(), + _ => unimplemented!("received an unexpected request"), + } +} From 1dfb26382a6ab639d6c89d6e4c4829acc128c097 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 5 Jan 2024 13:16:50 +0200 Subject: [PATCH 161/192] refactor NetworkMessage Signed-off-by: Andrei Sandu --- .../subsystem-bench/src/availability/mod.rs | 70 ++-- .../subsystem-bench/src/core/configuration.rs | 4 +- .../subsystem-bench/src/core/environment.rs | 17 +- .../src/core/mock/network_bridge.rs | 18 +- .../node/subsystem-bench/src/core/network.rs | 305 ++++++++++-------- 5 files changed, 223 insertions(+), 191 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 24be6e647191..280392a430b7 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -53,7 +53,7 @@ use polkadot_node_network_protocol::{ }, IncomingRequest, OutgoingRequest, ReqProtocolNames, Requests, }, - BitfieldDistributionMessage, OurView, Versioned, View, + BitfieldDistributionMessage, OurView, Versioned, VersionedValidationProtocol, View, }; use sc_network::request_responses::IncomingRequest as RawIncomingRequest; @@ -145,14 +145,14 @@ pub struct NetworkAvailabilityState { pub chunks: Vec>, } -impl HandlePeerMessage for NetworkAvailabilityState { +impl HandleNetworkMessage for NetworkAvailabilityState { fn handle( &self, - message: PeerMessage, - node_sender: &mut futures::channel::mpsc::UnboundedSender, - ) -> Option { + message: NetworkMessage, + node_sender: &mut futures::channel::mpsc::UnboundedSender, + ) -> Option { match message { - PeerMessage::RequestFromNode(peer, request) => match request { + NetworkMessage::RequestFromNode(peer, request) => match request { Requests::ChunkFetchingV1(outgoing_request) => { gum::debug!(target: LOG_TARGET, request = ?outgoing_request, "Received `RequestFromNode`"); let validator_index: usize = outgoing_request.payload.index.0 as usize; @@ -175,10 +175,6 @@ impl HandlePeerMessage for NetworkAvailabilityState { if let Err(err) = outgoing_request.pending_response.send(response) { gum::error!(target: LOG_TARGET, "Failed to send `ChunkFetchingResponse`"); } - // let _ = outgoing_request - // .pending_response - // .send(response) - // .expect("Response is always sent succesfully"); None }, @@ -203,7 +199,7 @@ impl HandlePeerMessage for NetworkAvailabilityState { .expect("Response is always sent succesfully"); None }, - _ => Some(PeerMessage::RequestFromNode(peer, request)), + _ => Some(NetworkMessage::RequestFromNode(peer, request)), }, message => Some(message), @@ -211,7 +207,7 @@ impl HandlePeerMessage for NetworkAvailabilityState { } } -/// Takes a test configuration and uses it to creates the `TestEnvironment`. +/// Takes a test configuration and uses it to create the `TestEnvironment`. pub fn prepare_test( config: TestConfiguration, state: &mut TestState, @@ -647,7 +643,7 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: for index in 1..config.n_validators { let (pending_response, pending_response_receiver) = oneshot::channel(); - let message = PeerMessage::RequestFromPeer(RawIncomingRequest { + let request = RawIncomingRequest { peer: PeerId::random(), payload: ChunkFetchingRequest { candidate_hash: state.backed_candidates()[block_num].hash(), @@ -655,16 +651,16 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: } .encode(), pending_response, - }); + }; let peer = env .authorities() .validator_authority_id .get(index) - .expect("all validators have keys") - .clone(); + .expect("all validators have keys"); + + env.network().send_request_from_peer(peer, request); - env.network_mut().submit_peer_action(NetworkAction { message, peer }); receivers.push(pending_response_receiver); } @@ -684,19 +680,17 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: let authorities = env.authorities().clone(); let n_validators = config.n_validators; - // Spawn a task that will generate `n_validator` - 1 signed bitfiends and // send them from the emulated peers to the subsystem. - env.spawn("send-bitfields", async move { + env.spawn_blocking("send-bitfields", async move { for index in 1..n_validators { - let validator_public = authorities - .validator_public - .get(index) - .expect("All validator keys are known"); + let validator_public = + authorities.validator_public.get(index).expect("All validator keys are known"); // Node has all the chunks in the world. let payload: AvailabilityBitfield = AvailabilityBitfield(bitvec![u8, bitvec::order::Lsb0; 1u8; 32]); + // TODO(soon): Use pre-signed messages. This is quite intensive on the CPU. let signed_bitfield = Signed::::sign( &authorities.keyring.keystore(), payload, @@ -708,15 +702,12 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: .flatten() .expect("should be signed"); - let from_peer = authorities.validator_authority_id[index].clone(); - // Send to our node. - let to_peer = authorities.validator_authority_id[0].clone(); + let from_peer = &authorities.validator_authority_id[index]; - let action = - send_peer_bitfield(relay_block_hash, signed_bitfield, &from_peer, &to_peer); + let message = peer_bitfield_message_v2(relay_block_hash, signed_bitfield); // Send the action to `to_peer`. - network.submit_peer_action(action); + network.send_message_from_peer(from_peer, message); } gum::info!("Waiting for {} bitfields to be received and processed", n_validators - 1); @@ -728,7 +719,7 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: (config.n_validators - 1) * (block_num), ) .await; - + gum::info!("All bitfields processed"); let block_time = Instant::now().sub(block_start_ts).as_millis() as u64; @@ -755,25 +746,16 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: env.stop().await; } -pub fn send_peer_bitfield( +pub fn peer_bitfield_message_v2( relay_hash: H256, signed_bitfield: Signed, - from_peer: &AuthorityDiscoveryId, - to_peer: &AuthorityDiscoveryId, -) -> NetworkAction { +) -> VersionedValidationProtocol { let bitfield = polkadot_node_network_protocol::v2::BitfieldDistributionMessage::Bitfield( relay_hash, signed_bitfield.into(), ); - let to_peer = to_peer.clone(); - - let message = PeerMessage::Message( - to_peer, - Versioned::V2( - polkadot_node_network_protocol::v2::ValidationProtocol::BitfieldDistribution(bitfield), - ), - ); - - NetworkAction { peer: from_peer.clone(), message } + Versioned::V2(polkadot_node_network_protocol::v2::ValidationProtocol::BitfieldDistribution( + bitfield, + )) } diff --git a/polkadot/node/subsystem-bench/src/core/configuration.rs b/polkadot/node/subsystem-bench/src/core/configuration.rs index a5e4110bc595..fb4b4e668c5e 100644 --- a/polkadot/node/subsystem-bench/src/core/configuration.rs +++ b/polkadot/node/subsystem-bench/src/core/configuration.rs @@ -37,9 +37,9 @@ fn random_uniform_sample + From>(min_value: T, max_value: /// Peer response latency configuration. #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct PeerLatency { - /// Min latency for `NetworkAction` completion. + /// Min latency of the peer. pub min_latency: Duration, - /// Max latency or `NetworkAction` completion. + /// Max latency or the peer. pub max_latency: Duration, } diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 16883350f6ac..e89d93eefc3c 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -252,11 +252,6 @@ impl TestEnvironment { &self.network } - /// Returns a mutable reference to the inner network emulator handle. - pub fn network_mut(&mut self) -> &mut NetworkEmulatorHandle { - &mut self.network - } - /// Returns the Prometheus registry. pub fn registry(&self) -> &Registry { &self.dependencies.registry @@ -270,6 +265,18 @@ impl TestEnvironment { .spawn(name, "test-environment", task); } + /// Spawn a blocking named task in the `test-environment` task group. + pub fn spawn_blocking( + &self, + name: &'static str, + task: impl Future + Send + 'static, + ) { + self.dependencies.task_manager.spawn_handle().spawn_blocking( + name, + "test-environment", + task, + ); + } /// Returns a reference to the test environment metrics instance pub fn metrics(&self) -> &TestEnvironmentMetrics { &self.metrics diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index 4edc1c9e08c4..280c17aa9631 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -52,19 +52,17 @@ use polkadot_primitives::AuthorityDiscoveryId; use crate::core::{ configuration::{random_error, random_latency, TestConfiguration}, - network::{ - NetworkAction, NetworkEmulatorHandle, NetworkInterfaceReceiver, PeerMessage, RateLimit, - }, + network::{NetworkEmulatorHandle, NetworkInterfaceReceiver, NetworkMessage, RateLimit}, }; -const LOG_TARGET: &str = "subsystem-bench::network-bridge-tx-mock"; +const LOG_TARGET: &str = "subsystem-bench::network-bridge"; /// A mock of the network bridge tx subsystem. pub struct MockNetworkBridgeTx { /// A network emulator handle network: NetworkEmulatorHandle, /// A channel to the network interface, - to_network_interface: UnboundedSender, + to_network_interface: UnboundedSender, } /// A mock of the network bridge tx subsystem. @@ -78,7 +76,7 @@ pub struct MockNetworkBridgeRx { impl MockNetworkBridgeTx { pub fn new( network: NetworkEmulatorHandle, - to_network_interface: UnboundedSender, + to_network_interface: UnboundedSender, ) -> MockNetworkBridgeTx { Self { network, to_network_interface } } @@ -150,7 +148,7 @@ impl MockNetworkBridgeTx { let peer_id = request.authority_id().expect("all nodes are authorities").clone(); let peer_message = - PeerMessage::RequestFromNode(peer_id.clone(), request); + NetworkMessage::RequestFromNode(peer_id.clone(), request); let _ = self.to_network_interface.unbounded_send(peer_message); } }, @@ -176,7 +174,7 @@ impl MockNetworkBridgeRx { maybe_peer_message = from_network_interface.next() => { if let Some(message) = maybe_peer_message { match message { - PeerMessage::Message(peer, message) => match message { + NetworkMessage::MessageFromPeer(message) => match message { Versioned::V2( polkadot_node_network_protocol::v2::ValidationProtocol::BitfieldDistribution( bitfield, @@ -190,7 +188,7 @@ impl MockNetworkBridgeRx { unimplemented!("We only talk v2 network protocol") }, }, - PeerMessage::RequestFromPeer(request) => { + NetworkMessage::RequestFromPeer(request) => { if let Some(protocol) = self.chunk_request_sender.as_mut() { if let Some(inbound_queue) = protocol.inbound_queue.as_ref() { let _ = inbound_queue @@ -201,7 +199,7 @@ impl MockNetworkBridgeRx { } }, _ => { - panic!("1PeerMessage::RequestFromNode1 is not expected to be received from a peer") + panic!("NetworkMessage::RequestFromNode is not expected to be received from a peer") } } } diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 95a98f010a5b..74a4e02d3020 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -16,6 +16,19 @@ //! //! Implements network emulation and interfaces to control and specialize //! network peer behaviour. +//! +//! Peer1 Peer2 Peer3 Peer4 +//! \ | | / +//! \ | | / +//! \ | | / +//! \ | | / +//! \ | | / +//! [Network Interface] +//! | +//! [Emulated Network Bridge] +//! | +//! Subsystems under test + use super::{ configuration::{TestAuthorities, TestConfiguration}, @@ -131,32 +144,40 @@ impl RateLimit { /// A wrapper for both gossip and request/response protocols along with the destination /// peer(`AuthorityDiscoveryId``). -pub enum PeerMessage { - /// A gossip message - Message(AuthorityDiscoveryId, VersionedValidationProtocol), +pub enum NetworkMessage { + /// A gossip message from peer to node. + MessageFromPeer(VersionedValidationProtocol), + /// A gossip message from node to a peer. + MessageFromNode(AuthorityDiscoveryId, VersionedValidationProtocol), /// A request originating from our node RequestFromNode(AuthorityDiscoveryId, Requests), /// A request originating from an emultated peer RequestFromPeer(IncomingRequest), } -impl PeerMessage { +impl NetworkMessage { /// Returns the size of the encoded message or request pub fn size(&self) -> usize { match &self { - PeerMessage::Message(_peer_id, Versioned::V2(message)) => message.encoded_size(), - PeerMessage::Message(_peer_id, Versioned::V1(message)) => message.encoded_size(), - PeerMessage::Message(_peer_id, Versioned::VStaging(message)) => message.encoded_size(), - PeerMessage::RequestFromNode(_peer_id, incoming) => request_size(incoming), - PeerMessage::RequestFromPeer(request) => request.payload.encoded_size(), + NetworkMessage::MessageFromPeer(Versioned::V2(message)) => message.encoded_size(), + NetworkMessage::MessageFromPeer(Versioned::V1(message)) => message.encoded_size(), + NetworkMessage::MessageFromPeer(Versioned::VStaging(message)) => message.encoded_size(), + NetworkMessage::MessageFromNode(_peer_id, Versioned::V2(message)) => + message.encoded_size(), + NetworkMessage::MessageFromNode(_peer_id, Versioned::V1(message)) => + message.encoded_size(), + NetworkMessage::MessageFromNode(_peer_id, Versioned::VStaging(message)) => + message.encoded_size(), + NetworkMessage::RequestFromNode(_peer_id, incoming) => request_size(incoming), + NetworkMessage::RequestFromPeer(request) => request.payload.encoded_size(), } } - /// Returns the destination peer from the message + /// Returns the destination peer from the message or `None` if it originates from a peer. pub fn peer(&self) -> Option<&AuthorityDiscoveryId> { match &self { - PeerMessage::Message(peer_id, _) | PeerMessage::RequestFromNode(peer_id, _) => - Some(peer_id), + NetworkMessage::MessageFromNode(peer_id, _) | + NetworkMessage::RequestFromNode(peer_id, _) => Some(peer_id), _ => None, } } @@ -165,18 +186,13 @@ impl PeerMessage { /// A network interface of the node under test. /// TODO(soon): Implement latency and connection errors here, instead of doing it on the peers. pub struct NetworkInterface { - // Used to receive traffic from the `network-bridge-tx` subsystem. - // The network action is forwarded via `network` to the relevant peer(s). - // from_netowork_bridge: Receiver, // Sender for subsystems. - bridge_to_interface_sender: UnboundedSender, - // A sender to forward actions to the network bridge subsystem. - interface_to_bridge_sender: UnboundedSender, + bridge_to_interface_sender: UnboundedSender, } // Wraps the receiving side of a interface to bridge channel. It is a required // parameter of the `network-bridge` mock. -pub struct NetworkInterfaceReceiver(pub UnboundedReceiver); +pub struct NetworkInterfaceReceiver(pub UnboundedReceiver); struct ProxiedRequest { sender: Option>, @@ -216,22 +232,22 @@ impl NetworkInterface { pub fn new( spawn_task_handle: SpawnTaskHandle, mut network: NetworkEmulatorHandle, - bandiwdth_bps: usize, - mut from_network: UnboundedReceiver, + bandwidth_bps: usize, + mut from_network: UnboundedReceiver, ) -> (NetworkInterface, NetworkInterfaceReceiver) { - let mut rx_limiter = RateLimit::new(10, bandiwdth_bps); + let mut rx_limiter = RateLimit::new(10, bandwidth_bps); // We need to share the transimit limiter as we handle incoming request/response on rx // thread. - let mut tx_limiter = Arc::new(Mutex::new(RateLimit::new(10, bandiwdth_bps))); + let mut tx_limiter = Arc::new(Mutex::new(RateLimit::new(10, bandwidth_bps))); let mut proxied_requests = FuturesUnordered::new(); - // The sender (`egress_tx`) is + // Channel for receiving messages from the network bridge subsystem. let (bridge_to_interface_sender, mut bridge_to_interface_receiver) = - mpsc::unbounded::(); + mpsc::unbounded::(); - // Channel for forwarding actions to the bridge. + // Channel for forwarding messages to the network bridge subsystem. let (interface_to_bridge_sender, interface_to_bridge_receiver) = - mpsc::unbounded::(); + mpsc::unbounded::(); let mut rx_network = network.clone(); let mut tx_network = network; @@ -240,7 +256,7 @@ impl NetworkInterface { let rx_task_tx_limiter = tx_limiter.clone(); let tx_task_tx_limiter = tx_limiter; - // Spawn the network interface task. + // A task that forwards messages from emulated peers to the node (emulated network bridge). let rx_task = async move { loop { let mut from_network = from_network.next().fuse(); @@ -248,22 +264,23 @@ impl NetworkInterface { maybe_peer_message = from_network => { if let Some(peer_message) = maybe_peer_message { let size = peer_message.size(); - // TODO (maybe): Implement limiter as part of something like `RateLimitedChannel`. rx_limiter.reap(size).await; rx_network.inc_received(size); - // We act as an incoming request proxy, so we'll craft a new request and wait for - // the answer from subsystem, and only then, after ratelimiting we send back a response - // to the peer - if let PeerMessage::RequestFromPeer(request) = peer_message { + // To be able to apply the configured bandwidth limits for responses being sent + // over channels, we need to implement a simple proxy that allows this loop + // to receive the response and enforce the configured bandwidth before + // sending it to the original recipient. + if let NetworkMessage::RequestFromPeer(request) = peer_message { let (response_sender, response_receiver) = oneshot::channel(); + // Create a new `IncomingRequest` that we forward to the network bridge. let new_request = IncomingRequest {payload: request.payload, peer: request.peer, pending_response: response_sender}; - proxied_requests.push(ProxiedRequest {sender: Some(request.pending_response), receiver: response_receiver}); - // Send the new message to network bridge rx + + // Send the new message to network bridge subsystem. rx_task_bridge_sender - .unbounded_send(PeerMessage::RequestFromPeer(new_request)) + .unbounded_send(NetworkMessage::RequestFromPeer(new_request)) .expect("network bridge subsystem is alive"); continue } @@ -284,9 +301,13 @@ impl NetworkInterface { let bytes = result.encoded_size(); gum::trace!(target: LOG_TARGET, size = bytes, "proxied request completed"); + // Enforce bandwidth based on the response the node has sent. + // TODO: Fix the stall of RX when TX lock() takes a while to refill + // the token bucket. rx_task_tx_limiter.lock().await.reap(bytes).await; rx_network.inc_sent(bytes); + // Forward the response to original recipient. proxied_request.sender.send( OutgoingResponse { reputation_changes: Vec::new(), @@ -309,18 +330,24 @@ impl NetworkInterface { } .boxed(); + // A task that forwards messages from the node to emulated peers. let tx_task = async move { loop { if let Some(peer_message) = bridge_to_interface_receiver.next().await { let size = peer_message.size(); + // Ensure bandwidth used is limited. tx_task_tx_limiter.lock().await.reap(size).await; - let dst_peer = peer_message - .peer() - .expect( - "Node always needs to specify destination peer when sending a message", - ) - .clone(); - tx_network.submit_peer_message(&dst_peer, peer_message); + + match peer_message { + NetworkMessage::MessageFromNode(peer, message) => + tx_network.send_message_to_peer(&peer, message), + NetworkMessage::RequestFromNode(peer, request) => + tx_network.send_request_to_peer(&peer, request), + _ => panic!( + "Unexpected network message received from emulated network bridge" + ), + } + tx_network.inc_sent(size); } else { gum::info!(target: LOG_TARGET, "Downlink channel closed, network interface task exiting"); @@ -334,120 +361,111 @@ impl NetworkInterface { spawn_task_handle.spawn("network-interface-tx", "test-environment", tx_task); ( - Self { bridge_to_interface_sender, interface_to_bridge_sender }, + Self { bridge_to_interface_sender }, NetworkInterfaceReceiver(interface_to_bridge_receiver), ) } /// Get a sender that can be used by a subsystem to send network actions to the network. - pub fn subsystem_sender(&self) -> UnboundedSender { + pub fn subsystem_sender(&self) -> UnboundedSender { self.bridge_to_interface_sender.clone() } - - /// Get a sender that can be used by the Network to send network actions to the network. - pub fn network_sender(&self) -> UnboundedSender { - self.interface_to_bridge_sender.clone() - } } -/// An emulated peer network interface. -#[derive(Debug)] -pub struct PeerNetworkInterface { - /// Receive rate limiter. - rx_limiter: RateLimit, - /// Transmit rate limiter - tx_limiter: RateLimit, - /// Network receive queue. - /// This is paired to a `Sender` in `NetworkEmulator`. - /// The network interface task will forward messages/requests from the node over - /// this channel. - rx_queue: UnboundedReceiver, - /// Network send queue. - /// Paired to the `rx_queue` receiver on the `NetworkInterface`. - tx_queue: UnboundedSender, -} - -/// A handle send messages and actions to an emulated peer. +/// A handle for controlling an emulated peer. #[derive(Clone)] pub struct EmulatedPeerHandle { - /// Send messages to the peer emulator task - messages_tx: UnboundedSender, - /// Send actions to the peer emulator task - actions_tx: UnboundedSender, + /// Send messages to be processed by the peer. + messages_tx: UnboundedSender, + /// Send actions to be performed by the peer. + actions_tx: UnboundedSender, } impl EmulatedPeerHandle { - /// Send a message to the peer. - pub fn send_message(&self, message: PeerMessage) { + /// Receive and process a message + pub fn receive(&self, message: NetworkMessage) { let _ = self .messages_tx .unbounded_send(message) - .expect("Sending message to the peer never fails"); + .expect("Sending action to the peer never fails"); + } + + pub fn send_message(&self, message: VersionedValidationProtocol) { + let _ = self + .actions_tx + .unbounded_send(NetworkMessage::MessageFromPeer(message)) + .expect("Sending action to the peer never fails"); } - /// Send a message to the peer. - pub fn send_action(&self, action: NetworkAction) { + /// Send a `request` to the node. + pub fn send_request(&self, request: IncomingRequest) { let _ = self .actions_tx - .unbounded_send(action) + .unbounded_send(NetworkMessage::RequestFromPeer(request)) .expect("Sending action to the peer never fails"); } } -/// A network peer emulator. Receives `PeerMessages` and `NetworkActions`. Tipically -/// these actions send a message to the node under test. -pub struct EmulatedPeer { - to_node: UnboundedSender, +// A network peer emulator. +struct EmulatedPeer { + to_node: UnboundedSender, tx_limiter: RateLimit, rx_limiter: RateLimit, } impl EmulatedPeer { - pub async fn send_message(&mut self, message: PeerMessage) { + /// Send a message to the node. + pub async fn send_message(&mut self, message: NetworkMessage) { self.tx_limiter.reap(message.size()).await; let _ = self.to_node.unbounded_send(message).expect("Sending to the node never fails"); } + /// Returns the rx bandwidth limiter. pub fn rx_limiter(&mut self) -> &mut RateLimit { &mut self.rx_limiter } + /// Returns the tx bandwidth limiter. pub fn tx_limiter(&mut self) -> &mut RateLimit { &mut self.tx_limiter } } /// Interceptor pattern for handling messages. -pub trait HandlePeerMessage { - // Returns `None` if the message was handled, or the `message` - // otherwise. +pub trait HandleNetworkMessage { + /// Returns `None` if the message was handled, or the `message` + /// otherwise. + /// + /// `node_sender` allows sending of messages to the node in response + /// to the handled message. fn handle( &self, - message: PeerMessage, - node_sender: &mut UnboundedSender, - ) -> Option; + message: NetworkMessage, + node_sender: &mut UnboundedSender, + ) -> Option; } -impl HandlePeerMessage for Arc +impl HandleNetworkMessage for Arc where - T: HandlePeerMessage, + T: HandleNetworkMessage, { fn handle( &self, - message: PeerMessage, - node_sender: &mut UnboundedSender, - ) -> Option { + message: NetworkMessage, + node_sender: &mut UnboundedSender, + ) -> Option { self.as_ref().handle(message, node_sender) } } +// This loop is responsible for handling of messages/requests between the peer and the node. async fn emulated_peer_loop( - handlers: Vec>, + handlers: Vec>, stats: Arc, mut emulated_peer: EmulatedPeer, - messages_rx: UnboundedReceiver, - actions_rx: UnboundedReceiver, - mut to_network_interface: UnboundedSender, + messages_rx: UnboundedReceiver, + actions_rx: UnboundedReceiver, + mut to_network_interface: UnboundedSender, ) { let mut proxied_requests = FuturesUnordered::new(); let mut messages_rx = messages_rx.fuse(); @@ -461,8 +479,12 @@ async fn emulated_peer_loop( stats.inc_received(size); let mut message = Some(peer_message); + + // Try all handlers until the message gets processed. + // Panic if the message is not consumed. for handler in handlers.iter() { - // The check below guarantees that message is always `Some`. + // The check below guarantees that message is always `Some`: we are still + // inside the loop. message = handler.handle(message.unwrap(), &mut to_network_interface); if message.is_none() { break @@ -477,24 +499,23 @@ async fn emulated_peer_loop( } }, maybe_action = actions_rx.next() => { - if let Some(action) = maybe_action { + match maybe_action { // We proxy any request being sent to the node to limit bandwidth as we // do in the `NetworkInterface` task. - if let PeerMessage::RequestFromPeer(request) = action.message { + Some(NetworkMessage::RequestFromPeer(request)) => { let (response_sender, response_receiver) = oneshot::channel(); // Create a new `IncomingRequest` that we forward to the network interface. let new_request = IncomingRequest {payload: request.payload, peer: request.peer, pending_response: response_sender}; proxied_requests.push(ProxiedRequest {sender: Some(request.pending_response), receiver: response_receiver}); - emulated_peer.send_message(PeerMessage::RequestFromPeer(new_request)).await; - continue + emulated_peer.send_message(NetworkMessage::RequestFromPeer(new_request)).await; + }, + Some(message) => emulated_peer.send_message(message).await, + None => { + gum::debug!(target: LOG_TARGET, "Action channel closed, peer task exiting"); + break } - - emulated_peer.send_message(action.message).await; - } else { - gum::debug!(target: LOG_TARGET, "Action channel closed, peer task exiting"); - break } }, proxied_request = proxied_requests.next() => { @@ -528,12 +549,12 @@ async fn emulated_peer_loop( pub fn new_peer( bandwidth: usize, spawn_task_handle: SpawnTaskHandle, - handlers: Vec>, + handlers: Vec>, stats: Arc, - mut to_network_interface: UnboundedSender, + mut to_network_interface: UnboundedSender, ) -> EmulatedPeerHandle { - let (messages_tx, mut messages_rx) = mpsc::unbounded::(); - let (actions_tx, mut actions_rx) = mpsc::unbounded::(); + let (messages_tx, mut messages_rx) = mpsc::unbounded::(); + let (actions_tx, mut actions_rx) = mpsc::unbounded::(); let rx_limiter = RateLimit::new(10, bandwidth); let tx_limiter = RateLimit::new(10, bandwidth); @@ -557,14 +578,6 @@ pub fn new_peer( EmulatedPeerHandle { messages_tx, actions_tx } } -/// A network action to be completed by an emulator task. -pub struct NetworkAction { - /// The message to be sent by the peer. - pub message: PeerMessage, - /// Peer which should run the action. - pub peer: AuthorityDiscoveryId, -} - /// Book keeping of sent and received bytes. pub struct PeerEmulatorStats { metrics: Metrics, @@ -652,7 +665,7 @@ pub fn new_network( config: &TestConfiguration, dependencies: &TestEnvironmentDependencies, authorities: &TestAuthorities, - handlers: Vec>, + handlers: Vec>, ) -> (NetworkEmulatorHandle, NetworkInterface, NetworkInterfaceReceiver) { let n_peers = config.n_validators; gum::info!(target: LOG_TARGET, "{}",format!("Initializing emulation for a {} peer network.", n_peers).bright_blue()); @@ -717,24 +730,56 @@ impl NetworkEmulatorHandle { self.peer(peer).is_connected() } - /// Forward `message`` to an emulated `peer`. + /// Forward notification `message` to an emulated `peer`. /// Panics if peer is not connected. - pub fn submit_peer_message(&self, peer: &AuthorityDiscoveryId, message: PeerMessage) { - let peer = self.peer(peer); + pub fn send_message_to_peer( + &self, + peer_id: &AuthorityDiscoveryId, + message: VersionedValidationProtocol, + ) { + let peer = self.peer(peer_id); assert!(peer.is_connected(), "forward message only for connected peers."); - peer.handle().send_message(message); + peer.handle().receive(NetworkMessage::MessageFromNode(peer_id.clone(), message)); + } + + /// Forward a `request`` to an emulated `peer`. + /// Panics if peer is not connected. + pub fn send_request_to_peer(&self, peer_id: &AuthorityDiscoveryId, request: Requests) { + let peer = self.peer(peer_id); + assert!(peer.is_connected(), "forward request only for connected peers."); + peer.handle().receive(NetworkMessage::RequestFromNode(peer_id.clone(), request)); + } + + /// Send a message from a peer to the node. + pub fn send_message_from_peer( + &self, + from_peer: &AuthorityDiscoveryId, + message: VersionedValidationProtocol, + ) { + let dst_peer = self.peer(&from_peer); + + if !dst_peer.is_connected() { + gum::warn!(target: LOG_TARGET, "Attempted to send message from a peer not connected to our node, operation ignored"); + return + } + + dst_peer.handle().send_message(message); } - /// Run a `NetworkAction` in the context of an emulated peer. - pub fn submit_peer_action(&mut self, action: NetworkAction) { - let dst_peer = self.peer(&action.peer); + /// Send a request from a peer to the node. + pub fn send_request_from_peer( + &self, + from_peer: &AuthorityDiscoveryId, + request: IncomingRequest, + ) { + let dst_peer = self.peer(&from_peer); if !dst_peer.is_connected() { - gum::warn!(target: LOG_TARGET, "Attempted to send data from a disconnected peer, operation ignored"); + gum::warn!(target: LOG_TARGET, "Attempted to send request from a peer not connected to our node, operation ignored"); return } - dst_peer.handle().send_action(action); + dst_peer.handle().send_request(request); } // Returns the sent/received stats for `peer_index`. From f7a2948726df3afcfcd98be8e43d3d090e13f7d1 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 8 Jan 2024 10:10:44 +0200 Subject: [PATCH 162/192] Run in CI Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/test.yml | 20 + Cargo.lock | 599 +++++++++++++----- .../node/subsystem-bench/src/approval/mod.rs | 5 +- 3 files changed, 480 insertions(+), 144 deletions(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 359d5b4dbcd0..77dbf2c997de 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -403,6 +403,26 @@ test-linux-stable-int: - WASM_BUILD_NO_COLOR=1 time cargo test -p staging-node-cli --release --locked -- --ignored +test-linux-subsystem-bench: + stage: test + extends: + - .docker-env + - .common-refs + - .run-immediately + - .pipeline-stopper-artifacts + variables: + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-C debug-assertions -D warnings" + RUST_BACKTRACE: 1 + WASM_BUILD_NO_COLOR: 1 + WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings" + # Ensure we run the UI tests. + RUN_UI_TESTS: 1 + script: + - WASM_BUILD_NO_COLOR=1 + time cargo run -p polkadot-subsystem-bench --release -- test-sequence --path polkadot/node/subsystem-bench/examples/approvals_throughput.yaml + # more information about this job can be found here: # https://github.com/paritytech/substrate/pull/6916 check-tracing: diff --git a/Cargo.lock b/Cargo.lock index aff3af9193d6..bed77ee5f755 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -125,14 +125,15 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" dependencies = [ "cfg-if", "getrandom 0.2.10", "once_cell", "version_check", + "zerocopy", ] [[package]] @@ -190,7 +191,7 @@ checksum = "c0391754c09fab4eae3404d19d0d297aa1c670c1775ab51d8a5312afeca23157" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -205,7 +206,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", "syn-solidity", "tiny-keccak", ] @@ -1226,7 +1227,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -1243,7 +1244,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -1311,7 +1312,7 @@ dependencies = [ "cfg-if", "libc", "miniz_oxide", - "object 0.32.0", + "object 0.32.2", "rustc-demangle", ] @@ -1425,7 +1426,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -2638,9 +2639,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.11" +version = "4.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +checksum = "52bdc885e4cacc7f7c9eedc1ef6da641603180c783c41a15c264944deeaab642" dependencies = [ "clap_builder", "clap_derive 4.4.7", @@ -2657,9 +2658,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.11" +version = "4.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9" dependencies = [ "anstream", "anstyle", @@ -2674,7 +2675,7 @@ version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "586a385f7ef2f8b4d86bddaa0c094794e7ccbfe5ffef1f434fe928143fc783a5" dependencies = [ - "clap 4.4.11", + "clap 4.4.13", ] [[package]] @@ -2699,7 +2700,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -3412,7 +3413,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.4.11", + "clap 4.4.13", "criterion-plot", "futures", "is-terminal", @@ -3575,7 +3576,7 @@ dependencies = [ name = "cumulus-client-cli" version = "0.1.0" dependencies = [ - "clap 4.4.11", + "clap 4.4.13", "parity-scale-codec", "sc-chain-spec", "sc-cli", @@ -3625,9 +3626,9 @@ dependencies = [ "cumulus-client-collator", "cumulus-client-consensus-common", "cumulus-client-consensus-proposer", + "cumulus-client-parachain-inherent", "cumulus-primitives-aura", "cumulus-primitives-core", - "cumulus-primitives-parachain-inherent", "cumulus-relay-chain-interface", "futures", "parity-scale-codec", @@ -3759,6 +3760,29 @@ dependencies = [ "url", ] +[[package]] +name = "cumulus-client-parachain-inherent" +version = "0.1.0" +dependencies = [ + "async-trait", + "cumulus-primitives-core", + "cumulus-primitives-parachain-inherent", + "cumulus-relay-chain-interface", + "cumulus-test-relay-sproof-builder", + "parity-scale-codec", + "sc-client-api", + "scale-info", + "sp-api", + "sp-core", + "sp-inherents", + "sp-runtime", + "sp-state-machine", + "sp-std 8.0.0", + "sp-storage 13.0.0", + "sp-trie", + "tracing", +] + [[package]] name = "cumulus-client-pov-recovery" version = "0.1.0" @@ -3909,7 +3933,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -4032,20 +4056,14 @@ version = "0.1.0" dependencies = [ "async-trait", "cumulus-primitives-core", - "cumulus-relay-chain-interface", - "cumulus-test-relay-sproof-builder", "parity-scale-codec", - "sc-client-api", "scale-info", - "sp-api", "sp-core", "sp-inherents", "sp-runtime", "sp-state-machine", "sp-std 8.0.0", - "sp-storage 13.0.0", "sp-trie", - "tracing", ] [[package]] @@ -4297,16 +4315,16 @@ name = "cumulus-test-service" version = "0.1.0" dependencies = [ "async-trait", - "clap 4.4.11", + "clap 4.4.13", "criterion 0.5.1", "cumulus-client-cli", "cumulus-client-consensus-common", "cumulus-client-consensus-relay-chain", + "cumulus-client-parachain-inherent", "cumulus-client-pov-recovery", "cumulus-client-service", "cumulus-pallet-parachain-system", "cumulus-primitives-core", - "cumulus-primitives-parachain-inherent", "cumulus-relay-chain-inprocess-interface", "cumulus-relay-chain-interface", "cumulus-relay-chain-minimal-node", @@ -4420,7 +4438,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -4460,7 +4478,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -4477,7 +4495,7 @@ checksum = "50c49547d73ba8dcfd4ad7325d64c6d5391ff4224d498fc39a6f3f49825a530d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -4487,7 +4505,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd72493923899c6f10c641bdbdeddc7183d6396641d99c1a0d1597f37f92e28" dependencies = [ "cfg-if", - "hashbrown 0.14.0", + "hashbrown 0.14.3", "lock_api", "once_cell", "parking_lot_core 0.9.8", @@ -4685,7 +4703,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -4746,7 +4764,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.41", + "syn 2.0.48", "termcolor", "toml 0.7.8", "walkdir", @@ -4864,7 +4882,7 @@ checksum = "7d9ce6874da5d4415896cd45ffbc4d1cfc0c4f9c079427bd870742c30f2f65a9" dependencies = [ "curve25519-dalek 4.1.1", "ed25519", - "hashbrown 0.14.0", + "hashbrown 0.14.3", "hex", "rand_core 0.6.4", "sha2 0.10.7", @@ -4974,7 +4992,7 @@ checksum = "5e9a1f9f7d83e59740248a6e14ecf93929ade55027844dfcea78beafccc15745" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -4985,7 +5003,7 @@ checksum = "c2ad8cef1d801a4686bfd8919f0b30eac4c8e48968c437a6405ded4fb5272d2b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -5153,7 +5171,7 @@ dependencies = [ "fs-err", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -5178,6 +5196,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + [[package]] name = "fastrand" version = "1.9.0" @@ -5475,7 +5499,7 @@ dependencies = [ "Inflector", "array-bytes 6.1.0", "chrono", - "clap 4.4.11", + "clap 4.4.13", "comfy-table", "frame-benchmarking", "frame-support", @@ -5541,7 +5565,7 @@ dependencies = [ "quote", "scale-info", "sp-arithmetic", - "syn 2.0.41", + "syn 2.0.48", "trybuild", ] @@ -5567,7 +5591,7 @@ dependencies = [ name = "frame-election-solution-type-fuzzer" version = "2.0.0-alpha.5" dependencies = [ - "clap 4.4.11", + "clap 4.4.13", "frame-election-provider-solution-type", "frame-election-provider-support", "frame-support", @@ -5694,7 +5718,7 @@ dependencies = [ "quote", "regex", "sp-core-hashing", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -5705,7 +5729,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -5714,7 +5738,7 @@ version = "3.0.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -5947,7 +5971,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -6107,7 +6131,7 @@ version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" dependencies = [ - "fallible-iterator", + "fallible-iterator 0.2.0", "indexmap 1.9.3", "stable_deref_trait", ] @@ -6117,6 +6141,10 @@ name = "gimli" version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +dependencies = [ + "fallible-iterator 0.3.0", + "stable_deref_trait", +] [[package]] name = "glob" @@ -6262,16 +6290,16 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.7", ] [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.7", "allocator-api2", "serde", ] @@ -6282,7 +6310,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown 0.14.0", + "hashbrown 0.14.3", ] [[package]] @@ -6642,7 +6670,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.14.3", ] [[package]] @@ -7020,6 +7048,8 @@ dependencies = [ "pallet-babe", "pallet-bags-list", "pallet-balances", + "pallet-beefy", + "pallet-beefy-mmr", "pallet-bounties", "pallet-broker", "pallet-child-bounties", @@ -7093,6 +7123,7 @@ dependencies = [ "sp-authority-discovery", "sp-block-builder", "sp-consensus-babe", + "sp-consensus-beefy", "sp-consensus-grandpa", "sp-core", "sp-genesis-builder", @@ -7165,15 +7196,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "layout-rs" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1164ef87cb9607c2d887216eca79f0fc92895affe1789bba805dd38d829584e0" -dependencies = [ - "log", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -7884,7 +7906,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -7898,7 +7920,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -7909,7 +7931,7 @@ checksum = "9ea73aa640dc01d62a590d48c0c3521ed739d53b27f919b25c3551e233481654" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -7920,7 +7942,7 @@ checksum = "ef9d79ae96aaba821963320eb2b6e34d17df1e5a83d8a1985c29cc5be59577b3" dependencies = [ "macro_magic_core", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -8085,7 +8107,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" name = "minimal-node" version = "4.0.0-dev" dependencies = [ - "clap 4.4.11", + "clap 4.4.13", "frame", "futures", "futures-timer", @@ -8549,7 +8571,7 @@ name = "node-bench" version = "0.9.0-dev" dependencies = [ "array-bytes 6.1.0", - "clap 4.4.11", + "clap 4.4.13", "derive_more", "fs_extra", "futures", @@ -8600,6 +8622,8 @@ dependencies = [ "sc-client-api", "sc-consensus-babe", "sc-consensus-babe-rpc", + "sc-consensus-beefy", + "sc-consensus-beefy-rpc", "sc-consensus-grandpa", "sc-consensus-grandpa-rpc", "sc-mixnet", @@ -8624,7 +8648,7 @@ dependencies = [ name = "node-runtime-generate-bags" version = "3.0.0" dependencies = [ - "clap 4.4.11", + "clap 4.4.13", "generate-bags", "kitchensink-runtime", ] @@ -8633,7 +8657,7 @@ dependencies = [ name = "node-template" version = "4.0.0-dev" dependencies = [ - "clap 4.4.11", + "clap 4.4.13", "frame-benchmarking", "frame-benchmarking-cli", "frame-system", @@ -8677,7 +8701,7 @@ dependencies = [ name = "node-template-release" version = "3.0.0" dependencies = [ - "clap 4.4.11", + "clap 4.4.13", "flate2", "fs_extra", "glob", @@ -8908,9 +8932,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.0" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] @@ -8983,12 +9007,9 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d035b1f968d91a826f2e34a9d6d02cb2af5aa7ca39ebd27922d850ab4b2dd2c6" dependencies = [ - "anyhow", "expander 2.0.0", - "fs-err", "indexmap 2.0.0", "itertools 0.11.0", - "layout-rs", "petgraph", "proc-macro-crate 1.3.1", "proc-macro2", @@ -9601,9 +9622,9 @@ name = "pallet-contracts-fixtures" version = "1.0.0" dependencies = [ "anyhow", - "cfg-if", "frame-system", "parity-wasm", + "polkavm-linker", "sp-runtime", "tempfile", "toml 0.8.2", @@ -9611,10 +9632,6 @@ dependencies = [ "wat", ] -[[package]] -name = "pallet-contracts-fixtures-common" -version = "1.0.0" - [[package]] name = "pallet-contracts-mock-network" version = "1.0.0" @@ -9659,7 +9676,7 @@ version = "4.0.0-dev" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -9669,6 +9686,7 @@ dependencies = [ "bitflags 1.3.2", "parity-scale-codec", "paste", + "polkavm-derive", "scale-info", ] @@ -10819,7 +10837,7 @@ dependencies = [ "proc-macro2", "quote", "sp-runtime", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -11225,7 +11243,7 @@ dependencies = [ name = "parachain-template-node" version = "0.1.0" dependencies = [ - "clap 4.4.11", + "clap 4.4.13", "color-print", "cumulus-client-cli", "cumulus-client-collator", @@ -11353,6 +11371,7 @@ dependencies = [ "pallet-balances", "pallet-collator-selection", "pallet-message-queue", + "pallet-xcm", "parity-scale-codec", "polkadot-core-primitives", "polkadot-primitives", @@ -11367,6 +11386,7 @@ dependencies = [ "staging-parachain-info", "staging-xcm", "staging-xcm-builder", + "staging-xcm-executor", "substrate-wasm-builder", "westend-runtime-constants", ] @@ -11676,6 +11696,222 @@ dependencies = [ "substrate-wasm-builder", ] +[[package]] +name = "people-rococo-emulated-chain" +version = "0.1.0" +dependencies = [ + "cumulus-primitives-core", + "emulated-integration-tests-common", + "frame-support", + "parachains-common", + "people-rococo-runtime", + "rococo-emulated-chain", + "serde_json", + "sp-core", + "sp-runtime", +] + +[[package]] +name = "people-rococo-integration-tests" +version = "0.1.0" +dependencies = [ + "assert_matches", + "asset-test-utils", + "emulated-integration-tests-common", + "frame-support", + "pallet-asset-conversion", + "pallet-assets", + "pallet-balances", + "pallet-identity", + "pallet-message-queue", + "pallet-xcm", + "parachains-common", + "parity-scale-codec", + "penpal-runtime", + "people-rococo-runtime", + "polkadot-primitives", + "polkadot-runtime-common", + "rococo-runtime", + "rococo-runtime-constants", + "rococo-system-emulated-network", + "sp-runtime", + "staging-xcm", + "staging-xcm-executor", +] + +[[package]] +name = "people-rococo-runtime" +version = "0.1.0" +dependencies = [ + "cumulus-pallet-aura-ext", + "cumulus-pallet-dmp-queue", + "cumulus-pallet-parachain-system", + "cumulus-pallet-session-benchmarking", + "cumulus-pallet-xcm", + "cumulus-pallet-xcmp-queue", + "cumulus-primitives-core", + "cumulus-primitives-utility", + "enumflags2", + "frame-benchmarking", + "frame-executive", + "frame-support", + "frame-system", + "frame-system-benchmarking", + "frame-system-rpc-runtime-api", + "frame-try-runtime", + "hex-literal", + "log", + "pallet-aura", + "pallet-authorship", + "pallet-balances", + "pallet-collator-selection", + "pallet-identity", + "pallet-message-queue", + "pallet-multisig", + "pallet-session", + "pallet-timestamp", + "pallet-transaction-payment", + "pallet-transaction-payment-rpc-runtime-api", + "pallet-utility", + "pallet-xcm", + "pallet-xcm-benchmarks", + "parachains-common", + "parity-scale-codec", + "polkadot-core-primitives", + "polkadot-parachain-primitives", + "polkadot-runtime-common", + "rococo-runtime-constants", + "scale-info", + "serde", + "smallvec", + "sp-api", + "sp-block-builder", + "sp-consensus-aura", + "sp-core", + "sp-genesis-builder", + "sp-inherents", + "sp-offchain", + "sp-runtime", + "sp-session", + "sp-std 8.0.0", + "sp-storage 13.0.0", + "sp-transaction-pool", + "sp-version", + "staging-parachain-info", + "staging-xcm", + "staging-xcm-builder", + "staging-xcm-executor", + "substrate-wasm-builder", +] + +[[package]] +name = "people-westend-emulated-chain" +version = "0.1.0" +dependencies = [ + "cumulus-primitives-core", + "emulated-integration-tests-common", + "frame-support", + "parachains-common", + "people-westend-runtime", + "serde_json", + "sp-core", + "sp-runtime", + "westend-emulated-chain", +] + +[[package]] +name = "people-westend-integration-tests" +version = "0.1.0" +dependencies = [ + "assert_matches", + "asset-test-utils", + "emulated-integration-tests-common", + "frame-support", + "pallet-asset-conversion", + "pallet-assets", + "pallet-balances", + "pallet-identity", + "pallet-message-queue", + "pallet-xcm", + "parachains-common", + "parity-scale-codec", + "penpal-runtime", + "people-westend-runtime", + "polkadot-primitives", + "polkadot-runtime-common", + "sp-runtime", + "staging-xcm", + "staging-xcm-executor", + "westend-runtime", + "westend-runtime-constants", + "westend-system-emulated-network", +] + +[[package]] +name = "people-westend-runtime" +version = "0.1.0" +dependencies = [ + "cumulus-pallet-aura-ext", + "cumulus-pallet-dmp-queue", + "cumulus-pallet-parachain-system", + "cumulus-pallet-session-benchmarking", + "cumulus-pallet-xcm", + "cumulus-pallet-xcmp-queue", + "cumulus-primitives-core", + "cumulus-primitives-utility", + "enumflags2", + "frame-benchmarking", + "frame-executive", + "frame-support", + "frame-system", + "frame-system-benchmarking", + "frame-system-rpc-runtime-api", + "frame-try-runtime", + "hex-literal", + "log", + "pallet-aura", + "pallet-authorship", + "pallet-balances", + "pallet-collator-selection", + "pallet-identity", + "pallet-message-queue", + "pallet-multisig", + "pallet-session", + "pallet-timestamp", + "pallet-transaction-payment", + "pallet-transaction-payment-rpc-runtime-api", + "pallet-utility", + "pallet-xcm", + "pallet-xcm-benchmarks", + "parachains-common", + "parity-scale-codec", + "polkadot-core-primitives", + "polkadot-parachain-primitives", + "polkadot-runtime-common", + "scale-info", + "serde", + "smallvec", + "sp-api", + "sp-block-builder", + "sp-consensus-aura", + "sp-core", + "sp-genesis-builder", + "sp-inherents", + "sp-offchain", + "sp-runtime", + "sp-session", + "sp-std 8.0.0", + "sp-storage 13.0.0", + "sp-transaction-pool", + "sp-version", + "staging-parachain-info", + "staging-xcm", + "staging-xcm-builder", + "staging-xcm-executor", + "substrate-wasm-builder", + "westend-runtime-constants", +] + [[package]] name = "percent-encoding" version = "2.3.0" @@ -11712,7 +11948,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -11753,7 +11989,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -11966,7 +12202,7 @@ name = "polkadot-cli" version = "1.1.0" dependencies = [ "cfg-if", - "clap 4.4.11", + "clap 4.4.13", "frame-benchmarking-cli", "futures", "log", @@ -12807,7 +13043,7 @@ dependencies = [ "async-trait", "bridge-hub-rococo-runtime", "bridge-hub-westend-runtime", - "clap 4.4.11", + "clap 4.4.13", "collectives-westend-runtime", "color-print", "contracts-rococo-runtime", @@ -12819,10 +13055,10 @@ dependencies = [ "cumulus-client-consensus-common", "cumulus-client-consensus-proposer", "cumulus-client-consensus-relay-chain", + "cumulus-client-parachain-inherent", "cumulus-client-service", "cumulus-primitives-aura", "cumulus-primitives-core", - "cumulus-primitives-parachain-inherent", "cumulus-relay-chain-interface", "frame-benchmarking", "frame-benchmarking-cli", @@ -12841,6 +13077,8 @@ dependencies = [ "parachains-common", "parity-scale-codec", "penpal-runtime", + "people-rococo-runtime", + "people-westend-runtime", "polkadot-cli", "polkadot-primitives", "polkadot-service", @@ -13316,7 +13554,7 @@ dependencies = [ "assert_matches", "async-trait", "bincode", - "clap 4.4.11", + "clap 4.4.13", "clap-num", "color-eyre", "colored", @@ -13405,7 +13643,7 @@ version = "1.0.0" dependencies = [ "assert_matches", "async-trait", - "clap 4.4.11", + "clap 4.4.13", "color-eyre", "futures", "futures-timer", @@ -13552,12 +13790,60 @@ dependencies = [ name = "polkadot-voter-bags" version = "1.0.0" dependencies = [ - "clap 4.4.11", + "clap 4.4.13", "generate-bags", "sp-io", "westend-runtime", ] +[[package]] +name = "polkavm-common" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01363cf0a778e8d93eff31e8a03bc59992cba35faa419ea4f3e80146b69195ba" + +[[package]] +name = "polkavm-common" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88e869d66a254db6c7069992f240626416aba8e87d65c00e4be443135babfe82" + +[[package]] +name = "polkavm-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26501292b2cb980cbeaac3304f0fc4480ff1bac2473045453d7333d775658b6a" +dependencies = [ + "polkavm-derive-impl", + "syn 2.0.48", +] + +[[package]] +name = "polkavm-derive-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "903e16ad3ed768f35e6f40acff2e8aaf6afb9f2889b0a8982dd43dcbee29db2d" +dependencies = [ + "polkavm-common 0.2.0", + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "polkavm-linker" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f8719d37effca6df1cecf5c816d84ab09b7d18e960511f61c254a7581fa50c3" +dependencies = [ + "gimli 0.28.0", + "hashbrown 0.14.3", + "log", + "object 0.32.2", + "polkavm-common 0.3.0", + "rustc-demangle", +] + [[package]] name = "polling" version = "2.8.0" @@ -13730,7 +14016,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" dependencies = [ "proc-macro2", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -13822,14 +14108,14 @@ checksum = "9b698b0b09d40e9b7c1a47b132d66a8b54bcd20583d9b6d06e4535e383b4405c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] @@ -13894,7 +14180,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -14092,9 +14378,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -14312,7 +14598,7 @@ checksum = "7f7473c2cfcf90008193dd0e3e16599455cb601a9fce322b5bb55de799664925" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -14381,7 +14667,7 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" name = "remote-ext-tests-bags-list" version = "1.0.0" dependencies = [ - "clap 4.4.11", + "clap 4.4.13", "frame-system", "log", "pallet-bags-list-remote-tests", @@ -14713,6 +14999,7 @@ dependencies = [ "bridge-hub-rococo-emulated-chain", "emulated-integration-tests-common", "penpal-emulated-chain", + "people-rococo-emulated-chain", "rococo-emulated-chain", ] @@ -15146,7 +15433,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -15156,7 +15443,7 @@ dependencies = [ "array-bytes 6.1.0", "bip39", "chrono", - "clap 4.4.11", + "clap 4.4.13", "fdlimit", "futures", "futures-timer", @@ -15465,7 +15752,7 @@ dependencies = [ name = "sc-consensus-grandpa" version = "0.10.0-dev" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.7", "array-bytes 6.1.0", "assert_matches", "async-trait", @@ -15849,7 +16136,7 @@ dependencies = [ name = "sc-network-gossip" version = "0.10.0-dev" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.7", "async-trait", "futures", "futures-timer", @@ -16300,7 +16587,7 @@ dependencies = [ name = "sc-storage-monitor" version = "0.1.0" dependencies = [ - "clap 4.4.11", + "clap 4.4.13", "fs4", "log", "sc-client-db", @@ -16402,7 +16689,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -16534,7 +16821,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "772575a524feeb803e5b0fcbc6dd9f367e579488197c94c6e4023aad2305774d" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.7", "cfg-if", "hashbrown 0.13.2", ] @@ -16787,9 +17074,9 @@ checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" [[package]] name = "serde" -version = "1.0.193" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] @@ -16814,13 +17101,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -16845,9 +17132,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -16910,7 +17197,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -17188,7 +17475,7 @@ dependencies = [ "fnv", "futures-lite", "futures-util", - "hashbrown 0.14.0", + "hashbrown 0.14.3", "hex", "hmac 0.12.1", "itertools 0.11.0", @@ -17237,7 +17524,7 @@ dependencies = [ "futures-channel", "futures-lite", "futures-util", - "hashbrown 0.14.0", + "hashbrown 0.14.3", "hex", "itertools 0.11.0", "log", @@ -17700,7 +17987,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -18035,7 +18322,7 @@ version = "9.0.0" dependencies = [ "quote", "sp-core-hashing", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -18093,7 +18380,7 @@ version = "8.0.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -18103,7 +18390,7 @@ source = "git+https://github.com/paritytech/polkadot-sdk#82912acb33a9030c0ef3bf5 dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -18262,7 +18549,7 @@ dependencies = [ name = "sp-npos-elections-fuzzer" version = "2.0.0-alpha.5" dependencies = [ - "clap 4.4.11", + "clap 4.4.13", "honggfuzz", "rand 0.8.5", "sp-npos-elections", @@ -18376,7 +18663,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -18388,7 +18675,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -18611,7 +18898,7 @@ dependencies = [ name = "sp-trie" version = "22.0.0" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.7", "array-bytes 6.1.0", "criterion 0.4.0", "hash-db", @@ -18659,7 +18946,7 @@ dependencies = [ "proc-macro2", "quote", "sp-version", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -18783,7 +19070,7 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" name = "staging-chain-spec-builder" version = "2.0.0" dependencies = [ - "clap 4.4.11", + "clap 4.4.13", "log", "sc-chain-spec", "serde_json", @@ -18796,7 +19083,7 @@ version = "3.0.0-dev" dependencies = [ "array-bytes 6.1.0", "assert_cmd", - "clap 4.4.11", + "clap 4.4.13", "clap_complete", "criterion 0.4.0", "frame-benchmarking", @@ -18808,6 +19095,7 @@ dependencies = [ "jsonrpsee", "kitchensink-runtime", "log", + "mmr-gadget", "nix 0.26.2", "node-primitives", "node-rpc", @@ -18838,6 +19126,7 @@ dependencies = [ "sc-client-db", "sc-consensus", "sc-consensus-babe", + "sc-consensus-beefy", "sc-consensus-epochs", "sc-consensus-grandpa", "sc-consensus-slots", @@ -18869,6 +19158,7 @@ dependencies = [ "sp-blockchain", "sp-consensus", "sp-consensus-babe", + "sp-consensus-beefy", "sp-consensus-grandpa", "sp-core", "sp-externalities 0.19.0", @@ -18877,6 +19167,7 @@ dependencies = [ "sp-keyring", "sp-keystore", "sp-mixnet", + "sp-mmr-primitives", "sp-runtime", "sp-state-machine", "sp-statement-store", @@ -18901,7 +19192,7 @@ dependencies = [ name = "staging-node-inspect" version = "0.9.0-dev" dependencies = [ - "clap 4.4.11", + "clap 4.4.13", "parity-scale-codec", "sc-cli", "sc-client-api", @@ -18935,6 +19226,7 @@ version = "1.0.0" name = "staging-xcm" version = "1.0.0" dependencies = [ + "array-bytes 6.1.0", "bounded-collections", "derivative", "environmental", @@ -19105,7 +19397,7 @@ dependencies = [ name = "subkey" version = "3.0.0" dependencies = [ - "clap 4.4.11", + "clap 4.4.13", "sc-cli", ] @@ -19147,7 +19439,7 @@ dependencies = [ name = "substrate-frame-cli" version = "4.0.0-dev" dependencies = [ - "clap 4.4.11", + "clap 4.4.13", "frame-support", "frame-system", "sc-cli", @@ -19493,9 +19785,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.41" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -19511,7 +19803,7 @@ dependencies = [ "paste", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -19625,7 +19917,7 @@ dependencies = [ name = "test-parachain-adder-collator" version = "1.0.0" dependencies = [ - "clap 4.4.11", + "clap 4.4.13", "futures", "futures-timer", "log", @@ -19673,7 +19965,7 @@ dependencies = [ name = "test-parachain-undying-collator" version = "1.0.0" dependencies = [ - "clap 4.4.11", + "clap 4.4.13", "futures", "futures-timer", "log", @@ -19762,7 +20054,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -19923,7 +20215,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -20129,7 +20421,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -20171,7 +20463,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -20324,7 +20616,7 @@ version = "0.10.0-dev" dependencies = [ "assert_cmd", "async-trait", - "clap 4.4.11", + "clap 4.4.13", "frame-remote-externalities", "frame-try-runtime", "hex", @@ -20701,7 +20993,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", "wasm-bindgen-shared", ] @@ -20735,7 +21027,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -21317,6 +21609,7 @@ dependencies = [ "collectives-westend-emulated-chain", "emulated-integration-tests-common", "penpal-emulated-chain", + "people-westend-emulated-chain", "westend-emulated-chain", ] @@ -21702,7 +21995,7 @@ dependencies = [ "proc-macro2", "quote", "staging-xcm", - "syn 2.0.41", + "syn 2.0.48", "trybuild", ] @@ -21805,6 +22098,26 @@ dependencies = [ "time", ] +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "zeroize" version = "1.6.0" @@ -21822,7 +22135,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 01982656d145..fbba473bbce6 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -32,7 +32,10 @@ use crate::{ core::{ configuration::{TestAuthorities, TestConfiguration}, environment::{TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT}, - mock::{dummy_builder, AlwaysSupportsParachains, MockNetworkBridgeTx, TestSyncOracle}, + mock::{ + dummy_builder, network_bridge::MockNetworkBridgeTx, AlwaysSupportsParachains, + TestSyncOracle, + }, network::{NetworkAction, NetworkEmulator}, }, }; From 272280bcfa059c83a26447137326c0ff5c0370da Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 8 Jan 2024 10:30:02 +0200 Subject: [PATCH 163/192] Add throughput scenarios Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/test.yml | 9 ++++- .../examples/approvals_throughput.yaml | 2 +- .../approvals_throughput_12_seconds.yaml | 31 ++++++++++++++ ...hroughput_12_seconds_no_optimisations.yaml | 31 ++++++++++++++ ...s_throughput_no_optimisations_enabled.yaml | 31 ++++++++++++++ .../node/subsystem-bench/src/approval/mod.rs | 40 +++++++++---------- 6 files changed, 121 insertions(+), 23 deletions(-) create mode 100644 polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml create mode 100644 polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml create mode 100644 polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 77dbf2c997de..d05e470a1e46 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -420,8 +420,13 @@ test-linux-subsystem-bench: # Ensure we run the UI tests. RUN_UI_TESTS: 1 script: - - WASM_BUILD_NO_COLOR=1 - time cargo run -p polkadot-subsystem-bench --release -- test-sequence --path polkadot/node/subsystem-bench/examples/approvals_throughput.yaml + - nproc + - cat /proc/cpuinfo + - cat /proc/meminfo + - time cargo run -p polkadot-subsystem-bench --release -- test-sequence --path polkadot/node/subsystem-bench/examples/approvals_throughput.yaml + - time cargo run -p polkadot-subsystem-bench --release -- test-sequence --path polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml + - time cargo run -p polkadot-subsystem-bench --release -- test-sequence --path polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml + - time cargo run -p polkadot-subsystem-bench --release -- test-sequence --path polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml # more information about this job can be found here: # https://github.com/paritytech/substrate/pull/6916 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml index f086ac5eff57..d2dabad316d1 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml @@ -5,7 +5,7 @@ TestConfiguration: min_coalesce: 1 max_coalesce: 6 enable_assignments_v2: true - send_till_tranche: 60 + send_till_tranche: 89 stop_when_approved: false coalesce_tranche_diff: 12 workdir_prefix: "/tmp/" diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml new file mode 100644 index 000000000000..f8f8aacb3320 --- /dev/null +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml @@ -0,0 +1,31 @@ +TestConfiguration: +# Test 1 +- objective: !ApprovalsTest + last_considered_tranche: 89 + min_coalesce: 1 + max_coalesce: 6 + enable_assignments_v2: true + send_till_tranche: 89 + stop_when_approved: false + coalesce_tranche_diff: 12 + workdir_prefix: "/tmp/" + num_no_shows_per_candidate: 0 + approval_distribution_expected_tof: 6.0 + approval_distribution_cpu_ms: 3.0 + approval_voting_cpu_ms: 4.30 + n_validators: 500 + n_cores: 100 + n_included_candidates: 50 + min_pov_size: 1120 + max_pov_size: 5120 + peer_bandwidth: 524288000000 + bandwidth: 524288000000 + latency: + min_latency: + secs: 0 + nanos: 1000000 + max_latency: + secs: 0 + nanos: 100000000 + error: 0 + num_blocks: 10 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml new file mode 100644 index 000000000000..7fb68425ee5d --- /dev/null +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml @@ -0,0 +1,31 @@ +TestConfiguration: +# Test 1 +- objective: !ApprovalsTest + last_considered_tranche: 89 + min_coalesce: 1 + max_coalesce: 1 + enable_assignments_v2: false + send_till_tranche: 89 + stop_when_approved: false + coalesce_tranche_diff: 12 + workdir_prefix: "/tmp/" + num_no_shows_per_candidate: 0 + approval_distribution_expected_tof: 6.0 + approval_distribution_cpu_ms: 3.0 + approval_voting_cpu_ms: 4.30 + n_validators: 500 + n_cores: 100 + n_included_candidates: 50 + min_pov_size: 1120 + max_pov_size: 5120 + peer_bandwidth: 524288000000 + bandwidth: 524288000000 + latency: + min_latency: + secs: 0 + nanos: 1000000 + max_latency: + secs: 0 + nanos: 100000000 + error: 0 + num_blocks: 10 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml new file mode 100644 index 000000000000..cc7f538c3b68 --- /dev/null +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml @@ -0,0 +1,31 @@ +TestConfiguration: +# Test 1 +- objective: !ApprovalsTest + last_considered_tranche: 89 + min_coalesce: 1 + max_coalesce: 1 + enable_assignments_v2: false + send_till_tranche: 89 + stop_when_approved: false + coalesce_tranche_diff: 12 + workdir_prefix: "/tmp/" + num_no_shows_per_candidate: 0 + approval_distribution_expected_tof: 6.0 + approval_distribution_cpu_ms: 3.0 + approval_voting_cpu_ms: 4.30 + n_validators: 500 + n_cores: 100 + n_included_candidates: 100 + min_pov_size: 1120 + max_pov_size: 5120 + peer_bandwidth: 524288000000 + bandwidth: 524288000000 + latency: + min_latency: + secs: 0 + nanos: 1000000 + max_latency: + secs: 0 + nanos: 100000000 + error: 0 + num_blocks: 10 diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index fbba473bbce6..48726fdc3eda 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -939,24 +939,24 @@ pub async fn bench_approvals_run( ); gum::info!("{}", &env); - assert!(env.bucket_metric_lower_than( - "polkadot_parachain_subsystem_bounded_tof_bucket", - "subsystem_name", - "approval-distribution-subsystem", - state.options.approval_distribution_expected_tof - )); - - assert!(env.metric_with_label_lower_than( - "substrate_tasks_polling_duration_sum", - "task_group", - "approval-voting", - state.options.approval_voting_cpu_ms * state.blocks.len() as f64 - )); - - assert!(env.metric_with_label_lower_than( - "substrate_tasks_polling_duration_sum", - "task_group", - "approval-distribution", - state.options.approval_distribution_cpu_ms * state.blocks.len() as f64 - )); + // assert!(env.bucket_metric_lower_than( + // "polkadot_parachain_subsystem_bounded_tof_bucket", + // "subsystem_name", + // "approval-distribution-subsystem", + // state.options.approval_distribution_expected_tof + // )); + + // assert!(env.metric_with_label_lower_than( + // "substrate_tasks_polling_duration_sum", + // "task_group", + // "approval-voting", + // state.options.approval_voting_cpu_ms * state.blocks.len() as f64 + // )); + + // assert!(env.metric_with_label_lower_than( + // "substrate_tasks_polling_duration_sum", + // "task_group", + // "approval-distribution", + // state.options.approval_distribution_cpu_ms * state.blocks.len() as f64 + // )); } From 794a9b8b128ebb2a8a6c12d330af56c4eb91fbd7 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 8 Jan 2024 10:40:54 +0200 Subject: [PATCH 164/192] Fix clippy unused Signed-off-by: Alexandru Gheorghe --- .../node/subsystem-bench/src/approval/mod.rs | 43 ++++++++++--------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 48726fdc3eda..918a6a424a0b 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -939,24 +939,27 @@ pub async fn bench_approvals_run( ); gum::info!("{}", &env); - // assert!(env.bucket_metric_lower_than( - // "polkadot_parachain_subsystem_bounded_tof_bucket", - // "subsystem_name", - // "approval-distribution-subsystem", - // state.options.approval_distribution_expected_tof - // )); - - // assert!(env.metric_with_label_lower_than( - // "substrate_tasks_polling_duration_sum", - // "task_group", - // "approval-voting", - // state.options.approval_voting_cpu_ms * state.blocks.len() as f64 - // )); - - // assert!(env.metric_with_label_lower_than( - // "substrate_tasks_polling_duration_sum", - // "task_group", - // "approval-distribution", - // state.options.approval_distribution_cpu_ms * state.blocks.len() as f64 - // )); + assert!(env.bucket_metric_lower_than( + "polkadot_parachain_subsystem_bounded_tof_bucket", + "subsystem_name", + "approval-distribution-subsystem", + // state.options.approval_distribution_expected_tof + 12000.0 + )); + + assert!(env.metric_with_label_lower_than( + "substrate_tasks_polling_duration_sum", + "task_group", + "approval-voting", + //state.options.approval_voting_cpu_ms * state.blocks.len() as f64 + 12000.0 + )); + + assert!(env.metric_with_label_lower_than( + "substrate_tasks_polling_duration_sum", + "task_group", + "approval-distribution", + //state.options.approval_distribution_cpu_ms * state.blocks.len() as f64 + 12000.00 + )); } From 34c60baa2eb5eb735067717f47a410c192be65bb Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 8 Jan 2024 11:23:42 +0200 Subject: [PATCH 165/192] Retrigger CI From 4c9205e9ec5c6d4affee86d0455dc51e45f0be2d Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 9 Jan 2024 12:53:39 +0200 Subject: [PATCH 166/192] Add benchamarks Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/test.yml | 2 ++ .../examples/approvals_no_shows.yaml | 31 ++++++++++++++++++ .../examples/approvals_throughput.yaml | 6 ++-- .../approvals_throughput_12_seconds.yaml | 6 ++-- ...hroughput_12_seconds_no_optimisations.yaml | 6 ++-- .../approvals_throughput_best_case.yaml | 31 ++++++++++++++++++ ...s_throughput_no_optimisations_enabled.yaml | 6 ++-- .../node/subsystem-bench/src/approval/mod.rs | 32 ++++++++----------- 8 files changed, 90 insertions(+), 30 deletions(-) create mode 100644 polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml create mode 100644 polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index d05e470a1e46..3349ae50f313 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -427,6 +427,8 @@ test-linux-subsystem-bench: - time cargo run -p polkadot-subsystem-bench --release -- test-sequence --path polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml - time cargo run -p polkadot-subsystem-bench --release -- test-sequence --path polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml - time cargo run -p polkadot-subsystem-bench --release -- test-sequence --path polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml + - time cargo run -p polkadot-subsystem-bench --release -- test-sequence --path polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml + - time cargo run -p polkadot-subsystem-bench --release -- test-sequence --path polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml # more information about this job can be found here: # https://github.com/paritytech/substrate/pull/6916 diff --git a/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml b/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml new file mode 100644 index 000000000000..0bb46666555d --- /dev/null +++ b/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml @@ -0,0 +1,31 @@ +TestConfiguration: +# Test 1 +- objective: !ApprovalsTest + last_considered_tranche: 89 + min_coalesce: 1 + max_coalesce: 6 + enable_assignments_v2: true + send_till_tranche: 89 + stop_when_approved: true + coalesce_tranche_diff: 12 + workdir_prefix: "/tmp/" + num_no_shows_per_candidate: 10 + approval_distribution_expected_tof: 60.0 + approval_distribution_cpu_ms: 3.0 + approval_voting_cpu_ms: 4.30 + n_validators: 500 + n_cores: 100 + n_included_candidates: 100 + min_pov_size: 1120 + max_pov_size: 5120 + peer_bandwidth: 524288000000 + bandwidth: 524288000000 + latency: + min_latency: + secs: 0 + nanos: 1000000 + max_latency: + secs: 0 + nanos: 100000000 + error: 0 + num_blocks: 10 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml index d2dabad316d1..60b97c7c4c78 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml @@ -10,9 +10,9 @@ TestConfiguration: coalesce_tranche_diff: 12 workdir_prefix: "/tmp/" num_no_shows_per_candidate: 0 - approval_distribution_expected_tof: 6.0 - approval_distribution_cpu_ms: 3.0 - approval_voting_cpu_ms: 4.30 + approval_distribution_expected_tof: 60.0 + approval_distribution_cpu_ms: 7.70 + approval_voting_cpu_ms: 9.90 n_validators: 500 n_cores: 100 n_included_candidates: 100 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml index f8f8aacb3320..8b3d8da6198a 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml @@ -10,9 +10,9 @@ TestConfiguration: coalesce_tranche_diff: 12 workdir_prefix: "/tmp/" num_no_shows_per_candidate: 0 - approval_distribution_expected_tof: 6.0 - approval_distribution_cpu_ms: 3.0 - approval_voting_cpu_ms: 4.30 + approval_distribution_expected_tof: 60.0 + approval_distribution_cpu_ms: 3.50 + approval_voting_cpu_ms: 5.50 n_validators: 500 n_cores: 100 n_included_candidates: 50 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml index 7fb68425ee5d..5fd62058a94f 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml @@ -10,9 +10,9 @@ TestConfiguration: coalesce_tranche_diff: 12 workdir_prefix: "/tmp/" num_no_shows_per_candidate: 0 - approval_distribution_expected_tof: 6.0 - approval_distribution_cpu_ms: 3.0 - approval_voting_cpu_ms: 4.30 + approval_distribution_expected_tof: 60.0 + approval_distribution_cpu_ms: 4.70 + approval_voting_cpu_ms: 6.30 n_validators: 500 n_cores: 100 n_included_candidates: 50 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml new file mode 100644 index 000000000000..d92609ae9a86 --- /dev/null +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml @@ -0,0 +1,31 @@ +TestConfiguration: +# Test 1 +- objective: !ApprovalsTest + last_considered_tranche: 89 + min_coalesce: 1 + max_coalesce: 6 + enable_assignments_v2: true + send_till_tranche: 89 + stop_when_approved: true + coalesce_tranche_diff: 12 + workdir_prefix: "/tmp/" + num_no_shows_per_candidate: 0 + approval_distribution_expected_tof: 60.0 + approval_distribution_cpu_ms: 1.0 + approval_voting_cpu_ms: 1.0 + n_validators: 500 + n_cores: 100 + n_included_candidates: 100 + min_pov_size: 1120 + max_pov_size: 5120 + peer_bandwidth: 524288000000 + bandwidth: 524288000000 + latency: + min_latency: + secs: 0 + nanos: 1000000 + max_latency: + secs: 0 + nanos: 100000000 + error: 0 + num_blocks: 10 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml index cc7f538c3b68..0526c80f6673 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml @@ -10,9 +10,9 @@ TestConfiguration: coalesce_tranche_diff: 12 workdir_prefix: "/tmp/" num_no_shows_per_candidate: 0 - approval_distribution_expected_tof: 6.0 - approval_distribution_cpu_ms: 3.0 - approval_voting_cpu_ms: 4.30 + approval_distribution_expected_tof: 60.0 + approval_distribution_cpu_ms: 10.50 + approval_voting_cpu_ms: 13.00 n_validators: 500 n_cores: 100 n_included_candidates: 100 diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 918a6a424a0b..08f12841fd62 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -527,12 +527,11 @@ impl PeerMessageProducer { .unwrap_or_default() { let bundle = progressing_iterator.next().unwrap(); - re_process_skipped = re_process_skipped || - self.process_message( - bundle, - &mut per_candidate_data, - &mut skipped_messages, - ); + re_process_skipped = self.process_message( + bundle, + &mut per_candidate_data, + &mut skipped_messages, + ) || re_process_skipped; } sleep(Duration::from_millis(50)).await; } @@ -939,27 +938,24 @@ pub async fn bench_approvals_run( ); gum::info!("{}", &env); - assert!(env.bucket_metric_lower_than( - "polkadot_parachain_subsystem_bounded_tof_bucket", - "subsystem_name", - "approval-distribution-subsystem", - // state.options.approval_distribution_expected_tof - 12000.0 - )); - assert!(env.metric_with_label_lower_than( "substrate_tasks_polling_duration_sum", "task_group", "approval-voting", - //state.options.approval_voting_cpu_ms * state.blocks.len() as f64 - 12000.0 + state.options.approval_voting_cpu_ms * state.blocks.len() as f64 )); assert!(env.metric_with_label_lower_than( "substrate_tasks_polling_duration_sum", "task_group", "approval-distribution", - //state.options.approval_distribution_cpu_ms * state.blocks.len() as f64 - 12000.00 + state.options.approval_distribution_cpu_ms * state.blocks.len() as f64 + )); + + assert!(env.bucket_metric_lower_than( + "polkadot_parachain_subsystem_bounded_tof_bucket", + "subsystem_name", + "approval-distribution-subsystem", + state.options.approval_distribution_expected_tof )); } From 9fbb29d77f14788f9b05a0dde9bd5c114f20de5e Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 10 Jan 2024 11:51:47 +0200 Subject: [PATCH 167/192] Fixup channel size Signed-off-by: Alexandru Gheorghe --- polkadot/node/overseer/src/lib.rs | 2 +- .../node/subsystem-bench/src/approval/mod.rs | 28 +++++++++++-------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/polkadot/node/overseer/src/lib.rs b/polkadot/node/overseer/src/lib.rs index f687cb4f89e0..f4eddf1f41ce 100644 --- a/polkadot/node/overseer/src/lib.rs +++ b/polkadot/node/overseer/src/lib.rs @@ -577,7 +577,7 @@ pub struct Overseer { ])] collator_protocol: CollatorProtocol, - #[subsystem(blocking, message_capacity: 6400000, ApprovalDistributionMessage, sends: [ + #[subsystem(blocking, message_capacity: 64000, ApprovalDistributionMessage, sends: [ NetworkBridgeTxMessage, ApprovalVotingMessage, ])] diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 08f12841fd62..42b7a85d0ce3 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -483,17 +483,23 @@ impl PeerMessageProducer { while all_messages.peek().is_some() { let current_slot = tick_to_slot_number(SLOT_DURATION_MILLIS, system_clock.tick_now()); - let block_info = self.state.get_info_by_slot(current_slot).cloned(); - - if let Some(block_info) = block_info { - if !initialized_blocks.contains(&block_info.hash) && - !TestEnvironment::metric_lower_than( - &self.registry, - "polkadot_parachain_imported_candidates_total", - (block_info.total_candidates_before + - block_info.candidates.len() as u64 - - 1) as f64, - ) { + let block_to_initialize = self + .state + .blocks + .iter() + .filter(|block_info| { + block_info.slot <= current_slot && + !initialized_blocks.contains(&block_info.hash) + }) + .cloned() + .collect_vec(); + for block_info in block_to_initialize { + if !TestEnvironment::metric_lower_than( + &self.registry, + "polkadot_parachain_imported_candidates_total", + (block_info.total_candidates_before + block_info.candidates.len() as u64 - + 1) as f64, + ) { initialized_blocks.insert(block_info.hash); self.initialize_block(&block_info).await; } From 87892fd0dec41c57732f578478d40807875a1901 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 10 Jan 2024 12:12:14 +0200 Subject: [PATCH 168/192] refactor Signed-off-by: Andrei Sandu --- Cargo.lock | 1 + polkadot/node/subsystem-bench/Cargo.toml | 3 + polkadot/node/subsystem-bench/README.md | 40 ++++--- .../src/availability/av_store_helpers.rs | 9 +- .../subsystem-bench/src/availability/mod.rs | 19 ++- .../subsystem-bench/src/core/configuration.rs | 61 ++++------ .../node/subsystem-bench/src/core/display.rs | 2 +- .../subsystem-bench/src/core/environment.rs | 5 +- .../node/subsystem-bench/src/core/keyring.rs | 8 +- .../src/core/mock/network_bridge.rs | 33 ++++-- .../src/core/mock/runtime_api.rs | 3 +- .../node/subsystem-bench/src/core/network.rs | 110 ++++++++++-------- .../subsystem-bench/src/subsystem-bench.rs | 33 +++--- 13 files changed, 170 insertions(+), 157 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be72f4b6d877..4953b5ce82b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13313,6 +13313,7 @@ dependencies = [ "polkadot-primitives-test-helpers", "prometheus", "rand 0.8.5", + "rand_distr", "sc-keystore", "sc-network", "sc-service", diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index 3f4e724dca11..7e8de2213903 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -43,6 +43,9 @@ polkadot-erasure-coding = { package = "polkadot-erasure-coding", path = "../../e log = "0.4.17" env_logger = "0.9.0" rand = "0.8.5" +# `rand` only supports uniform distribution, we need normal distribution for latency. +rand_distr = "0.4.3" + parity-scale-codec = { version = "3.6.1", features = ["std", "derive"] } tokio = "1.24.2" clap-num = "1.0.2" diff --git a/polkadot/node/subsystem-bench/README.md b/polkadot/node/subsystem-bench/README.md index f4ea04662f9e..322f592499e6 100644 --- a/polkadot/node/subsystem-bench/README.md +++ b/polkadot/node/subsystem-bench/README.md @@ -92,20 +92,30 @@ Note: `test-sequence` is a special test objective that wraps up an arbitrary num ``` Options: - --network The type of network to be emulated [default: ideal] [possible values: - ideal, healthy, degraded] - --n-cores Number of cores to fetch availability for [default: 100] - --n-validators Number of validators to fetch chunks from [default: 500] - --min-pov-size The minimum pov size in KiB [default: 5120] - --max-pov-size The maximum pov size bytes [default: 5120] - -n, --num-blocks The number of blocks the test is going to run [default: 1] - -p, --peer-bandwidth The bandwidth of simulated remote peers in KiB - -b, --bandwidth The bandwidth of our simulated node in KiB - --peer-error Simulated conection error ratio [0-100] - --peer-min-latency Minimum remote peer latency in milliseconds [0-5000] - --peer-max-latency Maximum remote peer latency in milliseconds [0-5000] - -h, --help Print help - -V, --version Print version + --network + The type of network to be emulated [default: ideal] [possible values: ideal, healthy, degraded] + --n-cores + Number of cores to fetch availability for [default: 100] + --n-validators + Number of validators to fetch chunks from [default: 500] + --min-pov-size + The minimum pov size in KiB [default: 5120] + --max-pov-size + The maximum pov size bytes [default: 5120] + -n, --num-blocks + The number of blocks the test is going to run [default: 1] + -p, --peer-bandwidth + The bandwidth of emulated remote peers in KiB + -b, --bandwidth + The bandwidth of our node in KiB + --connectivity + Emulated peer connection ratio [0-100] + --peer-mean-latency + Mean remote peer latency in milliseconds [0-5000] + --peer-latency-std-dev + Remote peer latency standard deviation + -h, --help + Print help ``` These apply to all test objectives, except `test-sequence` which relies on the values being specified in a file. @@ -154,7 +164,7 @@ node validator network. ``` target/testnet/subsystem-bench --n-cores 10 data-availability-read [2023-11-28T09:01:59Z INFO subsystem_bench::core::display] n_validators = 500, n_cores = 10, pov_size = 5120 - 5120, - error = 0, latency = None + latency = None [2023-11-28T09:01:59Z INFO subsystem-bench::availability] Generating template candidate index=0 pov_size=5242880 [2023-11-28T09:01:59Z INFO subsystem-bench::availability] Created test environment. [2023-11-28T09:01:59Z INFO subsystem-bench::availability] Pre-generating 10 candidates. diff --git a/polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs b/polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs index 12c86aa7e546..18ea2f72891f 100644 --- a/polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs +++ b/polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs @@ -19,16 +19,9 @@ use super::*; use polkadot_node_metrics::metrics::Metrics; use polkadot_node_core_av_store::Config; -use polkadot_node_subsystem_test_helpers as test_helpers; -use polkadot_node_subsystem_util::{database::Database, TimeoutExt}; -use polkadot_primitives::{ - CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, HeadData, Header, - PersistedValidationData, ValidatorId, -}; +use polkadot_node_subsystem_util::database::Database; use polkadot_node_core_av_store::AvailabilityStoreSubsystem; -use sp_keyring::Sr25519Keyring; -use std::time::Duration; mod columns { pub const DATA: u32 = 0; diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 280392a430b7..abf74a2c1d2c 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -24,20 +24,17 @@ use polkadot_node_subsystem_types::{ messages::{AvailabilityStoreMessage, NetworkBridgeEvent}, Span, }; -use polkadot_primitives::AuthorityDiscoveryId; - use polkadot_overseer::Handle as OverseerHandle; use sc_network::{request_responses::ProtocolConfig, PeerId}; use sp_core::H256; -use std::{collections::HashMap, iter::Cycle, ops::Sub, pin::Pin, sync::Arc, time::Instant}; +use std::{collections::HashMap, iter::Cycle, ops::Sub, sync::Arc, time::Instant}; use av_store_helpers::new_av_store; -use futures::{channel::oneshot, stream::FuturesUnordered, Future, StreamExt}; +use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; use polkadot_availability_distribution::{ AvailabilityDistributionSubsystem, IncomingRequestReceivers, }; use polkadot_node_metrics::metrics::Metrics; -use polkadot_node_subsystem::TimeoutExt; use polkadot_availability_recovery::AvailabilityRecoverySubsystem; use polkadot_node_primitives::{AvailableData, ErasureChunk}; @@ -57,7 +54,7 @@ use polkadot_node_network_protocol::{ }; use sc_network::request_responses::IncomingRequest as RawIncomingRequest; -use polkadot_node_primitives::{BlockData, PoV, Proof}; +use polkadot_node_primitives::{BlockData, PoV}; use polkadot_node_subsystem::messages::{AllMessages, AvailabilityRecoveryMessage}; use crate::core::{ @@ -659,9 +656,9 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: .get(index) .expect("all validators have keys"); - env.network().send_request_from_peer(peer, request); - - receivers.push(pending_response_receiver); + if env.network().send_request_from_peer(peer, request).is_ok() { + receivers.push(pending_response_receiver); + } } gum::info!(target: LOG_TARGET, "Waiting for all emulated peers to receive their chunk from us ..."); @@ -676,7 +673,7 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: gum::info!("All chunks received in {}ms", chunk_fetch_duration); let signing_context = SigningContext { session_index: 0, parent_hash: relay_block_hash }; - let mut network = env.network().clone(); + let network = env.network().clone(); let authorities = env.authorities().clone(); let n_validators = config.n_validators; @@ -707,7 +704,7 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: let message = peer_bitfield_message_v2(relay_block_hash, signed_bitfield); // Send the action to `to_peer`. - network.send_message_from_peer(from_peer, message); + let _ = network.send_message_from_peer(from_peer, message); } gum::info!("Waiting for {} bitfields to be received and processed", n_validators - 1); diff --git a/polkadot/node/subsystem-bench/src/core/configuration.rs b/polkadot/node/subsystem-bench/src/core/configuration.rs index fb4b4e668c5e..440b507ebcb0 100644 --- a/polkadot/node/subsystem-bench/src/core/configuration.rs +++ b/polkadot/node/subsystem-bench/src/core/configuration.rs @@ -17,11 +17,13 @@ //! Test configuration definition and helpers. use super::*; use keyring::Keyring; -use std::{path::Path, time::Duration}; +use std::path::Path; pub use crate::cli::TestObjective; use polkadot_primitives::{AuthorityDiscoveryId, ValidatorId}; -use rand::{distributions::Uniform, prelude::Distribution, thread_rng}; +use rand::thread_rng; +use rand_distr::{Distribution, Normal, Uniform}; + use serde::{Deserialize, Serialize}; pub fn random_pov_size(min_pov_size: usize, max_pov_size: usize) -> usize { @@ -34,13 +36,13 @@ fn random_uniform_sample + From>(min_value: T, max_value: .into() } -/// Peer response latency configuration. +/// Peer networking latency configuration. #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct PeerLatency { - /// Min latency of the peer. - pub min_latency: Duration, - /// Max latency or the peer. - pub max_latency: Duration, + /// The mean latency(milliseconds) of the peers. + pub mean_latency_ms: usize, + /// The standard deviation + pub std_dev: f64, } // Default PoV size in KiB. @@ -90,12 +92,9 @@ pub struct TestConfiguration { /// The amount of bandiwdth our node has. #[serde(default = "default_bandwidth")] pub bandwidth: usize, - /// Optional peer emulation latency + /// Optional peer emulation latency (round trip time) wrt node under test #[serde(default)] pub latency: Option, - /// Error probability, applies to sending messages to the emulated network peers - #[serde(default)] - pub error: usize, /// Connectivity ratio, the percentage of peers we are not connected to, but ar part of /// the topology. #[serde(default = "default_connectivity")] @@ -192,7 +191,6 @@ impl TestConfiguration { peer_bandwidth: 50 * 1024 * 1024, // No latency latency: None, - error: 0, num_blocks, min_pov_size, max_pov_size, @@ -216,11 +214,7 @@ impl TestConfiguration { pov_sizes: generate_pov_sizes(n_cores, min_pov_size, max_pov_size), bandwidth: 50 * 1024 * 1024, peer_bandwidth: 50 * 1024 * 1024, - latency: Some(PeerLatency { - min_latency: Duration::from_millis(1), - max_latency: Duration::from_millis(100), - }), - error: 3, + latency: Some(PeerLatency { mean_latency_ms: 50, std_dev: 12.5 }), num_blocks, min_pov_size, max_pov_size, @@ -244,11 +238,7 @@ impl TestConfiguration { pov_sizes: generate_pov_sizes(n_cores, min_pov_size, max_pov_size), bandwidth: 50 * 1024 * 1024, peer_bandwidth: 50 * 1024 * 1024, - latency: Some(PeerLatency { - min_latency: Duration::from_millis(10), - max_latency: Duration::from_millis(500), - }), - error: 33, + latency: Some(PeerLatency { mean_latency_ms: 150, std_dev: 40.0 }), num_blocks, min_pov_size, max_pov_size, @@ -257,21 +247,14 @@ impl TestConfiguration { } } -/// Produce a randomized duration between `min` and `max`. -/// TODO: Use normal distribution. -pub fn random_latency(maybe_peer_latency: Option<&PeerLatency>) -> Option { - if let Some(peer_latency) = maybe_peer_latency { - Some( - Uniform::from(peer_latency.min_latency..=peer_latency.max_latency) - .sample(&mut thread_rng()), - ) - } else { - None - } -} - -/// Generate a random error based on `probability`. -/// `probability` should be a number between 0 and 100. -pub fn random_error(probability: usize) -> bool { - Uniform::from(0..=99).sample(&mut thread_rng()) < probability +/// Sample latency (in milliseconds) from a normal distribution with parameters +/// specified in `maybe_peer_latency`. +pub fn random_latency(maybe_peer_latency: Option<&PeerLatency>) -> usize { + maybe_peer_latency + .map(|latency_config| { + Normal::new(latency_config.mean_latency_ms as f64, latency_config.std_dev) + .expect("normal distribution parameters are good") + .sample(&mut thread_rng()) + }) + .unwrap_or(0.0) as usize } diff --git a/polkadot/node/subsystem-bench/src/core/display.rs b/polkadot/node/subsystem-bench/src/core/display.rs index b9ff82d1c06a..0efd19171a07 100644 --- a/polkadot/node/subsystem-bench/src/core/display.rs +++ b/polkadot/node/subsystem-bench/src/core/display.rs @@ -185,7 +185,7 @@ pub fn display_configuration(test_config: &TestConfiguration) { format!("n_cores = {}", test_config.n_cores).blue(), format!("pov_size = {} - {}", test_config.min_pov_size, test_config.max_pov_size) .bright_black(), - format!("error = {}", test_config.error).bright_black(), + format!("connectivity = {}", test_config.connectivity).bright_black(), format!("latency = {:?}", test_config.latency).bright_black(), ); } diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index e89d93eefc3c..5887f467423c 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -30,10 +30,7 @@ use polkadot_node_subsystem_util::metrics::prometheus::{ }; use sc_service::{SpawnTaskHandle, TaskManager}; -use std::{ - fmt::Display, - net::{Ipv4Addr, SocketAddr}, -}; +use std::net::{Ipv4Addr, SocketAddr}; use tokio::runtime::Handle; const LOG_TARGET: &str = "subsystem-bench::environment"; diff --git a/polkadot/node/subsystem-bench/src/core/keyring.rs b/polkadot/node/subsystem-bench/src/core/keyring.rs index a8aba1aec850..6cf031f5712f 100644 --- a/polkadot/node/subsystem-bench/src/core/keyring.rs +++ b/polkadot/node/subsystem-bench/src/core/keyring.rs @@ -18,12 +18,8 @@ use polkadot_primitives::ValidatorId; use sc_keystore::LocalKeystore; use sp_application_crypto::AppCrypto; pub use sp_core::sr25519; -use sp_core::{ - sr25519::{Pair, Public}, - Pair as PairT, -}; -use sp_keyring::Sr25519Keyring; -use sp_keystore::{Keystore, KeystorePtr}; +use sp_core::sr25519::Public; +use sp_keystore::Keystore; use std::sync::Arc; /// Set of test accounts generated and kept safe by a keystore. diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index 280c17aa9631..c08d07d08027 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -45,15 +45,12 @@ use polkadot_node_subsystem::{ }; use polkadot_node_network_protocol::{ - request_response::{self as req_res, v1::ChunkResponse, Recipient, Requests}, + request_response::{ v1::ChunkResponse, Recipient, Requests, ResponseSender}, Versioned, }; use polkadot_primitives::AuthorityDiscoveryId; -use crate::core::{ - configuration::{random_error, random_latency, TestConfiguration}, - network::{NetworkEmulatorHandle, NetworkInterfaceReceiver, NetworkMessage, RateLimit}, -}; +use crate::core::network::{NetworkEmulatorHandle, NetworkInterfaceReceiver, NetworkMessage}; const LOG_TARGET: &str = "subsystem-bench::network-bridge"; @@ -109,11 +106,13 @@ impl MockNetworkBridgeRx { } } -trait RequestAuthority { +// Helper trait for `Requests`. +trait RequestExt { fn authority_id(&self) -> Option<&AuthorityDiscoveryId>; + fn into_response_sender(self) -> ResponseSender; } -impl RequestAuthority for Requests { +impl RequestExt for Requests { fn authority_id(&self) -> Option<&AuthorityDiscoveryId> { match self { Requests::ChunkFetchingV1(request) => { @@ -128,6 +127,18 @@ impl RequestAuthority for Requests { }, } } + + fn into_response_sender(self) -> ResponseSender { + match self { + Requests::ChunkFetchingV1(outgoing_request) => { + outgoing_request.pending_response + }, + Requests::AvailableDataFetchingV1(outgoing_request) => { + outgoing_request.pending_response + } + _ => unimplemented!("unsupported request type") + } + } } #[overseer::contextbounds(NetworkBridgeTx, prefix = self::overseer)] @@ -147,8 +158,16 @@ impl MockNetworkBridgeTx { gum::debug!(target: LOG_TARGET, request = ?request, "Processing request"); let peer_id = request.authority_id().expect("all nodes are authorities").clone(); + + if !self.network.is_peer_connected(&peer_id) { + // Attempting to send a request to a disconnected peer. + let _ = request.into_response_sender().send(Err(RequestFailure::NotConnected)).expect("send never fails"); + continue + } + let peer_message = NetworkMessage::RequestFromNode(peer_id.clone(), request); + let _ = self.to_network_interface.unbounded_send(peer_message); } }, diff --git a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs index dba9c0e66f4b..f01f2c0d29e9 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs @@ -17,8 +17,7 @@ //! A generic runtime api subsystem mockup suitable to be used in benchmarks. use polkadot_primitives::{ - CandidateHash, CandidateReceipt, CoreState, GroupIndex, IndexedVec, OccupiedCore, SessionInfo, - ValidatorIndex, + CandidateReceipt, CoreState, GroupIndex, IndexedVec, OccupiedCore, SessionInfo, ValidatorIndex, }; use bitvec::prelude::BitVec; diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 74a4e02d3020..222f36d040ae 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -17,6 +17,13 @@ //! Implements network emulation and interfaces to control and specialize //! network peer behaviour. //! +//! High level component wiring chart: +//! +//! [TestEnvironment] +//! [NetworkEmulatorHandle] +//! || +//! +-------+--||--+-------+ +//! | | | | //! Peer1 Peer2 Peer3 Peer4 //! \ | | / //! \ | | / @@ -24,11 +31,12 @@ //! \ | | / //! \ | | / //! [Network Interface] -//! | +//! | //! [Emulated Network Bridge] //! | //! Subsystems under test +use crate::core::configuration::random_latency; use super::{ configuration::{TestAuthorities, TestConfiguration}, @@ -38,14 +46,10 @@ use super::{ use colored::Colorize; use futures::{ channel::{mpsc, oneshot}, - future::FusedFuture, lock::Mutex, stream::FuturesUnordered, }; -use net_protocol::{ - request_response::{OutgoingRequest, Requests}, - VersionedValidationProtocol, -}; +use net_protocol::{request_response::Requests, VersionedValidationProtocol}; use parity_scale_codec::Encode; use polkadot_primitives::AuthorityDiscoveryId; use prometheus_endpoint::U64; @@ -57,12 +61,7 @@ use sc_network::{ use sc_service::SpawnTaskHandle; use std::{ collections::HashMap, - ops::DerefMut, - pin::Pin, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, - }, + sync::Arc, time::{Duration, Instant}, }; @@ -184,7 +183,6 @@ impl NetworkMessage { } /// A network interface of the node under test. -/// TODO(soon): Implement latency and connection errors here, instead of doing it on the peers. pub struct NetworkInterface { // Sender for subsystems. bridge_to_interface_sender: UnboundedSender, @@ -408,27 +406,38 @@ impl EmulatedPeerHandle { // A network peer emulator. struct EmulatedPeer { + spawn_handle: SpawnTaskHandle, to_node: UnboundedSender, tx_limiter: RateLimit, rx_limiter: RateLimit, + latency_ms: usize, } impl EmulatedPeer { /// Send a message to the node. pub async fn send_message(&mut self, message: NetworkMessage) { self.tx_limiter.reap(message.size()).await; - let _ = self.to_node.unbounded_send(message).expect("Sending to the node never fails"); + + if self.latency_ms == 0 { + let _ = self.to_node.unbounded_send(message).expect("Sending to the node never fails"); + } else { + let mut to_node = self.to_node.clone(); + let latency_ms = std::time::Duration::from_millis(self.latency_ms as u64); + + // Emulate RTT latency + self.spawn_handle + .spawn("peer-latency-emulator", "test-environment", async move { + tokio::time::sleep(latency_ms).await; + let _ = + to_node.unbounded_send(message).expect("Sending to the node never fails"); + }); + } } /// Returns the rx bandwidth limiter. pub fn rx_limiter(&mut self) -> &mut RateLimit { &mut self.rx_limiter } - - /// Returns the tx bandwidth limiter. - pub fn tx_limiter(&mut self) -> &mut RateLimit { - &mut self.tx_limiter - } } /// Interceptor pattern for handling messages. @@ -470,11 +479,13 @@ async fn emulated_peer_loop( let mut proxied_requests = FuturesUnordered::new(); let mut messages_rx = messages_rx.fuse(); let mut actions_rx = actions_rx.fuse(); + loop { futures::select! { maybe_peer_message = messages_rx.next() => { if let Some(peer_message) = maybe_peer_message { let size = peer_message.size(); + emulated_peer.rx_limiter().reap(size).await; stats.inc_received(size); @@ -552,14 +563,20 @@ pub fn new_peer( handlers: Vec>, stats: Arc, mut to_network_interface: UnboundedSender, + latency_ms: usize, ) -> EmulatedPeerHandle { let (messages_tx, mut messages_rx) = mpsc::unbounded::(); let (actions_tx, mut actions_rx) = mpsc::unbounded::(); let rx_limiter = RateLimit::new(10, bandwidth); let tx_limiter = RateLimit::new(10, bandwidth); - let mut emulated_peer = - EmulatedPeer { rx_limiter, tx_limiter, to_node: to_network_interface.clone() }; + let mut emulated_peer = EmulatedPeer { + spawn_handle: spawn_task_handle.clone(), + rx_limiter, + tx_limiter, + to_node: to_network_interface.clone(), + latency_ms, + }; spawn_task_handle.clone().spawn( "peer-emulator", @@ -640,12 +657,6 @@ impl Peer { Peer::Disconnected(ref emulator) => emulator, } } - pub fn handle_mut(&mut self) -> &mut EmulatedPeerHandle { - match self { - Peer::Connected(ref mut emulator) => emulator, - Peer::Disconnected(ref mut emulator) => emulator, - } - } } /// A ha emulated network implementation. @@ -669,7 +680,7 @@ pub fn new_network( ) -> (NetworkEmulatorHandle, NetworkInterface, NetworkInterfaceReceiver) { let n_peers = config.n_validators; gum::info!(target: LOG_TARGET, "{}",format!("Initializing emulation for a {} peer network.", n_peers).bright_blue()); - gum::info!(target: LOG_TARGET, "{}",format!("connectivity {}%, error {}%", config.connectivity, config.error).bright_black()); + gum::info!(target: LOG_TARGET, "{}",format!("connectivity {}%, latency {:?}%", config.connectivity, config.latency).bright_black()); let metrics = Metrics::new(&dependencies.registry).expect("Metrics always register succesfully"); @@ -692,6 +703,7 @@ pub fn new_network( handlers.clone(), stats, to_network_interface.clone(), + random_latency(config.latency.as_ref()), )), ) }) @@ -725,6 +737,11 @@ pub fn new_network( (handle, network_interface, network_interface_receiver) } +/// Errors that can happen when sending data to emulated peers. +pub enum EmulatedPeerError { + NotConnected +} + impl NetworkEmulatorHandle { pub fn is_peer_connected(&self, peer: &AuthorityDiscoveryId) -> bool { self.peer(peer).is_connected() @@ -755,15 +772,16 @@ impl NetworkEmulatorHandle { &self, from_peer: &AuthorityDiscoveryId, message: VersionedValidationProtocol, - ) { + ) -> Result<(), EmulatedPeerError> { let dst_peer = self.peer(&from_peer); if !dst_peer.is_connected() { gum::warn!(target: LOG_TARGET, "Attempted to send message from a peer not connected to our node, operation ignored"); - return + return Err(EmulatedPeerError::NotConnected) } dst_peer.handle().send_message(message); + Ok(()) } /// Send a request from a peer to the node. @@ -771,15 +789,16 @@ impl NetworkEmulatorHandle { &self, from_peer: &AuthorityDiscoveryId, request: IncomingRequest, - ) { + ) -> Result<(), EmulatedPeerError> { let dst_peer = self.peer(&from_peer); if !dst_peer.is_connected() { gum::warn!(target: LOG_TARGET, "Attempted to send request from a peer not connected to our node, operation ignored"); - return + return Err(EmulatedPeerError::NotConnected) } dst_peer.handle().send_request(request); + Ok(()) } // Returns the sent/received stats for `peer_index`. @@ -800,13 +819,6 @@ impl NetworkEmulatorHandle { &self.peers[self.peer_index(peer)] } - // Returns the sent/received stats for `peer`. - pub fn peer_stats_by_id(&mut self, peer: &AuthorityDiscoveryId) -> Arc { - let peer_index = self.peer_index(peer); - - self.stats[peer_index].clone() - } - // Increment bytes sent by our node (the node that contains the subsystem under test) pub fn inc_sent(&self, bytes: usize) { // Our node is always peer 0. @@ -874,6 +886,16 @@ impl Metrics { } } +/// A helper to determine the request payload size. +pub fn request_size(request: &Requests) -> usize { + match request { + Requests::ChunkFetchingV1(outgoing_request) => outgoing_request.payload.encoded_size(), + Requests::AvailableDataFetchingV1(outgoing_request) => + outgoing_request.payload.encoded_size(), + _ => unimplemented!("received an unexpected request"), + } +} + #[cfg(test)] mod tests { use std::time::Instant; @@ -910,13 +932,3 @@ mod tests { assert!(total_sent as u128 <= upper_bound); } } - -/// A helper to determine the request payload size. -pub fn request_size(request: &Requests) -> usize { - match request { - Requests::ChunkFetchingV1(outgoing_request) => outgoing_request.payload.encoded_size(), - Requests::AvailableDataFetchingV1(outgoing_request) => - outgoing_request.payload.encoded_size(), - _ => unimplemented!("received an unexpected request"), - } -} diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 195316871050..99673b8fe1bf 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -20,7 +20,7 @@ use clap::Parser; use color_eyre::eyre; use colored::Colorize; -use std::{path::Path, time::Duration}; +use std::path::Path; pub(crate) mod availability; pub(crate) mod cli; @@ -57,24 +57,24 @@ struct BenchCli { pub standard_configuration: cli::StandardTestOptions, #[clap(short, long)] - /// The bandwidth of simulated remote peers in KiB + /// The bandwidth of emulated remote peers in KiB pub peer_bandwidth: Option, #[clap(short, long)] - /// The bandwidth of our simulated node in KiB + /// The bandwidth of our node in KiB pub bandwidth: Option, #[clap(long, value_parser=le_100)] - /// Simulated conection error ratio [0-100]. - pub peer_error: Option, + /// Emulated peer connection ratio [0-100]. + pub connectivity: Option, #[clap(long, value_parser=le_5000)] - /// Minimum remote peer latency in milliseconds [0-5000]. - pub peer_min_latency: Option, + /// Mean remote peer latency in milliseconds [0-5000]. + pub peer_mean_latency: Option, #[clap(long, value_parser=le_5000)] - /// Maximum remote peer latency in milliseconds [0-5000]. - pub peer_max_latency: Option, + /// Remote peer latency standard deviation + pub peer_latency_std_dev: Option, #[command(subcommand)] pub objective: cli::TestObjective, @@ -141,16 +141,19 @@ impl BenchCli { let mut latency_config = test_config.latency.clone().unwrap_or_default(); - if let Some(latency) = self.peer_min_latency { - latency_config.min_latency = Duration::from_millis(latency); + if let Some(latency) = self.peer_mean_latency { + latency_config.mean_latency_ms = latency; } - if let Some(latency) = self.peer_max_latency { - latency_config.max_latency = Duration::from_millis(latency); + if let Some(std_dev) = self.peer_latency_std_dev { + latency_config.std_dev = std_dev; } - if let Some(error) = self.peer_error { - test_config.error = error; + // Write back the updated latency. + test_config.latency = Some(latency_config); + + if let Some(connectivity) = self.connectivity { + test_config.connectivity = connectivity; } if let Some(bandwidth) = self.peer_bandwidth { From 3d80227eb62467db8c47875eca2b39988b5116eb Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 12 Jan 2024 10:33:29 +0200 Subject: [PATCH 169/192] Fix warnings Signed-off-by: Alexandru Gheorghe --- .../src/approval/message_generator.rs | 9 ++--- .../node/subsystem-bench/src/approval/mod.rs | 31 +++++++---------- .../src/approval/test_message.rs | 13 ++----- .../subsystem-bench/src/availability/mod.rs | 23 ++++--------- .../subsystem-bench/src/core/configuration.rs | 4 +-- .../subsystem-bench/src/core/environment.rs | 27 ++------------- .../node/subsystem-bench/src/core/keyring.rs | 2 +- .../src/core/mock/chain_api.rs | 4 +-- .../src/core/mock/network_bridge.rs | 34 +++++-------------- .../node/subsystem-bench/src/core/network.rs | 26 +++++++------- 10 files changed, 53 insertions(+), 120 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/approval/message_generator.rs b/polkadot/node/subsystem-bench/src/approval/message_generator.rs index b95f203bd844..dfbf23f83897 100644 --- a/polkadot/node/subsystem-bench/src/approval/message_generator.rs +++ b/polkadot/node/subsystem-bench/src/approval/message_generator.rs @@ -39,8 +39,7 @@ use polkadot_node_primitives::approval::{ }; use polkadot_primitives::{ vstaging::ApprovalVoteMultipleCandidates, CandidateEvent, CandidateHash, CandidateIndex, - CoreIndex, SessionInfo, Slot, ValidatorId, ValidatorIndex, ValidatorPair, - ASSIGNMENT_KEY_TYPE_ID, + CoreIndex, SessionInfo, Slot, ValidatorId, ValidatorIndex, ASSIGNMENT_KEY_TYPE_ID, }; use rand::{seq::SliceRandom, RngCore, SeedableRng}; use rand_chacha::ChaCha20Rng; @@ -49,7 +48,6 @@ use sc_network::PeerId; use sha1::Digest; use sp_application_crypto::AppCrypto; use sp_consensus_babe::SlotDuration; -use sp_core::Pair; use sp_keystore::Keystore; use sp_timestamp::Timestamp; @@ -63,10 +61,7 @@ use crate::{ GeneratedState, BUFFER_FOR_GENERATION_MILLIS, LOG_TARGET, NODE_UNDER_TEST, SLOT_DURATION_MILLIS, }, - core::{ - configuration::{TestAuthorities, TestConfiguration, TestObjective}, - keyring::Keyring, - }, + core::configuration::{TestAuthorities, TestConfiguration, TestObjective}, }; use polkadot_node_network_protocol::v3 as protocol_v3; use polkadot_primitives::Hash; diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index d58f384d015f..f16c545a03f1 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -43,7 +43,7 @@ use crate::{ }, }; use colored::Colorize; -use futures::{channel::oneshot, FutureExt}; +use futures::channel::oneshot; use itertools::Itertools; use orchestra::TimeoutExt; use overseer::{metrics::Metrics as OverseerMetrics, MetricsTrait}; @@ -53,7 +53,7 @@ use polkadot_node_core_approval_voting::{ time::{slot_number_to_tick, tick_to_slot_number, Clock, ClockExt, SystemClock}, ApprovalVotingSubsystem, Config as ApprovalVotingConfig, Metrics as ApprovalVotingMetrics, }; -use polkadot_node_network_protocol::{v3 as protocol_v3, Versioned}; +use polkadot_node_network_protocol::v3 as protocol_v3; use polkadot_node_primitives::approval::{self, v1::RelayVRFStory}; use polkadot_node_subsystem::{overseer, AllMessages, Overseer, OverseerConnector, SpawnGlue}; use polkadot_node_subsystem_test_helpers::mock::new_block_import_info; @@ -405,7 +405,7 @@ impl ApprovalTestState { &mut self, network_emulator: &NetworkEmulatorHandle, overseer_handle: OverseerHandleReal, - spawn_task_handle: &SpawnTaskHandle, + env: &TestEnvironment, registry: Registry, ) -> oneshot::Receiver<()> { gum::info!(target: LOG_TARGET, "Start assignments/approvals production"); @@ -421,7 +421,7 @@ impl ApprovalTestState { }; peer_message_source - .produce_messages(spawn_task_handle, self.generated_state.all_messages.take().unwrap()); + .produce_messages(env, self.generated_state.all_messages.take().unwrap()); producer_rx } } @@ -452,8 +452,8 @@ impl ApprovalTestState { impl HandleNetworkMessage for ApprovalTestState { fn handle( &self, - message: crate::core::network::NetworkMessage, - node_sender: &mut futures::channel::mpsc::UnboundedSender< + _message: crate::core::network::NetworkMessage, + _node_sender: &mut futures::channel::mpsc::UnboundedSender< crate::core::network::NetworkMessage, >, ) -> Option { @@ -479,12 +479,8 @@ struct PeerMessageProducer { impl PeerMessageProducer { /// Generates messages by spawning a blocking task in the background which begins creating /// the assignments/approvals and peer view changes at the begining of each block. - fn produce_messages( - mut self, - spawn_task_handle: &SpawnTaskHandle, - all_messages: Vec, - ) { - spawn_task_handle.spawn_blocking("produce-messages", "produce-messages", async move { + fn produce_messages(mut self, env: &TestEnvironment, all_messages: Vec) { + env.spawn_blocking("produce-messages", async move { let mut initialized_blocks = HashSet::new(); let mut per_candidate_data: HashMap<(Hash, CandidateIndex), CandidateTestData> = self.initialize_candidates_test_data(); @@ -603,7 +599,7 @@ impl PeerMessageProducer { .fetch_add(1, std::sync::atomic::Ordering::SeqCst); for (peer, messages) in message.split_by_peer_id(&self.state.test_authorities) { for message in messages { - let latency = message.get_latency(); + let _latency = message.get_latency(); self.state .total_sent_messages .as_ref() @@ -673,8 +669,8 @@ impl PeerMessageProducer { async fn send_message( &mut self, message: AllMessages, - sent_by: ValidatorIndex, - latency: Option, + _sent_by: ValidatorIndex, + _latency: Option, ) { self.overseer_handle .send_msg(message, LOG_TARGET) @@ -732,7 +728,7 @@ impl PeerMessageProducer { fn build_overseer( state: &ApprovalTestState, network: &NetworkEmulatorHandle, - config: &TestConfiguration, + _config: &TestConfiguration, dependencies: &TestEnvironmentDependencies, network_interface: &NetworkInterface, network_receiver: NetworkInterfaceReceiver, @@ -826,7 +822,6 @@ fn prepare_test_inner( overseer, overseer_handle, state.test_authorities.clone(), - network_interface, ), state, ) @@ -837,7 +832,7 @@ pub async fn bench_approvals(env: &mut TestEnvironment, mut state: ApprovalTestS .start_message_production( env.network(), env.overseer_handle().clone(), - &env.spawn_handle(), + &env, env.registry().clone(), ) .await; diff --git a/polkadot/node/subsystem-bench/src/approval/test_message.rs b/polkadot/node/subsystem-bench/src/approval/test_message.rs index 5fb37a5160f0..d8cb3c73d6a5 100644 --- a/polkadot/node/subsystem-bench/src/approval/test_message.rs +++ b/polkadot/node/subsystem-bench/src/approval/test_message.rs @@ -18,9 +18,9 @@ use super::{ApprovalsOptions, BlockTestData, CandidateTestData}; use crate::core::configuration::TestAuthorities; use itertools::Itertools; use parity_scale_codec::{Decode, Encode}; -use polkadot_node_network_protocol::{v3 as protocol_v3, Versioned}; -use polkadot_node_subsystem_types::messages::{ApprovalDistributionMessage, NetworkBridgeEvent}; -use polkadot_overseer::AllMessages; +use polkadot_node_network_protocol::{v3 as protocol_v3}; + + use polkadot_primitives::{CandidateIndex, Hash, ValidatorIndex}; use sc_network::PeerId; use std::{ @@ -125,13 +125,6 @@ impl MessagesBundle { } impl TestMessageInfo { - /// Converts message to `AllMessages` to be sent to overseer. - pub fn into_all_messages_from_peer(self, peer: PeerId) -> AllMessages { - AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate( - NetworkBridgeEvent::PeerMessage(peer, Versioned::V3(self.msg)), - )) - } - /// Gives us the latency for this message. pub fn get_latency(&self) -> Option { match &self.msg { diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 94dd65be04d8..f3a379d1b6b3 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -40,7 +40,6 @@ use polkadot_availability_recovery::AvailabilityRecoverySubsystem; use polkadot_node_primitives::{AvailableData, ErasureChunk}; use crate::GENESIS_HASH; -use futures::FutureExt; use parity_scale_codec::Encode; use polkadot_node_network_protocol::{ request_response::{ @@ -48,9 +47,9 @@ use polkadot_node_network_protocol::{ AvailableDataFetchingResponse, ChunkFetchingRequest, ChunkFetchingResponse, ChunkResponse, }, - IncomingRequest, OutgoingRequest, ReqProtocolNames, Requests, + IncomingRequest, ReqProtocolNames, Requests, }, - BitfieldDistributionMessage, OurView, Versioned, VersionedValidationProtocol, View, + OurView, Versioned, VersionedValidationProtocol, }; use sc_network::request_responses::IncomingRequest as RawIncomingRequest; @@ -152,7 +151,7 @@ impl HandleNetworkMessage for NetworkAvailabilityState { fn handle( &self, message: NetworkMessage, - node_sender: &mut futures::channel::mpsc::UnboundedSender, + _node_sender: &mut futures::channel::mpsc::UnboundedSender, ) -> Option { match message { NetworkMessage::RequestFromNode(peer, request) => match request { @@ -171,14 +170,14 @@ impl HandleNetworkMessage for NetworkAvailabilityState { [validator_index] .clone() .into(); - let mut size = chunk.encoded_size(); + let _size = chunk.encoded_size(); let response = Ok(( ChunkFetchingResponse::from(Some(chunk)).encode(), ProtocolName::from(""), )); - if let Err(err) = outgoing_request.pending_response.send(response) { + if let Err(_err) = outgoing_request.pending_response.send(response) { gum::error!(target: LOG_TARGET, "Failed to send `ChunkFetchingResponse`"); } @@ -195,7 +194,7 @@ impl HandleNetworkMessage for NetworkAvailabilityState { let available_data = self.available_data.get(*candidate_index as usize).unwrap().clone(); - let size = available_data.encoded_size(); + let _size = available_data.encoded_size(); let response = Ok(( AvailableDataFetchingResponse::from(Some(available_data)).encode(), @@ -377,7 +376,6 @@ fn prepare_test_inner( overseer, overseer_handle, test_authorities, - network_interface, ), req_cfgs, ) @@ -404,8 +402,6 @@ pub struct TestState { chunks: Vec>, // Availability distribution chunk_request_protocol: Option, - // Availability distribution. - pov_request_protocol: Option, // Per relay chain block - candidate backed by our backing group backed_candidates: Vec, } @@ -507,7 +503,6 @@ impl TestState { candidate_hashes: HashMap::new(), candidates: Vec::new().into_iter().cycle(), chunk_request_protocol: None, - pov_request_protocol: None, backed_candidates: Vec::new(), }; @@ -523,10 +518,6 @@ impl TestState { self.chunk_request_protocol = Some(config); } - pub fn set_pov_request_protocol(&mut self, config: ProtocolConfig) { - self.pov_request_protocol = Some(config); - } - pub fn chunk_request_protocol(&self) -> Option { self.chunk_request_protocol.clone() } @@ -638,7 +629,7 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: env.import_block(new_block_import_info(relay_block_hash, block_num as BlockNumber)) .await; - let chunk_request_protocol = + let _chunk_request_protocol = state.chunk_request_protocol().expect("No chunk fetching protocol configured"); // Inform bitfield distribution about our view of current test block diff --git a/polkadot/node/subsystem-bench/src/core/configuration.rs b/polkadot/node/subsystem-bench/src/core/configuration.rs index fb1bc76369a5..abf50f6705b5 100644 --- a/polkadot/node/subsystem-bench/src/core/configuration.rs +++ b/polkadot/node/subsystem-bench/src/core/configuration.rs @@ -21,10 +21,10 @@ use keyring::Keyring; use rand_distr::Normal; use sc_network::PeerId; use sp_consensus_babe::AuthorityId; -use std::{collections::HashMap, path::Path, time::Duration}; +use std::{collections::HashMap, path::Path}; pub use crate::cli::TestObjective; -use polkadot_primitives::{AssignmentId, AuthorityDiscoveryId, HashT, ValidatorId}; +use polkadot_primitives::{AssignmentId, AuthorityDiscoveryId, ValidatorId}; use rand::{distributions::Uniform, prelude::Distribution, thread_rng}; use serde::{Deserialize, Serialize}; diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index c404cb5a64e6..40280d0fcbb4 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -28,16 +28,10 @@ use polkadot_node_subsystem_util::metrics::prometheus::{ use sc_network::peer_store::LOG_TARGET; use sc_service::{SpawnTaskHandle, TaskManager}; -use std::{ - fmt::Display, - net::{Ipv4Addr, SocketAddr}, -}; +use std::net::{Ipv4Addr, SocketAddr}; use tokio::runtime::Handle; -use super::{ - configuration::TestAuthorities, - network::{NetworkEmulatorHandle, NetworkInterface}, -}; +use super::{configuration::TestAuthorities, network::NetworkEmulatorHandle}; const MIB: f64 = 1024.0 * 1024.0; @@ -197,8 +191,6 @@ pub struct TestEnvironment { metrics: TestEnvironmentMetrics, /// Test authorities generated from the configuration. authorities: TestAuthorities, - /// The network interface used by the node. - network_interface: NetworkInterface, } impl TestEnvironment { @@ -210,7 +202,6 @@ impl TestEnvironment { overseer: Overseer, AlwaysSupportsParachains>, overseer_handle: OverseerHandle, authorities: TestAuthorities, - network_interface: NetworkInterface, ) -> Self { let metrics = TestEnvironmentMetrics::new(&dependencies.registry) .expect("Metrics need to be registered"); @@ -240,7 +231,6 @@ impl TestEnvironment { network, metrics, authorities, - network_interface, } } @@ -259,24 +249,11 @@ impl TestEnvironment { &self.overseer_handle } - /// Returns a reference to the spawn task handle. - pub fn spawn_handle(&self) -> SpawnTaskHandle { - self.dependencies.task_manager.spawn_handle() - } - /// Returns the Prometheus registry. pub fn registry(&self) -> &Registry { &self.dependencies.registry } - /// Spawn a named task in the `test-environment` task group. - pub fn spawn(&self, name: &'static str, task: impl Future + Send + 'static) { - self.dependencies - .task_manager - .spawn_handle() - .spawn(name, "test-environment", task); - } - /// Spawn a blocking named task in the `test-environment` task group. pub fn spawn_blocking( &self, diff --git a/polkadot/node/subsystem-bench/src/core/keyring.rs b/polkadot/node/subsystem-bench/src/core/keyring.rs index 00d5cf25f449..f6b4b9878462 100644 --- a/polkadot/node/subsystem-bench/src/core/keyring.rs +++ b/polkadot/node/subsystem-bench/src/core/keyring.rs @@ -41,7 +41,7 @@ impl Keyring { .sr25519_generate_new(ValidatorId::ID, Some(seed)) .expect("Insert key into keystore"); - let assignment_id = self + let _assignment_id = self .keystore .sr25519_generate_new(ASSIGNMENT_KEY_TYPE_ID, Some(seed)) .expect("should not fail"); diff --git a/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs b/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs index ba6d53366bc5..5acb3547e076 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs @@ -16,7 +16,7 @@ //! //! A generic runtime api subsystem mockup suitable to be used in benchmarks. -use polkadot_primitives::{GroupIndex, Header, IndexedVec, SessionInfo, ValidatorIndex}; +use polkadot_primitives::{Header}; use polkadot_node_subsystem::{ messages::ChainApiMessage, overseer, SpawnedSubsystem, SubsystemError, @@ -77,7 +77,7 @@ impl MockChainApi { .expect("Relay chain block hashes are known"), ))); }, - ChainApiMessage::Ancestors { hash, k, response_channel } => { + ChainApiMessage::Ancestors { hash: _, k: _, response_channel } => { // For our purposes, no ancestors is fine. let _ = response_channel.send(Ok(Vec::new())); }, diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index ecc76b03b920..44d3c1c15b2e 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -16,37 +16,23 @@ //! //! Mocked `network-bridge` subsystems that uses a `NetworkInterface` to access //! the emulated network. -use futures::{ - channel::{ - mpsc::{UnboundedReceiver, UnboundedSender}, - oneshot, - }, - Future, -}; -use overseer::AllMessages; -use parity_scale_codec::Encode; +use futures::channel::mpsc::UnboundedSender; + use polkadot_node_subsystem_types::{ messages::{ApprovalDistributionMessage, BitfieldDistributionMessage, NetworkBridgeEvent}, OverseerSignal, }; -use std::{collections::HashMap, f32::consts::E, pin::Pin}; -use futures::{FutureExt, Stream, StreamExt}; +use futures::{FutureExt, StreamExt}; -use polkadot_primitives::CandidateHash; -use sc_network::{ - network_state::Peer, - protocol_controller::Message, - request_responses::{IncomingRequest, ProtocolConfig}, - OutboundFailure, PeerId, RequestFailure, -}; +use sc_network::{request_responses::ProtocolConfig, RequestFailure}; use polkadot_node_subsystem::{ messages::NetworkBridgeTxMessage, overseer, SpawnedSubsystem, SubsystemError, }; use polkadot_node_network_protocol::{ - request_response::{v1::ChunkResponse, Recipient, Requests, ResponseSender}, + request_response::{Recipient, Requests, ResponseSender}, Versioned, }; use polkadot_primitives::AuthorityDiscoveryId; @@ -183,20 +169,18 @@ impl MockNetworkBridgeTx { }, NetworkBridgeTxMessage::SendValidationMessage(peers, message) => { for peer in peers { - self.to_network_interface.unbounded_send( - NetworkMessage::MessageFromNode( + self.to_network_interface + .unbounded_send(NetworkMessage::MessageFromNode( self.test_authorithies .peer_id_to_authority .get(&peer) .unwrap() .clone(), message.clone(), - ) - .expect("Should not fail"), - ); + )) + .expect("Should not fail"); } }, - NetworkBridgeTxMessage::ReportPeer(_) => {}, NetworkBridgeTxMessage::DisconnectPeer(_, _) => todo!(), NetworkBridgeTxMessage::SendCollationMessage(_, _) => todo!(), NetworkBridgeTxMessage::SendValidationMessages(_) => todo!(), diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 0091303ac219..fcd96bb83b55 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -66,15 +66,12 @@ use std::{ }; use polkadot_node_network_protocol::{ - self as net_protocol, - peer_set::{ProtocolVersion, ValidationVersion}, - v1 as protocol_v1, v2 as protocol_v2, v3 as protocol_v3, OurView, PeerId, - UnifiedReputationChange as Rep, Versioned, View, + self as net_protocol, PeerId, Versioned, }; use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender}; -use futures::{Future, FutureExt, Stream, StreamExt}; +use futures::{Future, FutureExt, StreamExt}; // An emulated node egress traffic rate_limiter. #[derive(Debug)] pub struct RateLimit { @@ -229,14 +226,14 @@ impl NetworkInterface { /// Create a new `NetworkInterface` pub fn new( spawn_task_handle: SpawnTaskHandle, - mut network: NetworkEmulatorHandle, + network: NetworkEmulatorHandle, bandwidth_bps: usize, mut from_network: UnboundedReceiver, ) -> (NetworkInterface, NetworkInterfaceReceiver) { let mut rx_limiter = RateLimit::new(10, bandwidth_bps); // We need to share the transimit limiter as we handle incoming request/response on rx // thread. - let mut tx_limiter = Arc::new(Mutex::new(RateLimit::new(10, bandwidth_bps))); + let tx_limiter = Arc::new(Mutex::new(RateLimit::new(10, bandwidth_bps))); let mut proxied_requests = FuturesUnordered::new(); // Channel for receiving messages from the network bridge subsystem. @@ -247,8 +244,8 @@ impl NetworkInterface { let (interface_to_bridge_sender, interface_to_bridge_receiver) = mpsc::unbounded::(); - let mut rx_network = network.clone(); - let mut tx_network = network; + let rx_network = network.clone(); + let tx_network = network; let rx_task_bridge_sender = interface_to_bridge_sender.clone(); let rx_task_tx_limiter = tx_limiter.clone(); @@ -422,7 +419,7 @@ impl EmulatedPeer { if self.latency_ms == 0 { let _ = self.to_node.unbounded_send(message).expect("Sending to the node never fails"); } else { - let mut to_node = self.to_node.clone(); + let to_node = self.to_node.clone(); let latency_ms = std::time::Duration::from_millis(self.latency_ms as u64); // Emulate RTT latency @@ -563,16 +560,16 @@ pub fn new_peer( spawn_task_handle: SpawnTaskHandle, handlers: Vec>, stats: Arc, - mut to_network_interface: UnboundedSender, + to_network_interface: UnboundedSender, latency_ms: usize, peer_id: PeerId, ) -> EmulatedPeerHandle { - let (messages_tx, mut messages_rx) = mpsc::unbounded::(); - let (actions_tx, mut actions_rx) = mpsc::unbounded::(); + let (messages_tx, messages_rx) = mpsc::unbounded::(); + let (actions_tx, actions_rx) = mpsc::unbounded::(); let rx_limiter = RateLimit::new(10, bandwidth); let tx_limiter = RateLimit::new(10, bandwidth); - let mut emulated_peer = EmulatedPeer { + let emulated_peer = EmulatedPeer { spawn_handle: spawn_task_handle.clone(), rx_limiter, tx_limiter, @@ -741,6 +738,7 @@ pub fn new_network( } /// Errors that can happen when sending data to emulated peers. +#[derive(Clone, Debug)] pub enum EmulatedPeerError { NotConnected, } From c47a96f41ede424f0e469bb98c7cc9dc4c79d7dd Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 12 Jan 2024 10:55:41 +0200 Subject: [PATCH 170/192] Cleanup1 Signed-off-by: Alexandru Gheorghe --- .../subsystem-bench/src/approval/helpers.rs | 34 +---- .../src/approval/message_generator.rs | 9 +- .../src/approval/mock_chain_api.rs | 80 ----------- .../src/approval/mock_runtime_api.rs | 89 ------------ .../node/subsystem-bench/src/approval/mod.rs | 129 ++++++++++++------ .../src/approval/test_message.rs | 16 +-- .../src/availability/av_store_helpers.rs | 16 +-- .../subsystem-bench/src/availability/mod.rs | 2 + .../subsystem-bench/src/core/configuration.rs | 2 +- .../node/subsystem-bench/src/core/keyring.rs | 14 +- .../src/core/mock/chain_api.rs | 50 ++++++- .../src/core/mock/runtime_api.rs | 122 ++++++++++++----- .../subsystem-bench/src/subsystem-bench.rs | 1 - 13 files changed, 240 insertions(+), 324 deletions(-) delete mode 100644 polkadot/node/subsystem-bench/src/approval/mock_chain_api.rs delete mode 100644 polkadot/node/subsystem-bench/src/approval/mock_runtime_api.rs diff --git a/polkadot/node/subsystem-bench/src/approval/helpers.rs b/polkadot/node/subsystem-bench/src/approval/helpers.rs index f2d55197b6ed..9cdbf0066953 100644 --- a/polkadot/node/subsystem-bench/src/approval/helpers.rs +++ b/polkadot/node/subsystem-bench/src/approval/helpers.rs @@ -11,7 +11,7 @@ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. -use crate::core::configuration::{TestAuthorities, TestConfiguration}; +use crate::core::configuration::TestAuthorities; use itertools::Itertools; use polkadot_node_core_approval_voting::time::{Clock, SystemClock, Tick}; use polkadot_node_network_protocol::{ @@ -25,7 +25,7 @@ use polkadot_node_subsystem_types::messages::{ use polkadot_overseer::AllMessages; use polkadot_primitives::{ BlockNumber, CandidateEvent, CandidateReceipt, CoreIndex, GroupIndex, Hash, Header, - Id as ParaId, IndexedVec, SessionInfo, Slot, ValidatorIndex, + Id as ParaId, Slot, ValidatorIndex, }; use polkadot_primitives_test_helpers::dummy_candidate_receipt_bad_sig; use rand::{seq::SliceRandom, SeedableRng}; @@ -40,8 +40,6 @@ use sp_keyring::sr25519::Keyring as Sr25519Keyring; use sp_runtime::{Digest, DigestItem}; use std::sync::{atomic::AtomicU64, Arc}; -use super::NEEDED_APPROVALS; - // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . @@ -162,34 +160,6 @@ pub fn generate_peer_view_change_for(block_hash: Hash, peer_id: PeerId) -> AllMe AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate(network)) } -/// Generates a test session info with all passed authorities as consensus validators. -pub fn session_info_for_peers( - configuration: &TestConfiguration, - authorities: TestAuthorities, -) -> SessionInfo { - SessionInfo { - validators: authorities.validator_public.iter().cloned().collect(), - discovery_keys: authorities.validator_authority_id.iter().cloned().collect(), - assignment_keys: authorities.validator_assignment_id.iter().cloned().collect(), - validator_groups: IndexedVec::>::from( - (0..authorities.validator_authority_id.len()) - .map(|index| vec![ValidatorIndex(index as u32)]) - .collect_vec(), - ), - n_cores: configuration.n_cores as u32, - needed_approvals: NEEDED_APPROVALS, - zeroth_delay_tranche_width: 0, - relay_vrf_modulo_samples: 6, - n_delay_tranches: 89, - no_show_slots: 3, - active_validator_indices: (0..authorities.validator_authority_id.len()) - .map(|index| ValidatorIndex(index as u32)) - .collect_vec(), - dispute_period: 6, - random_seed: [0u8; 32], - } -} - /// Helper function to create a a signature for the block header. fn garbage_vrf_signature() -> VrfSignature { let transcript = VrfTranscript::new(b"test-garbage", &[]); diff --git a/polkadot/node/subsystem-bench/src/approval/message_generator.rs b/polkadot/node/subsystem-bench/src/approval/message_generator.rs index dfbf23f83897..f84e2bcb0d3c 100644 --- a/polkadot/node/subsystem-bench/src/approval/message_generator.rs +++ b/polkadot/node/subsystem-bench/src/approval/message_generator.rs @@ -57,11 +57,14 @@ use super::{ }; use crate::{ approval::{ - helpers::{generate_babe_epoch, generate_topology, session_info_for_peers}, + helpers::{generate_babe_epoch, generate_topology}, GeneratedState, BUFFER_FOR_GENERATION_MILLIS, LOG_TARGET, NODE_UNDER_TEST, SLOT_DURATION_MILLIS, }, - core::configuration::{TestAuthorities, TestConfiguration, TestObjective}, + core::{ + configuration::{TestAuthorities, TestConfiguration, TestObjective}, + mock::session_info_for_peers, + }, }; use polkadot_node_network_protocol::v3 as protocol_v3; use polkadot_primitives::Hash; @@ -191,7 +194,7 @@ impl PeerMessagesGenerator { ); let babe_epoch = generate_babe_epoch(initial_slot, test_authorities.clone()); - let session_info = session_info_for_peers(configuration, test_authorities.clone()); + let session_info = session_info_for_peers(configuration, &test_authorities); let blocks = ApprovalTestState::generate_blocks_information( configuration, &babe_epoch, diff --git a/polkadot/node/subsystem-bench/src/approval/mock_chain_api.rs b/polkadot/node/subsystem-bench/src/approval/mock_chain_api.rs deleted file mode 100644 index 4e490e60f34c..000000000000 --- a/polkadot/node/subsystem-bench/src/approval/mock_chain_api.rs +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use super::ApprovalTestState; -use futures::FutureExt; -use itertools::Itertools; -use polkadot_node_subsystem::{overseer, SpawnedSubsystem, SubsystemError}; -use polkadot_node_subsystem_types::messages::ChainApiMessage; - -/// Mock ChainApi subsystem used to answer request made by the approval-voting subsystem, during -/// benchmark. All the necessary information to answer the requests is stored in the `state` -pub struct MockChainApi { - pub state: ApprovalTestState, -} -#[overseer::subsystem(ChainApi, error=SubsystemError, prefix=self::overseer)] -impl MockChainApi { - fn start(self, ctx: Context) -> SpawnedSubsystem { - let future = self.run(ctx).map(|_| Ok(())).boxed(); - - SpawnedSubsystem { name: "chain-api-subsystem", future } - } -} - -#[overseer::contextbounds(ChainApi, prefix = self::overseer)] -impl MockChainApi { - async fn run(self, mut ctx: Context) { - loop { - let msg = ctx.recv().await.expect("Should not fail"); - match msg { - orchestra::FromOrchestra::Signal(_) => {}, - orchestra::FromOrchestra::Communication { msg } => match msg { - ChainApiMessage::FinalizedBlockNumber(val) => { - val.send(Ok(0)).unwrap(); - }, - ChainApiMessage::BlockHeader(requested_hash, sender) => { - let info = self.state.get_info_by_hash(requested_hash); - sender.send(Ok(Some(info.header.clone()))).unwrap(); - }, - ChainApiMessage::FinalizedBlockHash(requested_number, sender) => { - let hash = self.state.get_info_by_number(requested_number).hash; - sender.send(Ok(Some(hash))).unwrap(); - }, - ChainApiMessage::BlockNumber(requested_hash, sender) => { - sender - .send(Ok(Some( - self.state.get_info_by_hash(requested_hash).block_number, - ))) - .unwrap(); - }, - ChainApiMessage::Ancestors { hash, k: _, response_channel } => { - let position = self - .state - .blocks - .iter() - .find_position(|block_info| block_info.hash == hash) - .unwrap(); - let (ancestors, _) = self.state.blocks.split_at(position.0); - - let ancestors = ancestors.iter().rev().map(|val| val.hash).collect_vec(); - response_channel.send(Ok(ancestors)).unwrap(); - }, - _ => {}, - }, - } - } - } -} diff --git a/polkadot/node/subsystem-bench/src/approval/mock_runtime_api.rs b/polkadot/node/subsystem-bench/src/approval/mock_runtime_api.rs deleted file mode 100644 index b590ab9e2a13..000000000000 --- a/polkadot/node/subsystem-bench/src/approval/mock_runtime_api.rs +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use super::ApprovalTestState; -use futures::FutureExt; -use polkadot_node_subsystem::{overseer, SpawnedSubsystem, SubsystemError}; -use polkadot_node_subsystem_types::messages::{RuntimeApiMessage, RuntimeApiRequest}; -use polkadot_primitives::{vstaging::NodeFeatures, ExecutorParams}; - -/// Mock RuntimeApi subsystem used to answer request made by the approval-voting subsystem, -/// during benchmark. All the necessary information to answer the requests is stored in the `state` -pub struct MockRuntimeApi { - pub state: ApprovalTestState, -} -#[overseer::subsystem(RuntimeApi, error=SubsystemError, prefix=self::overseer)] -impl MockRuntimeApi { - fn start(self, ctx: Context) -> SpawnedSubsystem { - let future = self.run(ctx).map(|_| Ok(())).boxed(); - - SpawnedSubsystem { name: "runtime-api-subsystem", future } - } -} - -#[overseer::contextbounds(RuntimeApi, prefix = self::overseer)] -impl MockRuntimeApi { - async fn run(self, mut ctx: Context) { - loop { - let msg = ctx.recv().await.expect("Should not fail"); - - match msg { - orchestra::FromOrchestra::Signal(_) => {}, - orchestra::FromOrchestra::Communication { msg } => match msg { - RuntimeApiMessage::Request( - request, - RuntimeApiRequest::CandidateEvents(sender), - ) => { - let candidate_events = - self.state.get_info_by_hash(request).candidates.clone(); - let _ = sender.send(Ok(candidate_events)); - }, - RuntimeApiMessage::Request( - _request, - RuntimeApiRequest::SessionIndexForChild(sender), - ) => { - let _ = sender.send(Ok(1)); - }, - RuntimeApiMessage::Request( - _request, - RuntimeApiRequest::SessionInfo(_session_index, sender), - ) => { - let _ = sender.send(Ok(Some(self.state.session_info.clone()))); - }, - RuntimeApiMessage::Request( - _request, - RuntimeApiRequest::NodeFeatures(_session_index, sender), - ) => { - let _ = sender.send(Ok(NodeFeatures::EMPTY)); - }, - RuntimeApiMessage::Request( - _request, - RuntimeApiRequest::SessionExecutorParams(_session_index, sender), - ) => { - let _ = sender.send(Ok(Some(ExecutorParams::default()))); - }, - RuntimeApiMessage::Request( - _request, - RuntimeApiRequest::CurrentBabeEpoch(sender), - ) => { - let _ = sender.send(Ok(self.state.babe_epoch.clone())); - }, - _ => {}, - }, - } - } - } -} diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index f16c545a03f1..b916b570a9f7 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -22,19 +22,18 @@ use crate::{ approval::{ helpers::{ generate_babe_epoch, generate_new_session_topology, generate_peer_connected, - generate_peer_view_change_for, session_info_for_peers, PastSystemClock, + generate_peer_view_change_for, PastSystemClock, }, message_generator::PeerMessagesGenerator, - mock_chain_api::MockChainApi, mock_chain_selection::MockChainSelection, - mock_runtime_api::MockRuntimeApi, }, core::{ configuration::{TestAuthorities, TestConfiguration}, environment::{TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT}, mock::{ dummy_builder, network_bridge::MockNetworkBridgeTx, AlwaysSupportsParachains, - MockNetworkBridgeRx, TestSyncOracle, + ChainApiState, MockChainApi, MockNetworkBridgeRx, MockRuntimeApi, TestSyncOracle, + NEEDED_APPROVALS, }, network::{ new_network, HandleNetworkMessage, NetworkEmulatorHandle, NetworkInterface, @@ -61,13 +60,15 @@ use polkadot_node_subsystem_types::messages::{ApprovalDistributionMessage, Appro use polkadot_node_subsystem_util::metrics::Metrics; use polkadot_overseer::Handle as OverseerHandleReal; use polkadot_primitives::{ - BlockNumber, CandidateEvent, CandidateIndex, Hash, Header, SessionInfo, Slot, ValidatorIndex, + BlockNumber, CandidateEvent, CandidateIndex, CandidateReceipt, Hash, Header, Slot, + ValidatorIndex, }; use prometheus::Registry; use sc_keystore::LocalKeystore; use sc_service::SpawnTaskHandle; use serde::{Deserialize, Serialize}; use sp_consensus_babe::Epoch as BabeEpoch; +use sp_core::H256; use std::{ cmp::max, collections::{HashMap, HashSet}, @@ -84,9 +85,7 @@ use tokio::time::sleep; mod helpers; mod message_generator; -mod mock_chain_api; mod mock_chain_selection; -mod mock_runtime_api; mod test_message; pub const LOG_TARGET: &str = "bench::approval"; @@ -99,7 +98,6 @@ pub(crate) const TEST_CONFIG: ApprovalVotingConfig = ApprovalVotingConfig { }; pub const NODE_UNDER_TEST: u32 = 0; -pub const NEEDED_APPROVALS: u32 = 30; /// Start generating messages for a slot into the future, so that the /// generation nevers falls behind the current slot. @@ -264,16 +262,16 @@ pub struct ApprovalTestState { blocks: Vec, /// The babe epoch used during testing. babe_epoch: BabeEpoch, - /// The session info used during testing. - session_info: SessionInfo, /// The slot at which this benchamrk begins. generated_state: GeneratedState, /// The test authorities test_authorities: TestAuthorities, /// Last approved block number. last_approved_block: Arc, - /// Total sent messages. - total_sent_messages: Arc, + /// Total sent messages from peers to node + total_sent_messages_to_node: Arc, + /// Total sent messages from test node to other peers + total_sent_messages_from_node: Arc, /// Total unique sent messages. total_unique_messages: Arc, /// Approval voting metrics. @@ -319,7 +317,6 @@ impl ApprovalTestState { let babe_epoch = generate_babe_epoch(generated_state.initial_slot, test_authorities.clone()); - let session_info = session_info_for_peers(configuration, test_authorities.clone()); let blocks = Self::generate_blocks_information( configuration, &babe_epoch, @@ -329,11 +326,11 @@ impl ApprovalTestState { let state = ApprovalTestState { blocks, babe_epoch: babe_epoch.clone(), - session_info: session_info.clone(), generated_state, test_authorities, last_approved_block: Arc::new(AtomicU32::new(0)), - total_sent_messages: Arc::new(AtomicU64::new(0)), + total_sent_messages_to_node: Arc::new(AtomicU64::new(0)), + total_sent_messages_from_node: Arc::new(AtomicU64::new(0)), total_unique_messages: Arc::new(AtomicU64::new(0)), options, approval_voting_metrics: ApprovalVotingMetrics::try_register(&dependencies.registry) @@ -355,7 +352,7 @@ impl ApprovalTestState { ) -> Vec { let mut per_block_heads: Vec = Vec::new(); let mut prev_candidates = 0; - for block_number in 1..configuration.num_blocks + 1 { + for block_number in 1..=configuration.num_blocks { let block_hash = Hash::repeat_byte(block_number as u8); let parent_hash = per_block_heads.last().map(|val| val.hash).unwrap_or(Hash::repeat_byte(0xde)); @@ -424,6 +421,43 @@ impl ApprovalTestState { .produce_messages(env, self.generated_state.all_messages.take().unwrap()); producer_rx } + + // Generates a ChainApiState used for driving MockChainApi + fn build_chain_api_state(&self) -> ChainApiState { + ChainApiState { + block_headers: self + .blocks + .iter() + .map(|block| (block.hash, block.header.clone())) + .collect(), + } + } + + // Builds a map with the list of candidate events per-block. + fn candidate_events_by_block(&self) -> HashMap> { + self.blocks.iter().map(|block| (block.hash, block.candidates.clone())).collect() + } + + // Builds a map with the list of candidate hashes per-block. + fn candidate_hashes_by_block(&self) -> HashMap> { + self.blocks + .iter() + .map(|block| { + ( + block.hash, + block + .candidates + .iter() + .map(|candidate_event| match candidate_event { + CandidateEvent::CandidateBacked(_, _, _, _) => todo!(), + CandidateEvent::CandidateIncluded(receipt, _, _, _) => receipt.clone(), + CandidateEvent::CandidateTimedOut(_, _, _) => todo!(), + }) + .collect_vec(), + ) + }) + .collect() + } } impl ApprovalTestState { @@ -435,14 +469,6 @@ impl ApprovalTestState { .expect("Mocks should not use unknown hashes") } - /// Returns test data for the given block number - fn get_info_by_number(&self, requested_number: u32) -> &BlockTestData { - self.blocks - .iter() - .find(|block| block.block_number == requested_number) - .expect("Mocks should not use unknown numbers") - } - /// Returns test data for the given slot fn get_info_by_slot(&self, slot: Slot) -> Option<&BlockTestData> { self.blocks.iter().find(|block| block.slot == slot) @@ -457,6 +483,9 @@ impl HandleNetworkMessage for ApprovalTestState { crate::core::network::NetworkMessage, >, ) -> Option { + self.total_sent_messages_from_node + .as_ref() + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); None } } @@ -472,7 +501,10 @@ struct PeerMessageProducer { /// A handle to the overseer, used for sending messages to the node /// under test. overseer_handle: OverseerHandleReal, + /// Channel for producer to notify main loop it finished sending + /// all messages and they have been processed. notify_done: oneshot::Sender<()>, + /// The metrics registry. registry: Registry, } @@ -561,14 +593,20 @@ impl PeerMessageProducer { sleep(Duration::from_secs(6)).await; let (tx, rx) = oneshot::channel(); let msg = ApprovalDistributionMessage::GetApprovalSignatures(HashSet::new(), tx); - self.send_message(AllMessages::ApprovalDistribution(msg), ValidatorIndex(0), None) - .await; + self.send_overseer_message( + AllMessages::ApprovalDistribution(msg), + ValidatorIndex(0), + None, + ) + .await; rx.await.expect("Failed to get signatures"); self.notify_done.send(()).expect("Failed to notify main loop"); gum::info!("All messages processed "); }); } + // Processes a single message bundle and queue the messages to be sent by the peers that would + // send the message in our simulation. pub fn process_message( &mut self, bundle: MessagesBundle, @@ -599,9 +637,8 @@ impl PeerMessageProducer { .fetch_add(1, std::sync::atomic::Ordering::SeqCst); for (peer, messages) in message.split_by_peer_id(&self.state.test_authorities) { for message in messages { - let _latency = message.get_latency(); self.state - .total_sent_messages + .total_sent_messages_to_node .as_ref() .fetch_add(1, std::sync::atomic::Ordering::SeqCst); self.queue_message_from_peer(message, peer.0) @@ -616,6 +653,7 @@ impl PeerMessageProducer { reprocess_skipped } + // Tells if it is the time to process a message. pub fn time_to_process_message( &self, bundle: &MessagesBundle, @@ -628,7 +666,7 @@ impl PeerMessageProducer { self.state.get_info_by_hash(bundle.assignments.first().unwrap().block_hash); let tranche_now = system_clock.tranche_now(SLOT_DURATION_MILLIS, block_info.slot); - Self::time_to_send( + Self::is_past_tranche( bundle, tranche_now, current_slot, @@ -637,7 +675,8 @@ impl PeerMessageProducer { ) || !bundle.bundle_needed(per_candidate_data, &self.options) } - pub fn time_to_send( + // Tells if the tranche where the bundle should be sent has passed. + pub fn is_past_tranche( bundle: &MessagesBundle, tranche_now: u32, current_slot: Slot, @@ -649,6 +688,7 @@ impl PeerMessageProducer { block_initialized } + // Queue message to be sent by validator `sent_by` fn queue_message_from_peer(&mut self, message: TestMessageInfo, sent_by: ValidatorIndex) { let peer_authority_id = self .state @@ -665,8 +705,8 @@ impl PeerMessageProducer { .expect("Network should be up and running"); } - /// Queues a message to be sent by the peer identified by the `sent_by` value. - async fn send_message( + // Queues a message to be sent by the peer identified by the `sent_by` value. + async fn send_overseer_message( &mut self, message: AllMessages, _sent_by: ValidatorIndex, @@ -681,6 +721,8 @@ impl PeerMessageProducer { }); } + // Sends the messages needed by approval-distribution and approval-voting for processing a + // message. E.g: PeerViewChange. async fn initialize_block(&mut self, block_info: &BlockTestData) { gum::info!("Initialize block {:?}", block_info.hash); let (tx, rx) = oneshot::channel(); @@ -696,10 +738,12 @@ impl PeerMessageProducer { let view_update = generate_peer_view_change_for(block_info.hash, *peer_id); - self.send_message(view_update, validator, None).await; + self.send_overseer_message(view_update, validator, None).await; } } + // Initializes the candidates test data. This is used for bookeeping if more assignments and + // approvals would be needed. fn initialize_candidates_test_data( &self, ) -> HashMap<(Hash, CandidateIndex), CandidateTestData> { @@ -728,7 +772,7 @@ impl PeerMessageProducer { fn build_overseer( state: &ApprovalTestState, network: &NetworkEmulatorHandle, - _config: &TestConfiguration, + config: &TestConfiguration, dependencies: &TestEnvironmentDependencies, network_interface: &NetworkInterface, network_receiver: NetworkInterfaceReceiver, @@ -755,9 +799,15 @@ fn build_overseer( let approval_distribution = ApprovalDistribution::new(Metrics::register(Some(&dependencies.registry)).unwrap()); - let mock_chain_api = MockChainApi { state: state.clone() }; + let mock_chain_api = MockChainApi::new(state.build_chain_api_state()); let mock_chain_selection = MockChainSelection { state: state.clone(), clock: system_clock }; - let mock_runtime_api = MockRuntimeApi { state: state.clone() }; + let mock_runtime_api = MockRuntimeApi::new( + config.clone(), + state.test_authorities.clone(), + state.candidate_hashes_by_block(), + state.candidate_events_by_block(), + Some(state.babe_epoch.clone()), + ); let mock_tx_bridge = MockNetworkBridgeTx::new( network.clone(), network_interface.subsystem_sender(), @@ -910,7 +960,7 @@ pub async fn bench_approvals_run( gum::info!( "Waiting for all blocks to be approved current approved {:} num_sent {:} num_unique {:}", state.last_approved_block.load(std::sync::atomic::Ordering::SeqCst), - state.total_sent_messages.load(std::sync::atomic::Ordering::SeqCst), + state.total_sent_messages_to_node.load(std::sync::atomic::Ordering::SeqCst), state.total_unique_messages.load(std::sync::atomic::Ordering::SeqCst) ); tokio::time::sleep(Duration::from_secs(6)).await; @@ -966,9 +1016,10 @@ pub async fn bench_approvals_run( let duration: u128 = start_marker.elapsed().as_millis(); gum::info!( - "All blocks processed in {} num_messages {} num_unique_messages {}", + "All blocks processed in {} total_sent_messages_to_node {} total_sent_messages_from_node {} num_unique_messages {}", format!("{:?}ms", duration).cyan(), - state.total_sent_messages.load(std::sync::atomic::Ordering::SeqCst), + state.total_sent_messages_to_node.load(std::sync::atomic::Ordering::SeqCst), + state.total_sent_messages_from_node.load(std::sync::atomic::Ordering::SeqCst), state.total_unique_messages.load(std::sync::atomic::Ordering::SeqCst) ); diff --git a/polkadot/node/subsystem-bench/src/approval/test_message.rs b/polkadot/node/subsystem-bench/src/approval/test_message.rs index d8cb3c73d6a5..785bfd530510 100644 --- a/polkadot/node/subsystem-bench/src/approval/test_message.rs +++ b/polkadot/node/subsystem-bench/src/approval/test_message.rs @@ -18,15 +18,11 @@ use super::{ApprovalsOptions, BlockTestData, CandidateTestData}; use crate::core::configuration::TestAuthorities; use itertools::Itertools; use parity_scale_codec::{Decode, Encode}; -use polkadot_node_network_protocol::{v3 as protocol_v3}; - +use polkadot_node_network_protocol::v3 as protocol_v3; use polkadot_primitives::{CandidateIndex, Hash, ValidatorIndex}; use sc_network::PeerId; -use std::{ - collections::{HashMap, HashSet}, - time::Duration, -}; +use std::collections::{HashMap, HashSet}; #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] pub struct TestMessageInfo { @@ -125,14 +121,6 @@ impl MessagesBundle { } impl TestMessageInfo { - /// Gives us the latency for this message. - pub fn get_latency(&self) -> Option { - match &self.msg { - protocol_v3::ApprovalDistributionMessage::Assignments(_) => None, - protocol_v3::ApprovalDistributionMessage::Approvals(_) => None, - } - } - /// Tells if the message is an approval. fn is_approval(&self) -> bool { match self.msg { diff --git a/polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs b/polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs index 18ea2f72891f..e6827f1d8aea 100644 --- a/polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs +++ b/polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use crate::core::mock::TestSyncOracle; + use super::*; use polkadot_node_metrics::metrics::Metrics; @@ -31,22 +33,10 @@ mod columns { const TEST_CONFIG: Config = Config { col_data: columns::DATA, col_meta: columns::META }; -struct DumbOracle; - -impl sp_consensus::SyncOracle for DumbOracle { - fn is_major_syncing(&self) -> bool { - false - } - - fn is_offline(&self) -> bool { - unimplemented!("oh no!") - } -} - pub fn new_av_store(dependencies: &TestEnvironmentDependencies) -> AvailabilityStoreSubsystem { let metrics = Metrics::try_register(&dependencies.registry).unwrap(); - AvailabilityStoreSubsystem::new(test_store(), TEST_CONFIG, Box::new(DumbOracle), metrics) + AvailabilityStoreSubsystem::new(test_store(), TEST_CONFIG, Box::new(TestSyncOracle {}), metrics) } fn test_store() -> Arc { diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index f3a379d1b6b3..edfc6289d56d 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -257,6 +257,8 @@ fn prepare_test_inner( config.clone(), test_authorities.clone(), candidate_hashes, + Default::default(), + Default::default(), ); let availability_state = NetworkAvailabilityState { diff --git a/polkadot/node/subsystem-bench/src/core/configuration.rs b/polkadot/node/subsystem-bench/src/core/configuration.rs index abf50f6705b5..9b9b2fede81b 100644 --- a/polkadot/node/subsystem-bench/src/core/configuration.rs +++ b/polkadot/node/subsystem-bench/src/core/configuration.rs @@ -176,7 +176,7 @@ impl TestConfiguration { .map(|seed| keyring.sr25519_new(seed.as_str())) .collect::>(); - // Generate `AuthorityDiscoveryId`` for each peer + // Generate keys and peers ids in each of the format needed by the tests. let validator_public: Vec = keys.iter().map(|key| key.clone().into()).collect::>(); diff --git a/polkadot/node/subsystem-bench/src/core/keyring.rs b/polkadot/node/subsystem-bench/src/core/keyring.rs index f6b4b9878462..6d646443b6bc 100644 --- a/polkadot/node/subsystem-bench/src/core/keyring.rs +++ b/polkadot/node/subsystem-bench/src/core/keyring.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use polkadot_primitives::{ValidatorId, ASSIGNMENT_KEY_TYPE_ID}; +use polkadot_primitives::ValidatorId; use sc_keystore::LocalKeystore; use sp_application_crypto::AppCrypto; pub use sp_core::sr25519; @@ -36,17 +36,9 @@ impl Default for Keyring { impl Keyring { pub fn sr25519_new(&self, seed: &str) -> Public { - let validator_id = self - .keystore + self.keystore .sr25519_generate_new(ValidatorId::ID, Some(seed)) - .expect("Insert key into keystore"); - - let _assignment_id = self - .keystore - .sr25519_generate_new(ASSIGNMENT_KEY_TYPE_ID, Some(seed)) - .expect("should not fail"); - - validator_id + .expect("Insert key into keystore") } pub fn keystore(&self) -> Arc { diff --git a/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs b/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs index 5acb3547e076..6fc8d805319b 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs @@ -16,7 +16,8 @@ //! //! A generic runtime api subsystem mockup suitable to be used in benchmarks. -use polkadot_primitives::{Header}; +use itertools::Itertools; +use polkadot_primitives::Header; use polkadot_node_subsystem::{ messages::ChainApiMessage, overseer, SpawnedSubsystem, SubsystemError, @@ -38,6 +39,12 @@ pub struct MockChainApi { state: ChainApiState, } +impl ChainApiState { + fn get_header_by_number(&self, requested_number: u32) -> Option<&Header> { + self.block_headers.values().find(|header| header.number == requested_number) + } +} + impl MockChainApi { pub fn new(state: ChainApiState) -> MockChainApi { Self { state } @@ -77,9 +84,44 @@ impl MockChainApi { .expect("Relay chain block hashes are known"), ))); }, - ChainApiMessage::Ancestors { hash: _, k: _, response_channel } => { - // For our purposes, no ancestors is fine. - let _ = response_channel.send(Ok(Vec::new())); + ChainApiMessage::FinalizedBlockNumber(val) => { + val.send(Ok(0)).unwrap(); + }, + ChainApiMessage::FinalizedBlockHash(requested_number, sender) => { + let hash = self + .state + .get_header_by_number(requested_number) + .expect("Unknow block number") + .hash(); + sender.send(Ok(Some(hash))).unwrap(); + }, + ChainApiMessage::BlockNumber(requested_hash, sender) => { + sender + .send(Ok(Some( + self.state + .block_headers + .get(&requested_hash) + .expect("Unknown block hash") + .number, + ))) + .unwrap(); + }, + ChainApiMessage::Ancestors { hash, k: _, response_channel } => { + let block_number = self + .state + .block_headers + .get(&hash) + .expect("Unknown block hash") + .number; + let ancestors = self + .state + .block_headers + .iter() + .filter(|(_, header)| header.number < block_number) + .sorted_by(|a, b| a.1.number.cmp(&b.1.number)) + .map(|(hash, _)| *hash) + .collect_vec(); + response_channel.send(Ok(ancestors)).unwrap(); }, _ => { unimplemented!("Unexpected chain-api message") diff --git a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs index f01f2c0d29e9..5e0c5b13d677 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs @@ -16,16 +16,18 @@ //! //! A generic runtime api subsystem mockup suitable to be used in benchmarks. -use polkadot_primitives::{ - CandidateReceipt, CoreState, GroupIndex, IndexedVec, OccupiedCore, SessionInfo, ValidatorIndex, -}; - use bitvec::prelude::BitVec; +use itertools::Itertools; use polkadot_node_subsystem::{ messages::{RuntimeApiMessage, RuntimeApiRequest}, overseer, SpawnedSubsystem, SubsystemError, }; use polkadot_node_subsystem_types::OverseerSignal; +use polkadot_primitives::{ + vstaging::NodeFeatures, CandidateEvent, CandidateReceipt, CoreState, GroupIndex, IndexedVec, + OccupiedCore, SessionInfo, ValidatorIndex, +}; +use sp_consensus_babe::Epoch as BabeEpoch; use sp_core::H256; use std::collections::HashMap; @@ -38,8 +40,11 @@ const LOG_TARGET: &str = "subsystem-bench::runtime-api-mock"; pub struct RuntimeApiState { // All authorities in the test, authorities: TestAuthorities, - // Candidate + // Candidate hashes per block candidate_hashes: HashMap>, + // Included candidates per bock + included_candidates: HashMap>, + babe_epoch: Option, } /// A mocked `runtime-api` subsystem. @@ -53,35 +58,57 @@ impl MockRuntimeApi { config: TestConfiguration, authorities: TestAuthorities, candidate_hashes: HashMap>, + included_candidates: HashMap>, + babe_epoch: Option, ) -> MockRuntimeApi { - Self { state: RuntimeApiState { authorities, candidate_hashes }, config } + Self { + state: RuntimeApiState { + authorities, + candidate_hashes, + included_candidates, + babe_epoch, + }, + config, + } } fn session_info(&self) -> SessionInfo { - let all_validators = (0..self.config.n_validators) - .map(|i| ValidatorIndex(i as _)) - .collect::>(); - - let validator_groups = all_validators - .chunks(self.config.max_validators_per_core) - .map(|x| Vec::from(x)) - .collect::>(); - - SessionInfo { - validators: self.state.authorities.validator_public.clone().into(), - discovery_keys: self.state.authorities.validator_authority_id.clone(), - validator_groups: IndexedVec::>::from(validator_groups), - assignment_keys: vec![], - n_cores: self.config.n_cores as u32, - zeroth_delay_tranche_width: 0, - relay_vrf_modulo_samples: 0, - n_delay_tranches: 0, - no_show_slots: 0, - needed_approvals: 0, - active_validator_indices: vec![], - dispute_period: 6, - random_seed: [0u8; 32], - } + session_info_for_peers(&self.config, &self.state.authorities) + } +} + +pub const NEEDED_APPROVALS: u32 = 30; + +/// Generates a test session info with all passed authorities as consensus validators. +pub fn session_info_for_peers( + configuration: &TestConfiguration, + authorities: &TestAuthorities, +) -> SessionInfo { + let all_validators = (0..configuration.n_validators) + .map(|i| ValidatorIndex(i as _)) + .collect::>(); + + let validator_groups = all_validators + .chunks(configuration.max_validators_per_core) + .map(|x| Vec::from(x)) + .collect::>(); + + SessionInfo { + validators: authorities.validator_public.iter().cloned().collect(), + discovery_keys: authorities.validator_authority_id.iter().cloned().collect(), + assignment_keys: authorities.validator_assignment_id.iter().cloned().collect(), + validator_groups: IndexedVec::>::from(validator_groups), + n_cores: configuration.n_cores as u32, + needed_approvals: NEEDED_APPROVALS, + zeroth_delay_tranche_width: 0, + relay_vrf_modulo_samples: 6, + n_delay_tranches: 89, + no_show_slots: 3, + active_validator_indices: (0..authorities.validator_authority_id.len()) + .map(|index| ValidatorIndex(index as u32)) + .collect_vec(), + dispute_period: 6, + random_seed: [0u8; 32], } } @@ -111,6 +138,17 @@ impl MockRuntimeApi { gum::debug!(target: LOG_TARGET, msg=?msg, "recv message"); match msg { + RuntimeApiMessage::Request( + request, + RuntimeApiRequest::CandidateEvents(sender), + ) => { + let candidate_events = self + .state + .included_candidates + .get(&request) + .expect("Unknown block hash"); + let _ = sender.send(Ok(candidate_events.clone())); + }, RuntimeApiMessage::Request( _block_hash, RuntimeApiRequest::SessionInfo(_session_index, sender), @@ -124,24 +162,24 @@ impl MockRuntimeApi { let _ = sender.send(Ok(Some(Default::default()))); }, RuntimeApiMessage::Request( - _block_hash, - RuntimeApiRequest::Validators(sender), + _request, + RuntimeApiRequest::NodeFeatures(_session_index, sender), ) => { - let _ = - sender.send(Ok(self.state.authorities.validator_public.clone())); + let _ = sender.send(Ok(NodeFeatures::EMPTY)); }, RuntimeApiMessage::Request( _block_hash, - RuntimeApiRequest::CandidateEvents(sender), + RuntimeApiRequest::Validators(sender), ) => { - let _ = sender.send(Ok(Default::default())); + let _ = + sender.send(Ok(self.state.authorities.validator_public.clone())); }, RuntimeApiMessage::Request( _block_hash, RuntimeApiRequest::SessionIndexForChild(sender), ) => { // Session is always the same. - let _ = sender.send(Ok(0)); + let _ = sender.send(Ok(1)); }, RuntimeApiMessage::Request( block_hash, @@ -176,6 +214,16 @@ impl MockRuntimeApi { let _ = sender.send(Ok(cores)); }, + RuntimeApiMessage::Request( + _request, + RuntimeApiRequest::CurrentBabeEpoch(sender), + ) => { + let _ = sender.send(Ok(self + .state + .babe_epoch + .clone() + .expect("Babe epoch unpopulated"))); + }, // Long term TODO: implement more as needed. _ => { unimplemented!("Unexpected runtime-api message") diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index cd18ed709a3b..e2741f308788 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -217,7 +217,6 @@ fn main() -> eyre::Result<()> { .filter(Some("hyper"), log::LevelFilter::Info) // Avoid `Terminating due to subsystem exit subsystem` warnings .filter(Some("polkadot_overseer"), log::LevelFilter::Error) - .filter(Some("parachain"), log::LevelFilter::Info) .filter(None, log::LevelFilter::Info) .format_timestamp_millis() // .filter(None, log::LevelFilter::Trace) From 9514b47c1513c87f0b09c779e664769e0c4af8ec Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 12 Jan 2024 16:26:20 +0200 Subject: [PATCH 171/192] Fix running on a versi node Signed-off-by: Alexandru Gheorghe --- polkadot/node/subsystem-bench/src/approval/mod.rs | 8 ++++++++ polkadot/node/subsystem-bench/src/core/keyring.rs | 1 - 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index b916b570a9f7..04c5742be26b 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -969,6 +969,14 @@ pub async fn bench_approvals_run( gum::info!("Awaiting producer to signal done"); producer_rx.await.expect("Failed to receive done from message producer"); + + gum::info!("Awaiting polkadot_parachain_subsystem_bounded_received to tells us the messages have been processed"); + + env.wait_until_metric_ge( + "polkadot_parachain_subsystem_bounded_received", + state.total_sent_messages_to_node.load(std::sync::atomic::Ordering::SeqCst) as usize, + ) + .await; gum::info!("Requesting approval votes ms"); for info in &state.blocks { diff --git a/polkadot/node/subsystem-bench/src/core/keyring.rs b/polkadot/node/subsystem-bench/src/core/keyring.rs index 6d646443b6bc..c290d30b46fb 100644 --- a/polkadot/node/subsystem-bench/src/core/keyring.rs +++ b/polkadot/node/subsystem-bench/src/core/keyring.rs @@ -17,7 +17,6 @@ use polkadot_primitives::ValidatorId; use sc_keystore::LocalKeystore; use sp_application_crypto::AppCrypto; -pub use sp_core::sr25519; use sp_core::sr25519::Public; use sp_keystore::Keystore; use std::sync::Arc; From 83709afd80dd0eef55c5dd5db782a47b23b960d6 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 12 Jan 2024 16:51:39 +0200 Subject: [PATCH 172/192] Add label to wait_until_metric_ge Signed-off-by: Alexandru Gheorghe --- polkadot/node/subsystem-bench/src/approval/mod.rs | 1 + .../node/subsystem-bench/src/availability/mod.rs | 1 + .../node/subsystem-bench/src/core/environment.rs | 14 ++++++++++++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 04c5742be26b..cd0c8bb2a1ab 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -974,6 +974,7 @@ pub async fn bench_approvals_run( env.wait_until_metric_ge( "polkadot_parachain_subsystem_bounded_received", + Some(("subsystem_name", "approval-distribution")), state.total_sent_messages_to_node.load(std::sync::atomic::Ordering::SeqCst) as usize, ) .await; diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index edfc6289d56d..4c6e53e2414b 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -720,6 +720,7 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: // Wait for all bitfields to be processed. env.wait_until_metric_ge( "polkadot_parachain_received_availabilty_bitfields_total", + None, (config.n_validators - 1) * (block_num), ) .await; diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 40280d0fcbb4..594e76696bb2 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -340,10 +340,20 @@ impl TestEnvironment { } /// Blocks until `metric_name` >= `value` - pub async fn wait_until_metric_ge(&self, metric_name: &str, value: usize) { + pub async fn wait_until_metric_ge( + &self, + metric_name: &str, + label: Option<(&str, &str)>, + value: usize, + ) { let value = value as f64; loop { - let test_metrics = super::display::parse_metrics(self.registry()); + let test_metrics = if let Some((label_name, label_value)) = label { + super::display::parse_metrics(self.registry()) + .subset_with_label_value(label_name, label_value) + } else { + super::display::parse_metrics(self.registry()) + }; let current_value = test_metrics.sum_by(metric_name); gum::debug!(target: LOG_TARGET, metric_name, current_value, value, "Waiting for metric"); From 57714db871960a3bdb10fceedbca096b2f482127 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 12 Jan 2024 17:00:06 +0200 Subject: [PATCH 173/192] Fixup run Signed-off-by: Alexandru Gheorghe --- polkadot/node/subsystem-bench/src/approval/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index cd0c8bb2a1ab..08494ff77a09 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -974,7 +974,7 @@ pub async fn bench_approvals_run( env.wait_until_metric_ge( "polkadot_parachain_subsystem_bounded_received", - Some(("subsystem_name", "approval-distribution")), + Some(("subsystem_name", "approval-distribution-subsystem")), state.total_sent_messages_to_node.load(std::sync::atomic::Ordering::SeqCst) as usize, ) .await; From 21f36450ab9989b32bf45d51e930c74f66a6c170 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 15 Jan 2024 09:09:21 +0200 Subject: [PATCH 174/192] Fixup clippy Signed-off-by: Alexandru Gheorghe --- .../examples/approvals_no_shows.yaml | 11 ++------ .../examples/approvals_throughput.yaml | 4 +-- .../approvals_throughput_12_seconds.yaml | 11 ++------ ...hroughput_12_seconds_no_optimisations.yaml | 11 ++------ .../approvals_throughput_best_case.yaml | 4 +-- ...s_throughput_no_optimisations_enabled.yaml | 11 ++------ .../src/approval/message_generator.rs | 6 ++-- .../node/subsystem-bench/src/approval/mod.rs | 2 +- .../subsystem-bench/src/availability/mod.rs | 22 +++++++-------- .../subsystem-bench/src/core/configuration.rs | 8 +++--- .../subsystem-bench/src/core/environment.rs | 6 ++-- .../src/core/mock/network_bridge.rs | 4 +-- .../src/core/mock/runtime_api.rs | 8 +++--- polkadot/node/subsystem-bench/src/core/mod.rs | 1 + .../node/subsystem-bench/src/core/network.rs | 28 +++++++------------ .../subsystem-bench/src/subsystem-bench.rs | 2 +- 16 files changed, 51 insertions(+), 88 deletions(-) diff --git a/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml b/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml index 0bb46666555d..d6753ca6f1b3 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml @@ -11,8 +11,8 @@ TestConfiguration: workdir_prefix: "/tmp/" num_no_shows_per_candidate: 10 approval_distribution_expected_tof: 60.0 - approval_distribution_cpu_ms: 3.0 - approval_voting_cpu_ms: 4.30 + approval_distribution_cpu_ms: 14.00 + approval_voting_cpu_ms: 14.00 n_validators: 500 n_cores: 100 n_included_candidates: 100 @@ -20,12 +20,5 @@ TestConfiguration: max_pov_size: 5120 peer_bandwidth: 524288000000 bandwidth: 524288000000 - latency: - min_latency: - secs: 0 - nanos: 1000000 - max_latency: - secs: 0 - nanos: 100000000 error: 0 num_blocks: 10 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml index c043abd31b1b..4214da69d895 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml @@ -11,8 +11,8 @@ TestConfiguration: workdir_prefix: "/tmp/" num_no_shows_per_candidate: 0 approval_distribution_expected_tof: 60.0 - approval_distribution_cpu_ms: 7.70 - approval_voting_cpu_ms: 9.90 + approval_distribution_cpu_ms: 14.00 + approval_voting_cpu_ms: 14.00 n_validators: 500 n_cores: 100 n_included_candidates: 100 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml index 8b3d8da6198a..8cefefe76960 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml @@ -11,8 +11,8 @@ TestConfiguration: workdir_prefix: "/tmp/" num_no_shows_per_candidate: 0 approval_distribution_expected_tof: 60.0 - approval_distribution_cpu_ms: 3.50 - approval_voting_cpu_ms: 5.50 + approval_distribution_cpu_ms: 14.00 + approval_voting_cpu_ms: 14.00 n_validators: 500 n_cores: 100 n_included_candidates: 50 @@ -20,12 +20,5 @@ TestConfiguration: max_pov_size: 5120 peer_bandwidth: 524288000000 bandwidth: 524288000000 - latency: - min_latency: - secs: 0 - nanos: 1000000 - max_latency: - secs: 0 - nanos: 100000000 error: 0 num_blocks: 10 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml index 5fd62058a94f..16bf322f3e22 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml @@ -11,8 +11,8 @@ TestConfiguration: workdir_prefix: "/tmp/" num_no_shows_per_candidate: 0 approval_distribution_expected_tof: 60.0 - approval_distribution_cpu_ms: 4.70 - approval_voting_cpu_ms: 6.30 + approval_distribution_cpu_ms: 14.00 + approval_voting_cpu_ms: 14.00 n_validators: 500 n_cores: 100 n_included_candidates: 50 @@ -20,12 +20,5 @@ TestConfiguration: max_pov_size: 5120 peer_bandwidth: 524288000000 bandwidth: 524288000000 - latency: - min_latency: - secs: 0 - nanos: 1000000 - max_latency: - secs: 0 - nanos: 100000000 error: 0 num_blocks: 10 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml index 65d7a1702cd6..b7351d4b5245 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml @@ -11,8 +11,8 @@ TestConfiguration: workdir_prefix: "/tmp/" num_no_shows_per_candidate: 0 approval_distribution_expected_tof: 60.0 - approval_distribution_cpu_ms: 1.0 - approval_voting_cpu_ms: 1.0 + approval_distribution_cpu_ms: 14.00 + approval_voting_cpu_ms: 14.00 n_validators: 500 n_cores: 100 n_included_candidates: 100 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml index 0526c80f6673..49ae1a4dc292 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml @@ -11,8 +11,8 @@ TestConfiguration: workdir_prefix: "/tmp/" num_no_shows_per_candidate: 0 approval_distribution_expected_tof: 60.0 - approval_distribution_cpu_ms: 10.50 - approval_voting_cpu_ms: 13.00 + approval_distribution_cpu_ms: 14.00 + approval_voting_cpu_ms: 14.00 n_validators: 500 n_cores: 100 n_included_candidates: 100 @@ -20,12 +20,5 @@ TestConfiguration: max_pov_size: 5120 peer_bandwidth: 524288000000 bandwidth: 524288000000 - latency: - min_latency: - secs: 0 - nanos: 1000000 - max_latency: - secs: 0 - nanos: 100000000 error: 0 num_blocks: 10 diff --git a/polkadot/node/subsystem-bench/src/approval/message_generator.rs b/polkadot/node/subsystem-bench/src/approval/message_generator.rs index f84e2bcb0d3c..d5fa530346c3 100644 --- a/polkadot/node/subsystem-bench/src/approval/message_generator.rs +++ b/polkadot/node/subsystem-bench/src/approval/message_generator.rs @@ -194,7 +194,7 @@ impl PeerMessagesGenerator { ); let babe_epoch = generate_babe_epoch(initial_slot, test_authorities.clone()); - let session_info = session_info_for_peers(configuration, &test_authorities); + let session_info = session_info_for_peers(configuration, test_authorities); let blocks = ApprovalTestState::generate_blocks_information( configuration, &babe_epoch, @@ -391,7 +391,7 @@ fn issue_approvals( { approvals_to_create.push(TestSignInfo::sign_candidates( &mut queued_to_sign, - &validator_ids, + validator_ids, block_hash, num_coalesce, store, @@ -427,7 +427,7 @@ fn issue_approvals( if !queued_to_sign.is_empty() { messages.push(TestSignInfo::sign_candidates( &mut queued_to_sign, - &validator_ids, + validator_ids, block_hash, num_coalesce, store, diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 08494ff77a09..a80292663868 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -882,7 +882,7 @@ pub async fn bench_approvals(env: &mut TestEnvironment, mut state: ApprovalTestS .start_message_production( env.network(), env.overseer_handle().clone(), - &env, + env, env.registry().clone(), ) .await; diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 4c6e53e2414b..4369ab1963e3 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -109,6 +109,7 @@ fn build_overseer_for_availability_read( (overseer, OverseerHandle::new(raw_handle)) } +#[allow(clippy::too_many_arguments)] fn build_overseer_for_availability_write( spawn_task_handle: SpawnTaskHandle, runtime_api: MockRuntimeApi, @@ -166,10 +167,8 @@ impl HandleNetworkMessage for NetworkAvailabilityState { .expect("candidate was generated previously; qed"); gum::warn!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); - let chunk: ChunkResponse = self.chunks.get(*candidate_index as usize).unwrap() - [validator_index] - .clone() - .into(); + let chunk: ChunkResponse = + self.chunks.get(*candidate_index).unwrap()[validator_index].clone().into(); let _size = chunk.encoded_size(); let response = Ok(( @@ -192,7 +191,7 @@ impl HandleNetworkMessage for NetworkAvailabilityState { gum::debug!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); let available_data = - self.available_data.get(*candidate_index as usize).unwrap().clone(); + self.available_data.get(*candidate_index).unwrap().clone(); let _size = available_data.encoded_size(); @@ -200,7 +199,7 @@ impl HandleNetworkMessage for NetworkAvailabilityState { AvailableDataFetchingResponse::from(Some(available_data)).encode(), ProtocolName::from(""), )); - let _ = outgoing_request + outgoing_request .pending_response .send(response) .expect("Response is always sent succesfully"); @@ -270,14 +269,14 @@ fn prepare_test_inner( let mut req_cfgs = Vec::new(); let (collation_req_receiver, collation_req_cfg) = - IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); req_cfgs.push(collation_req_cfg); let (pov_req_receiver, pov_req_cfg) = - IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); let (chunk_req_receiver, chunk_req_cfg) = - IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); req_cfgs.push(pov_req_cfg); let (network, network_interface, network_receiver) = @@ -614,8 +613,7 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: )) .await; - let _ = rx - .await + rx.await .unwrap() .expect("Test candidates are stored nicely in availability store"); } @@ -700,7 +698,7 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: payload, &signing_context, ValidatorIndex(index as u32), - &validator_public.clone().into(), + &validator_public.clone(), ) .ok() .flatten() diff --git a/polkadot/node/subsystem-bench/src/core/configuration.rs b/polkadot/node/subsystem-bench/src/core/configuration.rs index 9b9b2fede81b..2a9cd67ea9f2 100644 --- a/polkadot/node/subsystem-bench/src/core/configuration.rs +++ b/polkadot/node/subsystem-bench/src/core/configuration.rs @@ -178,16 +178,16 @@ impl TestConfiguration { // Generate keys and peers ids in each of the format needed by the tests. let validator_public: Vec = - keys.iter().map(|key| key.clone().into()).collect::>(); + keys.iter().map(|key| (*key).into()).collect::>(); let validator_authority_id: Vec = - keys.iter().map(|key| key.clone().into()).collect::>().into(); + keys.iter().map(|key| (*key).into()).collect::>(); let validator_babe_id: Vec = - keys.iter().map(|key| key.clone().into()).collect::>().into(); + keys.iter().map(|key| (*key).into()).collect::>(); let validator_assignment_id: Vec = - keys.iter().map(|key| key.clone().into()).collect::>().into(); + keys.iter().map(|key| (*key).into()).collect::>(); let peer_ids: Vec = keys.iter().map(|_| PeerId::random()).collect::>(); let peer_id_to_authority = peer_ids diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 594e76696bb2..fd9c80de9bec 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -391,18 +391,18 @@ impl TestEnvironment { pub fn display_cpu_usage(&self, subsystems_under_test: &[&str]) { let test_metrics = super::display::parse_metrics(self.registry()); - for subsystem in subsystems_under_test.into_iter() { + for subsystem in subsystems_under_test.iter() { let subsystem_cpu_metrics = test_metrics.subset_with_label_value("task_group", subsystem); let total_cpu = subsystem_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); println!( "{} CPU usage {}", - format!("{}", subsystem).bright_green(), + subsystem.to_string().bright_green(), format!("{:.3}s", total_cpu).bright_purple() ); println!( "{} CPU usage per block {}", - format!("{}", subsystem).bright_green(), + subsystem.to_string().bright_green(), format!("{:.3}s", total_cpu / self.config().num_blocks as f64).bright_purple() ); } diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index 44d3c1c15b2e..7f5c80577f52 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -151,7 +151,7 @@ impl MockNetworkBridgeTx { if !self.network.is_peer_connected(&peer_id) { // Attempting to send a request to a disconnected peer. - let _ = request + request .into_response_sender() .send(Err(RequestFailure::NotConnected)) .expect("send never fails"); @@ -234,7 +234,7 @@ impl MockNetworkBridgeRx { NetworkMessage::RequestFromPeer(request) => { if let Some(protocol) = self.chunk_request_sender.as_mut() { if let Some(inbound_queue) = protocol.inbound_queue.as_ref() { - let _ = inbound_queue + inbound_queue .send(request) .await .expect("Forwarding requests to subsystem never fails"); diff --git a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs index 5e0c5b13d677..e35f0353cea8 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs @@ -90,13 +90,13 @@ pub fn session_info_for_peers( let validator_groups = all_validators .chunks(configuration.max_validators_per_core) - .map(|x| Vec::from(x)) + .map(Vec::from) .collect::>(); SessionInfo { validators: authorities.validator_public.iter().cloned().collect(), - discovery_keys: authorities.validator_authority_id.iter().cloned().collect(), - assignment_keys: authorities.validator_assignment_id.iter().cloned().collect(), + discovery_keys: authorities.validator_authority_id.to_vec(), + assignment_keys: authorities.validator_assignment_id.to_vec(), validator_groups: IndexedVec::>::from(validator_groups), n_cores: configuration.n_cores as u32, needed_approvals: NEEDED_APPROVALS, @@ -193,7 +193,7 @@ impl MockRuntimeApi { // All cores are always occupied. let cores = candidate_hashes - .into_iter() + .iter() .enumerate() .map(|(index, candidate_receipt)| { // Ensure test breaks if badly configured. diff --git a/polkadot/node/subsystem-bench/src/core/mod.rs b/polkadot/node/subsystem-bench/src/core/mod.rs index 282788d143b4..22de9ad5a330 100644 --- a/polkadot/node/subsystem-bench/src/core/mod.rs +++ b/polkadot/node/subsystem-bench/src/core/mod.rs @@ -21,4 +21,5 @@ pub mod display; pub mod environment; pub mod keyring; pub mod mock; +#[allow(clippy::tabs_in_doc_comments)] pub mod network; diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index fcd96bb83b55..d5b46031e310 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -10,7 +10,6 @@ // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. - // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . //! @@ -35,7 +34,6 @@ //! [Emulated Network Bridge] //! | //! Subsystems under test - use crate::core::configuration::random_latency; use super::{ @@ -65,9 +63,7 @@ use std::{ time::{Duration, Instant}, }; -use polkadot_node_network_protocol::{ - self as net_protocol, PeerId, Versioned, -}; +use polkadot_node_network_protocol::{self as net_protocol, PeerId, Versioned}; use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender}; @@ -380,23 +376,20 @@ pub struct EmulatedPeerHandle { impl EmulatedPeerHandle { /// Receive and process a message pub fn receive(&self, message: NetworkMessage) { - let _ = self - .messages_tx + self.messages_tx .unbounded_send(message) .expect("Sending action to the peer never fails"); } pub fn send_message(&self, message: VersionedValidationProtocol) { - let _ = self - .actions_tx + self.actions_tx .unbounded_send(NetworkMessage::MessageFromPeer(self.peer_id, message)) .expect("Sending action to the peer never fails"); } /// Send a `request` to the node. pub fn send_request(&self, request: IncomingRequest) { - let _ = self - .actions_tx + self.actions_tx .unbounded_send(NetworkMessage::RequestFromPeer(request)) .expect("Sending action to the peer never fails"); } @@ -417,7 +410,7 @@ impl EmulatedPeer { self.tx_limiter.reap(message.size()).await; if self.latency_ms == 0 { - let _ = self.to_node.unbounded_send(message).expect("Sending to the node never fails"); + self.to_node.unbounded_send(message).expect("Sending to the node never fails"); } else { let to_node = self.to_node.clone(); let latency_ms = std::time::Duration::from_millis(self.latency_ms as u64); @@ -426,8 +419,7 @@ impl EmulatedPeer { self.spawn_handle .spawn("peer-latency-emulator", "test-environment", async move { tokio::time::sleep(latency_ms).await; - let _ = - to_node.unbounded_send(message).expect("Sending to the node never fails"); + to_node.unbounded_send(message).expect("Sending to the node never fails"); }); } } @@ -690,7 +682,7 @@ pub fn new_network( // Create a `PeerEmulator` for each peer. let (stats, mut peers): (_, Vec<_>) = (0..n_peers) - .zip(authorities.validator_authority_id.clone().into_iter()) + .zip(authorities.validator_authority_id.clone()) .map(|(peer_index, authority_id)| { validator_authority_id_mapping.insert(authority_id, peer_index); let stats = Arc::new(PeerEmulatorStats::new(peer_index, metrics.clone())); @@ -774,7 +766,7 @@ impl NetworkEmulatorHandle { from_peer: &AuthorityDiscoveryId, message: VersionedValidationProtocol, ) -> Result<(), EmulatedPeerError> { - let dst_peer = self.peer(&from_peer); + let dst_peer = self.peer(from_peer); if !dst_peer.is_connected() { gum::warn!(target: LOG_TARGET, "Attempted to send message from a peer not connected to our node, operation ignored"); @@ -791,7 +783,7 @@ impl NetworkEmulatorHandle { from_peer: &AuthorityDiscoveryId, request: IncomingRequest, ) -> Result<(), EmulatedPeerError> { - let dst_peer = self.peer(&from_peer); + let dst_peer = self.peer(from_peer); if !dst_peer.is_connected() { gum::warn!(target: LOG_TARGET, "Attempted to send request from a peer not connected to our node, operation ignored"); @@ -915,7 +907,7 @@ mod tests { let mut reap_amount = 0; while rate_limiter.total_ticks < tick_rate { reap_amount += 1; - reap_amount = reap_amount % 100; + reap_amount %= 100; rate_limiter.reap(reap_amount).await; total_sent += reap_amount; diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index e2741f308788..4fcc87a08d0f 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -213,7 +213,7 @@ impl BenchCli { fn main() -> eyre::Result<()> { color_eyre::install()?; - let _ = env_logger::builder() + env_logger::builder() .filter(Some("hyper"), log::LevelFilter::Info) // Avoid `Terminating due to subsystem exit subsystem` warnings .filter(Some("polkadot_overseer"), log::LevelFilter::Error) From 2a146f002900e4bc955cb79884858c18ccd6055c Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 15 Jan 2024 09:41:36 +0200 Subject: [PATCH 175/192] Cargo fmt Signed-off-by: Alexandru Gheorghe --- polkadot/node/subsystem-bench/src/availability/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 4369ab1963e3..7af90d5b94b7 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -190,8 +190,7 @@ impl HandleNetworkMessage for NetworkAvailabilityState { .expect("candidate was generated previously; qed"); gum::debug!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); - let available_data = - self.available_data.get(*candidate_index).unwrap().clone(); + let available_data = self.available_data.get(*candidate_index).unwrap().clone(); let _size = available_data.encoded_size(); From 22a1bf911131e5b8f37725641522b2c85515aa10 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 17 Jan 2024 13:45:35 +0200 Subject: [PATCH 176/192] Finish network refactoring and fix benchmarks Signed-off-by: Andrei Sandu --- .../subsystem-bench/src/availability/mod.rs | 57 ++--- .../subsystem-bench/src/core/configuration.rs | 4 + .../node/subsystem-bench/src/core/display.rs | 3 +- .../subsystem-bench/src/core/environment.rs | 6 +- .../src/core/mock/chain_api.rs | 4 +- .../src/core/mock/network_bridge.rs | 74 +------ .../node/subsystem-bench/src/core/network.rs | 206 ++++++++++++++---- .../subsystem-bench/src/subsystem-bench.rs | 21 +- 8 files changed, 218 insertions(+), 157 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index abf74a2c1d2c..240d69381f31 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -40,7 +40,6 @@ use polkadot_availability_recovery::AvailabilityRecoverySubsystem; use polkadot_node_primitives::{AvailableData, ErasureChunk}; use crate::GENESIS_HASH; -use futures::FutureExt; use parity_scale_codec::Encode; use polkadot_node_network_protocol::{ request_response::{ @@ -48,9 +47,9 @@ use polkadot_node_network_protocol::{ AvailableDataFetchingResponse, ChunkFetchingRequest, ChunkFetchingResponse, ChunkResponse, }, - IncomingRequest, OutgoingRequest, ReqProtocolNames, Requests, + IncomingRequest, ReqProtocolNames, Requests, }, - BitfieldDistributionMessage, OurView, Versioned, VersionedValidationProtocol, View, + OurView, Versioned, VersionedValidationProtocol, }; use sc_network::request_responses::IncomingRequest as RawIncomingRequest; @@ -146,7 +145,7 @@ impl HandleNetworkMessage for NetworkAvailabilityState { fn handle( &self, message: NetworkMessage, - node_sender: &mut futures::channel::mpsc::UnboundedSender, + _node_sender: &mut futures::channel::mpsc::UnboundedSender, ) -> Option { match message { NetworkMessage::RequestFromNode(peer, request) => match request { @@ -165,12 +164,10 @@ impl HandleNetworkMessage for NetworkAvailabilityState { [validator_index] .clone() .into(); - let mut size = chunk.encoded_size(); - let response = Ok(ChunkFetchingResponse::from(Some(chunk)).encode()); if let Err(err) = outgoing_request.pending_response.send(response) { - gum::error!(target: LOG_TARGET, "Failed to send `ChunkFetchingResponse`"); + gum::error!(target: LOG_TARGET, ?err, "Failed to send `ChunkFetchingResponse`"); } None @@ -186,8 +183,6 @@ impl HandleNetworkMessage for NetworkAvailabilityState { let available_data = self.available_data.get(*candidate_index as usize).unwrap().clone(); - let size = available_data.encoded_size(); - let response = Ok(AvailableDataFetchingResponse::from(Some(available_data)).encode()); let _ = outgoing_request @@ -281,11 +276,8 @@ fn prepare_test_inner( state.set_chunk_request_protocol(chunk_req_cfg); let (overseer, overseer_handle) = match &state.config().objective { - TestObjective::DataAvailabilityRead(_options) => { - let use_fast_path = match &state.config().objective { - TestObjective::DataAvailabilityRead(options) => options.fetch_from_backers, - _ => panic!("Unexpected objective"), - }; + TestObjective::DataAvailabilityRead(options) => { + let use_fast_path = options.fetch_from_backers; let subsystem = if use_fast_path { AvailabilityRecoverySubsystem::with_fast_path( @@ -300,7 +292,6 @@ fn prepare_test_inner( }; // Use a mocked av-store. - // TODO: switch to real av-store. let av_store = av_store::MockAvailabilityStore::new( state.chunks.clone(), state.candidate_hashes.clone(), @@ -390,8 +381,6 @@ pub struct TestState { chunks: Vec>, // Availability distribution chunk_request_protocol: Option, - // Availability distribution. - pov_request_protocol: Option, // Per relay chain block - candidate backed by our backing group backed_candidates: Vec, } @@ -493,7 +482,6 @@ impl TestState { candidate_hashes: HashMap::new(), candidates: Vec::new().into_iter().cycle(), chunk_request_protocol: None, - pov_request_protocol: None, backed_candidates: Vec::new(), }; @@ -508,14 +496,6 @@ impl TestState { pub fn set_chunk_request_protocol(&mut self, config: ProtocolConfig) { self.chunk_request_protocol = Some(config); } - - pub fn set_pov_request_protocol(&mut self, config: ProtocolConfig) { - self.pov_request_protocol = Some(config); - } - - pub fn chunk_request_protocol(&self) -> Option { - self.chunk_request_protocol.clone() - } } pub async fn benchmark_availability_read(env: &mut TestEnvironment, mut state: TestState) { @@ -624,9 +604,6 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: env.import_block(new_block_import_info(relay_block_hash, block_num as BlockNumber)) .await; - let chunk_request_protocol = - state.chunk_request_protocol().expect("No chunk fetching protocol configured"); - // Inform bitfield distribution about our view of current test block let message = polkadot_node_subsystem_types::messages::BitfieldDistributionMessage::NetworkBridgeUpdate( NetworkBridgeEvent::OurViewChange(OurView::new(vec![(relay_block_hash, Arc::new(Span::Disabled))], 0)) @@ -656,7 +633,9 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: .get(index) .expect("all validators have keys"); - if env.network().send_request_from_peer(peer, request).is_ok() { + if env.network().is_peer_connected(peer) && + env.network().send_request_from_peer(peer, request).is_ok() + { receivers.push(pending_response_receiver); } } @@ -679,6 +658,7 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: // Spawn a task that will generate `n_validator` - 1 signed bitfiends and // send them from the emulated peers to the subsystem. + // TODO: Implement topology. env.spawn_blocking("send-bitfields", async move { for index in 1..n_validators { let validator_public = @@ -703,17 +683,22 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: let message = peer_bitfield_message_v2(relay_block_hash, signed_bitfield); - // Send the action to `to_peer`. - let _ = network.send_message_from_peer(from_peer, message); + // Send the action from peer only if it is connected to our node. + if network.is_peer_connected(&from_peer) { + let _ = network.send_message_from_peer(from_peer, message); + } } - - gum::info!("Waiting for {} bitfields to be received and processed", n_validators - 1); }); + gum::info!( + "Waiting for {} bitfields to be received and processed", + config.connected_count() + ); + // Wait for all bitfields to be processed. - env.wait_until_metric_ge( + env.wait_until_metric_eq( "polkadot_parachain_received_availabilty_bitfields_total", - (config.n_validators - 1) * (block_num), + config.connected_count() * block_num, ) .await; diff --git a/polkadot/node/subsystem-bench/src/core/configuration.rs b/polkadot/node/subsystem-bench/src/core/configuration.rs index 440b507ebcb0..9d54ea21b5d4 100644 --- a/polkadot/node/subsystem-bench/src/core/configuration.rs +++ b/polkadot/node/subsystem-bench/src/core/configuration.rs @@ -153,6 +153,10 @@ impl TestConfiguration { pub fn pov_sizes(&self) -> &[usize] { &self.pov_sizes } + /// Return the number of peers connected to our node. + pub fn connected_count(&self) -> usize { + ((self.n_validators - 1) as f64 / (100.0 / self.connectivity as f64)) as usize + } /// Generates the authority keys we need for the network emulation. pub fn generate_authorities(&self) -> TestAuthorities { diff --git a/polkadot/node/subsystem-bench/src/core/display.rs b/polkadot/node/subsystem-bench/src/core/display.rs index 0efd19171a07..e5824c6fc274 100644 --- a/polkadot/node/subsystem-bench/src/core/display.rs +++ b/polkadot/node/subsystem-bench/src/core/display.rs @@ -180,7 +180,8 @@ pub fn parse_metrics(registry: &Registry) -> MetricCollection { pub fn display_configuration(test_config: &TestConfiguration) { gum::info!( - "{}, {}, {}, {}, {}", + "[] {}, {}, {}, {}, {}", + format!("objective = {:?}", test_config.objective).green(), format!("n_validators = {}", test_config.n_validators).blue(), format!("n_cores = {}", test_config.n_cores).blue(), format!("pov_size = {} - {}", test_config.min_pov_size, test_config.max_pov_size) diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 5887f467423c..7e3ccab458e3 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -36,8 +36,6 @@ use tokio::runtime::Handle; const LOG_TARGET: &str = "subsystem-bench::environment"; use super::{configuration::TestAuthorities, network::NetworkInterface}; -const MIB: usize = 1024 * 1024; - /// Test environment/configuration metrics #[derive(Clone)] pub struct TestEnvironmentMetrics { @@ -317,14 +315,14 @@ impl TestEnvironment { } /// Blocks until `metric_name` >= `value` - pub async fn wait_until_metric_ge(&self, metric_name: &str, value: usize) { + pub async fn wait_until_metric_eq(&self, metric_name: &str, value: usize) { let value = value as f64; loop { let test_metrics = super::display::parse_metrics(self.registry()); let current_value = test_metrics.sum_by(metric_name); gum::debug!(target: LOG_TARGET, metric_name, current_value, value, "Waiting for metric"); - if current_value >= value { + if current_value == value { break } diff --git a/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs b/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs index ba6d53366bc5..a440a3155d50 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs @@ -16,7 +16,7 @@ //! //! A generic runtime api subsystem mockup suitable to be used in benchmarks. -use polkadot_primitives::{GroupIndex, Header, IndexedVec, SessionInfo, ValidatorIndex}; +use polkadot_primitives::Header; use polkadot_node_subsystem::{ messages::ChainApiMessage, overseer, SpawnedSubsystem, SubsystemError, @@ -77,7 +77,7 @@ impl MockChainApi { .expect("Relay chain block hashes are known"), ))); }, - ChainApiMessage::Ancestors { hash, k, response_channel } => { + ChainApiMessage::Ancestors { hash: _hash, k: _k, response_channel } => { // For our purposes, no ancestors is fine. let _ = response_channel.send(Ok(Vec::new())); }, diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index c08d07d08027..68a8dce3bc70 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -16,41 +16,23 @@ //! //! Mocked `network-bridge` subsystems that uses a `NetworkInterface` to access //! the emulated network. -use futures::{ - channel::{ - mpsc::{UnboundedReceiver, UnboundedSender}, - oneshot, - }, - Future, -}; -use overseer::AllMessages; -use parity_scale_codec::Encode; +use futures::{channel::mpsc::UnboundedSender, FutureExt, StreamExt}; use polkadot_node_subsystem_types::{ messages::{BitfieldDistributionMessage, NetworkBridgeEvent}, OverseerSignal, }; -use std::{collections::HashMap, f32::consts::E, pin::Pin}; - -use futures::{FutureExt, Stream, StreamExt}; -use polkadot_primitives::CandidateHash; -use sc_network::{ - network_state::Peer, - request_responses::{IncomingRequest, ProtocolConfig}, - OutboundFailure, PeerId, RequestFailure, -}; +use sc_network::{request_responses::ProtocolConfig, PeerId, RequestFailure}; use polkadot_node_subsystem::{ messages::NetworkBridgeTxMessage, overseer, SpawnedSubsystem, SubsystemError, }; -use polkadot_node_network_protocol::{ - request_response::{ v1::ChunkResponse, Recipient, Requests, ResponseSender}, - Versioned, -}; -use polkadot_primitives::AuthorityDiscoveryId; +use polkadot_node_network_protocol::Versioned; -use crate::core::network::{NetworkEmulatorHandle, NetworkInterfaceReceiver, NetworkMessage}; +use crate::core::network::{ + NetworkEmulatorHandle, NetworkInterfaceReceiver, NetworkMessage, RequestExt, +}; const LOG_TARGET: &str = "subsystem-bench::network-bridge"; @@ -106,41 +88,6 @@ impl MockNetworkBridgeRx { } } -// Helper trait for `Requests`. -trait RequestExt { - fn authority_id(&self) -> Option<&AuthorityDiscoveryId>; - fn into_response_sender(self) -> ResponseSender; -} - -impl RequestExt for Requests { - fn authority_id(&self) -> Option<&AuthorityDiscoveryId> { - match self { - Requests::ChunkFetchingV1(request) => { - if let Recipient::Authority(authority_id) = &request.peer { - Some(authority_id) - } else { - None - } - }, - request => { - unimplemented!("RequestAuthority not implemented for {:?}", request) - }, - } - } - - fn into_response_sender(self) -> ResponseSender { - match self { - Requests::ChunkFetchingV1(outgoing_request) => { - outgoing_request.pending_response - }, - Requests::AvailableDataFetchingV1(outgoing_request) => { - outgoing_request.pending_response - } - _ => unimplemented!("unsupported request type") - } - } -} - #[overseer::contextbounds(NetworkBridgeTx, prefix = self::overseer)] impl MockNetworkBridgeTx { async fn run(self, mut ctx: Context) { @@ -161,13 +108,16 @@ impl MockNetworkBridgeTx { if !self.network.is_peer_connected(&peer_id) { // Attempting to send a request to a disconnected peer. - let _ = request.into_response_sender().send(Err(RequestFailure::NotConnected)).expect("send never fails"); + let _ = request + .into_response_sender() + .send(Err(RequestFailure::NotConnected)) + .expect("send never fails"); continue } - + let peer_message = NetworkMessage::RequestFromNode(peer_id.clone(), request); - + let _ = self.to_network_interface.unbounded_send(peer_message); } }, diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 222f36d040ae..763a346e9558 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -49,7 +49,11 @@ use futures::{ lock::Mutex, stream::FuturesUnordered, }; -use net_protocol::{request_response::Requests, VersionedValidationProtocol}; + +use net_protocol::{ + request_response::{Recipient, Requests, ResponseSender}, + VersionedValidationProtocol, +}; use parity_scale_codec::Encode; use polkadot_primitives::AuthorityDiscoveryId; use prometheus_endpoint::U64; @@ -65,16 +69,11 @@ use std::{ time::{Duration, Instant}, }; -use polkadot_node_network_protocol::{ - self as net_protocol, - peer_set::{ProtocolVersion, ValidationVersion}, - v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, OurView, PeerId, - UnifiedReputationChange as Rep, Versioned, View, -}; +use polkadot_node_network_protocol::{self as net_protocol, Versioned}; use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender}; -use futures::{Future, FutureExt, Stream, StreamExt}; +use futures::{Future, FutureExt, StreamExt}; // An emulated node egress traffic rate_limiter. #[derive(Debug)] pub struct RateLimit { @@ -167,7 +166,7 @@ impl NetworkMessage { message.encoded_size(), NetworkMessage::MessageFromNode(_peer_id, Versioned::VStaging(message)) => message.encoded_size(), - NetworkMessage::RequestFromNode(_peer_id, incoming) => request_size(incoming), + NetworkMessage::RequestFromNode(_peer_id, incoming) => incoming.size(), NetworkMessage::RequestFromPeer(request) => request.payload.encoded_size(), } } @@ -229,15 +228,12 @@ impl NetworkInterface { /// Create a new `NetworkInterface` pub fn new( spawn_task_handle: SpawnTaskHandle, - mut network: NetworkEmulatorHandle, + network: NetworkEmulatorHandle, bandwidth_bps: usize, mut from_network: UnboundedReceiver, ) -> (NetworkInterface, NetworkInterfaceReceiver) { - let mut rx_limiter = RateLimit::new(10, bandwidth_bps); - // We need to share the transimit limiter as we handle incoming request/response on rx - // thread. - let mut tx_limiter = Arc::new(Mutex::new(RateLimit::new(10, bandwidth_bps))); - let mut proxied_requests = FuturesUnordered::new(); + let rx_limiter = Arc::new(Mutex::new(RateLimit::new(10, bandwidth_bps))); + let tx_limiter = Arc::new(Mutex::new(RateLimit::new(10, bandwidth_bps))); // Channel for receiving messages from the network bridge subsystem. let (bridge_to_interface_sender, mut bridge_to_interface_receiver) = @@ -247,22 +243,25 @@ impl NetworkInterface { let (interface_to_bridge_sender, interface_to_bridge_receiver) = mpsc::unbounded::(); - let mut rx_network = network.clone(); - let mut tx_network = network; + let rx_network = network.clone(); + let tx_network = network; let rx_task_bridge_sender = interface_to_bridge_sender.clone(); - let rx_task_tx_limiter = tx_limiter.clone(); - let tx_task_tx_limiter = tx_limiter; + + let task_rx_limiter = rx_limiter.clone(); + let task_tx_limiter = tx_limiter.clone(); // A task that forwards messages from emulated peers to the node (emulated network bridge). let rx_task = async move { + let mut proxied_requests = FuturesUnordered::new(); + loop { let mut from_network = from_network.next().fuse(); futures::select! { maybe_peer_message = from_network => { if let Some(peer_message) = maybe_peer_message { let size = peer_message.size(); - rx_limiter.reap(size).await; + task_rx_limiter.lock().await.reap(size).await; rx_network.inc_received(size); // To be able to apply the configured bandwidth limits for responses being sent @@ -301,8 +300,8 @@ impl NetworkInterface { // Enforce bandwidth based on the response the node has sent. // TODO: Fix the stall of RX when TX lock() takes a while to refill - // the token bucket. - rx_task_tx_limiter.lock().await.reap(bytes).await; + // the token bucket. Good idea would be to create a task for each request. + task_tx_limiter.lock().await.reap(bytes).await; rx_network.inc_sent(bytes); // Forward the response to original recipient. @@ -328,19 +327,38 @@ impl NetworkInterface { } .boxed(); + let task_spawn_handle = spawn_task_handle.clone(); + let task_rx_limiter = rx_limiter.clone(); + let task_tx_limiter = tx_limiter.clone(); + // A task that forwards messages from the node to emulated peers. let tx_task = async move { + // Wrap it in an `Arc` to avoid `clone()` the inner data as we need to share it across + // many send tasks. + let tx_network = Arc::new(tx_network); + loop { if let Some(peer_message) = bridge_to_interface_receiver.next().await { let size = peer_message.size(); // Ensure bandwidth used is limited. - tx_task_tx_limiter.lock().await.reap(size).await; + task_tx_limiter.lock().await.reap(size).await; match peer_message { NetworkMessage::MessageFromNode(peer, message) => tx_network.send_message_to_peer(&peer, message), - NetworkMessage::RequestFromNode(peer, request) => - tx_network.send_request_to_peer(&peer, request), + NetworkMessage::RequestFromNode(peer, request) => { + // Send request through a proxy so we can account and limit bandwidth + // usage for the node. + let send_task = Self::proxy_send_request( + peer.clone(), + request, + tx_network.clone(), + task_rx_limiter.clone(), + ) + .boxed(); + + task_spawn_handle.spawn("request-proxy", "test-environment", send_task); + }, _ => panic!( "Unexpected network message received from emulated network bridge" ), @@ -368,6 +386,43 @@ impl NetworkInterface { pub fn subsystem_sender(&self) -> UnboundedSender { self.bridge_to_interface_sender.clone() } + + /// Helper method that proxies a request from node to peer and implements rate limiting and + /// accounting. + async fn proxy_send_request( + peer: AuthorityDiscoveryId, + mut request: Requests, + tx_network: Arc, + task_rx_limiter: Arc>, + ) { + let (proxy_sender, proxy_receiver) = oneshot::channel(); + + // Modify the request response sender so we can intercept the answer + let sender = request.swap_response_sender(proxy_sender); + + // Send the modified request to the peer. + tx_network.send_request_to_peer(&peer, request); + + // Wait for answer (intercept the response). + match proxy_receiver.await { + Err(_) => { + panic!("Emulated peer hangup"); + }, + Ok(Err(err)) => { + sender.send(Err(err)).expect("Oneshot send always works."); + }, + Ok(Ok(response)) => { + let response_size = response.encoded_size(); + task_rx_limiter.lock().await.reap(response_size).await; + tx_network.inc_received(response_size); + + // Send the response to the original request sender. + if let Err(_) = sender.send(Ok(response)) { + gum::warn!(target: LOG_TARGET, response_size, "response oneshot canceled by node") + } + }, + }; + } } /// A handle for controlling an emulated peer. @@ -380,19 +435,17 @@ pub struct EmulatedPeerHandle { } impl EmulatedPeerHandle { - /// Receive and process a message + /// Receive and process a message from the node. pub fn receive(&self, message: NetworkMessage) { - let _ = self - .messages_tx - .unbounded_send(message) - .expect("Sending action to the peer never fails"); + let _ = self.messages_tx.unbounded_send(message).expect("Peer message channel hangup"); } + /// Send a message to the node. pub fn send_message(&self, message: VersionedValidationProtocol) { let _ = self .actions_tx .unbounded_send(NetworkMessage::MessageFromPeer(message)) - .expect("Sending action to the peer never fails"); + .expect("Peer action channel hangup"); } /// Send a `request` to the node. @@ -400,7 +453,7 @@ impl EmulatedPeerHandle { let _ = self .actions_tx .unbounded_send(NetworkMessage::RequestFromPeer(request)) - .expect("Sending action to the peer never fails"); + .expect("Peer action channel hangup"); } } @@ -421,7 +474,7 @@ impl EmulatedPeer { if self.latency_ms == 0 { let _ = self.to_node.unbounded_send(message).expect("Sending to the node never fails"); } else { - let mut to_node = self.to_node.clone(); + let to_node = self.to_node.clone(); let latency_ms = std::time::Duration::from_millis(self.latency_ms as u64); // Emulate RTT latency @@ -557,20 +610,21 @@ async fn emulated_peer_loop( } } +/// Creates a new peer emulator task and returns a handle to it. pub fn new_peer( bandwidth: usize, spawn_task_handle: SpawnTaskHandle, handlers: Vec>, stats: Arc, - mut to_network_interface: UnboundedSender, + to_network_interface: UnboundedSender, latency_ms: usize, ) -> EmulatedPeerHandle { - let (messages_tx, mut messages_rx) = mpsc::unbounded::(); - let (actions_tx, mut actions_rx) = mpsc::unbounded::(); + let (messages_tx, messages_rx) = mpsc::unbounded::(); + let (actions_tx, actions_rx) = mpsc::unbounded::(); let rx_limiter = RateLimit::new(10, bandwidth); let tx_limiter = RateLimit::new(10, bandwidth); - let mut emulated_peer = EmulatedPeer { + let emulated_peer = EmulatedPeer { spawn_handle: spawn_task_handle.clone(), rx_limiter, tx_limiter, @@ -709,10 +763,9 @@ pub fn new_network( }) .unzip(); - let connected_count = config.n_validators as f64 / (100.0 / config.connectivity as f64); + let connected_count = config.connected_count(); - let (_connected, to_disconnect) = - peers.partial_shuffle(&mut thread_rng(), connected_count as usize); + let (_connected, to_disconnect) = peers.partial_shuffle(&mut thread_rng(), connected_count); for peer in to_disconnect { peer.disconnect(); @@ -739,10 +792,11 @@ pub fn new_network( /// Errors that can happen when sending data to emulated peers. pub enum EmulatedPeerError { - NotConnected + NotConnected, } impl NetworkEmulatorHandle { + /// Returns true if the emulated peer is connected to the node under test. pub fn is_peer_connected(&self, peer: &AuthorityDiscoveryId) -> bool { self.peer(peer).is_connected() } @@ -886,13 +940,69 @@ impl Metrics { } } -/// A helper to determine the request payload size. -pub fn request_size(request: &Requests) -> usize { - match request { - Requests::ChunkFetchingV1(outgoing_request) => outgoing_request.payload.encoded_size(), - Requests::AvailableDataFetchingV1(outgoing_request) => - outgoing_request.payload.encoded_size(), - _ => unimplemented!("received an unexpected request"), +// Helper trait for low level access to `Requests` variants. +pub trait RequestExt { + /// Get the authority id if any from the request. + fn authority_id(&self) -> Option<&AuthorityDiscoveryId>; + /// Consume self and return the response sender. + fn into_response_sender(self) -> ResponseSender; + /// Allows to change the `ResponseSender` in place. + fn swap_response_sender(&mut self, new_sender: ResponseSender) -> ResponseSender; + /// Returns the size in bytes of the request payload. + fn size(&self) -> usize; +} + +impl RequestExt for Requests { + fn authority_id(&self) -> Option<&AuthorityDiscoveryId> { + match self { + Requests::ChunkFetchingV1(request) => { + if let Recipient::Authority(authority_id) = &request.peer { + Some(authority_id) + } else { + None + } + }, + Requests::AvailableDataFetchingV1(request) => { + if let Recipient::Authority(authority_id) = &request.peer { + Some(authority_id) + } else { + None + } + }, + request => { + unimplemented!("RequestAuthority not implemented for {:?}", request) + }, + } + } + + fn into_response_sender(self) -> ResponseSender { + match self { + Requests::ChunkFetchingV1(outgoing_request) => outgoing_request.pending_response, + Requests::AvailableDataFetchingV1(outgoing_request) => + outgoing_request.pending_response, + _ => unimplemented!("unsupported request type"), + } + } + + /// Swaps the `ResponseSender` and returns the previous value. + fn swap_response_sender(&mut self, new_sender: ResponseSender) -> ResponseSender { + match self { + Requests::ChunkFetchingV1(outgoing_request) => + std::mem::replace(&mut outgoing_request.pending_response, new_sender), + Requests::AvailableDataFetchingV1(outgoing_request) => + std::mem::replace(&mut outgoing_request.pending_response, new_sender), + _ => unimplemented!("unsupported request type"), + } + } + + /// Returns the size in bytes of the request payload. + fn size(&self) -> usize { + match self { + Requests::ChunkFetchingV1(outgoing_request) => outgoing_request.payload.encoded_size(), + Requests::AvailableDataFetchingV1(outgoing_request) => + outgoing_request.payload.encoded_size(), + _ => unimplemented!("received an unexpected request"), + } } } diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 99673b8fe1bf..affcdc322f79 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -128,10 +128,23 @@ impl BenchCli { gum::info!("{}", format!("Step {}/{}", index + 1, num_steps).bright_purple(),); display_configuration(&test_config); - let mut state = TestState::new(&test_config); - let (mut env, _protocol_config) = prepare_test(test_config, &mut state); - env.runtime() - .block_on(availability::benchmark_availability_read(&mut env, state)); + match test_config.objective { + TestObjective::DataAvailabilityRead(ref _opts) => { + let mut state = TestState::new(&test_config); + let (mut env, _protocol_config) = prepare_test(test_config, &mut state); + env.runtime().block_on(availability::benchmark_availability_read( + &mut env, state, + )); + }, + TestObjective::DataAvailabilityWrite => { + let mut state = TestState::new(&test_config); + let (mut env, _protocol_config) = prepare_test(test_config, &mut state); + env.runtime().block_on(availability::benchmark_availability_write( + &mut env, state, + )); + }, + _ => gum::error!("Invalid test objective in sequence"), + } } return Ok(()) }, From 196c4267065ef0d3f3da7cc0946c9e9971d0185f Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 17 Jan 2024 13:45:57 +0200 Subject: [PATCH 177/192] update examples Signed-off-by: Andrei Sandu --- .../examples/availability_read.yaml | 36 +++++++------------ .../examples/availability_write.yaml | 15 ++++++++ 2 files changed, 27 insertions(+), 24 deletions(-) create mode 100644 polkadot/node/subsystem-bench/examples/availability_write.yaml diff --git a/polkadot/node/subsystem-bench/examples/availability_read.yaml b/polkadot/node/subsystem-bench/examples/availability_read.yaml index 311ea972141f..82355b0e2973 100644 --- a/polkadot/node/subsystem-bench/examples/availability_read.yaml +++ b/polkadot/node/subsystem-bench/examples/availability_read.yaml @@ -1,7 +1,7 @@ TestConfiguration: # Test 1 - objective: !DataAvailabilityRead - fetch_from_backers: false + fetch_from_backers: true n_validators: 300 n_cores: 20 min_pov_size: 5120 @@ -9,18 +9,14 @@ TestConfiguration: peer_bandwidth: 52428800 bandwidth: 52428800 latency: - min_latency: - secs: 0 - nanos: 1000000 - max_latency: - secs: 0 - nanos: 100000000 - error: 3 + mean_latency_ms: 100 + std_dev: 1 num_blocks: 3 + connectivity: 90 # Test 2 - objective: !DataAvailabilityRead - fetch_from_backers: false + fetch_from_backers: true n_validators: 500 n_cores: 20 min_pov_size: 5120 @@ -28,18 +24,14 @@ TestConfiguration: peer_bandwidth: 52428800 bandwidth: 52428800 latency: - min_latency: - secs: 0 - nanos: 1000000 - max_latency: - secs: 0 - nanos: 100000000 - error: 3 + mean_latency_ms: 100 + std_dev: 1 num_blocks: 3 + connectivity: 90 # Test 3 - objective: !DataAvailabilityRead - fetch_from_backers: false + fetch_from_backers: true n_validators: 1000 n_cores: 20 min_pov_size: 5120 @@ -47,11 +39,7 @@ TestConfiguration: peer_bandwidth: 52428800 bandwidth: 52428800 latency: - min_latency: - secs: 0 - nanos: 1000000 - max_latency: - secs: 0 - nanos: 100000000 - error: 3 + mean_latency_ms: 100 + std_dev: 1 num_blocks: 3 + connectivity: 90 diff --git a/polkadot/node/subsystem-bench/examples/availability_write.yaml b/polkadot/node/subsystem-bench/examples/availability_write.yaml new file mode 100644 index 000000000000..64e07d769692 --- /dev/null +++ b/polkadot/node/subsystem-bench/examples/availability_write.yaml @@ -0,0 +1,15 @@ +TestConfiguration: +# Test 1kV, 200 cores, max Pov +- objective: DataAvailabilityWrite + n_validators: 1000 + n_cores: 200 + max_validators_per_core: 5 + min_pov_size: 5120 + max_pov_size: 5120 + peer_bandwidth: 52428800 + bandwidth: 52428800 + latency: + mean_latency_ms: 30 + std_dev: 2.0 + connectivity: 75 + num_blocks: 3 From 7d63bd95ea6235927787d0ae5ded494b6a0dea0b Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 17 Jan 2024 17:34:19 +0200 Subject: [PATCH 178/192] merge fixes Signed-off-by: Andrei Sandu --- .../examples/availability_read.yaml | 45 ------------------- .../subsystem-bench/src/availability/mod.rs | 14 ++++-- .../node/subsystem-bench/src/core/display.rs | 2 +- .../subsystem-bench/src/core/mock/av_store.rs | 5 +-- .../src/core/mock/runtime_api.rs | 10 ++++- .../node/subsystem-bench/src/core/network.rs | 8 ++-- .../subsystem-bench/src/subsystem-bench.rs | 3 +- 7 files changed, 26 insertions(+), 61 deletions(-) diff --git a/polkadot/node/subsystem-bench/examples/availability_read.yaml b/polkadot/node/subsystem-bench/examples/availability_read.yaml index 2de04cdc75e0..82355b0e2973 100644 --- a/polkadot/node/subsystem-bench/examples/availability_read.yaml +++ b/polkadot/node/subsystem-bench/examples/availability_read.yaml @@ -1,11 +1,7 @@ TestConfiguration: # Test 1 - objective: !DataAvailabilityRead -<<<<<<< HEAD fetch_from_backers: true -======= - fetch_from_backers: false ->>>>>>> c01dbebeaa6394691974de46dd2d41a582f6a4c2 n_validators: 300 n_cores: 20 min_pov_size: 5120 @@ -13,7 +9,6 @@ TestConfiguration: peer_bandwidth: 52428800 bandwidth: 52428800 latency: -<<<<<<< HEAD mean_latency_ms: 100 std_dev: 1 num_blocks: 3 @@ -22,20 +17,6 @@ TestConfiguration: # Test 2 - objective: !DataAvailabilityRead fetch_from_backers: true -======= - min_latency: - secs: 0 - nanos: 1000000 - max_latency: - secs: 0 - nanos: 100000000 - error: 3 - num_blocks: 3 - -# Test 2 -- objective: !DataAvailabilityRead - fetch_from_backers: false ->>>>>>> c01dbebeaa6394691974de46dd2d41a582f6a4c2 n_validators: 500 n_cores: 20 min_pov_size: 5120 @@ -43,7 +24,6 @@ TestConfiguration: peer_bandwidth: 52428800 bandwidth: 52428800 latency: -<<<<<<< HEAD mean_latency_ms: 100 std_dev: 1 num_blocks: 3 @@ -52,20 +32,6 @@ TestConfiguration: # Test 3 - objective: !DataAvailabilityRead fetch_from_backers: true -======= - min_latency: - secs: 0 - nanos: 1000000 - max_latency: - secs: 0 - nanos: 100000000 - error: 3 - num_blocks: 3 - -# Test 3 -- objective: !DataAvailabilityRead - fetch_from_backers: false ->>>>>>> c01dbebeaa6394691974de46dd2d41a582f6a4c2 n_validators: 1000 n_cores: 20 min_pov_size: 5120 @@ -73,18 +39,7 @@ TestConfiguration: peer_bandwidth: 52428800 bandwidth: 52428800 latency: -<<<<<<< HEAD mean_latency_ms: 100 std_dev: 1 num_blocks: 3 connectivity: 90 -======= - min_latency: - secs: 0 - nanos: 1000000 - max_latency: - secs: 0 - nanos: 100000000 - error: 3 - num_blocks: 3 ->>>>>>> c01dbebeaa6394691974de46dd2d41a582f6a4c2 diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 7aa3fb14d6b0..e2c179bffdd3 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -25,7 +25,7 @@ use polkadot_node_subsystem_types::{ Span, }; use polkadot_overseer::Handle as OverseerHandle; -use sc_network::{request_responses::ProtocolConfig, PeerId}; +use sc_network::{request_responses::ProtocolConfig, PeerId, ProtocolName}; use sp_core::H256; use std::{collections::HashMap, iter::Cycle, ops::Sub, sync::Arc, time::Instant}; @@ -164,7 +164,10 @@ impl HandleNetworkMessage for NetworkAvailabilityState { [validator_index] .clone() .into(); - let response = Ok(ChunkFetchingResponse::from(Some(chunk)).encode()); + let response = Ok(( + ChunkFetchingResponse::from(Some(chunk)).encode(), + ProtocolName::Static("dummy"), + )); if let Err(err) = outgoing_request.pending_response.send(response) { gum::error!(target: LOG_TARGET, ?err, "Failed to send `ChunkFetchingResponse`"); @@ -183,8 +186,10 @@ impl HandleNetworkMessage for NetworkAvailabilityState { let available_data = self.available_data.get(*candidate_index as usize).unwrap().clone(); - let response = - Ok(AvailableDataFetchingResponse::from(Some(available_data)).encode()); + let response = Ok(( + AvailableDataFetchingResponse::from(Some(available_data)).encode(), + ProtocolName::Static("dummy"), + )); let _ = outgoing_request .pending_response .send(response) @@ -483,6 +488,7 @@ impl TestState { candidates: Vec::new().into_iter().cycle(), chunk_request_protocol: None, backed_candidates: Vec::new(), + }; _self.generate_candidates(); _self diff --git a/polkadot/node/subsystem-bench/src/core/display.rs b/polkadot/node/subsystem-bench/src/core/display.rs index e5824c6fc274..c983faf7ce56 100644 --- a/polkadot/node/subsystem-bench/src/core/display.rs +++ b/polkadot/node/subsystem-bench/src/core/display.rs @@ -180,7 +180,7 @@ pub fn parse_metrics(registry: &Registry) -> MetricCollection { pub fn display_configuration(test_config: &TestConfiguration) { gum::info!( - "[] {}, {}, {}, {}, {}", + "[{}] {}, {}, {}, {}, {}", format!("objective = {:?}", test_config.objective).green(), format!("n_validators = {}", test_config.n_validators).blue(), format!("n_cores = {}", test_config.n_cores).blue(), diff --git a/polkadot/node/subsystem-bench/src/core/mock/av_store.rs b/polkadot/node/subsystem-bench/src/core/mock/av_store.rs index 257cbfccf76c..470cdc723199 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/av_store.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/av_store.rs @@ -123,9 +123,8 @@ impl MockAvailabilityStore { .expect("candidate was generated previously; qed"); gum::debug!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); - let chunk_size = self.state.chunks.get(*candidate_index).unwrap() - [0] - .encoded_size(); + let chunk_size = + self.state.chunks.get(*candidate_index).unwrap()[0].encoded_size(); let _ = tx.send(Some(chunk_size)); }, AvailabilityStoreMessage::StoreChunk { candidate_hash, chunk, tx } => { diff --git a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs index f01f2c0d29e9..71d84e9025d4 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs @@ -176,9 +176,15 @@ impl MockRuntimeApi { let _ = sender.send(Ok(cores)); }, + RuntimeApiMessage::Request( + _block_hash, + RuntimeApiRequest::NodeFeatures(_session_index, sender), + ) => { + let _ = sender.send(Ok(Default::default())); + }, // Long term TODO: implement more as needed. - _ => { - unimplemented!("Unexpected runtime-api message") + message => { + unimplemented!("Unexpected runtime-api message: {:?}", message) }, } }, diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 763a346e9558..4c072c628a8a 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -159,12 +159,12 @@ impl NetworkMessage { match &self { NetworkMessage::MessageFromPeer(Versioned::V2(message)) => message.encoded_size(), NetworkMessage::MessageFromPeer(Versioned::V1(message)) => message.encoded_size(), - NetworkMessage::MessageFromPeer(Versioned::VStaging(message)) => message.encoded_size(), + NetworkMessage::MessageFromPeer(Versioned::V3(message)) => message.encoded_size(), NetworkMessage::MessageFromNode(_peer_id, Versioned::V2(message)) => message.encoded_size(), NetworkMessage::MessageFromNode(_peer_id, Versioned::V1(message)) => message.encoded_size(), - NetworkMessage::MessageFromNode(_peer_id, Versioned::VStaging(message)) => + NetworkMessage::MessageFromNode(_peer_id, Versioned::V3(message)) => message.encoded_size(), NetworkMessage::RequestFromNode(_peer_id, incoming) => incoming.size(), NetworkMessage::RequestFromPeer(request) => request.payload.encoded_size(), @@ -411,13 +411,13 @@ impl NetworkInterface { Ok(Err(err)) => { sender.send(Err(err)).expect("Oneshot send always works."); }, - Ok(Ok(response)) => { + Ok(Ok((response, protocol_name))) => { let response_size = response.encoded_size(); task_rx_limiter.lock().await.reap(response_size).await; tx_network.inc_received(response_size); // Send the response to the original request sender. - if let Err(_) = sender.send(Ok(response)) { + if let Err(_) = sender.send(Ok((response, protocol_name))) { gum::warn!(target: LOG_TARGET, response_size, "response oneshot canceled by node") } }, diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 31aafad43ee2..6dfff274a3c7 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -80,7 +80,7 @@ struct BenchCli { #[clap(long, value_parser=le_5000)] /// Remote peer latency standard deviation pub peer_latency_std_dev: Option, - + #[clap(long, default_value_t = false)] /// Enable CPU Profiling with Pyroscope pub profile: bool, @@ -149,7 +149,6 @@ impl BenchCli { None }; - let configuration = self.standard_configuration; let mut test_config = match self.objective { TestObjective::TestSequence(options) => { let test_sequence = From 67c4c1e20dd07d6d2b80d5241a703274aef1a920 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 17 Jan 2024 19:02:46 +0200 Subject: [PATCH 179/192] clippy & fmt Signed-off-by: Andrei Sandu --- .../subsystem-bench/src/availability/mod.rs | 30 ++++------ .../subsystem-bench/src/core/configuration.rs | 10 ++-- .../node/subsystem-bench/src/core/display.rs | 10 ++-- .../subsystem-bench/src/core/environment.rs | 13 ++--- .../subsystem-bench/src/core/mock/av_store.rs | 8 +-- .../src/core/mock/chain_api.rs | 8 +-- .../src/core/mock/network_bridge.rs | 17 +++--- .../src/core/mock/runtime_api.rs | 13 ++--- .../node/subsystem-bench/src/core/network.rs | 57 +++++++++---------- .../subsystem-bench/src/subsystem-bench.rs | 6 +- 10 files changed, 77 insertions(+), 95 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index e2c179bffdd3..b60b8f9c4d56 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -160,10 +160,8 @@ impl HandleNetworkMessage for NetworkAvailabilityState { .expect("candidate was generated previously; qed"); gum::warn!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); - let chunk: ChunkResponse = self.chunks.get(*candidate_index as usize).unwrap() - [validator_index] - .clone() - .into(); + let chunk: ChunkResponse = + self.chunks.get(*candidate_index).unwrap()[validator_index].clone().into(); let response = Ok(( ChunkFetchingResponse::from(Some(chunk)).encode(), ProtocolName::Static("dummy"), @@ -183,14 +181,13 @@ impl HandleNetworkMessage for NetworkAvailabilityState { .expect("candidate was generated previously; qed"); gum::debug!(target: LOG_TARGET, ?candidate_hash, candidate_index, "Candidate mapped to index"); - let available_data = - self.available_data.get(*candidate_index as usize).unwrap().clone(); + let available_data = self.available_data.get(*candidate_index).unwrap().clone(); let response = Ok(( AvailableDataFetchingResponse::from(Some(available_data)).encode(), ProtocolName::Static("dummy"), )); - let _ = outgoing_request + outgoing_request .pending_response .send(response) .expect("Response is always sent succesfully"); @@ -258,14 +255,14 @@ fn prepare_test_inner( let mut req_cfgs = Vec::new(); let (collation_req_receiver, collation_req_cfg) = - IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); req_cfgs.push(collation_req_cfg); let (pov_req_receiver, pov_req_cfg) = - IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); let (chunk_req_receiver, chunk_req_cfg) = - IncomingRequest::get_config_receiver(&ReqProtocolNames::new(&GENESIS_HASH, None)); + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); req_cfgs.push(pov_req_cfg); let (network, network_interface, network_receiver) = @@ -359,7 +356,6 @@ fn prepare_test_inner( overseer, overseer_handle, test_authorities, - network_interface, ), req_cfgs, ) @@ -474,20 +470,19 @@ impl TestState { candidate_receipt_templates.push(candidate_receipt); } - let pov_sizes = config.pov_sizes().to_vec().into_iter().cycle(); gum::info!(target: LOG_TARGET, "{}","Created test environment.".bright_blue()); let mut _self = Self { - config, available_data, candidate_receipt_templates, chunks, pov_size_to_candidate, - pov_sizes, + pov_sizes: Vec::from(config.pov_sizes()).into_iter().cycle(), candidate_hashes: HashMap::new(), candidates: Vec::new().into_iter().cycle(), chunk_request_protocol: None, backed_candidates: Vec::new(), + config, }; _self.generate_candidates(); @@ -592,8 +587,7 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: )) .await; - let _ = rx - .await + rx.await .unwrap() .expect("Test candidates are stored nicely in availability store"); } @@ -678,7 +672,7 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: payload, &signing_context, ValidatorIndex(index as u32), - &validator_public.clone().into(), + validator_public, ) .ok() .flatten() @@ -689,7 +683,7 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: let message = peer_bitfield_message_v2(relay_block_hash, signed_bitfield); // Send the action from peer only if it is connected to our node. - if network.is_peer_connected(&from_peer) { + if network.is_peer_connected(from_peer) { let _ = network.send_message_from_peer(from_peer, message); } } diff --git a/polkadot/node/subsystem-bench/src/core/configuration.rs b/polkadot/node/subsystem-bench/src/core/configuration.rs index 9d54ea21b5d4..66da8a1db45d 100644 --- a/polkadot/node/subsystem-bench/src/core/configuration.rs +++ b/polkadot/node/subsystem-bench/src/core/configuration.rs @@ -114,7 +114,7 @@ pub struct TestSequence { } impl TestSequence { - pub fn to_vec(self) -> Vec { + pub fn into_vec(self) -> Vec { self.test_configurations .into_iter() .map(|mut config| { @@ -128,7 +128,7 @@ impl TestSequence { impl TestSequence { pub fn new_from_file(path: &Path) -> std::io::Result { - let string = String::from_utf8(std::fs::read(&path)?).expect("File is valid UTF8"); + let string = String::from_utf8(std::fs::read(path)?).expect("File is valid UTF8"); Ok(serde_yaml::from_str(&string).expect("File is valid test sequence YA")) } } @@ -163,15 +163,15 @@ impl TestConfiguration { let keyring = Keyring::default(); let keys = (0..self.n_validators) - .map(|peer_index| keyring.sr25519_new(format!("Node{}", peer_index).into())) + .map(|peer_index| keyring.sr25519_new(format!("Node{}", peer_index))) .collect::>(); // Generate `AuthorityDiscoveryId`` for each peer let validator_public: Vec = - keys.iter().map(|key| key.clone().into()).collect::>(); + keys.iter().map(|key| (*key).into()).collect::>(); let validator_authority_id: Vec = - keys.iter().map(|key| key.clone().into()).collect::>().into(); + keys.iter().map(|key| (*key).into()).collect::>(); TestAuthorities { keyring, validator_public, validator_authority_id } } diff --git a/polkadot/node/subsystem-bench/src/core/display.rs b/polkadot/node/subsystem-bench/src/core/display.rs index c983faf7ce56..bca82d7b90ae 100644 --- a/polkadot/node/subsystem-bench/src/core/display.rs +++ b/polkadot/node/subsystem-bench/src/core/display.rs @@ -43,8 +43,8 @@ impl MetricCollection { /// Sums up all metrics with the given name in the collection pub fn sum_by(&self, name: &str) -> f64 { self.all() - .into_iter() - .filter(|metric| &metric.name == name) + .iter() + .filter(|metric| metric.name == name) .map(|metric| metric.value) .sum() } @@ -71,7 +71,7 @@ impl MetricCollection { impl Display for MetricCollection { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writeln!(f, "")?; + writeln!(f)?; let metrics = self.all(); for metric in metrics { writeln!(f, "{}", metric)?; @@ -150,7 +150,7 @@ pub fn parse_metrics(registry: &Registry) -> MetricCollection { }, MetricType::HISTOGRAM => { let h = m.get_histogram(); - let h_name = name.clone() + "_sum".into(); + let h_name = name.clone() + "_sum"; test_metrics.push(TestMetric { name: h_name, label_names: label_names.clone(), @@ -158,7 +158,7 @@ pub fn parse_metrics(registry: &Registry) -> MetricCollection { value: h.get_sample_sum(), }); - let h_name = name.clone() + "_count".into(); + let h_name = name.clone() + "_count"; test_metrics.push(TestMetric { name: h_name, label_names, diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 7e3ccab458e3..3ac83926400a 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -34,7 +34,7 @@ use std::net::{Ipv4Addr, SocketAddr}; use tokio::runtime::Handle; const LOG_TARGET: &str = "subsystem-bench::environment"; -use super::{configuration::TestAuthorities, network::NetworkInterface}; +use super::configuration::TestAuthorities; /// Test environment/configuration metrics #[derive(Clone)] @@ -190,8 +190,6 @@ pub struct TestEnvironment { metrics: TestEnvironmentMetrics, /// Test authorities generated from the configuration. authorities: TestAuthorities, - /// The network interface used by the node. - network_interface: NetworkInterface, } impl TestEnvironment { @@ -203,7 +201,6 @@ impl TestEnvironment { overseer: Overseer, AlwaysSupportsParachains>, overseer_handle: OverseerHandle, authorities: TestAuthorities, - network_interface: NetworkInterface, ) -> Self { let metrics = TestEnvironmentMetrics::new(&dependencies.registry) .expect("Metrics need to be registered"); @@ -233,7 +230,6 @@ impl TestEnvironment { network, metrics, authorities, - network_interface, } } @@ -253,6 +249,7 @@ impl TestEnvironment { } /// Spawn a named task in the `test-environment` task group. + #[allow(unused)] pub fn spawn(&self, name: &'static str, task: impl Future + Send + 'static) { self.dependencies .task_manager @@ -356,18 +353,18 @@ impl TestEnvironment { pub fn display_cpu_usage(&self, subsystems_under_test: &[&str]) { let test_metrics = super::display::parse_metrics(self.registry()); - for subsystem in subsystems_under_test.into_iter() { + for subsystem in subsystems_under_test.iter() { let subsystem_cpu_metrics = test_metrics.subset_with_label_value("task_group", subsystem); let total_cpu = subsystem_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); println!( "{} CPU usage {}", - format!("{}", subsystem).bright_green(), + subsystem.to_string().bright_green(), format!("{:.3}s", total_cpu).bright_purple() ); println!( "{} CPU usage per block {}", - format!("{}", subsystem).bright_green(), + subsystem.to_string().bright_green(), format!("{:.3}s", total_cpu / self.config().num_blocks as f64).bright_purple() ); } diff --git a/polkadot/node/subsystem-bench/src/core/mock/av_store.rs b/polkadot/node/subsystem-bench/src/core/mock/av_store.rs index 470cdc723199..3c6dbf32c681 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/av_store.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/av_store.rs @@ -96,10 +96,10 @@ impl MockAvailabilityStore { let msg = ctx.recv().await.expect("Overseer never fails us"); match msg { - orchestra::FromOrchestra::Signal(signal) => match signal { - OverseerSignal::Conclude => return, - _ => {}, - }, + orchestra::FromOrchestra::Signal(signal) => + if signal == OverseerSignal::Conclude { + return + }, orchestra::FromOrchestra::Communication { msg } => match msg { AvailabilityStoreMessage::QueryAvailableData(candidate_hash, tx) => { gum::debug!(target: LOG_TARGET, candidate_hash = ?candidate_hash, "Responding to QueryAvailableData"); diff --git a/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs b/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs index a440a3155d50..008d8eef106a 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs @@ -60,10 +60,10 @@ impl MockChainApi { let msg = ctx.recv().await.expect("Overseer never fails us"); match msg { - orchestra::FromOrchestra::Signal(signal) => match signal { - OverseerSignal::Conclude => return, - _ => {}, - }, + orchestra::FromOrchestra::Signal(signal) => + if signal == OverseerSignal::Conclude { + return + }, orchestra::FromOrchestra::Communication { msg } => { gum::debug!(target: LOG_TARGET, msg=?msg, "recv message"); diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index 68a8dce3bc70..ba3a19d628de 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -95,10 +95,10 @@ impl MockNetworkBridgeTx { loop { let subsystem_message = ctx.recv().await.expect("Overseer never fails us"); match subsystem_message { - orchestra::FromOrchestra::Signal(signal) => match signal { - OverseerSignal::Conclude => return, - _ => {}, - }, + orchestra::FromOrchestra::Signal(signal) => + if signal == OverseerSignal::Conclude { + return + }, orchestra::FromOrchestra::Communication { msg } => match msg { NetworkBridgeTxMessage::SendRequests(requests, _if_disconnected) => { for request in requests { @@ -108,7 +108,7 @@ impl MockNetworkBridgeTx { if !self.network.is_peer_connected(&peer_id) { // Attempting to send a request to a disconnected peer. - let _ = request + request .into_response_sender() .send(Err(RequestFailure::NotConnected)) .expect("send never fails"); @@ -160,7 +160,7 @@ impl MockNetworkBridgeRx { NetworkMessage::RequestFromPeer(request) => { if let Some(protocol) = self.chunk_request_sender.as_mut() { if let Some(inbound_queue) = protocol.inbound_queue.as_ref() { - let _ = inbound_queue + inbound_queue .send(request) .await .expect("Forwarding requests to subsystem never fails"); @@ -175,10 +175,7 @@ impl MockNetworkBridgeRx { }, subsystem_message = ctx.recv().fuse() => { match subsystem_message.expect("Overseer never fails us") { - orchestra::FromOrchestra::Signal(signal) => match signal { - OverseerSignal::Conclude => return, - _ => {}, - }, + orchestra::FromOrchestra::Signal(signal) => if signal == OverseerSignal::Conclude { return }, _ => { unimplemented!("Unexpected network bridge rx message") }, diff --git a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs index 71d84e9025d4..caefe068efff 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs @@ -64,9 +64,8 @@ impl MockRuntimeApi { let validator_groups = all_validators .chunks(self.config.max_validators_per_core) - .map(|x| Vec::from(x)) + .map(Vec::from) .collect::>(); - SessionInfo { validators: self.state.authorities.validator_public.clone().into(), discovery_keys: self.state.authorities.validator_authority_id.clone(), @@ -103,10 +102,10 @@ impl MockRuntimeApi { let msg = ctx.recv().await.expect("Overseer never fails us"); match msg { - orchestra::FromOrchestra::Signal(signal) => match signal { - OverseerSignal::Conclude => return, - _ => {}, - }, + orchestra::FromOrchestra::Signal(signal) => + if signal == OverseerSignal::Conclude { + return + }, orchestra::FromOrchestra::Communication { msg } => { gum::debug!(target: LOG_TARGET, msg=?msg, "recv message"); @@ -155,7 +154,7 @@ impl MockRuntimeApi { // All cores are always occupied. let cores = candidate_hashes - .into_iter() + .iter() .enumerate() .map(|(index, candidate_receipt)| { // Ensure test breaks if badly configured. diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 4c072c628a8a..2e3425b21c2d 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -16,25 +16,23 @@ //! //! Implements network emulation and interfaces to control and specialize //! network peer behaviour. -//! -//! High level component wiring chart: -//! -//! [TestEnvironment] -//! [NetworkEmulatorHandle] -//! || -//! +-------+--||--+-------+ -//! | | | | -//! Peer1 Peer2 Peer3 Peer4 -//! \ | | / -//! \ | | / -//! \ | | / -//! \ | | / -//! \ | | / -//! [Network Interface] -//! | -//! [Emulated Network Bridge] -//! | -//! Subsystems under test +// +// [TestEnvironment] +// [NetworkEmulatorHandle] +// || +// +-------+--||--+-------+ +// | | | | +// Peer1 Peer2 Peer3 Peer4 +// \ | | / +// \ | | / +// \ | | / +// \ | | / +// \ | | / +// [Network Interface] +// | +// [Emulated Network Bridge] +// | +// Subsystems under test use crate::core::configuration::random_latency; @@ -417,7 +415,7 @@ impl NetworkInterface { tx_network.inc_received(response_size); // Send the response to the original request sender. - if let Err(_) = sender.send(Ok((response, protocol_name))) { + if sender.send(Ok((response, protocol_name))).is_err() { gum::warn!(target: LOG_TARGET, response_size, "response oneshot canceled by node") } }, @@ -437,21 +435,19 @@ pub struct EmulatedPeerHandle { impl EmulatedPeerHandle { /// Receive and process a message from the node. pub fn receive(&self, message: NetworkMessage) { - let _ = self.messages_tx.unbounded_send(message).expect("Peer message channel hangup"); + self.messages_tx.unbounded_send(message).expect("Peer message channel hangup"); } /// Send a message to the node. pub fn send_message(&self, message: VersionedValidationProtocol) { - let _ = self - .actions_tx + self.actions_tx .unbounded_send(NetworkMessage::MessageFromPeer(message)) .expect("Peer action channel hangup"); } /// Send a `request` to the node. pub fn send_request(&self, request: IncomingRequest) { - let _ = self - .actions_tx + self.actions_tx .unbounded_send(NetworkMessage::RequestFromPeer(request)) .expect("Peer action channel hangup"); } @@ -472,7 +468,7 @@ impl EmulatedPeer { self.tx_limiter.reap(message.size()).await; if self.latency_ms == 0 { - let _ = self.to_node.unbounded_send(message).expect("Sending to the node never fails"); + self.to_node.unbounded_send(message).expect("Sending to the node never fails"); } else { let to_node = self.to_node.clone(); let latency_ms = std::time::Duration::from_millis(self.latency_ms as u64); @@ -481,8 +477,7 @@ impl EmulatedPeer { self.spawn_handle .spawn("peer-latency-emulator", "test-environment", async move { tokio::time::sleep(latency_ms).await; - let _ = - to_node.unbounded_send(message).expect("Sending to the node never fails"); + to_node.unbounded_send(message).expect("Sending to the node never fails"); }); } } @@ -745,7 +740,7 @@ pub fn new_network( // Create a `PeerEmulator` for each peer. let (stats, mut peers): (_, Vec<_>) = (0..n_peers) - .zip(authorities.validator_authority_id.clone().into_iter()) + .zip(authorities.validator_authority_id.clone()) .map(|(peer_index, authority_id)| { validator_authority_id_mapping.insert(authority_id, peer_index); let stats = Arc::new(PeerEmulatorStats::new(peer_index, metrics.clone())); @@ -827,7 +822,7 @@ impl NetworkEmulatorHandle { from_peer: &AuthorityDiscoveryId, message: VersionedValidationProtocol, ) -> Result<(), EmulatedPeerError> { - let dst_peer = self.peer(&from_peer); + let dst_peer = self.peer(from_peer); if !dst_peer.is_connected() { gum::warn!(target: LOG_TARGET, "Attempted to send message from a peer not connected to our node, operation ignored"); @@ -844,7 +839,7 @@ impl NetworkEmulatorHandle { from_peer: &AuthorityDiscoveryId, request: IncomingRequest, ) -> Result<(), EmulatedPeerError> { - let dst_peer = self.peer(&from_peer); + let dst_peer = self.peer(from_peer); if !dst_peer.is_connected() { gum::warn!(target: LOG_TARGET, "Attempted to send request from a peer not connected to our node, operation ignored"); diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 6dfff274a3c7..a79f708bf552 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -24,7 +24,7 @@ use color_eyre::eyre; use pyroscope::PyroscopeAgent; use pyroscope_pprofrs::{pprof_backend, PprofConfig}; -use std::{path::Path, time::Duration}; +use std::path::Path; pub(crate) mod availability; pub(crate) mod cli; @@ -154,7 +154,7 @@ impl BenchCli { let test_sequence = core::configuration::TestSequence::new_from_file(Path::new(&options.path)) .expect("File exists") - .to_vec(); + .into_vec(); let num_steps = test_sequence.len(); gum::info!( "{}", @@ -243,7 +243,7 @@ impl BenchCli { fn main() -> eyre::Result<()> { color_eyre::install()?; - let _ = env_logger::builder() + env_logger::builder() .filter(Some("hyper"), log::LevelFilter::Info) // Avoid `Terminating due to subsystem exit subsystem` warnings .filter(Some("polkadot_overseer"), log::LevelFilter::Error) From 8c00d70056cf0fd2c2fac6d89cb858d80979978c Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 17 Jan 2024 19:10:19 +0200 Subject: [PATCH 180/192] fix print Signed-off-by: Andrei Sandu --- polkadot/node/subsystem-bench/src/core/network.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 2e3425b21c2d..13742ec432f6 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -729,7 +729,7 @@ pub fn new_network( ) -> (NetworkEmulatorHandle, NetworkInterface, NetworkInterfaceReceiver) { let n_peers = config.n_validators; gum::info!(target: LOG_TARGET, "{}",format!("Initializing emulation for a {} peer network.", n_peers).bright_blue()); - gum::info!(target: LOG_TARGET, "{}",format!("connectivity {}%, latency {:?}%", config.connectivity, config.latency).bright_black()); + gum::info!(target: LOG_TARGET, "{}",format!("connectivity {}%, latency {:?}", config.connectivity, config.latency).bright_black()); let metrics = Metrics::new(&dependencies.registry).expect("Metrics always register succesfully"); From f839f7717fc944421228d86bb7252bc7fcd1df07 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 17 Jan 2024 19:16:57 +0200 Subject: [PATCH 181/192] Fix block time Signed-off-by: Andrei Sandu --- .../subsystem-bench/src/availability/mod.rs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index b60b8f9c4d56..a1bc9688ce9b 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -503,7 +503,7 @@ pub async fn benchmark_availability_read(env: &mut TestEnvironment, mut state: T env.import_block(new_block_import_info(Hash::repeat_byte(1), 1)).await; - let start_marker = Instant::now(); + let test_start = Instant::now(); let mut batch = FuturesUnordered::new(); let mut availability_bytes = 0u128; @@ -546,7 +546,7 @@ pub async fn benchmark_availability_read(env: &mut TestEnvironment, mut state: T gum::info!("All work for block completed in {}", format!("{:?}ms", block_time).cyan()); } - let duration: u128 = start_marker.elapsed().as_millis(); + let duration: u128 = test_start.elapsed().as_millis(); let availability_bytes = availability_bytes / 1024; gum::info!("All blocks processed in {}", format!("{:?}ms", duration).cyan()); gum::info!( @@ -554,9 +554,8 @@ pub async fn benchmark_availability_read(env: &mut TestEnvironment, mut state: T format!("{} KiB/block", availability_bytes / env.config().num_blocks as u128).bright_red() ); gum::info!( - "Block time: {}", - format!("{} ms", start_marker.elapsed().as_millis() / env.config().num_blocks as u128) - .red() + "Avg block time: {}", + format!("{} ms", test_start.elapsed().as_millis() / env.config().num_blocks as u128).red() ); env.display_network_usage(); @@ -566,7 +565,6 @@ pub async fn benchmark_availability_read(env: &mut TestEnvironment, mut state: T pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: TestState) { let config = env.config().clone(); - let start_marker = Instant::now(); env.metrics().set_n_validators(config.n_validators); env.metrics().set_n_cores(config.n_cores); @@ -594,6 +592,8 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: gum::info!("Done"); + let test_start = Instant::now(); + for block_num in 1..=env.config().num_blocks { gum::info!(target: LOG_TARGET, "Current block #{}", block_num); env.metrics().set_current_block(block_num); @@ -708,12 +708,11 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: gum::info!("All work for block completed in {}", format!("{:?}ms", block_time).cyan()); } - let duration: u128 = start_marker.elapsed().as_millis(); + let duration: u128 = test_start.elapsed().as_millis(); gum::info!("All blocks processed in {}", format!("{:?}ms", duration).cyan()); gum::info!( - "Block time: {}", - format!("{} ms", start_marker.elapsed().as_millis() / env.config().num_blocks as u128) - .red() + "Avg block time: {}", + format!("{} ms", test_start.elapsed().as_millis() / env.config().num_blocks as u128).red() ); env.display_network_usage(); From 48508ba8fd4b641ccb950044c7899ab9fcb7af0e Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 17 Jan 2024 19:50:20 +0200 Subject: [PATCH 182/192] clippy Signed-off-by: Andrei Sandu --- polkadot/node/subsystem-bench/src/core/network.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 13742ec432f6..38ff56cff6cd 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -1019,7 +1019,7 @@ mod tests { let mut reap_amount = 0; while rate_limiter.total_ticks < tick_rate { reap_amount += 1; - reap_amount = reap_amount % 100; + reap_amount %= 100; rate_limiter.reap(reap_amount).await; total_sent += reap_amount; From 28d9f7766a676f5c6d6112ebdf9c6045c9547a99 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 17 Jan 2024 21:57:30 +0200 Subject: [PATCH 183/192] Fix ASCII art Signed-off-by: Andrei Sandu --- polkadot/node/subsystem-bench/src/core/network.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 38ff56cff6cd..e2932bf0f51b 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -17,13 +17,13 @@ //! Implements network emulation and interfaces to control and specialize //! network peer behaviour. // -// [TestEnvironment] -// [NetworkEmulatorHandle] -// || +// [TestEnvironment] +// [NetworkEmulatorHandle] +// || // +-------+--||--+-------+ // | | | | // Peer1 Peer2 Peer3 Peer4 -// \ | | / +// \ | | / // \ | | / // \ | | / // \ | | / From ca6eb1818920dab70f0a6d056f3871dc97b42f3b Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 19 Jan 2024 14:38:25 +0200 Subject: [PATCH 184/192] Addressing review feedback Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/test.yml | 27 -- polkadot/node/overseer/src/metrics.rs | 4 +- polkadot/node/subsystem-bench/README.md | 85 ---- .../examples/approvals_no_shows.yaml | 10 +- .../examples/approvals_throughput.yaml | 14 +- .../approvals_throughput_12_seconds.yaml | 24 -- ...hroughput_12_seconds_no_optimisations.yaml | 24 -- .../approvals_throughput_best_case.yaml | 13 +- ...s_throughput_no_optimisations_enabled.yaml | 13 +- .../subsystem-bench/src/approval/helpers.rs | 22 +- .../src/approval/message_generator.rs | 386 +++++++++--------- .../node/subsystem-bench/src/approval/mod.rs | 135 +++--- .../src/approval/test_message.rs | 7 +- .../subsystem-bench/src/availability/mod.rs | 5 +- polkadot/node/subsystem-bench/src/cli.rs | 2 +- .../subsystem-bench/src/core/configuration.rs | 50 ++- .../node/subsystem-bench/src/core/display.rs | 58 +-- .../subsystem-bench/src/core/environment.rs | 22 +- .../node/subsystem-bench/src/core/mock/mod.rs | 2 +- .../src/core/mock/network_bridge.rs | 14 +- .../src/core/mock/runtime_api.rs | 12 +- polkadot/node/subsystem-bench/src/core/mod.rs | 1 + .../node/subsystem-bench/src/core/network.rs | 38 +- .../subsystem-bench/src/subsystem-bench.rs | 10 +- 24 files changed, 372 insertions(+), 606 deletions(-) delete mode 100644 polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml delete mode 100644 polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index dc1514171fe1..e75700ffddc4 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -365,33 +365,6 @@ test-linux-stable-int: - WASM_BUILD_NO_COLOR=1 time cargo test -p staging-node-cli --release --locked -- --ignored -test-linux-subsystem-bench: - stage: test - extends: - - .docker-env - - .common-refs - - .run-immediately - - .pipeline-stopper-artifacts - variables: - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-C debug-assertions -D warnings" - RUST_BACKTRACE: 1 - WASM_BUILD_NO_COLOR: 1 - WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings" - # Ensure we run the UI tests. - RUN_UI_TESTS: 1 - script: - - nproc - - cat /proc/cpuinfo - - cat /proc/meminfo - - time cargo run -p polkadot-subsystem-bench --release -- test-sequence --path polkadot/node/subsystem-bench/examples/approvals_throughput.yaml - - time cargo run -p polkadot-subsystem-bench --release -- test-sequence --path polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml - - time cargo run -p polkadot-subsystem-bench --release -- test-sequence --path polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml - - time cargo run -p polkadot-subsystem-bench --release -- test-sequence --path polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml - - time cargo run -p polkadot-subsystem-bench --release -- test-sequence --path polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml - - time cargo run -p polkadot-subsystem-bench --release -- test-sequence --path polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml - # more information about this job can be found here: # https://github.com/paritytech/substrate/pull/6916 check-tracing: diff --git a/polkadot/node/overseer/src/metrics.rs b/polkadot/node/overseer/src/metrics.rs index 18080468dc14..9b6053ccf769 100644 --- a/polkadot/node/overseer/src/metrics.rs +++ b/polkadot/node/overseer/src/metrics.rs @@ -168,7 +168,7 @@ impl MetricsTrait for Metrics { ) .buckets(vec![ 0.0001, 0.0004, 0.0016, 0.0064, 0.0256, 0.1024, 0.4096, 1.6384, 3.2768, - 4.9152, 6.1, 7.1, 8.0, 13.12, 26.25, + 4.9152, 6.5536, ]), &["subsystem_name"], )?, @@ -212,7 +212,7 @@ impl MetricsTrait for Metrics { ) .buckets(vec![ 0.0001, 0.0004, 0.0016, 0.0064, 0.0256, 0.1024, 0.4096, 1.6384, 3.2768, - 4.9152, 6.1, 7.1, 8.0, 13.12, 26.25, + 4.9152, 6.5536, ]), &["subsystem_name"], )?, diff --git a/polkadot/node/subsystem-bench/README.md b/polkadot/node/subsystem-bench/README.md index 136d67175d1f..34a7a0b6e55c 100644 --- a/polkadot/node/subsystem-bench/README.md +++ b/polkadot/node/subsystem-bench/README.md @@ -1,16 +1,5 @@ # Subsystem benchmark client -<<<<<<< HEAD -Run parachain consensus stress and performance tests on your development machine. - -## Motivation - -The parachain consensus node implementation spans across many modules which we call subsystems. Each subsystem is -responsible for a small part of logic of the parachain consensus pipeline, but in general the most load and -performance issues are localized in just a few core subsystems like `availability-recovery`, `approval-voting` or -`dispute-coordinator`. In the absence of such a tool, we would run large test nets to load/stress test these parts of -the system. Setting up and making sense of the amount of data produced by such a large test is very expensive, hard -======= Run parachain consensus stress and performance tests on your development machine. ## Motivation @@ -20,7 +9,6 @@ responsible for a small part of logic of the parachain consensus pipeline, but i performance issues are localized in just a few core subsystems like `availability-recovery`, `approval-voting` or `dispute-coordinator`. In the absence of such a tool, we would run large test nets to load/stress test these parts of the system. Setting up and making sense of the amount of data produced by such a large test is very expensive, hard ->>>>>>> origin/sandreim/availability-write-bench to orchestrate and is a huge development time sink. This tool aims to solve the problem by making it easy to: @@ -128,33 +116,6 @@ Note: `test-sequence` is a special test objective that wraps up an arbitrary num ### Standard test options ``` -<<<<<<< HEAD -Options: - --network - The type of network to be emulated [default: ideal] [possible values: ideal, healthy, degraded] - --n-cores - Number of cores to fetch availability for [default: 100] - --n-validators - Number of validators to fetch chunks from [default: 500] - --min-pov-size - The minimum pov size in KiB [default: 5120] - --max-pov-size - The maximum pov size bytes [default: 5120] - -n, --num-blocks - The number of blocks the test is going to run [default: 1] - -p, --peer-bandwidth - The bandwidth of emulated remote peers in KiB - -b, --bandwidth - The bandwidth of our node in KiB - --connectivity - Emulated peer connection ratio [0-100] - --peer-mean-latency - Mean remote peer latency in milliseconds [0-5000] - --peer-latency-std-dev - Remote peer latency standard deviation - -h, --help - Print help -======= --network The type of network to be emulated [default: ideal] [possible values: ideal, healthy, degraded] --n-cores Number of cores to fetch availability for [default: 100] @@ -172,7 +133,6 @@ Options: --pyroscope-sample-rate Pyroscope Sample Rate [default: 113] --cache-misses Enable Cache Misses Profiling with Valgrind. Linux only, Valgrind must be in the PATH -h, --help Print help ->>>>>>> origin/sandreim/availability-write-bench ``` These apply to all test objectives, except `test-sequence` which relies on the values being specified in a file. @@ -190,13 +150,8 @@ Benchmark availability recovery strategies Usage: subsystem-bench data-availability-read [OPTIONS] Options: -<<<<<<< HEAD - -f, --fetch-from-backers Turbo boost AD Read by fetching the full availability datafrom backers first. Saves CPU - as we don't need to re-construct from chunks. Tipically this is only faster if nodes -======= -f, --fetch-from-backers Turbo boost AD Read by fetching the full availability datafrom backers first. Saves CPU as we don't need to re-construct from chunks. Tipically this is only faster if nodes ->>>>>>> origin/sandreim/availability-write-bench have enough bandwidth -h, --help Print help ``` @@ -213,15 +168,9 @@ usage: - for how many blocks the test should run (`num_blocks`) From the perspective of the subsystem under test, this means that it will receive an `ActiveLeavesUpdate` signal -<<<<<<< HEAD -followed by an arbitrary amount of messages. This process repeats itself for `num_blocks`. The messages are generally -test payloads pre-generated before the test run, or constructed on pre-genereated payloads. For example the -`AvailabilityRecoveryMessage::RecoverAvailableData` message includes a `CandidateReceipt` which is generated before -======= followed by an arbitrary amount of messages. This process repeats itself for `num_blocks`. The messages are generally test payloads pre-generated before the test run, or constructed on pre-genereated payloads. For example the `AvailabilityRecoveryMessage::RecoverAvailableData` message includes a `CandidateReceipt` which is generated before ->>>>>>> origin/sandreim/availability-write-bench the test is started. ### Example run @@ -230,15 +179,9 @@ Let's run an availabilty read test which will recover availability for 10 cores node validator network. ``` -<<<<<<< HEAD - target/testnet/subsystem-bench --n-cores 10 data-availability-read -[2023-11-28T09:01:59Z INFO subsystem_bench::core::display] n_validators = 500, n_cores = 10, pov_size = 5120 - 5120, - error = 0, latency = None -======= target/testnet/subsystem-bench --n-cores 10 data-availability-read [2023-11-28T09:01:59Z INFO subsystem_bench::core::display] n_validators = 500, n_cores = 10, pov_size = 5120 - 5120, latency = None ->>>>>>> origin/sandreim/availability-write-bench [2023-11-28T09:01:59Z INFO subsystem-bench::availability] Generating template candidate index=0 pov_size=5242880 [2023-11-28T09:01:59Z INFO subsystem-bench::availability] Created test environment. [2023-11-28T09:01:59Z INFO subsystem-bench::availability] Pre-generating 10 candidates. @@ -251,13 +194,8 @@ node validator network. [2023-11-28T09:02:07Z INFO subsystem_bench::availability] All blocks processed in 6001ms [2023-11-28T09:02:07Z INFO subsystem_bench::availability] Throughput: 51200 KiB/block [2023-11-28T09:02:07Z INFO subsystem_bench::availability] Block time: 6001 ms -<<<<<<< HEAD -[2023-11-28T09:02:07Z INFO subsystem_bench::availability] - -======= [2023-11-28T09:02:07Z INFO subsystem_bench::availability] ->>>>>>> origin/sandreim/availability-write-bench Total received from network: 66 MiB Total sent to network: 58 KiB Total subsystem CPU usage 4.16s @@ -266,20 +204,12 @@ node validator network. CPU usage per block 0.00s ``` -<<<<<<< HEAD -`Block time` in the context of `data-availability-read` has a different meaning. It measures the amount of time it -======= `Block time` in the context of `data-availability-read` has a different meaning. It measures the amount of time it ->>>>>>> origin/sandreim/availability-write-bench took the subsystem to finish processing all of the messages sent in the context of the current test block. ### Test logs -<<<<<<< HEAD -You can select log target, subtarget and verbosity just like with Polkadot node CLI, simply setting -======= You can select log target, subtarget and verbosity just like with Polkadot node CLI, simply setting ->>>>>>> origin/sandreim/availability-write-bench `RUST_LOOG="parachain=debug"` turns on debug logs for all parachain consensus subsystems in the test. ### View test metrics @@ -290,8 +220,6 @@ view the test progress in real time by accessing [this link](http://localhost:30 Now run `target/testnet/subsystem-bench test-sequence --path polkadot/node/subsystem-bench/examples/availability_read.yaml` and view the metrics in real time and spot differences between different `n_validators` values. -<<<<<<< HEAD -======= ### Profiling cache misses @@ -334,36 +262,23 @@ This file is best interpreted with `cg_annotate --auto=yes cachegrind.out.` For finer profiling of cache misses, better use `perf` on a bare-metal machine. ->>>>>>> origin/sandreim/availability-write-bench ## Create new test objectives This tool is intended to make it easy to write new test objectives that focus individual subsystems, or even multiple subsystems (for example `approval-distribution` and `approval-voting`). A special kind of test objectives are performance regression tests for the CI pipeline. These should be sequences -<<<<<<< HEAD -of tests that check the performance characteristics (such as CPU usage, speed) of the subsystem under test in both -======= of tests that check the performance characteristics (such as CPU usage, speed) of the subsystem under test in both ->>>>>>> origin/sandreim/availability-write-bench happy and negative scenarios (low bandwidth, network errors and low connectivity). ### Reusable test components -<<<<<<< HEAD -To faster write a new test objective you need to use some higher level wrappers and logic: `TestEnvironment`, -======= To faster write a new test objective you need to use some higher level wrappers and logic: `TestEnvironment`, ->>>>>>> origin/sandreim/availability-write-bench `TestConfiguration`, `TestAuthorities`, `NetworkEmulator`. To create the `TestEnvironment` you will need to also build an `Overseer`, but that should be easy using the mockups for subsystems in`core::mock`. ### Mocking Ideally we want to have a single mock implementation for subsystems that can be minimally configured to -<<<<<<< HEAD -be used in different tests. A good example is `runtime-api` which currently only responds to session information -======= be used in different tests. A good example is `runtime-api` which currently only responds to session information ->>>>>>> origin/sandreim/availability-write-bench requests based on static data. It can be easily extended to service other requests. diff --git a/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml b/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml index d6753ca6f1b3..85b2544ded21 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml @@ -2,20 +2,14 @@ TestConfiguration: # Test 1 - objective: !ApprovalsTest last_considered_tranche: 89 - min_coalesce: 1 - max_coalesce: 6 - enable_assignments_v2: true - send_till_tranche: 89 + coalesce_mean: 3.0 + coalesce_std_dev: 1.0 stop_when_approved: true coalesce_tranche_diff: 12 workdir_prefix: "/tmp/" num_no_shows_per_candidate: 10 - approval_distribution_expected_tof: 60.0 - approval_distribution_cpu_ms: 14.00 - approval_voting_cpu_ms: 14.00 n_validators: 500 n_cores: 100 - n_included_candidates: 100 min_pov_size: 1120 max_pov_size: 5120 peer_bandwidth: 524288000000 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml index 4214da69d895..9eeeefc53a42 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml @@ -1,18 +1,14 @@ TestConfiguration: # Test 1 -- objective: !ApprovalsTest - last_considered_tranche: 89 - min_coalesce: 1 - max_coalesce: 6 +- objective: !ApprovalVoting + coalesce_mean: 3.0 + coalesce_std_dev: 1.0 enable_assignments_v2: true - send_till_tranche: 89 + last_considered_tranche: 89 stop_when_approved: false coalesce_tranche_diff: 12 - workdir_prefix: "/tmp/" + workdir_prefix: "/tmp" num_no_shows_per_candidate: 0 - approval_distribution_expected_tof: 60.0 - approval_distribution_cpu_ms: 14.00 - approval_voting_cpu_ms: 14.00 n_validators: 500 n_cores: 100 n_included_candidates: 100 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml deleted file mode 100644 index 8cefefe76960..000000000000 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds.yaml +++ /dev/null @@ -1,24 +0,0 @@ -TestConfiguration: -# Test 1 -- objective: !ApprovalsTest - last_considered_tranche: 89 - min_coalesce: 1 - max_coalesce: 6 - enable_assignments_v2: true - send_till_tranche: 89 - stop_when_approved: false - coalesce_tranche_diff: 12 - workdir_prefix: "/tmp/" - num_no_shows_per_candidate: 0 - approval_distribution_expected_tof: 60.0 - approval_distribution_cpu_ms: 14.00 - approval_voting_cpu_ms: 14.00 - n_validators: 500 - n_cores: 100 - n_included_candidates: 50 - min_pov_size: 1120 - max_pov_size: 5120 - peer_bandwidth: 524288000000 - bandwidth: 524288000000 - error: 0 - num_blocks: 10 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml deleted file mode 100644 index 16bf322f3e22..000000000000 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput_12_seconds_no_optimisations.yaml +++ /dev/null @@ -1,24 +0,0 @@ -TestConfiguration: -# Test 1 -- objective: !ApprovalsTest - last_considered_tranche: 89 - min_coalesce: 1 - max_coalesce: 1 - enable_assignments_v2: false - send_till_tranche: 89 - stop_when_approved: false - coalesce_tranche_diff: 12 - workdir_prefix: "/tmp/" - num_no_shows_per_candidate: 0 - approval_distribution_expected_tof: 60.0 - approval_distribution_cpu_ms: 14.00 - approval_voting_cpu_ms: 14.00 - n_validators: 500 - n_cores: 100 - n_included_candidates: 50 - min_pov_size: 1120 - max_pov_size: 5120 - peer_bandwidth: 524288000000 - bandwidth: 524288000000 - error: 0 - num_blocks: 10 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml index b7351d4b5245..6d817aed17a9 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml @@ -1,21 +1,16 @@ TestConfiguration: # Test 1 -- objective: !ApprovalsTest - last_considered_tranche: 89 - min_coalesce: 1 - max_coalesce: 6 +- objective: !ApprovalVoting + coalesce_mean: 3.0 + coalesce_std_dev: 1.0 enable_assignments_v2: true - send_till_tranche: 89 + last_considered_tranche: 89 stop_when_approved: true coalesce_tranche_diff: 12 workdir_prefix: "/tmp/" num_no_shows_per_candidate: 0 - approval_distribution_expected_tof: 60.0 - approval_distribution_cpu_ms: 14.00 - approval_voting_cpu_ms: 14.00 n_validators: 500 n_cores: 100 - n_included_candidates: 100 min_pov_size: 1120 max_pov_size: 5120 peer_bandwidth: 524288000000 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml index 49ae1a4dc292..23f932771dc0 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml @@ -1,21 +1,16 @@ TestConfiguration: # Test 1 -- objective: !ApprovalsTest - last_considered_tranche: 89 - min_coalesce: 1 - max_coalesce: 1 +- objective: !ApprovalVoting + coalesce_mean: 1.0 + coalesce_std_dev: 0.0 enable_assignments_v2: false - send_till_tranche: 89 + last_considered_tranche: 89 stop_when_approved: false coalesce_tranche_diff: 12 workdir_prefix: "/tmp/" num_no_shows_per_candidate: 0 - approval_distribution_expected_tof: 60.0 - approval_distribution_cpu_ms: 14.00 - approval_voting_cpu_ms: 14.00 n_validators: 500 n_cores: 100 - n_included_candidates: 100 min_pov_size: 1120 max_pov_size: 5120 peer_bandwidth: 524288000000 diff --git a/polkadot/node/subsystem-bench/src/approval/helpers.rs b/polkadot/node/subsystem-bench/src/approval/helpers.rs index 9cdbf0066953..6719b7ad6761 100644 --- a/polkadot/node/subsystem-bench/src/approval/helpers.rs +++ b/polkadot/node/subsystem-bench/src/approval/helpers.rs @@ -16,8 +16,7 @@ use itertools::Itertools; use polkadot_node_core_approval_voting::time::{Clock, SystemClock, Tick}; use polkadot_node_network_protocol::{ grid_topology::{SessionGridTopology, TopologyPeerInfo}, - peer_set::{ProtocolVersion, ValidationVersion}, - ObservedRole, View, + View, }; use polkadot_node_subsystem_types::messages::{ network_bridge_event::NewGossipTopology, ApprovalDistributionMessage, NetworkBridgeEvent, @@ -134,25 +133,6 @@ pub fn generate_new_session_topology( vec![AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate(event))] } -/// Generates peer_connected messages for all peers in `test_authorities` -pub fn generate_peer_connected(test_authorities: &TestAuthorities) -> Vec { - test_authorities - .peer_ids - .iter() - .map(|peer_id| { - let network = NetworkBridgeEvent::PeerConnected( - *peer_id, - ObservedRole::Full, - ProtocolVersion::from(ValidationVersion::V3), - None, - ); - AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate( - network, - )) - }) - .collect_vec() -} - /// Generates a peer view change for the passed `block_hash` pub fn generate_peer_view_change_for(block_hash: Hash, peer_id: PeerId) -> AllMessages { let network = NetworkBridgeEvent::PeerViewChange(peer_id, View::new([block_hash], 0)); diff --git a/polkadot/node/subsystem-bench/src/approval/message_generator.rs b/polkadot/node/subsystem-bench/src/approval/message_generator.rs index d5fa530346c3..6da5137543be 100644 --- a/polkadot/node/subsystem-bench/src/approval/message_generator.rs +++ b/polkadot/node/subsystem-bench/src/approval/message_generator.rs @@ -15,6 +15,7 @@ // along with Polkadot. If not, see . use std::{ + cmp::max, collections::{BTreeMap, HashSet}, fs, io::Write, @@ -34,7 +35,6 @@ use polkadot_node_network_protocol::grid_topology::{ }; use polkadot_node_primitives::approval::{ self, - v1::RelayVRFStory, v2::{CoreBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2}, }; use polkadot_primitives::{ @@ -43,6 +43,7 @@ use polkadot_primitives::{ }; use rand::{seq::SliceRandom, RngCore, SeedableRng}; use rand_chacha::ChaCha20Rng; +use rand_distr::{Distribution, Normal}; use sc_keystore::LocalKeystore; use sc_network::PeerId; use sha1::Digest; @@ -58,12 +59,12 @@ use super::{ use crate::{ approval::{ helpers::{generate_babe_epoch, generate_topology}, - GeneratedState, BUFFER_FOR_GENERATION_MILLIS, LOG_TARGET, NODE_UNDER_TEST, - SLOT_DURATION_MILLIS, + GeneratedState, BUFFER_FOR_GENERATION_MILLIS, LOG_TARGET, SLOT_DURATION_MILLIS, }, core::{ configuration::{TestAuthorities, TestConfiguration, TestObjective}, mock::session_info_for_peers, + NODE_UNDER_TEST, }, }; use polkadot_node_network_protocol::v3 as protocol_v3; @@ -90,16 +91,8 @@ pub struct PeerMessagesGenerator { pub session_info: SessionInfo, /// The blocks used for testing pub blocks: Vec, - /// If v2_assignments is enabled - pub enable_assignments_v2: bool, - /// Last considered tranche for generating assignments and approvals - pub last_considered_tranche: u32, - /// Minimum of approvals coalesced together. - pub min_coalesce: u32, - /// Maximum of approvals coalesced together. - pub max_coalesce: u32, - /// Maximum tranche diffs between two coalesced approvals. - pub coalesce_tranche_diff: u32, + /// Approval options params. + pub options: ApprovalsOptions, } impl PeerMessagesGenerator { @@ -107,20 +100,8 @@ impl PeerMessagesGenerator { /// the assignments/approvals and peer view changes at the begining of each block. pub fn generate_messages(mut self, spawn_task_handle: &SpawnTaskHandle) { spawn_task_handle.spawn_blocking("generate-messages", "generate-messages", async move { - for block_info in self.blocks { - let assignments = generate_assignments( - &block_info, - self.test_authorities.peer_ids.clone(), - self.test_authorities.key_seeds.clone(), - &self.session_info, - self.enable_assignments_v2, - &self.random_samplings, - self.validator_index.0, - &block_info.relay_vrf_story, - &self.topology_node_under_test, - &self.topology, - self.last_considered_tranche, - ); + for block_info in &self.blocks { + let assignments = self.generate_assignments(block_info); let bytes = self.validator_index.0.to_be_bytes(); let seed = [ @@ -134,9 +115,7 @@ impl PeerMessagesGenerator { block_info.hash, &self.test_authorities.validator_public, block_info.candidates.clone(), - self.min_coalesce, - self.max_coalesce, - self.coalesce_tranche_diff, + &self.options, &mut rand_chacha, self.test_authorities.keyring.keystore_ref(), ); @@ -204,7 +183,7 @@ impl PeerMessagesGenerator { gum::info!(target: LOG_TARGET, "Generate messages"); let topology = generate_topology(test_authorities); - let random_samplings = random_samplings_to_node_patterns( + let random_samplings = random_samplings_to_node( ValidatorIndex(NODE_UNDER_TEST), test_authorities.validator_public.len(), test_authorities.validator_public.len() * 2, @@ -216,7 +195,7 @@ impl PeerMessagesGenerator { let (tx, mut rx) = futures::channel::mpsc::unbounded(); // Spawn a thread to generate the messages for each validator, so that we speed up the - // generation a bit. + // generation. for current_validator_index in 1..test_authorities.validator_public.len() { let peer_message_source = PeerMessagesGenerator { topology_node_under_test: topology_node_under_test.clone(), @@ -227,11 +206,7 @@ impl PeerMessagesGenerator { blocks: blocks.clone(), tx_messages: tx.clone(), random_samplings: random_samplings.clone(), - enable_assignments_v2: options.enable_assignments_v2, - last_considered_tranche: options.last_considered_tranche, - min_coalesce: options.min_coalesce, - max_coalesce: options.max_coalesce, - coalesce_tranche_diff: options.coalesce_tranche_diff, + options: options.clone(), }; peer_message_source.generate_messages(spawn_task_handle); @@ -292,23 +267,166 @@ impl PeerMessagesGenerator { .expect("Could not update message file"); path.to_path_buf() } + + /// Generates assignments for the given `current_validator_index` + /// Returns a list of assignments to be sent sorted by tranche. + fn generate_assignments(&self, block_info: &BlockTestData) -> Vec { + let config = Config::from(&self.session_info); + + let leaving_cores = block_info + .candidates + .clone() + .into_iter() + .map(|candidate_event| { + if let CandidateEvent::CandidateIncluded(candidate, _, core_index, group_index) = + candidate_event + { + (candidate.hash(), core_index, group_index) + } else { + todo!("Variant is never created in this benchmark") + } + }) + .collect_vec(); + + let mut assignments_by_tranche = BTreeMap::new(); + + let bytes = self.validator_index.0.to_be_bytes(); + let seed = [ + bytes[0], bytes[1], bytes[2], bytes[3], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + let mut rand_chacha = ChaCha20Rng::from_seed(seed); + + let to_be_sent_by = neighbours_that_would_sent_message( + &self.test_authorities.peer_ids, + self.validator_index.0, + &self.topology_node_under_test, + &self.topology, + ); + + let leaving_cores = leaving_cores + .clone() + .into_iter() + .filter(|(_, core_index, _group_index)| core_index.0 != self.validator_index.0) + .collect_vec(); + + let store = LocalKeystore::in_memory(); + let _public = store + .sr25519_generate_new( + ASSIGNMENT_KEY_TYPE_ID, + Some(self.test_authorities.key_seeds[self.validator_index.0 as usize].as_str()), + ) + .expect("should not fail"); + let assignments = compute_assignments( + &store, + block_info.relay_vrf_story.clone(), + &config, + leaving_cores.clone(), + self.options.enable_assignments_v2, + ); + + let random_sending_nodes = self + .random_samplings + .get(rand_chacha.next_u32() as usize % self.random_samplings.len()) + .unwrap(); + let random_sending_peer_ids = random_sending_nodes + .iter() + .map(|validator| (*validator, self.test_authorities.peer_ids[validator.0 as usize])) + .collect_vec(); + + let mut unique_assignments = HashSet::new(); + for (core_index, assignment) in assignments { + let assigned_cores = match &assignment.cert().kind { + approval::v2::AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => + core_bitfield.iter_ones().map(|val| CoreIndex::from(val as u32)).collect_vec(), + approval::v2::AssignmentCertKindV2::RelayVRFDelay { core_index } => + vec![*core_index], + approval::v2::AssignmentCertKindV2::RelayVRFModulo { sample: _ } => + vec![core_index], + }; + + let bitfiled: CoreBitfield = assigned_cores.clone().try_into().unwrap(); + + // For the cases where tranch0 assignments are in a single certificate we need to make + // sure we create a single message. + if unique_assignments.insert(bitfiled) { + let this_tranche_assignments = + assignments_by_tranche.entry(assignment.tranche()).or_insert_with(Vec::new); + + this_tranche_assignments.push(( + IndirectAssignmentCertV2 { + block_hash: block_info.hash, + validator: self.validator_index, + cert: assignment.cert().clone(), + }, + block_info + .candidates + .iter() + .enumerate() + .filter(|(_index, candidate)| { + if let CandidateEvent::CandidateIncluded(_, _, core, _) = candidate { + assigned_cores.contains(core) + } else { + panic!("Should not happen"); + } + }) + .map(|(index, _)| index as u32) + .collect_vec() + .try_into() + .unwrap(), + to_be_sent_by + .iter() + .chain(random_sending_peer_ids.iter()) + .copied() + .collect::>(), + assignment.tranche(), + )); + } + } + + assignments_by_tranche + .into_values() + .flat_map(|assignments| assignments.into_iter()) + .map(|assignment| { + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( + assignment.0, + assignment.1, + )]); + TestMessageInfo { + msg, + sent_by: assignment + .2 + .into_iter() + .map(|(validator_index, _)| validator_index) + .collect_vec(), + tranche: assignment.3, + block_hash: block_info.hash, + } + }) + .collect_vec() + } } /// A list of random samplings that we use to determine which nodes should send a given message to /// the node under test. /// We can not sample every time for all the messages because that would be too expensive to /// perform, so pre-generate a list of samples for a given network size. -fn random_samplings_to_node_patterns( +/// - result[i] give us as a list of random nodes that would send a given message to the node under +/// test. +fn random_samplings_to_node( node_under_test: ValidatorIndex, num_validators: usize, - num_patterns: usize, + num_samplings: usize, ) -> Vec> { let seed = [7u8; 32]; let mut rand_chacha = ChaCha20Rng::from_seed(seed); - (0..num_patterns) + (0..num_samplings) .map(|_| { (0..num_validators) + .filter(|sending_validator_index| { + *sending_validator_index != NODE_UNDER_TEST as usize + }) .flat_map(|sending_validator_index| { let mut validators = (0..num_validators).collect_vec(); validators.shuffle(&mut rand_chacha); @@ -338,30 +456,33 @@ fn random_samplings_to_node_patterns( /// Helper function to randomly determine how many approvals we coalesce together in a single /// message. fn coalesce_approvals_len( - min_coalesce: u32, - max_coalesce: u32, + coalesce_mean: f32, + coalesce_std_dev: f32, rand_chacha: &mut ChaCha20Rng, ) -> usize { - let mut sampling: Vec = (min_coalesce as usize..max_coalesce as usize + 1).collect_vec(); - *(sampling.partial_shuffle(rand_chacha, 1).0.first().unwrap()) + max( + 1, + Normal::new(coalesce_mean, coalesce_std_dev) + .expect("normal distribution parameters are good") + .sample(rand_chacha) + .round() as i32, + ) as usize } /// Helper function to create approvals signatures for all assignments passed as arguments. /// Returns a list of Approvals messages that need to be sent. -#[allow(clippy::too_many_arguments)] fn issue_approvals( assignments: Vec, block_hash: Hash, validator_ids: &[ValidatorId], candidates: Vec, - min_coalesce: u32, - max_coalesce: u32, - coalesce_tranche_diff: u32, + options: &ApprovalsOptions, rand_chacha: &mut ChaCha20Rng, store: &LocalKeystore, ) -> Vec { let mut queued_to_sign: Vec = Vec::new(); - let mut num_coalesce = coalesce_approvals_len(min_coalesce, max_coalesce, rand_chacha); + let mut num_coalesce = + coalesce_approvals_len(options.coalesce_mean, options.coalesce_std_dev, rand_chacha); let result = assignments .iter() .enumerate() @@ -387,7 +508,7 @@ fn issue_approvals( if queued_to_sign.len() >= num_coalesce || (!queued_to_sign.is_empty() && current_validator_index != assignment.0.validator) || - message.tranche - earliest_tranche >= coalesce_tranche_diff + message.tranche - earliest_tranche >= options.coalesce_tranche_diff { approvals_to_create.push(TestSignInfo::sign_candidates( &mut queued_to_sign, @@ -396,7 +517,11 @@ fn issue_approvals( num_coalesce, store, )); - num_coalesce = coalesce_approvals_len(min_coalesce, max_coalesce, rand_chacha); + num_coalesce = coalesce_approvals_len( + options.coalesce_mean, + options.coalesce_std_dev, + rand_chacha, + ); } // If more that one candidate was in the assignment queue all of them for issuing @@ -530,11 +655,16 @@ fn neighbours_that_would_sent_message( topology_node_under_test.required_routing_by_index(**validator, false) == RequiredRouting::GridY }); + + assert!(originator_y != Some(&ValidatorIndex(NODE_UNDER_TEST))); + let originator_x = topology_originator.validator_indices_x.iter().find(|validator| { topology_node_under_test.required_routing_by_index(**validator, false) == RequiredRouting::GridX }); + assert!(originator_x != Some(&ValidatorIndex(NODE_UNDER_TEST))); + let is_neighbour = topology_originator .validator_indices_x .contains(&ValidatorIndex(NODE_UNDER_TEST)) || @@ -549,158 +679,8 @@ fn neighbours_that_would_sent_message( .collect_vec(); if is_neighbour { - to_be_sent_by.push((ValidatorIndex(NODE_UNDER_TEST), peer_ids[0])); - } - to_be_sent_by -} - -/// Generates assignments for the given `current_validator_index` -/// Returns a list of assignments to be sent sorted by tranche. -#[allow(clippy::too_many_arguments)] -fn generate_assignments( - block_info: &BlockTestData, - peer_ids: Vec, - seeds: Vec, - session_info: &SessionInfo, - generate_v2_assignments: bool, - random_samplings: &Vec>, - current_validator_index: u32, - relay_vrf_story: &RelayVRFStory, - topology_node_under_test: &GridNeighbors, - topology: &SessionGridTopology, - last_considered_tranche: u32, -) -> Vec { - let config = Config::from(session_info); - - let leaving_cores = block_info - .candidates - .clone() - .into_iter() - .map(|candidate_event| { - if let CandidateEvent::CandidateIncluded(candidate, _, core_index, group_index) = - candidate_event - { - (candidate.hash(), core_index, group_index) - } else { - todo!("Variant is never created in this benchmark") - } - }) - .collect_vec(); - - let mut assignments_by_tranche = BTreeMap::new(); - - let bytes = current_validator_index.to_be_bytes(); - let seed = [ - bytes[0], bytes[1], bytes[2], bytes[3], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]; - let mut rand_chacha = ChaCha20Rng::from_seed(seed); - - let to_be_sent_by = neighbours_that_would_sent_message( - &peer_ids, - current_validator_index, - topology_node_under_test, - topology, - ); - - let leaving_cores = leaving_cores - .clone() - .into_iter() - .filter(|(_, core_index, _group_index)| core_index.0 != current_validator_index) - .collect_vec(); - - let store = LocalKeystore::in_memory(); - let _public = store - .sr25519_generate_new( - ASSIGNMENT_KEY_TYPE_ID, - Some(seeds[current_validator_index as usize].as_str()), - ) - .expect("should not fail"); - let assignments = compute_assignments( - &store, - relay_vrf_story.clone(), - &config, - leaving_cores.clone(), - generate_v2_assignments, - ); - - let random_sending_nodes = random_samplings - .get(rand_chacha.next_u32() as usize % random_samplings.len()) - .unwrap(); - let random_sending_peer_ids = random_sending_nodes - .iter() - .map(|validator| (*validator, peer_ids[validator.0 as usize])) - .collect_vec(); - - let mut unique_assignments = HashSet::new(); - for (core_index, assignment) in assignments { - let assigned_cores = match &assignment.cert().kind { - approval::v2::AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => - core_bitfield.iter_ones().map(|val| CoreIndex::from(val as u32)).collect_vec(), - approval::v2::AssignmentCertKindV2::RelayVRFDelay { core_index } => vec![*core_index], - approval::v2::AssignmentCertKindV2::RelayVRFModulo { sample: _ } => vec![core_index], - }; - if assignment.tranche() > last_considered_tranche { - continue - } - - let bitfiled: CoreBitfield = assigned_cores.clone().try_into().unwrap(); - - // For the cases where tranch0 assignments are in a single certificate we need to make - // sure we create a single message. - if unique_assignments.insert(bitfiled) { - let this_tranche_assignments = - assignments_by_tranche.entry(assignment.tranche()).or_insert_with(Vec::new); - - this_tranche_assignments.push(( - IndirectAssignmentCertV2 { - block_hash: block_info.hash, - validator: ValidatorIndex(current_validator_index), - cert: assignment.cert().clone(), - }, - block_info - .candidates - .iter() - .enumerate() - .filter(|(_index, candidate)| { - if let CandidateEvent::CandidateIncluded(_, _, core, _) = candidate { - assigned_cores.contains(core) - } else { - panic!("Should not happen"); - } - }) - .map(|(index, _)| index as u32) - .collect_vec() - .try_into() - .unwrap(), - to_be_sent_by - .iter() - .chain(random_sending_peer_ids.iter()) - .copied() - .collect::>(), - assignment.tranche(), - )); - } + to_be_sent_by.push((ValidatorIndex(current_validator_index), peer_ids[0])); } - assignments_by_tranche - .into_values() - .flat_map(|assignments| assignments.into_iter()) - .map(|assignment| { - let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( - assignment.0, - assignment.1, - )]); - TestMessageInfo { - msg, - sent_by: assignment - .2 - .into_iter() - .map(|(validator_index, _)| validator_index) - .collect_vec(), - tranche: assignment.3, - block_hash: block_info.hash, - } - }) - .collect_vec() + to_be_sent_by } diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 795635a4d81f..6fb9f121ff2b 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -21,8 +21,8 @@ use self::{ use crate::{ approval::{ helpers::{ - generate_babe_epoch, generate_new_session_topology, generate_peer_connected, - generate_peer_view_change_for, PastSystemClock, + generate_babe_epoch, generate_new_session_topology, generate_peer_view_change_for, + PastSystemClock, }, message_generator::PeerMessagesGenerator, mock_chain_selection::MockChainSelection, @@ -33,12 +33,12 @@ use crate::{ mock::{ dummy_builder, network_bridge::MockNetworkBridgeTx, AlwaysSupportsParachains, ChainApiState, MockChainApi, MockNetworkBridgeRx, MockRuntimeApi, TestSyncOracle, - NEEDED_APPROVALS, }, network::{ new_network, HandleNetworkMessage, NetworkEmulatorHandle, NetworkInterface, NetworkInterfaceReceiver, }, + NODE_UNDER_TEST, }, }; use colored::Colorize; @@ -97,8 +97,6 @@ pub(crate) const TEST_CONFIG: ApprovalVotingConfig = ApprovalVotingConfig { slot_duration_millis: SLOT_DURATION_MILLIS, }; -pub const NODE_UNDER_TEST: u32 = 0; - /// Start generating messages for a slot into the future, so that the /// generation nevers falls behind the current slot. const BUFFER_FOR_GENERATION_MILLIS: u64 = 30_000; @@ -109,24 +107,19 @@ const BUFFER_FOR_GENERATION_MILLIS: u64 = 30_000; #[allow(missing_docs)] pub struct ApprovalsOptions { #[clap(short, long, default_value_t = 89)] - /// The last considered tranche for which we should generate message, this does not - /// mean the message is sent, because if the block is approved no other message is sent - /// anymore. + /// The last considered tranche for which we send the message. pub last_considered_tranche: u32, - #[clap(short, long, default_value_t = 1)] + #[clap(short, long, default_value_t = 1.0)] /// Min candidates to be signed in a single approval. - pub min_coalesce: u32, - #[clap(short, long, default_value_t = 1)] + pub coalesce_mean: f32, + #[clap(short, long, default_value_t = 1.0)] /// Max candidate to be signed in a single approval. - pub max_coalesce: u32, + pub coalesce_std_dev: f32, /// The maximum tranche diff between approvals coalesced toghther. pub coalesce_tranche_diff: u32, #[clap(short, long, default_value_t = false)] /// Enable assignments v2. pub enable_assignments_v2: bool, - #[clap(short, long, default_value_t = 89)] - /// Send messages till tranche - pub send_till_tranche: u32, #[clap(short, long, default_value_t = true)] /// Sends messages only till block is approved. pub stop_when_approved: bool, @@ -136,24 +129,14 @@ pub struct ApprovalsOptions { /// The number of no shows per candidate #[clap(short, long, default_value_t = 0)] pub num_no_shows_per_candidate: u32, - /// Max expected time of flight for approval-distribution. - #[clap(short, long, default_value_t = 6.0)] - pub approval_distribution_expected_tof: f64, - /// Max expected cpu usage by approval-distribution. - #[clap(short, long, default_value_t = 6.0)] - pub approval_distribution_cpu_ms: f64, - /// Max expected cpu usage by approval-voting. - #[clap(short, long, default_value_t = 6.0)] - pub approval_voting_cpu_ms: f64, } impl ApprovalsOptions { // Generates a fingerprint use to determine if messages need to be re-generated. fn fingerprint(&self) -> Vec { let mut bytes = Vec::new(); - bytes.extend(self.last_considered_tranche.to_be_bytes()); - bytes.extend(self.min_coalesce.to_be_bytes()); - bytes.extend(self.max_coalesce.to_be_bytes()); + bytes.extend(self.coalesce_mean.to_be_bytes()); + bytes.extend(self.coalesce_std_dev.to_be_bytes()); bytes.extend(self.coalesce_tranche_diff.to_be_bytes()); bytes.extend((self.enable_assignments_v2 as i32).to_be_bytes()); bytes @@ -204,12 +187,15 @@ struct CandidateTestData { num_no_shows: u32, /// The maximum tranche were we covered the needed approvals max_tranche: u32, + /// Minimum needed votes to approve candidate. + needed_approvals: u32, } impl CandidateTestData { /// If message in this tranche needs to be sent. fn should_send_tranche(&self, tranche: u32) -> bool { - self.sent_assignment <= NEEDED_APPROVALS || tranche <= self.max_tranche + self.num_no_shows + self.sent_assignment <= self.needed_approvals || + tranche <= self.max_tranche + self.num_no_shows } /// Sets max tranche @@ -224,8 +210,8 @@ impl CandidateTestData { } /// Marks an assignment sent. - fn sent_assignment(&mut self, tranche: u32) { - if self.sent_assignment < NEEDED_APPROVALS { + fn mark_sent_assignment(&mut self, tranche: u32) { + if self.sent_assignment < self.needed_approvals { self.set_max_tranche(tranche); } @@ -256,13 +242,15 @@ struct GeneratedState { /// updated between subsystems, they would have to be wrapped in Arc's. #[derive(Clone)] pub struct ApprovalTestState { + /// The main test configuration + configuration: TestConfiguration, /// The specific test configurations passed when starting the benchmark. options: ApprovalsOptions, /// The list of blocks used for testing. blocks: Vec, /// The babe epoch used during testing. babe_epoch: BabeEpoch, - /// The slot at which this benchamrk begins. + /// The pre-generated state. generated_state: GeneratedState, /// The test authorities test_authorities: TestAuthorities, @@ -336,6 +324,7 @@ impl ApprovalTestState { approval_voting_metrics: ApprovalVotingMetrics::try_register(&dependencies.registry) .unwrap(), delta_tick_from_generated: Arc::new(AtomicU64::new(630720000)), + configuration: configuration.clone(), }; gum::info!("Built testing state"); @@ -378,7 +367,7 @@ impl ApprovalTestState { block_hash, block_number as BlockNumber, configuration.n_cores as u32, - configuration.n_included_candidates as u32, + configuration.n_cores as u32, ), relay_vrf_story, approved: Arc::new(AtomicBool::new(false)), @@ -582,6 +571,7 @@ impl PeerMessageProducer { &mut skipped_messages, ) || re_process_skipped; } + // Sleep, so that we don't busy wait in this loop when don't have anything to send. sleep(Duration::from_millis(50)).await; } @@ -591,6 +581,9 @@ impl PeerMessageProducer { per_candidate_data.values().map(|data| data.last_tranche_with_no_show).max() ); sleep(Duration::from_secs(6)).await; + // Send an empty GetApprovalSignatures as the last message + // so when the approval-distribution answered to it, we know it doesn't have anything + // else to process. let (tx, rx) = oneshot::channel(); let msg = ApprovalDistributionMessage::GetApprovalSignatures(HashSet::new(), tx); self.send_overseer_message( @@ -619,7 +612,7 @@ impl PeerMessageProducer { .get_info_by_hash(bundle.assignments.first().unwrap().block_hash) .clone(); - if bundle.bundle_needed(per_candidate_data, &self.options) { + if bundle.should_send(per_candidate_data, &self.options) { bundle.record_sent_assignment(per_candidate_data); let assignments = bundle.assignments.clone(); @@ -635,7 +628,9 @@ impl PeerMessageProducer { .total_unique_messages .as_ref() .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - for (peer, messages) in message.split_by_peer_id(&self.state.test_authorities) { + for (peer, messages) in + message.clone().split_by_peer_id(&self.state.test_authorities) + { for message in messages { self.state .total_sent_messages_to_node @@ -672,7 +667,7 @@ impl PeerMessageProducer { current_slot, block_info, initialized_blocks.contains(&block_info.hash), - ) || !bundle.bundle_needed(per_candidate_data, &self.options) + ) || !bundle.should_send(per_candidate_data, &self.options) } // Tells if the tranche where the bundle should be sent has passed. @@ -690,9 +685,6 @@ impl PeerMessageProducer { // Queue message to be sent by validator `sent_by` fn queue_message_from_peer(&mut self, message: TestMessageInfo, sent_by: ValidatorIndex) { - if sent_by == ValidatorIndex(NODE_UNDER_TEST) { - return; - } let peer_authority_id = self .state .test_authorities @@ -706,7 +698,7 @@ impl PeerMessageProducer { &peer_authority_id, protocol_v3::ValidationProtocol::ApprovalDistribution(message.msg).into(), ) - .expect(format!("Network should be up and running {:?}", sent_by).as_str()); + .unwrap_or_else(|_| panic!("Network should be up and running {:?}", sent_by)); } // Queues a message to be sent by the peer identified by the `sent_by` value. @@ -739,7 +731,6 @@ impl PeerMessageProducer { for validator in 1..self.state.test_authorities.validator_authority_id.len() as u32 { let peer_id = self.state.test_authorities.peer_ids.get(validator as usize).unwrap(); let validator = ValidatorIndex(validator); - let view_update = generate_peer_view_change_for(block_info.hash, *peer_id); self.send_overseer_message(view_update, validator, None).await; @@ -763,6 +754,7 @@ impl PeerMessageProducer { sent_assignment: 0, num_no_shows: 0, max_tranche: 0, + needed_approvals: self.state.configuration.needed_approvals as u32, }, ); } @@ -906,7 +898,7 @@ pub async fn bench_approvals_run( // First create the initialization messages that make sure that then node under // tests receives notifications about the topology used and the connected peers. - let mut initialization_messages = generate_peer_connected(&state.test_authorities); + let mut initialization_messages = env.network().generate_peer_connected(); initialization_messages.extend(generate_new_session_topology( &state.test_authorities, ValidatorIndex(NODE_UNDER_TEST), @@ -975,11 +967,15 @@ pub async fn bench_approvals_run( producer_rx.await.expect("Failed to receive done from message producer"); gum::info!("Awaiting polkadot_parachain_subsystem_bounded_received to tells us the messages have been processed"); - - env.wait_until_metric_eq( + let at_least_messages = + state.total_sent_messages_to_node.load(std::sync::atomic::Ordering::SeqCst) as usize; + env.wait_until_metric( "polkadot_parachain_subsystem_bounded_received", Some(("subsystem_name", "approval-distribution-subsystem")), - state.total_sent_messages_to_node.load(std::sync::atomic::Ordering::SeqCst) as usize, + |value| { + gum::info!(target: LOG_TARGET, ?value, ?at_least_messages, "Waiting metric"); + value >= at_least_messages as f64 + }, ) .await; gum::info!("Requesting approval votes ms"); @@ -1014,12 +1010,36 @@ pub async fn bench_approvals_run( } } + gum::info!("Awaiting polkadot_parachain_subsystem_bounded_received to tells us the messages have been processed"); + let at_least_messages = + state.total_sent_messages_to_node.load(std::sync::atomic::Ordering::SeqCst) as usize; + env.wait_until_metric( + "polkadot_parachain_subsystem_bounded_received", + Some(("subsystem_name", "approval-distribution-subsystem")), + |value| { + gum::info!(target: LOG_TARGET, ?value, ?at_least_messages, "Waiting metric"); + value >= at_least_messages as f64 + }, + ) + .await; + for state in &state.blocks { - for (validator, votes) in state.votes.as_ref().iter().enumerate() { + for (validator, votes) in state + .votes + .as_ref() + .iter() + .enumerate() + .filter(|(validator, _)| *validator != NODE_UNDER_TEST as usize) + { for (index, candidate) in votes.iter().enumerate() { assert_eq!( - (validator, index, candidate.load(std::sync::atomic::Ordering::SeqCst)), - (validator, index, false) + ( + validator, + index, + candidate.load(std::sync::atomic::Ordering::SeqCst), + state.hash + ), + (validator, index, false, state.hash) ); } } @@ -1038,25 +1058,4 @@ pub async fn bench_approvals_run( env.display_network_usage(); env.display_cpu_usage(&["approval-distribution", "approval-voting"]); - - assert!(env.metric_with_label_lower_than( - "substrate_tasks_polling_duration_sum", - "task_group", - "approval-voting", - state.options.approval_voting_cpu_ms * state.blocks.len() as f64 - )); - - assert!(env.metric_with_label_lower_than( - "substrate_tasks_polling_duration_sum", - "task_group", - "approval-distribution", - state.options.approval_distribution_cpu_ms * state.blocks.len() as f64 - )); - - assert!(env.bucket_metric_lower_than( - "polkadot_parachain_subsystem_bounded_tof_bucket", - "subsystem_name", - "approval-distribution-subsystem", - state.options.approval_distribution_expected_tof - )); } diff --git a/polkadot/node/subsystem-bench/src/approval/test_message.rs b/polkadot/node/subsystem-bench/src/approval/test_message.rs index 785bfd530510..ea578776f261 100644 --- a/polkadot/node/subsystem-bench/src/approval/test_message.rs +++ b/polkadot/node/subsystem-bench/src/approval/test_message.rs @@ -90,13 +90,14 @@ impl MessagesBundle { /// Tells if the bundle is needed for sending. /// We either send it because we need more assignments and approvals to approve the candidates /// or because we configured the test to send messages untill a given tranche. - pub fn bundle_needed( + pub fn should_send( &self, candidates_test_data: &HashMap<(Hash, CandidateIndex), CandidateTestData>, options: &ApprovalsOptions, ) -> bool { self.needed_for_approval(candidates_test_data) || - (!options.stop_when_approved && self.min_tranche() <= options.send_till_tranche) + (!options.stop_when_approved && + self.min_tranche() <= options.last_considered_tranche) } /// Tells if the bundle is needed because we need more messages to approve the candidates. @@ -164,7 +165,7 @@ impl TestMessageInfo { let candidate_test_data = candidates_test_data .get_mut(&(assignment.block_hash, candidate_index as CandidateIndex)) .unwrap(); - candidate_test_data.sent_assignment(self.tranche) + candidate_test_data.mark_sent_assignment(self.tranche) } } }, diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index ab8a344f6d47..e0333c6781d3 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -109,6 +109,7 @@ fn build_overseer_for_availability_read( (overseer, OverseerHandle::new(raw_handle)) } +#[allow(clippy::too_many_arguments)] fn build_overseer_for_availability_write( spawn_task_handle: SpawnTaskHandle, runtime_api: MockRuntimeApi, @@ -706,10 +707,10 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: ); // Wait for all bitfields to be processed. - env.wait_until_metric_eq( + env.wait_until_metric( "polkadot_parachain_received_availabilty_bitfields_total", None, - config.connected_count() * block_num, + |value| value == (config.connected_count() * block_num) as f64, ) .await; diff --git a/polkadot/node/subsystem-bench/src/cli.rs b/polkadot/node/subsystem-bench/src/cli.rs index 4a845ade1fec..3a67fd1727d8 100644 --- a/polkadot/node/subsystem-bench/src/cli.rs +++ b/polkadot/node/subsystem-bench/src/cli.rs @@ -37,7 +37,7 @@ pub enum TestObjective { /// Run a test sequence specified in a file TestSequence(TestSequenceOptions), /// Benchmark the approval-voting and approval-distribution subsystems. - ApprovalsTest(ApprovalsOptions), + ApprovalVoting(ApprovalsOptions), Unimplemented, } diff --git a/polkadot/node/subsystem-bench/src/core/configuration.rs b/polkadot/node/subsystem-bench/src/core/configuration.rs index 103d3e7dec90..0c8a78c504c8 100644 --- a/polkadot/node/subsystem-bench/src/core/configuration.rs +++ b/polkadot/node/subsystem-bench/src/core/configuration.rs @@ -68,6 +68,25 @@ fn default_backing_group_size() -> usize { 5 } +// Default needed approvals +fn default_needed_approvals() -> usize { + 30 +} + +fn default_zeroth_delay_tranche_width() -> usize { + 0 +} +fn default_relay_vrf_modulo_samples() -> usize { + 6 +} + +fn default_n_delay_tranches() -> usize { + 89 +} +fn default_no_show_slots() -> usize { + 3 +} + /// The test input parameters #[derive(Clone, Debug, Serialize, Deserialize)] pub struct TestConfiguration { @@ -77,8 +96,17 @@ pub struct TestConfiguration { pub n_validators: usize, /// Number of cores pub n_cores: usize, - /// Number of included candidates - pub n_included_candidates: usize, + /// The number of needed votes to approve a candidate. + #[serde(default = "default_needed_approvals")] + pub needed_approvals: usize, + #[serde(default = "default_zeroth_delay_tranche_width")] + pub zeroth_delay_tranche_width: usize, + #[serde(default = "default_relay_vrf_modulo_samples")] + pub relay_vrf_modulo_samples: usize, + #[serde(default = "default_n_delay_tranches")] + pub n_delay_tranches: usize, + #[serde(default = "default_no_show_slots")] + pub no_show_slots: usize, /// Maximum backing group size #[serde(default = "default_backing_group_size")] pub max_validators_per_core: usize, @@ -225,7 +253,6 @@ impl TestConfiguration { Self { objective, n_cores, - n_included_candidates: n_cores, n_validators, max_validators_per_core: 5, pov_sizes: generate_pov_sizes(n_cores, min_pov_size, max_pov_size), @@ -237,6 +264,11 @@ impl TestConfiguration { min_pov_size, max_pov_size, connectivity: 100, + needed_approvals: default_needed_approvals(), + n_delay_tranches: default_n_delay_tranches(), + no_show_slots: default_no_show_slots(), + relay_vrf_modulo_samples: default_relay_vrf_modulo_samples(), + zeroth_delay_tranche_width: default_zeroth_delay_tranche_width(), } } @@ -251,7 +283,6 @@ impl TestConfiguration { Self { objective, n_cores, - n_included_candidates: n_cores, n_validators, max_validators_per_core: 5, pov_sizes: generate_pov_sizes(n_cores, min_pov_size, max_pov_size), @@ -262,6 +293,11 @@ impl TestConfiguration { min_pov_size, max_pov_size, connectivity: 95, + needed_approvals: default_needed_approvals(), + n_delay_tranches: default_n_delay_tranches(), + no_show_slots: default_no_show_slots(), + relay_vrf_modulo_samples: default_relay_vrf_modulo_samples(), + zeroth_delay_tranche_width: default_zeroth_delay_tranche_width(), } } @@ -276,7 +312,6 @@ impl TestConfiguration { Self { objective, n_cores, - n_included_candidates: n_cores, n_validators, max_validators_per_core: 5, pov_sizes: generate_pov_sizes(n_cores, min_pov_size, max_pov_size), @@ -287,6 +322,11 @@ impl TestConfiguration { min_pov_size, max_pov_size, connectivity: 67, + needed_approvals: default_needed_approvals(), + n_delay_tranches: default_n_delay_tranches(), + no_show_slots: default_no_show_slots(), + relay_vrf_modulo_samples: default_relay_vrf_modulo_samples(), + zeroth_delay_tranche_width: default_zeroth_delay_tranche_width(), } } } diff --git a/polkadot/node/subsystem-bench/src/core/display.rs b/polkadot/node/subsystem-bench/src/core/display.rs index fba0fbe833cb..48ec254c8dab 100644 --- a/polkadot/node/subsystem-bench/src/core/display.rs +++ b/polkadot/node/subsystem-bench/src/core/display.rs @@ -20,9 +20,8 @@ //! Currently histogram buckets are skipped. use super::{configuration::TestConfiguration, LOG_TARGET}; use colored::Colorize; -use itertools::Itertools; use prometheus::{ - proto::{Bucket, MetricFamily, MetricType}, + proto::{MetricFamily, MetricType}, Registry, }; use std::fmt::Display; @@ -50,53 +49,11 @@ impl MetricCollection { .sum() } - /// Filters metrics by name - pub fn subset_with_name(&self, name: &str) -> MetricCollection { - self.all() - .iter() - .filter(|metric| metric.name == name) - .cloned() - .collect::>() - .into() - } - /// Tells if entries in bucket metric is lower than `value` pub fn metric_lower_than(&self, metric_name: &str, value: f64) -> bool { self.sum_by(metric_name) < value } - /// Tells if entries in bucket metric is lower than `value` - pub fn bucket_metric_lower_than( - &self, - metric_name: &str, - label_name: &str, - label_value: &str, - value: f64, - ) -> bool { - let metric = self - .subset_with_name(metric_name) - .subset_with_label_value(label_name, label_value); - - metric.all().iter().all(|metric| { - metric - .bucket - .as_ref() - .map(|buckets| { - let mut prev_value = 0; - for bucket in buckets { - if value < bucket.get_upper_bound() && - prev_value != bucket.get_cumulative_count() - { - return false; - } - prev_value = bucket.get_cumulative_count(); - } - true - }) - .unwrap_or_default() - }) - } - pub fn subset_with_label_value(&self, label_name: &str, label_value: &str) -> MetricCollection { self.0 .iter() @@ -133,7 +90,6 @@ pub struct TestMetric { label_names: Vec, label_values: Vec, value: f64, - bucket: Option>, } impl Display for TestMetric { @@ -187,7 +143,6 @@ pub fn parse_metrics(registry: &Registry) -> MetricCollection { label_names, label_values, value: m.get_counter().get_value(), - bucket: None, }); }, MetricType::GAUGE => { @@ -196,7 +151,6 @@ pub fn parse_metrics(registry: &Registry) -> MetricCollection { label_names, label_values, value: m.get_gauge().get_value(), - bucket: None, }); }, MetricType::HISTOGRAM => { @@ -207,7 +161,6 @@ pub fn parse_metrics(registry: &Registry) -> MetricCollection { label_names: label_names.clone(), label_values: label_values.clone(), value: h.get_sample_sum(), - bucket: None, }); let h_name = name.clone() + "_count"; @@ -216,15 +169,6 @@ pub fn parse_metrics(registry: &Registry) -> MetricCollection { label_names: label_names.clone(), label_values: label_values.clone(), value: h.get_sample_count() as f64, - bucket: None, - }); - let h_name = name.clone() + "_bucket"; - test_metrics.push(TestMetric { - name: h_name, - label_names, - label_values, - value: 0.0, - bucket: Some(h.get_bucket().iter().cloned().collect_vec()), }); }, MetricType::SUMMARY => { diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index b04f9dc32dca..bba3cf4d1b25 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -331,31 +331,18 @@ impl TestEnvironment { test_metrics.metric_lower_than(metric_name, value) } - /// Tells if all entries in a Histogram are lower than value - pub fn bucket_metric_lower_than( - &self, - metric_name: &str, - label_name: &str, - label_value: &str, - value: f64, - ) -> bool { - let test_metrics = super::display::parse_metrics(self.registry()); - - test_metrics.bucket_metric_lower_than(metric_name, label_name, label_value, value) - } /// Stop overseer and subsystems. pub async fn stop(&mut self) { self.overseer_handle.stop().await; } /// Blocks until `metric_name` >= `value` - pub async fn wait_until_metric_eq( + pub async fn wait_until_metric( &self, metric_name: &str, label: Option<(&str, &str)>, - value: usize, + condition: impl Fn(f64) -> bool, ) { - let value = value as f64; loop { let test_metrics = if let Some((label_name, label_value)) = label { super::display::parse_metrics(self.registry()) @@ -365,11 +352,10 @@ impl TestEnvironment { }; let current_value = test_metrics.sum_by(metric_name); - gum::debug!(target: LOG_TARGET, metric_name, current_value, value, "Waiting for metric"); - if current_value == value { + gum::debug!(target: LOG_TARGET, metric_name, current_value, "Waiting for metric"); + if condition(current_value) { break } - // Check value every 50ms. tokio::time::sleep(std::time::Duration::from_millis(50)).await; } diff --git a/polkadot/node/subsystem-bench/src/core/mock/mod.rs b/polkadot/node/subsystem-bench/src/core/mock/mod.rs index 833043c4d35b..4e07ec2c20fa 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/mod.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/mod.rs @@ -88,6 +88,6 @@ impl SyncOracle for TestSyncOracle { } fn is_offline(&self) -> bool { - unimplemented!("should not be used by bridge") + unimplemented!("not used by subsystem benchmarks") } } diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index b0bf6d21094a..d73c07d1cc4e 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -142,19 +142,7 @@ impl MockNetworkBridgeTx { .expect("Should not fail"); } }, - NetworkBridgeTxMessage::DisconnectPeer(_, _) => todo!(), - NetworkBridgeTxMessage::SendCollationMessage(_, _) => todo!(), - NetworkBridgeTxMessage::SendValidationMessages(_) => todo!(), - NetworkBridgeTxMessage::SendCollationMessages(_) => todo!(), - NetworkBridgeTxMessage::ConnectToValidators { - validator_ids: _, - peer_set: _, - failed: _, - } => todo!(), - NetworkBridgeTxMessage::ConnectToResolvedValidators { - validator_addrs: _, - peer_set: _, - } => todo!(), + _ => todo!(), }, } } diff --git a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs index f768440b4b74..f286ccc62591 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs @@ -78,8 +78,6 @@ impl MockRuntimeApi { } } -pub const NEEDED_APPROVALS: u32 = 30; - /// Generates a test session info with all passed authorities as consensus validators. pub fn session_info_for_peers( configuration: &TestConfiguration, @@ -100,11 +98,11 @@ pub fn session_info_for_peers( assignment_keys: authorities.validator_assignment_id.to_vec(), validator_groups: IndexedVec::>::from(validator_groups), n_cores: configuration.n_cores as u32, - needed_approvals: NEEDED_APPROVALS, - zeroth_delay_tranche_width: 0, - relay_vrf_modulo_samples: 6, - n_delay_tranches: 89, - no_show_slots: 3, + needed_approvals: configuration.needed_approvals as u32, + zeroth_delay_tranche_width: configuration.zeroth_delay_tranche_width as u32, + relay_vrf_modulo_samples: configuration.relay_vrf_modulo_samples as u32, + n_delay_tranches: configuration.n_delay_tranches as u32, + no_show_slots: configuration.no_show_slots as u32, active_validator_indices: (0..authorities.validator_authority_id.len()) .map(|index| ValidatorIndex(index as u32)) .collect_vec(), diff --git a/polkadot/node/subsystem-bench/src/core/mod.rs b/polkadot/node/subsystem-bench/src/core/mod.rs index 22de9ad5a330..ca4a317b190e 100644 --- a/polkadot/node/subsystem-bench/src/core/mod.rs +++ b/polkadot/node/subsystem-bench/src/core/mod.rs @@ -15,6 +15,7 @@ // along with Polkadot. If not, see . const LOG_TARGET: &str = "subsystem-bench::core"; +pub const NODE_UNDER_TEST: u32 = 0; pub mod configuration; pub mod display; diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index 11a2f9df9e0d..b7564c792e6e 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -47,11 +47,15 @@ use futures::{ stream::FuturesUnordered, }; +use itertools::Itertools; use net_protocol::{ + peer_set::{ProtocolVersion, ValidationVersion}, request_response::{Recipient, Requests, ResponseSender}, - VersionedValidationProtocol, + ObservedRole, VersionedValidationProtocol, }; use parity_scale_codec::Encode; +use polkadot_node_subsystem_types::messages::{ApprovalDistributionMessage, NetworkBridgeEvent}; +use polkadot_overseer::AllMessages; use polkadot_primitives::AuthorityDiscoveryId; use prometheus_endpoint::U64; use rand::{seq::SliceRandom, thread_rng}; @@ -720,6 +724,28 @@ pub struct NetworkEmulatorHandle { validator_authority_ids: HashMap, } +impl NetworkEmulatorHandle { + /// Generates peer_connected messages for all peers in `test_authorities` + pub fn generate_peer_connected(&self) -> Vec { + self.peers + .iter() + .filter(|peer| peer.is_connected()) + .map(|peer| { + let network = NetworkBridgeEvent::PeerConnected( + peer.handle().peer_id, + ObservedRole::Full, + ProtocolVersion::from(ValidationVersion::V3), + None, + ); + + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate( + network, + )) + }) + .collect_vec() + } +} + /// Create a new emulated network based on `config`. /// Each emulated peer will run the specified `handlers` to process incoming messages. pub fn new_network( @@ -762,10 +788,14 @@ pub fn new_network( let connected_count = config.connected_count(); - let (_connected, to_disconnect) = peers.partial_shuffle(&mut thread_rng(), connected_count); + let mut peers_indicies = (0..n_peers).collect_vec(); + let (_connected, to_disconnect) = + peers_indicies.partial_shuffle(&mut thread_rng(), connected_count); - for peer in to_disconnect { - peer.disconnect(); + // Node under test is always mark as disconnected. + peers[NODE_UNDER_TEST].disconnect(); + for peer in to_disconnect.iter().skip(1) { + peers[*peer].disconnect(); } gum::info!(target: LOG_TARGET, "{}",format!("Network created, connected validator count {}", connected_count).bright_black()); diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 08100fae68c9..44103857b43b 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -42,9 +42,7 @@ use core::{ use clap_num::number_range; -use crate::approval::bench_approvals; -// const LOG_TARGET: &str = "subsystem-bench"; -use crate::core::display::display_configuration; +use crate::{approval::bench_approvals, core::display::display_configuration}; fn le_100(s: &str) -> Result { number_range(s, 0, 100) @@ -175,7 +173,7 @@ impl BenchCli { &mut env, state, )); }, - TestObjective::ApprovalsTest(ref options) => { + TestObjective::ApprovalVoting(ref options) => { let (mut env, state) = approval::prepare_test(test_config.clone(), options.clone()); @@ -198,7 +196,7 @@ impl BenchCli { }, TestObjective::DataAvailabilityRead(ref _options) => self.create_test_configuration(), TestObjective::DataAvailabilityWrite => self.create_test_configuration(), - TestObjective::ApprovalsTest(_) => todo!(), + TestObjective::ApprovalVoting(_) => todo!(), TestObjective::Unimplemented => todo!(), }; @@ -244,7 +242,7 @@ impl BenchCli { .block_on(availability::benchmark_availability_write(&mut env, state)); }, TestObjective::TestSequence(_options) => {}, - TestObjective::ApprovalsTest(_) => todo!(), + TestObjective::ApprovalVoting(_) => todo!(), TestObjective::Unimplemented => todo!(), } From 11e6dc1366789e5fea28f666b0cfc43610d68980 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 22 Jan 2024 09:45:27 +0200 Subject: [PATCH 185/192] Fixes Signed-off-by: Alexandru Gheorghe --- .../node/subsystem-bench/src/core/environment.rs | 13 ------------- polkadot/node/subsystem-bench/src/core/network.rs | 2 +- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index bba3cf4d1b25..b6e0bf240850 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -312,19 +312,6 @@ impl TestEnvironment { }); } - /// Tells if entries in bucket metric is lower than `value` - pub fn metric_with_label_lower_than( - &self, - metric_name: &str, - label_name: &str, - label_value: &str, - value: f64, - ) -> bool { - let test_metrics = super::display::parse_metrics(self.registry()) - .subset_with_label_value(label_name, label_value); - test_metrics.metric_lower_than(metric_name, value) - } - /// Tells if entries in bucket metric is lower than `value` pub fn metric_lower_than(registry: &Registry, metric_name: &str, value: f64) -> bool { let test_metrics = super::display::parse_metrics(registry); diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/core/network.rs index b7564c792e6e..8e7c28140635 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/core/network.rs @@ -793,7 +793,7 @@ pub fn new_network( peers_indicies.partial_shuffle(&mut thread_rng(), connected_count); // Node under test is always mark as disconnected. - peers[NODE_UNDER_TEST].disconnect(); + peers[NODE_UNDER_TEST as usize].disconnect(); for peer in to_disconnect.iter().skip(1) { peers[*peer].disconnect(); } From 2721a42d670b6ece21c3d8391888ccebef7f3900 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 29 Jan 2024 16:20:30 +0200 Subject: [PATCH 186/192] Minor fixes Signed-off-by: Alexandru Gheorghe --- polkadot/node/subsystem-bench/README.md | 26 +++++++++---------- .../src/core/mock/network_bridge.rs | 2 +- polkadot/node/subsystem-bench/src/core/mod.rs | 1 - 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/polkadot/node/subsystem-bench/README.md b/polkadot/node/subsystem-bench/README.md index 34a7a0b6e55c..e090a0392cb7 100644 --- a/polkadot/node/subsystem-bench/README.md +++ b/polkadot/node/subsystem-bench/README.md @@ -4,11 +4,11 @@ Run parachain consensus stress and performance tests on your development machine ## Motivation -The parachain consensus node implementation spans across many modules which we call subsystems. Each subsystem is -responsible for a small part of logic of the parachain consensus pipeline, but in general the most load and -performance issues are localized in just a few core subsystems like `availability-recovery`, `approval-voting` or -`dispute-coordinator`. In the absence of such a tool, we would run large test nets to load/stress test these parts of -the system. Setting up and making sense of the amount of data produced by such a large test is very expensive, hard +The parachain consensus node implementation spans across many modules which we call subsystems. Each subsystem is +responsible for a small part of logic of the parachain consensus pipeline, but in general the most load and +performance issues are localized in just a few core subsystems like `availability-recovery`, `approval-voting` or +`dispute-coordinator`. In the absence of such a tool, we would run large test nets to load/stress test these parts of +the system. Setting up and making sense of the amount of data produced by such a large test is very expensive, hard to orchestrate and is a huge development time sink. This tool aims to solve the problem by making it easy to: @@ -168,9 +168,9 @@ usage: - for how many blocks the test should run (`num_blocks`) From the perspective of the subsystem under test, this means that it will receive an `ActiveLeavesUpdate` signal -followed by an arbitrary amount of messages. This process repeats itself for `num_blocks`. The messages are generally -test payloads pre-generated before the test run, or constructed on pre-genereated payloads. For example the -`AvailabilityRecoveryMessage::RecoverAvailableData` message includes a `CandidateReceipt` which is generated before +followed by an arbitrary amount of messages. This process repeats itself for `num_blocks`. The messages are generally +test payloads pre-generated before the test run, or constructed on pre-genereated payloads. For example the +`AvailabilityRecoveryMessage::RecoverAvailableData` message includes a `CandidateReceipt` which is generated before the test is started. ### Example run @@ -204,12 +204,12 @@ node validator network. CPU usage per block 0.00s ``` -`Block time` in the context of `data-availability-read` has a different meaning. It measures the amount of time it +`Block time` in the context of `data-availability-read` has a different meaning. It measures the amount of time it took the subsystem to finish processing all of the messages sent in the context of the current test block. ### Test logs -You can select log target, subtarget and verbosity just like with Polkadot node CLI, simply setting +You can select log target, subtarget and verbosity just like with Polkadot node CLI, simply setting `RUST_LOOG="parachain=debug"` turns on debug logs for all parachain consensus subsystems in the test. ### View test metrics @@ -268,17 +268,17 @@ This tool is intended to make it easy to write new test objectives that focus in or even multiple subsystems (for example `approval-distribution` and `approval-voting`). A special kind of test objectives are performance regression tests for the CI pipeline. These should be sequences -of tests that check the performance characteristics (such as CPU usage, speed) of the subsystem under test in both +of tests that check the performance characteristics (such as CPU usage, speed) of the subsystem under test in both happy and negative scenarios (low bandwidth, network errors and low connectivity). ### Reusable test components -To faster write a new test objective you need to use some higher level wrappers and logic: `TestEnvironment`, +To faster write a new test objective you need to use some higher level wrappers and logic: `TestEnvironment`, `TestConfiguration`, `TestAuthorities`, `NetworkEmulator`. To create the `TestEnvironment` you will need to also build an `Overseer`, but that should be easy using the mockups for subsystems in`core::mock`. ### Mocking Ideally we want to have a single mock implementation for subsystems that can be minimally configured to -be used in different tests. A good example is `runtime-api` which currently only responds to session information +be used in different tests. A good example is `runtime-api` which currently only responds to session information requests based on static data. It can be easily extended to service other requests. diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs index a6b08a9bc3f1..a171deb2e715 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs @@ -144,7 +144,7 @@ impl MockNetworkBridgeTx { .expect("Should not fail"); } }, - _ => todo!(), + _ => unimplemented!("Unexpected network bridge message"), }, } } diff --git a/polkadot/node/subsystem-bench/src/core/mod.rs b/polkadot/node/subsystem-bench/src/core/mod.rs index 740b5534eff9..507dd1aa83f6 100644 --- a/polkadot/node/subsystem-bench/src/core/mod.rs +++ b/polkadot/node/subsystem-bench/src/core/mod.rs @@ -23,5 +23,4 @@ pub mod display; pub mod environment; pub mod keyring; pub mod mock; -#[allow(clippy::tabs_in_doc_comments)] pub mod network; From 296f7e7bd4d89eb9bdc542b3e7026fcf66c19e30 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 30 Jan 2024 09:18:40 +0200 Subject: [PATCH 187/192] Fix toml format Signed-off-by: Alexandru Gheorghe --- polkadot/node/subsystem-bench/Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index 9a24a3df8107..1cbefd19bf10 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -68,14 +68,14 @@ prometheus = { version = "0.13.0", default-features = false } serde = "1.0.195" serde_yaml = "0.9" -polkadot-node-core-approval-voting = { path = "../core/approval-voting"} -polkadot-approval-distribution = {path = "../network/approval-distribution"} +polkadot-node-core-approval-voting = { path = "../core/approval-voting" } +polkadot-approval-distribution = { path = "../network/approval-distribution" } sp-consensus-babe = { path = "../../../substrate/primitives/consensus/babe" } sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } sp-timestamp = { path = "../../../substrate/primitives/timestamp" } schnorrkel = { version = "0.9.1", default-features = false } -rand_core = "0.6.2" # should match schnorrkel +rand_core = "0.6.2" # should match schnorrkel rand_chacha = { version = "0.3.1" } paste = "1.0.14" orchestra = { version = "0.3.4", default-features = false, features = ["futures_channel"] } From bab7aa1cf996e4cade717b35dd30f54f0ea35b0c Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 30 Jan 2024 17:08:40 +0200 Subject: [PATCH 188/192] Remove commented code Signed-off-by: Alexandru Gheorghe --- polkadot/node/subsystem-bench/src/approval/mod.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 6fb9f121ff2b..87be762d880e 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -934,15 +934,10 @@ pub async fn bench_approvals_run( .await; } - // let block_time_delta = Duration::from_millis( - // (*current_slot + 1) * SLOT_DURATION_MILLIS - Timestamp::current().as_millis(), - // ); let block_time = Instant::now().sub(block_start_ts).as_millis() as u64; env.metrics().set_block_time(block_time); gum::info!("Block time {}", format!("{:?}ms", block_time).cyan()); - // gum::info!(target: LOG_TARGET,"{}", format!("Sleeping till end of block ({}ms)", - // block_time_delta.as_millis()).bright_black()); tokio::time::sleep(block_time_delta). - // await; + system_clock .wait(slot_number_to_tick(SLOT_DURATION_MILLIS, current_slot + 1)) .await; From 459a6284919f5a601c5b3a42c749b6dfa463cd6c Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 2 Feb 2024 12:32:06 +0200 Subject: [PATCH 189/192] Remove unused property Signed-off-by: Alexandru Gheorghe --- polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml | 1 - .../subsystem-bench/examples/approvals_throughput_best_case.yaml | 1 - .../examples/approvals_throughput_no_optimisations_enabled.yaml | 1 - 3 files changed, 3 deletions(-) diff --git a/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml b/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml index 7cf6be325af7..758c7fbbf112 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml @@ -15,5 +15,4 @@ TestConfiguration: max_pov_size: 5120 peer_bandwidth: 524288000000 bandwidth: 524288000000 - error: 0 num_blocks: 10 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml index 6d817aed17a9..370bb31a5c4c 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml @@ -15,5 +15,4 @@ TestConfiguration: max_pov_size: 5120 peer_bandwidth: 524288000000 bandwidth: 524288000000 - error: 0 num_blocks: 10 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml index 23f932771dc0..30b9ac8dc50f 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml @@ -15,5 +15,4 @@ TestConfiguration: max_pov_size: 5120 peer_bandwidth: 524288000000 bandwidth: 524288000000 - error: 0 num_blocks: 10 From 73fe67515f0fffe78a5d2316501e83a0a22782b7 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 2 Feb 2024 13:07:42 +0200 Subject: [PATCH 190/192] Address review findings Signed-off-by: Alexandru Gheorghe --- .../subsystem-bench/src/approval/message_generator.rs | 2 +- .../subsystem-bench/src/approval/mock_chain_selection.rs | 1 + polkadot/node/subsystem-bench/src/approval/mod.rs | 8 +++++--- polkadot/node/subsystem-bench/src/core/environment.rs | 3 +-- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/approval/message_generator.rs b/polkadot/node/subsystem-bench/src/approval/message_generator.rs index 6da5137543be..4318dcdf8902 100644 --- a/polkadot/node/subsystem-bench/src/approval/message_generator.rs +++ b/polkadot/node/subsystem-bench/src/approval/message_generator.rs @@ -99,7 +99,7 @@ impl PeerMessagesGenerator { /// Generates messages by spawning a blocking task in the background which begins creating /// the assignments/approvals and peer view changes at the begining of each block. pub fn generate_messages(mut self, spawn_task_handle: &SpawnTaskHandle) { - spawn_task_handle.spawn_blocking("generate-messages", "generate-messages", async move { + spawn_task_handle.spawn("generate-messages", "generate-messages", async move { for block_info in &self.blocks { let assignments = self.generate_assignments(block_info); diff --git a/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs b/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs index b7cff86f3dfe..ab23a8ced484 100644 --- a/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs +++ b/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs @@ -28,6 +28,7 @@ pub struct MockChainSelection { pub state: ApprovalTestState, pub clock: PastSystemClock, } + #[overseer::subsystem(ChainSelection, error=SubsystemError, prefix=self::overseer)] impl MockChainSelection { fn start(self, ctx: Context) -> SpawnedSubsystem { diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 87be762d880e..f9db8419ada1 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -31,8 +31,9 @@ use crate::{ configuration::{TestAuthorities, TestConfiguration}, environment::{TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT}, mock::{ - dummy_builder, network_bridge::MockNetworkBridgeTx, AlwaysSupportsParachains, - ChainApiState, MockChainApi, MockNetworkBridgeRx, MockRuntimeApi, TestSyncOracle, + dummy_builder, + network_bridge::{MockNetworkBridgeRx, MockNetworkBridgeTx}, + AlwaysSupportsParachains, ChainApiState, MockChainApi, MockRuntimeApi, TestSyncOracle, }, network::{ new_network, HandleNetworkMessage, NetworkEmulatorHandle, NetworkInterface, @@ -88,7 +89,7 @@ mod message_generator; mod mock_chain_selection; mod test_message; -pub const LOG_TARGET: &str = "bench::approval"; +pub const LOG_TARGET: &str = "subsystem-bench::approval"; const DATA_COL: u32 = 0; pub(crate) const NUM_COLUMNS: u32 = 1; pub(crate) const SLOT_DURATION_MILLIS: u64 = 6000; @@ -125,6 +126,7 @@ pub struct ApprovalsOptions { pub stop_when_approved: bool, #[clap(short, long)] /// Work directory. + #[clap(short, long, default_value_t = format!("/tmp"))] pub workdir_prefix: String, /// The number of no shows per candidate #[clap(short, long, default_value_t = 0)] diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 0bd35821a1d5..59bfed7f1120 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -117,8 +117,7 @@ fn new_runtime() -> tokio::runtime::Runtime { tokio::runtime::Builder::new_multi_thread() .thread_name("subsystem-bench") .enable_all() - .thread_stack_size(128 * 1024 * 1024) - .max_blocking_threads(4096) + .thread_stack_size(3 * 1024 * 1024) .build() .unwrap() } From c5cb34f3d7974a54bd0dc4540c2888080dad22d1 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 2 Feb 2024 15:20:30 +0200 Subject: [PATCH 191/192] Fixup Signed-off-by: Alexandru Gheorghe --- .../node/subsystem-bench/src/core/mock/runtime_api.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs index f286ccc62591..af4cd82f13b2 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs @@ -141,12 +141,8 @@ impl MockRuntimeApi { request, RuntimeApiRequest::CandidateEvents(sender), ) => { - let candidate_events = self - .state - .included_candidates - .get(&request) - .expect("Unknown block hash"); - let _ = sender.send(Ok(candidate_events.clone())); + let candidate_events = self.state.included_candidates.get(&request); + let _ = sender.send(Ok(candidate_events.cloned().unwrap_or_default())); }, RuntimeApiMessage::Request( _block_hash, From 8fe7026334034b82f75951909f69c996397286de Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 2 Feb 2024 16:18:15 +0200 Subject: [PATCH 192/192] Configurable session_index in mock Signed-off-by: Alexandru Gheorghe --- polkadot/node/subsystem-bench/src/approval/mod.rs | 1 + polkadot/node/subsystem-bench/src/availability/mod.rs | 3 ++- .../node/subsystem-bench/src/core/mock/runtime_api.rs | 8 ++++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index f9db8419ada1..3544ce74711e 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -805,6 +805,7 @@ fn build_overseer( state.candidate_hashes_by_block(), state.candidate_events_by_block(), Some(state.babe_epoch.clone()), + 1, ); let mock_tx_bridge = MockNetworkBridgeTx::new( network.clone(), diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 552284b24b5b..f7f1184448b3 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -180,6 +180,7 @@ fn prepare_test_inner( candidate_hashes, Default::default(), Default::default(), + 0, ); let availability_state = NetworkAvailabilityState { @@ -251,7 +252,7 @@ fn prepare_test_inner( Metrics::try_register(&dependencies.registry).unwrap(), ); - let block_headers = (0..=config.num_blocks) + let block_headers = (1..=config.num_blocks) .map(|block_number| { ( Hash::repeat_byte(block_number as u8), diff --git a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs index af4cd82f13b2..ca6896dbb29d 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs +++ b/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs @@ -19,7 +19,7 @@ use itertools::Itertools; use polkadot_primitives::{ vstaging::NodeFeatures, CandidateEvent, CandidateReceipt, CoreState, GroupIndex, IndexedVec, - OccupiedCore, SessionInfo, ValidatorIndex, + OccupiedCore, SessionIndex, SessionInfo, ValidatorIndex, }; use bitvec::prelude::BitVec; @@ -46,6 +46,8 @@ pub struct RuntimeApiState { // Included candidates per bock included_candidates: HashMap>, babe_epoch: Option, + // The session child index, + session_index: SessionIndex, } /// A mocked `runtime-api` subsystem. @@ -61,6 +63,7 @@ impl MockRuntimeApi { candidate_hashes: HashMap>, included_candidates: HashMap>, babe_epoch: Option, + session_index: SessionIndex, ) -> MockRuntimeApi { Self { state: RuntimeApiState { @@ -68,6 +71,7 @@ impl MockRuntimeApi { candidate_hashes, included_candidates, babe_epoch, + session_index, }, config, } @@ -174,7 +178,7 @@ impl MockRuntimeApi { RuntimeApiRequest::SessionIndexForChild(sender), ) => { // Session is always the same. - let _ = sender.send(Ok(1)); + let _ = sender.send(Ok(self.state.session_index)); }, RuntimeApiMessage::Request( block_hash,