Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

Approve multiple candidates with a single signature #7554

Open
wants to merge 62 commits into
base: sandreim/vrf_modulo_comapct_assignment
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 40 commits
Commits
Show all changes
62 commits
Select commit Hold shift + click to select a range
08c2381
Approve multiple candidates with a single signature
alexggh Jul 27, 2023
4901fee
Add Runtime configuration for max_approval_coalesce_count/wait_millis
alexggh Jul 28, 2023
1e3b511
Add zombienet for approval-coalescing
alexggh Jul 28, 2023
cf68a10
Refactor timers to address review comments
alexggh Jul 31, 2023
fd4b906
Make wait time for caching relative to no-show
alexggh Aug 1, 2023
2271c71
Enforce target candidate is part of the signature
alexggh Aug 2, 2023
df410ff
Merge remote-tracking branch 'origin/sandreim/vrf_modulo_comapct_assi…
alexggh Aug 2, 2023
f0f05c6
Fixup runtime configuration after rebase
alexggh Aug 2, 2023
76a10c6
Merge remote-tracking branch 'origin/sandreim/vrf_modulo_comapct_assi…
alexggh Aug 2, 2023
d31d1d0
Fixup build for unittests
alexggh Aug 2, 2023
6dfaecb
Fix bugs discovered during zombie-testing and unittest
alexggh Aug 3, 2023
1a2a5ec
Refactor the implementation a bit
alexggh Aug 3, 2023
81eb942
Make test better
alexggh Aug 8, 2023
ff0b35d
approval-voting: fix migration from v1 to v2
alexggh Aug 8, 2023
2570c1e
Fix-up bugs in assignnments_coalescing of tranche0
alexggh Aug 8, 2023
305a43a
Enable v2 network protocol
alexggh Aug 8, 2023
5bb1e91
Fixup test builds
alexggh Aug 8, 2023
a83cfb7
Enable assignments v2 computing
alexggh Aug 8, 2023
98705e2
Update test params
alexggh Aug 10, 2023
3f28a5d
Merge remote-tracking branch 'origin/sandreim/vrf_modulo_comapct_assi…
alexggh Aug 10, 2023
aae2a4a
Fixup logic for restarting the node
alexggh Aug 10, 2023
435d46a
Add metric to measure impact of coalescing of approvals
alexggh Aug 10, 2023
9016644
Add extra logs to improve debugability in versi
alexggh Aug 11, 2023
1662e14
Fixup sending approval before assignment
alexggh Aug 11, 2023
af57831
Count needed approvals by more than one third
alexggh Aug 15, 2023
be65f42
Add a few more metrics to understand versi behaviour
alexggh Aug 15, 2023
3ef84bf
Don't send assignments to peers that are not part of the authorities …
alexggh Aug 17, 2023
1dbcb90
Count all observe no-shows
alexggh Aug 17, 2023
3ac044e
Fix warning
alexggh Aug 17, 2023
2cb6756
Minor fixes for review
alexggh Aug 17, 2023
0429549
Fix approval-distribution tests
alexggh Aug 18, 2023
76823a4
Address initial review feedback
alexggh Aug 18, 2023
deff64f
Add zombient test 0006-approval-voting-coalescing to pipeline
alexggh Aug 18, 2023
62a57b5
Update implementers guide
alexggh Aug 18, 2023
f38697e
Fix importing of approval out-of-view
alexggh Aug 19, 2023
903cef2
PVF worker: Prevent access to env vars (#7330)
mrcnski Aug 21, 2023
ea027a8
Companion for substrate#14776 (#7648)
skunert Aug 21, 2023
0bbe0a7
PVF worker: random fixes (#7649)
mrcnski Aug 21, 2023
7bdf7dc
Merge remote-tracking branch 'origin/sandreim/vrf_modulo_comapct_assi…
alexggh Aug 22, 2023
e39c003
Add BEEFY capabilities to Westend and Kusama (#7591)
acatangiu Aug 22, 2023
8ce1716
Bump actions/setup-node from 3.8.0 to 3.8.1 (#7639)
dependabot[bot] Aug 22, 2023
3264a00
Merge remote-tracking branch 'origin/sandreim/vrf_modulo_comapct_assi…
alexggh Aug 22, 2023
9f1e9ea
Bound number of assets which can be withdrawn to pay for execution. (…
gavofyork Aug 22, 2023
d5fc80c
Enable approval-coalescing by default
alexggh Aug 22, 2023
301dab1
Increase num-workers
alexggh Aug 22, 2023
d861d81
Increase SMALL_POV_SIZE
alexggh Aug 22, 2023
a216036
Put logs on trace
alexggh Aug 22, 2023
759fe21
Fix xcm-builder mock (#7652)
ggwpez Aug 22, 2023
a908447
Merge remote-tracking branch 'origin/sandreim/vrf_modulo_comapct_assi…
alexggh Aug 23, 2023
e8ac355
Do not cleanup peer-knowledge on peer-connected/disconnected
alexggh Aug 23, 2023
7f0eb5f
Companion: restructure macro related exports (#7626)
juangirini Aug 23, 2023
7972d31
Revert some hacks
alexggh Aug 23, 2023
e988782
Merge remote-tracking branch 'origin/sandreim/vrf_modulo_comapct_assi…
alexggh Aug 23, 2023
feefbbb
Build image with network vstaging
alexggh Aug 23, 2023
71c08a4
Fixup sending approvals more than once
alexggh Aug 23, 2023
0349441
Fixup unify_with_peer
alexggh Aug 23, 2023
52034bd
Bump chevdor/srtool-actions from 0.7.0 to 0.8.0 (#7660)
dependabot[bot] Aug 24, 2023
ebb610f
Bump rustls-webpki from 0.101.2 to 0.101.4 (#7653)
dependabot[bot] Aug 24, 2023
f3da93d
extend abridged host config (#7659)
slumber Aug 24, 2023
598f4c6
cli: disallow BEEFY and warp sync together (#7661)
acatangiu Aug 25, 2023
52209dc
update readme: archived repo (#7654)
the-right-joyce Aug 25, 2023
a218e59
Merge remote-tracking branch 'origin/master' into approve_multiple_ca…
alexggh Aug 25, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 41 additions & 26 deletions node/core/approval-voting/src/approval_checking.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ pub enum RequiredTranches {
}

/// The result of a check.
#[derive(Debug, Clone, Copy)]
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum Check {
/// The candidate is unapproved.
Unapproved,
Expand Down Expand Up @@ -372,19 +372,22 @@ pub fn tranches_to_approve(
block_tick: Tick,
no_show_duration: Tick,
needed_approvals: usize,
) -> RequiredTranches {
) -> (RequiredTranches, usize) {
let tick_now = tranche_now as Tick + block_tick;
let n_validators = approval_entry.n_validators();

let initial_state = State {
assignments: 0,
depth: 0,
covered: 0,
covering: needed_approvals,
uncovered: 0,
next_no_show: None,
last_assignment_tick: None,
};
let initial_state = (
State {
assignments: 0,
depth: 0,
covered: 0,
covering: needed_approvals,
uncovered: 0,
next_no_show: None,
last_assignment_tick: None,
},
0usize,
);

// The `ApprovalEntry` doesn't have any data for empty tranches. We still want to iterate over
// these empty tranches, so we create an iterator to fill the gaps.
Expand All @@ -395,7 +398,7 @@ pub fn tranches_to_approve(
tranches_with_gaps_filled
.scan(Some(initial_state), |state, (tranche, assignments)| {
// The `Option` here is used for early exit.
let s = state.take()?;
let (s, prev_no_shows) = state.take()?;

let clock_drift = s.clock_drift(no_show_duration);
let drifted_tick_now = tick_now.saturating_sub(clock_drift);
Expand Down Expand Up @@ -444,11 +447,11 @@ pub fn tranches_to_approve(
RequiredTranches::Pending { .. } => {
// Pending results are only interesting when they are the last result of the iterator
// i.e. we never achieve a satisfactory level of assignment.
Some(s)
Some((s, no_shows + prev_no_shows))
}
};

Some(output)
Some((output, no_shows + prev_no_shows))
})
.last()
.expect("the underlying iterator is infinite, starts at 0, and never exits early before tranche 1; qed")
Expand Down Expand Up @@ -675,7 +678,8 @@ mod tests {
block_tick,
no_show_duration,
needed_approvals,
),
)
.0,
RequiredTranches::Exact {
needed: 1,
tolerated_missing: 0,
Expand Down Expand Up @@ -715,7 +719,8 @@ mod tests {
block_tick,
no_show_duration,
needed_approvals,
),
)
.0,
RequiredTranches::Pending {
considered: 2,
next_no_show: Some(block_tick + no_show_duration),
Expand Down Expand Up @@ -759,7 +764,8 @@ mod tests {
block_tick,
no_show_duration,
needed_approvals,
),
)
.0,
RequiredTranches::Pending {
considered: 11,
next_no_show: None,
Expand Down Expand Up @@ -807,7 +813,8 @@ mod tests {
block_tick,
no_show_duration,
needed_approvals,
),
)
.0,
RequiredTranches::Pending {
considered: 1,
next_no_show: None,
Expand All @@ -826,7 +833,8 @@ mod tests {
block_tick,
no_show_duration,
needed_approvals,
),
)
.0,
RequiredTranches::Pending {
considered: 1,
next_no_show: None,
Expand Down Expand Up @@ -879,7 +887,8 @@ mod tests {
block_tick,
no_show_duration,
needed_approvals,
),
)
.0,
RequiredTranches::Exact {
needed: 1,
tolerated_missing: 0,
Expand All @@ -898,7 +907,8 @@ mod tests {
block_tick,
no_show_duration,
needed_approvals,
),
)
.0,
RequiredTranches::Exact {
needed: 2,
tolerated_missing: 1,
Expand All @@ -917,7 +927,8 @@ mod tests {
block_tick,
no_show_duration,
needed_approvals,
),
)
.0,
RequiredTranches::Pending {
considered: 2,
next_no_show: None,
Expand Down Expand Up @@ -970,7 +981,8 @@ mod tests {
block_tick,
no_show_duration,
needed_approvals,
),
)
.0,
RequiredTranches::Exact {
needed: 2,
tolerated_missing: 1,
Expand All @@ -992,7 +1004,8 @@ mod tests {
block_tick,
no_show_duration,
needed_approvals,
),
)
.0,
RequiredTranches::Pending {
considered: 2,
next_no_show: None,
Expand All @@ -1013,7 +1026,8 @@ mod tests {
block_tick,
no_show_duration,
needed_approvals,
),
)
.0,
RequiredTranches::Exact {
needed: 3,
tolerated_missing: 2,
Expand Down Expand Up @@ -1068,7 +1082,8 @@ mod tests {
block_tick,
no_show_duration,
needed_approvals,
),
)
.0,
RequiredTranches::Pending {
considered: 10,
next_no_show: None,
Expand Down
34 changes: 30 additions & 4 deletions node/core/approval-voting/src/approval_db/v2/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,15 @@
//! Version 2 of the DB schema.

use parity_scale_codec::{Decode, Encode};
use polkadot_node_primitives::approval::{v1::DelayTranche, v2::AssignmentCertV2};
use polkadot_node_primitives::approval::{
v1::DelayTranche,
v2::{AssignmentCertV2, CandidateBitfield},
};
use polkadot_node_subsystem::{SubsystemError, SubsystemResult};
use polkadot_node_subsystem_util::database::{DBTransaction, Database};
use polkadot_primitives::{
BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, SessionIndex,
ValidatorIndex, ValidatorSignature,
BlockNumber, CandidateHash, CandidateIndex, CandidateReceipt, CoreIndex, GroupIndex, Hash,
SessionIndex, ValidatorIndex, ValidatorSignature,
};

use sp_consensus_slots::Slot;
Expand Down Expand Up @@ -197,14 +200,24 @@ pub struct TrancheEntry {
pub assignments: Vec<(ValidatorIndex, Tick)>,
}

/// Metadata about our approval signature
#[derive(Encode, Decode, Debug, Clone, PartialEq)]
pub struct OurApproval {
/// The signature for the candidates hashes pointed by indices.
pub signature: ValidatorSignature,
/// The indices of the candidates signed in this approval, an empty value means only
/// the candidate referred by this approval entry was signed.
pub signed_candidates_indices: Option<CandidateBitfield>,
}

/// Metadata regarding approval of a particular candidate within the context of some
/// particular block.
#[derive(Encode, Decode, Debug, Clone, PartialEq)]
pub struct ApprovalEntry {
pub tranches: Vec<TrancheEntry>,
pub backing_group: GroupIndex,
pub our_assignment: Option<OurAssignment>,
pub our_approval_sig: Option<ValidatorSignature>,
pub our_approval_sig: Option<OurApproval>,
// `n_validators` bits.
pub assigned_validators: Bitfield,
pub approved: bool,
Expand Down Expand Up @@ -241,12 +254,25 @@ pub struct BlockEntry {
// block. The block can be considered approved if the bitfield has all bits set to `true`.
pub approved_bitfield: Bitfield,
pub children: Vec<Hash>,
// A list of candidates that has been approved, but we didn't not sign and
// advertise the vote yet.
pub candidates_pending_signature: BTreeMap<CandidateIndex, CandidateSigningContext>,
// Assignments we already distributed. A 1 bit means the candidate index for which
// we already have sent out an assignment. We need this to avoid distributing
// multiple core assignments more than once.
pub distributed_assignments: Bitfield,
}

#[derive(Encode, Decode, Debug, Clone, PartialEq)]

/// Context needed for creating an approval signature for a given candidate.
pub struct CandidateSigningContext {
/// The candidate hash, to be included in the signature.
pub candidate_hash: CandidateHash,
/// The latest tick we have to create and send the approval.
pub send_no_later_than_tick: Tick,
}

impl From<crate::Tick> for Tick {
fn from(tick: crate::Tick) -> Tick {
Tick(tick)
Expand Down
1 change: 1 addition & 0 deletions node/core/approval-voting/src/approval_db/v2/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ fn make_block_entry(
approved_bitfield: make_bitvec(candidates.len()),
candidates,
children: Vec::new(),
candidates_pending_signature: Default::default(),
distributed_assignments: Default::default(),
}
}
Expand Down
2 changes: 1 addition & 1 deletion node/core/approval-voting/src/criteria.rs
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ impl AssignmentCriteria for RealAssignmentCriteria {
config: &Config,
leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>,
) -> HashMap<CoreIndex, OurAssignment> {
compute_assignments(keystore, relay_vrf_story, config, leaving_cores, false)
compute_assignments(keystore, relay_vrf_story, config, leaving_cores, true)
}

fn check_assignment_cert(
Expand Down
3 changes: 3 additions & 0 deletions node/core/approval-voting/src/import.rs
Original file line number Diff line number Diff line change
Expand Up @@ -513,6 +513,7 @@ pub(crate) async fn handle_new_head<Context, B: Backend>(
.collect(),
approved_bitfield,
children: Vec::new(),
candidates_pending_signature: Default::default(),
distributed_assignments: Default::default(),
};

Expand Down Expand Up @@ -639,6 +640,7 @@ pub(crate) mod tests {
clock: Box::new(MockClock::default()),
assignment_criteria: Box::new(MockAssignmentCriteria),
spans: HashMap::new(),
approval_voting_params_cache: None,
}
}

Expand Down Expand Up @@ -1267,6 +1269,7 @@ pub(crate) mod tests {
candidates: Vec::new(),
approved_bitfield: Default::default(),
children: Vec::new(),
candidates_pending_signature: Default::default(),
distributed_assignments: Default::default(),
}
.into(),
Expand Down
Loading
Loading