From a96b9aa54c342ba6515a50ec1e3c5f7ff9292d7b Mon Sep 17 00:00:00 2001 From: ss-es <155648797+ss-es@users.noreply.github.com> Date: Tue, 15 Oct 2024 11:27:49 -0400 Subject: [PATCH 01/15] add Leaf2 and QuorumProposal2 types --- crates/example-types/src/storage_types.rs | 29 +++++++- crates/types/src/data.rs | 91 +++++++++++++++++++++++ crates/types/src/traits/storage.rs | 14 +++- 3 files changed, 132 insertions(+), 2 deletions(-) diff --git a/crates/example-types/src/storage_types.rs b/crates/example-types/src/storage_types.rs index 5c6277f35c..256b035930 100644 --- a/crates/example-types/src/storage_types.rs +++ b/crates/example-types/src/storage_types.rs @@ -14,7 +14,7 @@ use async_lock::RwLock; use async_trait::async_trait; use hotshot_types::{ consensus::CommitmentMap, - data::{DaProposal, Leaf, QuorumProposal, VidDisperseShare}, + data::{DaProposal, Leaf, QuorumProposal, Leaf2, QuorumProposal2, VidDisperseShare}, event::HotShotAction, message::Proposal, simple_certificate::{QuorumCertificate, UpgradeCertificate}, @@ -38,6 +38,7 @@ pub struct TestStorageState { vids: VidShares, das: HashMap>>, proposals: BTreeMap>>, + proposals2: BTreeMap>>, high_qc: Option>, action: TYPES::Time, } @@ -48,6 +49,7 @@ impl Default for TestStorageState { vids: HashMap::new(), das: HashMap::new(), proposals: BTreeMap::new(), + proposals2: BTreeMap::new(), high_qc: None, action: TYPES::Time::genesis(), } @@ -142,6 +144,20 @@ impl Storage for TestStorage { .insert(proposal.data.view_number, proposal.clone()); Ok(()) } + async fn append_proposal2( + &self, + proposal: &Proposal>, + ) -> Result<()> { + if self.should_return_err { + bail!("Failed to append VID proposal to storage"); + } + Self::run_delay_settings_from_config(&self.delay_config).await; + let mut inner = self.inner.write().await; + inner + .proposals2 + .insert(proposal.data.view_number, proposal.clone()); + Ok(()) + } async fn record_action( &self, @@ -188,6 +204,17 @@ impl Storage for TestStorage { Self::run_delay_settings_from_config(&self.delay_config).await; Ok(()) } + async fn update_undecided_state2( + &self, + _leafs: CommitmentMap>, + _state: BTreeMap>, + ) -> Result<()> { + if self.should_return_err { + bail!("Failed to update high qc to storage"); + } + Self::run_delay_settings_from_config(&self.delay_config).await; + Ok(()) + } async fn update_decided_upgrade_certificate( &self, decided_upgrade_certificate: Option>, diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index d05760bf7a..bc5e7f81c9 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -368,6 +368,29 @@ pub struct QuorumProposal { pub proposal_certificate: Option>, } +/// Proposal to append a block. +#[derive(custom_debug::Debug, Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(bound(deserialize = ""))] +pub struct QuorumProposal2 { + /// The block header to append + pub block_header: TYPES::BlockHeader, + + /// CurView from leader when proposing leaf + pub view_number: TYPES::Time, + + /// Per spec, justification + pub justify_qc: LeafCertificate, + + /// Possible upgrade certificate, which the leader may optionally attach. + pub upgrade_certificate: Option>, + + /// Possible timeout or view sync certificate. + /// - A timeout certificate is only present if the justify_qc is not for the preceding view + /// - A view sync certificate is only present if the justify_qc and timeout_cert are not + /// present. + pub proposal_certificate: Option>, +} + impl HasViewNumber for DaProposal { fn view_number(&self) -> TYPES::Time { self.view_number @@ -392,6 +415,12 @@ impl HasViewNumber for QuorumProposal { } } +impl HasViewNumber for QuorumProposal2 { + fn view_number(&self) -> TYPES::Time { + self.view_number + } +} + impl HasViewNumber for UpgradeProposal { fn view_number(&self) -> TYPES::Time { self.view_number @@ -451,6 +480,68 @@ pub struct Leaf { block_payload: Option, } +/// This is the consensus-internal analogous concept to a block, and it contains the block proper, +/// as well as the hash of its parent `Leaf`. +/// NOTE: `State` is constrained to implementing `BlockContents`, is `TypeMap::BlockPayload` +#[derive(Serialize, Deserialize, Clone, Debug, Derivative, PartialEq, Eq)] +#[serde(bound(deserialize = ""))] +pub struct Leaf2 { + /// CurView from leader when proposing leaf + view_number: TYPES::Time, + + /// Per spec, justification + justify_qc: LeafCertificate, + + /// The hash of the parent `Leaf` + /// So we can ask if it extends + parent_commitment: Commitment, + + /// Block header. + block_header: TYPES::BlockHeader, + + /// Optional upgrade certificate, if one was attached to the quorum proposal for this view. + upgrade_certificate: Option>, + + /// Optional block payload. + /// + /// It may be empty for nodes not in the DA committee. + block_payload: Option, +} + +#[derive(Hash, Eq, Clone, PartialEq, Debug, Serialize, Deserialize)] +#[serde(bound(deserialize = ""))] +/// Certificates for the new `QuorumProposal2` type, +/// which may include either a QC or an eQC. +pub enum LeafCertificate { + /// a QC + Quorum(QuorumCertificate), + /// an eQC + Epoch, +} + +impl Committable for Leaf2 { + fn commit(&self) -> committable::Commitment { + match &self.justify_qc { + LeafCertificate::Quorum(justify_qc) => RawCommitmentBuilder::new("leaf commitment") + .u64_field("view number", *self.view_number) + .field("parent leaf commitment", self.parent_commitment) + .field("block header", self.block_header.commit()) + .field("justify qc", justify_qc.commit()) + .optional("upgrade certificate", &self.upgrade_certificate) + .finalize(), + LeafCertificate::Epoch => RawCommitmentBuilder::new("leaf commitment") + .u64_field("view number", *self.view_number) + .field("parent leaf commitment", self.parent_commitment) + .field("block header", self.block_header.commit()) + .u64_field("eqc", 0) + .optional("upgrade certificate", &self.upgrade_certificate) + .finalize(), + + } + } +} + + impl Leaf { #[allow(clippy::unused_async)] /// Calculate the leaf commitment, diff --git a/crates/types/src/traits/storage.rs b/crates/types/src/traits/storage.rs index c11ba771b3..404e7973e9 100644 --- a/crates/types/src/traits/storage.rs +++ b/crates/types/src/traits/storage.rs @@ -17,7 +17,7 @@ use async_trait::async_trait; use super::node_implementation::NodeType; use crate::{ consensus::{CommitmentMap, View}, - data::{DaProposal, Leaf, QuorumProposal, VidDisperseShare}, + data::{DaProposal, Leaf, QuorumProposal, Leaf2, QuorumProposal2, VidDisperseShare}, event::HotShotAction, message::Proposal, simple_certificate::{QuorumCertificate, UpgradeCertificate}, @@ -35,6 +35,11 @@ pub trait Storage: Send + Sync + Clone { &self, proposal: &Proposal>, ) -> Result<()>; + /// Add a proposal we sent to the store + async fn append_proposal2( + &self, + proposal: &Proposal>, + ) -> Result<()>; /// Record a HotShotAction taken. async fn record_action(&self, view: TYPES::Time, action: HotShotAction) -> Result<()>; /// Update the current high QC in storage. @@ -46,6 +51,13 @@ pub trait Storage: Send + Sync + Clone { leafs: CommitmentMap>, state: BTreeMap>, ) -> Result<()>; + /// Update the currently undecided state of consensus. This includes the undecided leaf chain, + /// and the undecided state. + async fn update_undecided_state2( + &self, + leafs: CommitmentMap>, + state: BTreeMap>, + ) -> Result<()>; /// Upgrade the current decided upgrade certificate in storage. async fn update_decided_upgrade_certificate( &self, From 409a06dac2973a99145f6a271786c621a6ed4629 Mon Sep 17 00:00:00 2001 From: ss-es <155648797+ss-es@users.noreply.github.com> Date: Tue, 15 Oct 2024 11:42:33 -0400 Subject: [PATCH 02/15] add From for QP2 --- crates/example-types/src/storage_types.rs | 2 +- crates/testing/tests/tests_1/test_success.rs | 12 ++--- crates/types/src/data.rs | 56 ++++++++++++-------- crates/types/src/traits/storage.rs | 2 +- 4 files changed, 41 insertions(+), 31 deletions(-) diff --git a/crates/example-types/src/storage_types.rs b/crates/example-types/src/storage_types.rs index 256b035930..de9d6d0437 100644 --- a/crates/example-types/src/storage_types.rs +++ b/crates/example-types/src/storage_types.rs @@ -14,7 +14,7 @@ use async_lock::RwLock; use async_trait::async_trait; use hotshot_types::{ consensus::CommitmentMap, - data::{DaProposal, Leaf, QuorumProposal, Leaf2, QuorumProposal2, VidDisperseShare}, + data::{DaProposal, Leaf, Leaf2, QuorumProposal, QuorumProposal2, VidDisperseShare}, event::HotShotAction, message::Proposal, simple_certificate::{QuorumCertificate, UpgradeCertificate}, diff --git a/crates/testing/tests/tests_1/test_success.rs b/crates/testing/tests/tests_1/test_success.rs index a2ee5f1572..853f823278 100644 --- a/crates/testing/tests/tests_1/test_success.rs +++ b/crates/testing/tests/tests_1/test_success.rs @@ -6,12 +6,12 @@ use std::time::Duration; -use hotshot_example_types::node_types::{ - Libp2pImpl, MemoryImpl, PushCdnImpl, TestConsecutiveLeaderTypes, TestTypes, - TestTypesRandomizedLeader, TestVersions, -}; -use hotshot_example_types::testable_delay::{ - DelayConfig, DelayOptions, DelaySettings, SupportedTraitTypesForAsyncDelay, +use hotshot_example_types::{ + node_types::{ + Libp2pImpl, MemoryImpl, PushCdnImpl, TestConsecutiveLeaderTypes, TestTypes, + TestTypesRandomizedLeader, TestVersions, + }, + testable_delay::{DelayConfig, DelayOptions, DelaySettings, SupportedTraitTypesForAsyncDelay}, }; use hotshot_macros::cross_tests; use hotshot_testing::{ diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index bc5e7f81c9..a4e871bf1c 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -391,6 +391,18 @@ pub struct QuorumProposal2 { pub proposal_certificate: Option>, } +impl From> for QuorumProposal2 { + fn from(quorum_proposal: QuorumProposal) -> Self { + Self { + block_header: quorum_proposal.block_header, + view_number: quorum_proposal.view_number, + justify_qc: LeafCertificate::Quorum(quorum_proposal.justify_qc), + upgrade_certificate: quorum_proposal.upgrade_certificate, + proposal_certificate: quorum_proposal.proposal_certificate, + } + } +} + impl HasViewNumber for DaProposal { fn view_number(&self) -> TYPES::Time { self.view_number @@ -510,38 +522,36 @@ pub struct Leaf2 { #[derive(Hash, Eq, Clone, PartialEq, Debug, Serialize, Deserialize)] #[serde(bound(deserialize = ""))] -/// Certificates for the new `QuorumProposal2` type, +/// Certificates for the new `QuorumProposal2` type, /// which may include either a QC or an eQC. pub enum LeafCertificate { - /// a QC - Quorum(QuorumCertificate), - /// an eQC - Epoch, + /// a QC + Quorum(QuorumCertificate), + /// an eQC + Epoch, } impl Committable for Leaf2 { fn commit(&self) -> committable::Commitment { - match &self.justify_qc { - LeafCertificate::Quorum(justify_qc) => RawCommitmentBuilder::new("leaf commitment") - .u64_field("view number", *self.view_number) - .field("parent leaf commitment", self.parent_commitment) - .field("block header", self.block_header.commit()) - .field("justify qc", justify_qc.commit()) - .optional("upgrade certificate", &self.upgrade_certificate) - .finalize(), - LeafCertificate::Epoch => RawCommitmentBuilder::new("leaf commitment") - .u64_field("view number", *self.view_number) - .field("parent leaf commitment", self.parent_commitment) - .field("block header", self.block_header.commit()) - .u64_field("eqc", 0) - .optional("upgrade certificate", &self.upgrade_certificate) - .finalize(), - - } + match &self.justify_qc { + LeafCertificate::Quorum(justify_qc) => RawCommitmentBuilder::new("leaf commitment") + .u64_field("view number", *self.view_number) + .field("parent leaf commitment", self.parent_commitment) + .field("block header", self.block_header.commit()) + .field("justify qc", justify_qc.commit()) + .optional("upgrade certificate", &self.upgrade_certificate) + .finalize(), + LeafCertificate::Epoch => RawCommitmentBuilder::new("leaf commitment") + .u64_field("view number", *self.view_number) + .field("parent leaf commitment", self.parent_commitment) + .field("block header", self.block_header.commit()) + .u64_field("eqc", 0) + .optional("upgrade certificate", &self.upgrade_certificate) + .finalize(), + } } } - impl Leaf { #[allow(clippy::unused_async)] /// Calculate the leaf commitment, diff --git a/crates/types/src/traits/storage.rs b/crates/types/src/traits/storage.rs index 404e7973e9..af5ea239ea 100644 --- a/crates/types/src/traits/storage.rs +++ b/crates/types/src/traits/storage.rs @@ -17,7 +17,7 @@ use async_trait::async_trait; use super::node_implementation::NodeType; use crate::{ consensus::{CommitmentMap, View}, - data::{DaProposal, Leaf, QuorumProposal, Leaf2, QuorumProposal2, VidDisperseShare}, + data::{DaProposal, Leaf, Leaf2, QuorumProposal, QuorumProposal2, VidDisperseShare}, event::HotShotAction, message::Proposal, simple_certificate::{QuorumCertificate, UpgradeCertificate}, From a9a7605d59f05251ef3778a0d534b44be1e54193 Mon Sep 17 00:00:00 2001 From: ss-es <155648797+ss-es@users.noreply.github.com> Date: Tue, 15 Oct 2024 14:57:01 -0400 Subject: [PATCH 03/15] add migration --- crates/example-types/src/storage_types.rs | 19 ++++++++++++++++++- crates/hotshot/src/lib.rs | 23 +++++++++++++++++++++-- crates/types/src/data.rs | 15 +++++++++++++++ crates/types/src/message.rs | 16 ++++++++++++++++ crates/types/src/traits/storage.rs | 8 ++++++++ 5 files changed, 78 insertions(+), 3 deletions(-) diff --git a/crates/example-types/src/storage_types.rs b/crates/example-types/src/storage_types.rs index de9d6d0437..df2b988435 100644 --- a/crates/example-types/src/storage_types.rs +++ b/crates/example-types/src/storage_types.rs @@ -32,7 +32,6 @@ type VidShares = HashMap< ::Time, HashMap<::SignatureKey, Proposal>>, >; - #[derive(Clone, Debug)] pub struct TestStorageState { vids: VidShares, @@ -223,4 +222,22 @@ impl Storage for TestStorage { Ok(()) } + + async fn migrate_consensus( + &self, + _convert_leaf: fn(Leaf) -> Leaf2, + convert_proposal: fn( + Proposal>, + ) -> Proposal>, + ) -> Result<()> { + let mut storage_writer = self.inner.write().await; + + for (view, proposal) in storage_writer.proposals.clone().iter() { + storage_writer + .proposals2 + .insert(*view, convert_proposal(proposal.clone())); + } + + Ok(()) + } } diff --git a/crates/hotshot/src/lib.rs b/crates/hotshot/src/lib.rs index eb63cb630b..ae0b3b305f 100644 --- a/crates/hotshot/src/lib.rs +++ b/crates/hotshot/src/lib.rs @@ -46,9 +46,9 @@ pub use hotshot_types::error::HotShotError; use hotshot_types::{ consensus::{Consensus, ConsensusMetricsValue, OuterConsensus, View, ViewInner}, constants::{EVENT_CHANNEL_SIZE, EXTERNAL_EVENT_CHANNEL_SIZE}, - data::{Leaf, QuorumProposal}, + data::{Leaf, Leaf2, QuorumProposal, QuorumProposal2}, event::{EventType, LeafInfo}, - message::{DataMessage, Message, MessageKind, Proposal}, + message::{convert_proposal, DataMessage, Message, MessageKind, Proposal}, simple_certificate::{QuorumCertificate, UpgradeCertificate}, traits::{ consensus_api::ConsensusApi, @@ -57,6 +57,7 @@ use hotshot_types::{ node_implementation::{ConsensusTime, NodeType}, signature_key::SignatureKey, states::ValidatedState, + storage::Storage, EncodeBytes, }, HotShotConfig, @@ -188,6 +189,10 @@ impl, V: Versions> SystemContext, V: Versions> SystemContext, ) -> Arc { + #[allow(clippy::panic)] + match storage + .migrate_consensus( + Into::>::into, + convert_proposal::, QuorumProposal2>, + ) + .await + { + Ok(()) => {} + Err(e) => { + panic!("Failed to migrate consensus storage: {e}"); + } + } + let interal_chan = broadcast(EVENT_CHANNEL_SIZE); let external_chan = broadcast(EXTERNAL_EVENT_CHANNEL_SIZE); diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index a4e871bf1c..1714c52d81 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -403,6 +403,21 @@ impl From> for QuorumProposal2 { } } +impl From> for Leaf2 { + fn from(leaf: Leaf) -> Self { + let bytes: [u8; 32] = leaf.parent_commitment.into(); + + Self { + view_number: leaf.view_number, + justify_qc: LeafCertificate::Quorum(leaf.justify_qc), + parent_commitment: Commitment::from_raw(bytes), + block_header: leaf.block_header, + upgrade_certificate: leaf.upgrade_certificate, + block_payload: leaf.block_payload, + } + } +} + impl HasViewNumber for DaProposal { fn view_number(&self) -> TYPES::Time { self.view_number diff --git a/crates/types/src/message.rs b/crates/types/src/message.rs index 407845b001..14f17589e4 100644 --- a/crates/types/src/message.rs +++ b/crates/types/src/message.rs @@ -349,6 +349,22 @@ pub struct Proposal + Deserializ pub _pd: PhantomData, } +/// Convert a `Proposal` by converting the underlying proposal type +pub fn convert_proposal( + proposal: Proposal, +) -> Proposal +where + TYPES: NodeType, + PROPOSAL: HasViewNumber + DeserializeOwned, + PROPOSAL2: HasViewNumber + DeserializeOwned + From, +{ + Proposal { + data: proposal.data.into(), + signature: proposal.signature, + _pd: proposal._pd, + } +} + impl Proposal> where TYPES: NodeType, diff --git a/crates/types/src/traits/storage.rs b/crates/types/src/traits/storage.rs index af5ea239ea..2d2b11c0c5 100644 --- a/crates/types/src/traits/storage.rs +++ b/crates/types/src/traits/storage.rs @@ -63,4 +63,12 @@ pub trait Storage: Send + Sync + Clone { &self, decided_upgrade_certificate: Option>, ) -> Result<()>; + /// Migrate leaves from `Leaf` to `Leaf2`, and proposals from `QuorumProposal` to `QuorumProposal2` + async fn migrate_consensus( + &self, + convert_leaf: fn(Leaf) -> Leaf2, + convert_proposal: fn( + Proposal>, + ) -> Proposal>, + ) -> Result<()>; } From 4cb30f543c943b655092b8e83a569634830f4f46 Mon Sep 17 00:00:00 2001 From: ss-es <155648797+ss-es@users.noreply.github.com> Date: Thu, 24 Oct 2024 16:45:41 -0400 Subject: [PATCH 04/15] add fields to qp2 --- crates/types/src/data.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index 26dc315277..f737c7725f 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -384,20 +384,27 @@ pub struct QuorumProposal2 { /// The block header to append pub block_header: TYPES::BlockHeader, - /// CurView from leader when proposing leaf + /// view number for the proposal pub view_number: TYPES::View, - /// Per spec, justification + /// certificate that the proposal is chaining from pub justify_qc: LeafCertificate, + /// a secondary certificate + /// Possible upgrade certificate, which the leader may optionally attach. pub upgrade_certificate: Option>, - /// Possible timeout or view sync certificate. - /// - A timeout certificate is only present if the justify_qc is not for the preceding view - /// - A view sync certificate is only present if the justify_qc and timeout_cert are not - /// present. + /// Possible timeout or view sync certificate. If the `justify_qc` is not for a proposal in the immediately preceding view, then either a timeout or view sync certificate must be attached. pub proposal_certificate: Option>, + + /// the DRB seed currently being calculated + #[serde(with = "serde_bytes")] + pub drb_seed: [u8; 96], + + /// the result of the DRB calculation + #[serde(with = "serde_bytes")] + pub drb_result: [u8; 32], } impl From> for QuorumProposal2 { @@ -408,6 +415,8 @@ impl From> for QuorumProposal2 { justify_qc: LeafCertificate::Quorum(quorum_proposal.justify_qc), upgrade_certificate: quorum_proposal.upgrade_certificate, proposal_certificate: quorum_proposal.proposal_certificate, + drb_seed: [0; 96], + drb_result: [0; 32], } } } From f3e887c3bca8faf01a3c0138b2c7abe7a253146f Mon Sep 17 00:00:00 2001 From: ss-es <155648797+ss-es@users.noreply.github.com> Date: Tue, 5 Nov 2024 11:16:56 -0500 Subject: [PATCH 05/15] add QC2 --- crates/types/src/data.rs | 13 +++++++++++-- crates/types/src/simple_certificate.rs | 4 +++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index 79d6375ded..45e0bc56ac 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -36,7 +36,8 @@ use vec1::Vec1; use crate::{ message::{Proposal, UpgradeLock}, simple_certificate::{ - QuorumCertificate, TimeoutCertificate, UpgradeCertificate, ViewSyncFinalizeCertificate2, + QuorumCertificate, QuorumCertificate2, TimeoutCertificate, UpgradeCertificate, + ViewSyncFinalizeCertificate2, }, simple_vote::{QuorumData, UpgradeProposalData, VersionedVoteData}, traits::{ @@ -527,7 +528,6 @@ pub struct Leaf { /// This is the consensus-internal analogous concept to a block, and it contains the block proper, /// as well as the hash of its parent `Leaf`. -/// NOTE: `State` is constrained to implementing `BlockContents`, is `TypeMap::BlockPayload` #[derive(Serialize, Deserialize, Clone, Debug, Derivative, PartialEq, Eq)] #[serde(bound(deserialize = ""))] pub struct Leaf2 { @@ -560,6 +560,8 @@ pub struct Leaf2 { pub enum LeafCertificate { /// a QC Quorum(QuorumCertificate), + /// a QC2 + Quorum2(QuorumCertificate2), /// an eQC Epoch, } @@ -574,6 +576,13 @@ impl Committable for Leaf2 { .field("justify qc", justify_qc.commit()) .optional("upgrade certificate", &self.upgrade_certificate) .finalize(), + LeafCertificate::Quorum2(justify_qc) => RawCommitmentBuilder::new("leaf commitment") + .u64_field("view number", *self.view_number) + .field("parent leaf commitment", self.parent_commitment) + .field("block header", self.block_header.commit()) + .field("justify qc", justify_qc.commit()) + .optional("upgrade certificate", &self.upgrade_certificate) + .finalize(), LeafCertificate::Epoch => RawCommitmentBuilder::new("leaf commitment") .u64_field("view number", *self.view_number) .field("parent leaf commitment", self.parent_commitment) diff --git a/crates/types/src/simple_certificate.rs b/crates/types/src/simple_certificate.rs index f7f58fd4e1..b765df2c48 100644 --- a/crates/types/src/simple_certificate.rs +++ b/crates/types/src/simple_certificate.rs @@ -249,8 +249,10 @@ impl UpgradeCertificate { } } -/// Type alias for a `QuorumCertificate`, which is a `SimpleCertificate` of `QuorumVotes` +/// Type alias for a `QuorumCertificate`, which is a `SimpleCertificate` over `QuorumData` pub type QuorumCertificate = SimpleCertificate, SuccessThreshold>; +/// Type alias for a `QuorumCertificate2`, which is a `SimpleCertificate` over `QuorumData2` +pub type QuorumCertificate2 = SimpleCertificate, SuccessThreshold>; /// Type alias for a DA certificate over `DaData` pub type DaCertificate = SimpleCertificate; /// Type alias for a Timeout certificate over a view number From 11af3b6e2911ceca29c877d26ef2c19048c170ea Mon Sep 17 00:00:00 2001 From: ss-es <155648797+ss-es@users.noreply.github.com> Date: Tue, 5 Nov 2024 11:20:40 -0500 Subject: [PATCH 06/15] add QuorumVote2 --- crates/types/src/simple_certificate.rs | 4 ++-- crates/types/src/simple_vote.rs | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/crates/types/src/simple_certificate.rs b/crates/types/src/simple_certificate.rs index b765df2c48..2cde2ccb9d 100644 --- a/crates/types/src/simple_certificate.rs +++ b/crates/types/src/simple_certificate.rs @@ -23,7 +23,7 @@ use crate::{ data::serialize_signature2, message::UpgradeLock, simple_vote::{ - DaData, QuorumData, TimeoutData, UpgradeProposalData, VersionedVoteData, + DaData, QuorumData, QuorumData2, TimeoutData, UpgradeProposalData, VersionedVoteData, ViewSyncCommitData, ViewSyncFinalizeData, ViewSyncPreCommitData, Voteable, }, traits::{ @@ -252,7 +252,7 @@ impl UpgradeCertificate { /// Type alias for a `QuorumCertificate`, which is a `SimpleCertificate` over `QuorumData` pub type QuorumCertificate = SimpleCertificate, SuccessThreshold>; /// Type alias for a `QuorumCertificate2`, which is a `SimpleCertificate` over `QuorumData2` -pub type QuorumCertificate2 = SimpleCertificate, SuccessThreshold>; +pub type QuorumCertificate2 = SimpleCertificate, SuccessThreshold>; /// Type alias for a DA certificate over `DaData` pub type DaCertificate = SimpleCertificate; /// Type alias for a Timeout certificate over a view number diff --git a/crates/types/src/simple_vote.rs b/crates/types/src/simple_vote.rs index f5111084b8..bcb7e0baab 100644 --- a/crates/types/src/simple_vote.rs +++ b/crates/types/src/simple_vote.rs @@ -14,7 +14,7 @@ use utils::anytrace::*; use vbs::version::Version; use crate::{ - data::Leaf, + data::{Leaf, Leaf2}, message::UpgradeLock, traits::{ node_implementation::{NodeType, Versions}, @@ -32,6 +32,13 @@ pub struct QuorumData { pub leaf_commit: Commitment>, } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Hash, Eq)] +/// Data used for a yes vote. +#[serde(bound(deserialize = ""))] +pub struct QuorumData2 { + /// Commitment to the leaf + pub leaf_commit: Commitment>, +} +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Hash, Eq)] /// Data used for a DA vote. pub struct DaData { /// Commitment to a block payload @@ -257,6 +264,14 @@ impl Committable for QuorumData { } } +impl Committable for QuorumData2 { + fn commit(&self) -> Commitment { + committable::RawCommitmentBuilder::new("Quorum data") + .var_size_bytes(self.leaf_commit.as_ref()) + .finalize() + } +} + impl Committable for TimeoutData { fn commit(&self) -> Commitment { committable::RawCommitmentBuilder::new("Timeout data") @@ -334,6 +349,9 @@ impl = SimpleVote>; +// Type aliases for simple use of all the main votes. We should never see `SimpleVote` outside this file +/// Quorum vote Alias +pub type QuorumVote2 = SimpleVote>; /// DA vote type alias pub type DaVote = SimpleVote; /// Timeout Vote type alias From 5f803e6aae93438c7c3659044d79654065c6b2e8 Mon Sep 17 00:00:00 2001 From: ss-es <155648797+ss-es@users.noreply.github.com> Date: Thu, 7 Nov 2024 14:12:39 -0500 Subject: [PATCH 07/15] add hotshot events --- crates/task-impls/src/events.rs | 59 ++++++++++++++++++++++++++++++--- crates/types/src/data.rs | 2 -- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/crates/task-impls/src/events.rs b/crates/task-impls/src/events.rs index 4aea9b5ec8..347257669a 100644 --- a/crates/task-impls/src/events.rs +++ b/crates/task-impls/src/events.rs @@ -11,8 +11,8 @@ use either::Either; use hotshot_task::task::TaskEvent; use hotshot_types::{ data::{ - DaProposal, Leaf, PackedBundle, QuorumProposal, UpgradeProposal, VidDisperse, - VidDisperseShare, + DaProposal, Leaf, PackedBundle, QuorumProposal, QuorumProposal2, UpgradeProposal, + VidDisperse, VidDisperseShare, }, message::Proposal, request_response::ProposalRequestPayload, @@ -21,8 +21,8 @@ use hotshot_types::{ ViewSyncCommitCertificate2, ViewSyncFinalizeCertificate2, ViewSyncPreCommitCertificate2, }, simple_vote::{ - DaVote, QuorumVote, TimeoutVote, UpgradeVote, ViewSyncCommitVote, ViewSyncFinalizeVote, - ViewSyncPreCommitVote, + DaVote, QuorumVote, QuorumVote2, TimeoutVote, UpgradeVote, ViewSyncCommitVote, + ViewSyncFinalizeVote, ViewSyncPreCommitVote, }, traits::{ block_contents::BuilderFee, network::DataRequest, node_implementation::NodeType, @@ -72,8 +72,12 @@ pub enum HotShotEvent { Shutdown, /// A quorum proposal has been received from the network; handled by the consensus task QuorumProposalRecv(Proposal>, TYPES::SignatureKey), + /// A quorum proposal has been received from the network; handled by the consensus task + QuorumProposal2Recv(Proposal>, TYPES::SignatureKey), /// A quorum vote has been received from the network; handled by the consensus task QuorumVoteRecv(QuorumVote), + /// A quorum vote has been received from the network; handled by the consensus task + QuorumVote2Recv(QuorumVote2), /// A timeout vote received from the network; handled by consensus task TimeoutVoteRecv(TimeoutVote), /// Send a timeout vote to the network; emitted by consensus task replicas @@ -90,8 +94,12 @@ pub enum HotShotEvent { DaCertificateValidated(DaCertificate), /// Send a quorum proposal to the network; emitted by the leader in the consensus task QuorumProposalSend(Proposal>, TYPES::SignatureKey), + /// Send a quorum proposal to the network; emitted by the leader in the consensus task + QuorumProposal2Send(Proposal>, TYPES::SignatureKey), /// Send a quorum vote to the next leader; emitted by a replica in the consensus task after seeing a valid quorum proposal QuorumVoteSend(QuorumVote), + /// Send a quorum vote to the next leader; emitted by a replica in the consensus task after seeing a valid quorum proposal + QuorumVote2Send(QuorumVote2), /// All dependencies for the quorum vote are validated. QuorumVoteDependenciesValidated(TYPES::View), /// A quorum proposal with the given parent leaf is validated. @@ -113,8 +121,12 @@ pub enum HotShotEvent { ), /// A quorum proposal was missing for a view. As the leader, we send a reply to the recipient with their key. QuorumProposalResponseSend(TYPES::SignatureKey, Proposal>), + /// A quorum proposal was missing for a view. As the leader, we send a reply to the recipient with their key. + QuorumProposal2ResponseSend(TYPES::SignatureKey, Proposal>), /// A quorum proposal was requested by a node for a view. QuorumProposalResponseRecv(Proposal>), + /// A quorum proposal was requested by a node for a view. + QuorumProposal2ResponseRecv(Proposal>), /// Send a DA proposal to the DA committee; emitted by the DA leader (which is the same node as the leader of view v + 1) in the DA task DaProposalSend(Proposal>, TYPES::SignatureKey), /// Send a DA vote to the DA leader; emitted by DA committee members in the DA task after seeing a valid DA proposal @@ -274,7 +286,16 @@ impl HotShotEvent { | HotShotEvent::QuorumProposalPreliminarilyValidated(proposal) => { Some(proposal.data.view_number()) } + HotShotEvent::QuorumProposal2Recv(proposal, _) + | HotShotEvent::QuorumProposal2Send(proposal, _) + | HotShotEvent::QuorumProposal2ResponseSend(_, proposal) + | HotShotEvent::QuorumProposal2ResponseRecv(proposal) => { + Some(proposal.data.view_number()) + } HotShotEvent::QuorumVoteSend(vote) => Some(vote.view_number()), + HotShotEvent::QuorumVote2Send(vote) | HotShotEvent::QuorumVote2Recv(vote) => { + Some(vote.view_number()) + } HotShotEvent::DaProposalRecv(proposal, _) | HotShotEvent::DaProposalValidated(proposal, _) | HotShotEvent::DaProposalSend(proposal, _) => Some(proposal.data.view_number()), @@ -351,9 +372,17 @@ impl Display for HotShotEvent { "QuorumProposalRecv(view_number={:?})", proposal.data.view_number() ), + HotShotEvent::QuorumProposal2Recv(proposal, _) => write!( + f, + "QuorumProposal2Recv(view_number={:?})", + proposal.data.view_number() + ), HotShotEvent::QuorumVoteRecv(v) => { write!(f, "QuorumVoteRecv(view_number={:?})", v.view_number()) } + HotShotEvent::QuorumVote2Recv(v) => { + write!(f, "QuorumVote2Recv(view_number={:?})", v.view_number()) + } HotShotEvent::TimeoutVoteRecv(v) => { write!(f, "TimeoutVoteRecv(view_number={:?})", v.view_number()) } @@ -386,9 +415,17 @@ impl Display for HotShotEvent { "QuorumProposalSend(view_number={:?})", proposal.data.view_number() ), + HotShotEvent::QuorumProposal2Send(proposal, _) => write!( + f, + "QuorumProposal2Send(view_number={:?})", + proposal.data.view_number() + ), HotShotEvent::QuorumVoteSend(vote) => { write!(f, "QuorumVoteSend(view_number={:?})", vote.view_number()) } + HotShotEvent::QuorumVote2Send(vote) => { + write!(f, "QuorumVote2Send(view_number={:?})", vote.view_number()) + } HotShotEvent::QuorumVoteDependenciesValidated(view_number) => { write!( f, @@ -565,6 +602,13 @@ impl Display for HotShotEvent { proposal.data.view_number ) } + HotShotEvent::QuorumProposal2ResponseSend(_, proposal) => { + write!( + f, + "QuorumProposal2ResponseSend(view_number={:?})", + proposal.data.view_number + ) + } HotShotEvent::QuorumProposalResponseRecv(proposal) => { write!( f, @@ -572,6 +616,13 @@ impl Display for HotShotEvent { proposal.data.view_number ) } + HotShotEvent::QuorumProposal2ResponseRecv(proposal) => { + write!( + f, + "QuorumProposal2ResponseRecv(view_number={:?})", + proposal.data.view_number + ) + } HotShotEvent::ValidatedStateUpdated(view_number, _) => { write!(f, "ValidatedStateUpdated(view_number={view_number:?})") } diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index 45e0bc56ac..6bd1252760 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -391,8 +391,6 @@ pub struct QuorumProposal2 { /// certificate that the proposal is chaining from pub justify_qc: LeafCertificate, - /// a secondary certificate - /// Possible upgrade certificate, which the leader may optionally attach. pub upgrade_certificate: Option>, From a287fca8f2f2b4c19a4ce37630a04ae7b828e5f4 Mon Sep 17 00:00:00 2001 From: ss-es <155648797+ss-es@users.noreply.github.com> Date: Fri, 8 Nov 2024 17:46:30 -0500 Subject: [PATCH 08/15] convert qc --- crates/examples/combined/all.rs | 2 +- crates/examples/push-cdn/all.rs | 2 +- crates/hotshot/src/lib.rs | 2 +- crates/libp2p-networking/src/network/mod.rs | 2 +- crates/task-impls/src/network.rs | 3 +- crates/task-impls/src/view_sync.rs | 3 +- crates/testing/src/completion_task.rs | 3 +- crates/testing/src/test_runner.rs | 3 +- crates/testing/src/test_task.rs | 2 +- crates/testing/src/txn_task.rs | 3 +- crates/types/src/data.rs | 84 ++++++++++----------- crates/types/src/simple_certificate.rs | 21 ++++++ crates/types/src/traits/network.rs | 12 ++- 13 files changed, 77 insertions(+), 65 deletions(-) diff --git a/crates/examples/combined/all.rs b/crates/examples/combined/all.rs index 1ccace8b48..ebfd828b0d 100644 --- a/crates/examples/combined/all.rs +++ b/crates/examples/combined/all.rs @@ -12,8 +12,8 @@ use std::path::Path; use cdn_broker::Broker; use cdn_marshal::Marshal; -use hotshot::helpers::initialize_logging; use hotshot::{ + helpers::initialize_logging, traits::implementations::{KeyPair, TestingDef, WrappedSignatureKey}, types::SignatureKey, }; diff --git a/crates/examples/push-cdn/all.rs b/crates/examples/push-cdn/all.rs index 726f41b22e..2767bbdaa3 100644 --- a/crates/examples/push-cdn/all.rs +++ b/crates/examples/push-cdn/all.rs @@ -12,8 +12,8 @@ use std::path::Path; use cdn_broker::{reexports::crypto::signature::KeyPair, Broker}; use cdn_marshal::Marshal; -use hotshot::helpers::initialize_logging; use hotshot::{ + helpers::initialize_logging, traits::implementations::{TestingDef, WrappedSignatureKey}, types::SignatureKey, }; diff --git a/crates/hotshot/src/lib.rs b/crates/hotshot/src/lib.rs index 5d23eca6e4..4791604aa9 100644 --- a/crates/hotshot/src/lib.rs +++ b/crates/hotshot/src/lib.rs @@ -42,7 +42,6 @@ use async_trait::async_trait; use futures::join; use hotshot_task::task::{ConsensusTaskRegistry, NetworkTaskRegistry}; use hotshot_task_impls::{events::HotShotEvent, helpers::broadcast_event}; -use tokio::{spawn, time::sleep}; // Internal /// Reexport error type pub use hotshot_types::error::HotShotError; @@ -69,6 +68,7 @@ use hotshot_types::{ // External /// Reexport rand crate pub use rand; +use tokio::{spawn, time::sleep}; use tracing::{debug, instrument, trace}; use crate::{ diff --git a/crates/libp2p-networking/src/network/mod.rs b/crates/libp2p-networking/src/network/mod.rs index 465e8ee25e..21a2811bb8 100644 --- a/crates/libp2p-networking/src/network/mod.rs +++ b/crates/libp2p-networking/src/network/mod.rs @@ -20,10 +20,10 @@ use std::{collections::HashSet, fmt::Debug}; use futures::channel::oneshot::Sender; use hotshot_types::traits::{network::NetworkError, node_implementation::NodeType}; -use libp2p::dns::tokio::Transport as DnsTransport; use libp2p::{ build_multiaddr, core::{muxing::StreamMuxerBox, transport::Boxed}, + dns::tokio::Transport as DnsTransport, gossipsub::Event as GossipEvent, identify::Event as IdentifyEvent, identity::Keypair, diff --git a/crates/task-impls/src/network.rs b/crates/task-impls/src/network.rs index 49ae29e964..4ffab75b68 100644 --- a/crates/task-impls/src/network.rs +++ b/crates/task-impls/src/network.rs @@ -33,8 +33,7 @@ use hotshot_types::{ }, vote::{HasViewNumber, Vote}, }; -use tokio::spawn; -use tokio::task::JoinHandle; +use tokio::{spawn, task::JoinHandle}; use tracing::instrument; use utils::anytrace::*; diff --git a/crates/task-impls/src/view_sync.rs b/crates/task-impls/src/view_sync.rs index 4b810196da..895c321983 100644 --- a/crates/task-impls/src/view_sync.rs +++ b/crates/task-impls/src/view_sync.rs @@ -31,8 +31,7 @@ use hotshot_types::{ }, vote::{Certificate, HasViewNumber, Vote}, }; -use tokio::task::JoinHandle; -use tokio::{spawn, time::sleep}; +use tokio::{spawn, task::JoinHandle, time::sleep}; use tracing::instrument; use utils::anytrace::*; diff --git a/crates/testing/src/completion_task.rs b/crates/testing/src/completion_task.rs index 96de8616e9..5806f8d1c8 100644 --- a/crates/testing/src/completion_task.rs +++ b/crates/testing/src/completion_task.rs @@ -8,8 +8,7 @@ use std::time::Duration; use async_broadcast::{Receiver, Sender}; use hotshot_task_impls::helpers::broadcast_event; -use tokio::task::JoinHandle; -use tokio::{spawn, time::timeout}; +use tokio::{spawn, task::JoinHandle, time::timeout}; use crate::test_task::TestEvent; diff --git a/crates/testing/src/test_runner.rs b/crates/testing/src/test_runner.rs index 1b55a2b841..2a08fda5e4 100644 --- a/crates/testing/src/test_runner.rs +++ b/crates/testing/src/test_runner.rs @@ -40,8 +40,7 @@ use hotshot_types::{ HotShotConfig, ValidatorConfig, }; use tide_disco::Url; -use tokio::spawn; -use tokio::task::JoinHandle; +use tokio::{spawn, task::JoinHandle}; #[allow(deprecated)] use tracing::info; diff --git a/crates/testing/src/test_task.rs b/crates/testing/src/test_task.rs index 036b0d6b5d..2791646cba 100644 --- a/crates/testing/src/test_task.rs +++ b/crates/testing/src/test_task.rs @@ -23,9 +23,9 @@ use hotshot_types::{ node_implementation::{NodeType, Versions}, }, }; -use tokio::task::JoinHandle; use tokio::{ spawn, + task::JoinHandle, time::{sleep, timeout}, }; use tracing::error; diff --git a/crates/testing/src/txn_task.rs b/crates/testing/src/txn_task.rs index 5d62d4232d..41b5ec3b14 100644 --- a/crates/testing/src/txn_task.rs +++ b/crates/testing/src/txn_task.rs @@ -11,8 +11,7 @@ use async_lock::RwLock; use hotshot::traits::TestableNodeImplementation; use hotshot_types::traits::node_implementation::{NodeType, Versions}; use rand::thread_rng; -use tokio::task::JoinHandle; -use tokio::{spawn, time::sleep}; +use tokio::{spawn, task::JoinHandle, time::sleep}; use crate::{test_runner::Node, test_task::TestEvent}; diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index 9fce602a37..81f8b9176a 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -33,8 +33,8 @@ use vec1::Vec1; use crate::{ message::{Proposal, UpgradeLock}, simple_certificate::{ - QuorumCertificate, QuorumCertificate2, TimeoutCertificate, UpgradeCertificate, - ViewSyncFinalizeCertificate2, + convert_quorum_certificate, QuorumCertificate, QuorumCertificate2, TimeoutCertificate, + UpgradeCertificate, ViewSyncFinalizeCertificate2, }, simple_vote::{QuorumData, UpgradeProposalData, VersionedVoteData}, traits::{ @@ -384,7 +384,7 @@ pub struct QuorumProposal2 { pub view_number: TYPES::View, /// certificate that the proposal is chaining from - pub justify_qc: LeafCertificate, + pub justify_qc: QuorumCertificate2, /// Possible upgrade certificate, which the leader may optionally attach. pub upgrade_certificate: Option>, @@ -406,7 +406,7 @@ impl From> for QuorumProposal2 { Self { block_header: quorum_proposal.block_header, view_number: quorum_proposal.view_number, - justify_qc: LeafCertificate::Quorum(quorum_proposal.justify_qc), + justify_qc: convert_quorum_certificate(quorum_proposal.justify_qc), upgrade_certificate: quorum_proposal.upgrade_certificate, proposal_certificate: quorum_proposal.proposal_certificate, drb_seed: [0; 96], @@ -421,7 +421,7 @@ impl From> for Leaf2 { Self { view_number: leaf.view_number, - justify_qc: LeafCertificate::Quorum(leaf.justify_qc), + justify_qc: convert_quorum_certificate(leaf.justify_qc), parent_commitment: Commitment::from_raw(bytes), block_header: leaf.block_header, upgrade_certificate: leaf.upgrade_certificate, @@ -528,7 +528,7 @@ pub struct Leaf2 { view_number: TYPES::View, /// Per spec, justification - justify_qc: LeafCertificate, + justify_qc: QuorumCertificate2, /// The hash of the parent `Leaf` /// So we can ask if it extends @@ -546,44 +546,15 @@ pub struct Leaf2 { block_payload: Option, } -#[derive(Hash, Eq, Clone, PartialEq, Debug, Serialize, Deserialize)] -#[serde(bound(deserialize = ""))] -/// Certificates for the new `QuorumProposal2` type, -/// which may include either a QC or an eQC. -pub enum LeafCertificate { - /// a QC - Quorum(QuorumCertificate), - /// a QC2 - Quorum2(QuorumCertificate2), - /// an eQC - Epoch, -} - impl Committable for Leaf2 { fn commit(&self) -> committable::Commitment { - match &self.justify_qc { - LeafCertificate::Quorum(justify_qc) => RawCommitmentBuilder::new("leaf commitment") - .u64_field("view number", *self.view_number) - .field("parent leaf commitment", self.parent_commitment) - .field("block header", self.block_header.commit()) - .field("justify qc", justify_qc.commit()) - .optional("upgrade certificate", &self.upgrade_certificate) - .finalize(), - LeafCertificate::Quorum2(justify_qc) => RawCommitmentBuilder::new("leaf commitment") - .u64_field("view number", *self.view_number) - .field("parent leaf commitment", self.parent_commitment) - .field("block header", self.block_header.commit()) - .field("justify qc", justify_qc.commit()) - .optional("upgrade certificate", &self.upgrade_certificate) - .finalize(), - LeafCertificate::Epoch => RawCommitmentBuilder::new("leaf commitment") - .u64_field("view number", *self.view_number) - .field("parent leaf commitment", self.parent_commitment) - .field("block header", self.block_header.commit()) - .u64_field("eqc", 0) - .optional("upgrade certificate", &self.upgrade_certificate) - .finalize(), - } + RawCommitmentBuilder::new("leaf commitment") + .u64_field("view number", *self.view_number) + .field("parent leaf commitment", self.parent_commitment) + .field("block header", self.block_header.commit()) + .field("justify qc", self.justify_qc.commit()) + .optional("upgrade certificate", &self.upgrade_certificate) + .finalize() } } @@ -895,6 +866,32 @@ impl Committable for Leaf { } } +impl Leaf2 { + /// Constructs a leaf from a given quorum proposal. + pub fn from_quorum_proposal(quorum_proposal: &QuorumProposal2) -> Self { + // WARNING: Do NOT change this to a wildcard match, or reference the fields directly in the construction of the leaf. + // The point of this match is that we will get a compile-time error if we add a field without updating this. + let QuorumProposal2 { + view_number, + justify_qc, + block_header, + upgrade_certificate, + proposal_certificate: _, + drb_seed: _, + drb_result: _, + } = quorum_proposal; + + Self { + view_number: *view_number, + justify_qc: justify_qc.clone(), + parent_commitment: justify_qc.data().leaf_commit, + block_header: block_header.clone(), + upgrade_certificate: upgrade_certificate.clone(), + block_payload: None, + } + } +} + impl Leaf { /// Constructs a leaf from a given quorum proposal. pub fn from_quorum_proposal(quorum_proposal: &QuorumProposal) -> Self { @@ -907,7 +904,8 @@ impl Leaf { upgrade_certificate, proposal_certificate: _, } = quorum_proposal; - Leaf { + + Self { view_number: *view_number, justify_qc: justify_qc.clone(), parent_commitment: justify_qc.data().leaf_commit, diff --git a/crates/types/src/simple_certificate.rs b/crates/types/src/simple_certificate.rs index 2cde2ccb9d..46684a8d77 100644 --- a/crates/types/src/simple_certificate.rs +++ b/crates/types/src/simple_certificate.rs @@ -249,6 +249,27 @@ impl UpgradeCertificate { } } +/// Convert a QC into a QC2 +pub fn convert_quorum_certificate( + quorum_certificate: QuorumCertificate, +) -> QuorumCertificate2 { + let bytes: [u8; 32] = quorum_certificate.data.leaf_commit.into(); + let data = QuorumData2 { + leaf_commit: Commitment::from_raw(bytes), + }; + + let bytes: [u8; 32] = quorum_certificate.vote_commitment.into(); + let vote_commitment = Commitment::from_raw(bytes); + + SimpleCertificate { + data, + vote_commitment, + view_number: quorum_certificate.view_number, + signatures: quorum_certificate.signatures.clone(), + _pd: PhantomData, + } +} + /// Type alias for a `QuorumCertificate`, which is a `SimpleCertificate` over `QuorumData` pub type QuorumCertificate = SimpleCertificate, SuccessThreshold>; /// Type alias for a `QuorumCertificate2`, which is a `SimpleCertificate` over `QuorumData2` diff --git a/crates/types/src/traits/network.rs b/crates/types/src/traits/network.rs index 7d03f17594..5a01560832 100644 --- a/crates/types/src/traits/network.rs +++ b/crates/types/src/traits/network.rs @@ -8,12 +8,6 @@ //! //! Contains types and traits used by `HotShot` to abstract over network access -use derivative::Derivative; -use dyn_clone::DynClone; -use futures::Future; -use thiserror::Error; -use tokio::{sync::mpsc::error::TrySendError, time::sleep}; - use std::{ collections::HashMap, fmt::{Debug, Display}, @@ -24,12 +18,16 @@ use std::{ }; use async_trait::async_trait; -use futures::future::join_all; +use derivative::Derivative; +use dyn_clone::DynClone; +use futures::{future::join_all, Future}; use rand::{ distributions::{Bernoulli, Uniform}, prelude::Distribution, }; use serde::{Deserialize, Serialize}; +use thiserror::Error; +use tokio::{sync::mpsc::error::TrySendError, time::sleep}; use super::{node_implementation::NodeType, signature_key::SignatureKey}; use crate::{ From 1a25769ccdf49539b529aaf56cbf6249f313d5fe Mon Sep 17 00:00:00 2001 From: ss-es <155648797+ss-es@users.noreply.github.com> Date: Mon, 11 Nov 2024 11:38:20 -0500 Subject: [PATCH 09/15] add tests --- crates/testing/tests/tests_1/message.rs | 85 +++++++++++++++++++++++++ crates/types/src/data.rs | 34 ++++++++++ 2 files changed, 119 insertions(+) diff --git a/crates/testing/tests/tests_1/message.rs b/crates/testing/tests/tests_1/message.rs index 92c8cb1da7..68e28c4e59 100644 --- a/crates/testing/tests/tests_1/message.rs +++ b/crates/testing/tests/tests_1/message.rs @@ -57,3 +57,88 @@ fn version_number_at_start_of_serialization() { assert_eq!(version.major, version_read.major); assert_eq!(version.minor, version_read.minor); } + +#[cfg(test)] +#[tokio::test(flavor = "multi_thread")] +async fn test_certificate2_validity() { + use futures::StreamExt; + use hotshot_example_types::node_types::{MemoryImpl, TestTypes, TestVersions}; + use hotshot_testing::{ + helpers::{build_payload_commitment, build_system_handle}, + view_generator::TestViewGenerator, + }; + use hotshot_types::{ + data::{EpochNumber, Leaf, Leaf2, ViewNumber}, + simple_certificate::convert_quorum_certificate, + traits::{election::Membership, node_implementation::ConsensusTime}, + vote::Certificate, + }; + + hotshot::helpers::initialize_logging(); + + let node_id = 1; + let handle = build_system_handle::(node_id) + .await + .0; + let quorum_membership = handle.hotshot.memberships.quorum_membership.clone(); + let da_membership = handle.hotshot.memberships.da_membership.clone(); + + let payload_commitment = build_payload_commitment::( + &quorum_membership, + ViewNumber::new(node_id), + EpochNumber::new(1), + ); + + let mut generator = TestViewGenerator::generate(quorum_membership.clone(), da_membership); + + let mut proposals = Vec::new(); + let mut leaders = Vec::new(); + let mut leaves = Vec::new(); + let mut vids = Vec::new(); + let mut vid_dispersals = Vec::new(); + let consensus = handle.hotshot.consensus(); + let mut consensus_writer = consensus.write().await; + for view in (&mut generator).take(4).collect::>().await { + proposals.push(view.quorum_proposal.clone()); + leaders.push(view.leader_public_key); + leaves.push(view.leaf.clone()); + vids.push(view.vid_proposal.clone()); + vid_dispersals.push(view.vid_disperse.clone()); + } + + let proposal = proposals[3].clone(); + let parent_proposal = proposals[2].clone(); + + // ensure that we don't break certificate validation + let qc = proposal.data.justify_qc.clone(); + let qc2 = convert_quorum_certificate(qc); + + assert!( + qc.is_valid_cert( + &quorum_membership, + EpochNumber::new(0), + &handle.hotshot.upgrade_lock + ) + .await + ); + + assert!( + qc2.is_valid_cert( + &quorum_membership, + EpochNumber::new(0), + &handle.hotshot.upgrade_lock + ) + .await + ); + + // ensure that we don't break the leaf commitment chain + let leaf = Leaf::from_quorum_proposal(&proposal.data); + let parent_leaf = Leaf::from_quorum_proposal(&parent_proposal.data); + + let leaf2 = Leaf2::from_quorum_proposal(&proposal.data.into()); + let parent_leaf2 = Leaf2::from_quorum_proposal(&parent_proposal.data.into()); + + assert!(leaf.parent_commitment() == parent_leaf.commit(&handle.hotshot.upgrade_lock).await); + + assert!(leaf2.parent_commitment() == parent_leaf2.commit()); +} diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index 81f8b9176a..f8ac0f54bd 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -546,6 +546,40 @@ pub struct Leaf2 { block_payload: Option, } +impl Leaf2 { + /// Time when this leaf was created. + pub fn view_number(&self) -> TYPES::View { + self.view_number + } + /// Height of this leaf in the chain. + /// + /// Equivalently, this is the number of leaves before this one in the chain. + pub fn height(&self) -> u64 { + self.block_header.block_number() + } + /// The QC linking this leaf to its parent in the chain. + pub fn justify_qc(&self) -> QuorumCertificate2 { + self.justify_qc.clone() + } + /// The QC linking this leaf to its parent in the chain. + pub fn upgrade_certificate(&self) -> Option> { + self.upgrade_certificate.clone() + } + /// Commitment to this leaf's parent. + pub fn parent_commitment(&self) -> Commitment { + self.parent_commitment + } + /// The block header contained in this leaf. + pub fn block_header(&self) -> &::BlockHeader { + &self.block_header + } + + /// Get a mutable reference to the block header contained in this leaf. + pub fn block_header_mut(&mut self) -> &mut ::BlockHeader { + &mut self.block_header + } +} + impl Committable for Leaf2 { fn commit(&self) -> committable::Commitment { RawCommitmentBuilder::new("leaf commitment") From 68e9a1e00521acbdddbe43e06aaaa0b5bd5799e1 Mon Sep 17 00:00:00 2001 From: ss-es <155648797+ss-es@users.noreply.github.com> Date: Mon, 11 Nov 2024 13:26:48 -0500 Subject: [PATCH 10/15] switch to new types --- crates/example-types/src/block_types.rs | 8 +- crates/example-types/src/state_types.rs | 4 +- crates/example-types/src/storage_types.rs | 31 ++++- crates/hotshot/src/lib.rs | 47 ++++--- crates/hotshot/src/types/handle.rs | 15 ++- crates/task-impls/src/consensus/handlers.rs | 4 +- crates/task-impls/src/consensus/mod.rs | 6 +- crates/task-impls/src/events.rs | 98 ++++---------- crates/task-impls/src/helpers.rs | 29 ++-- crates/task-impls/src/network.rs | 16 +-- .../src/quorum_proposal/handlers.rs | 22 +-- crates/task-impls/src/quorum_proposal/mod.rs | 4 +- .../src/quorum_proposal_recv/handlers.rs | 12 +- crates/task-impls/src/quorum_vote/handlers.rs | 19 +-- crates/task-impls/src/quorum_vote/mod.rs | 19 +-- crates/task-impls/src/vote_collection.rs | 33 ++++- .../src/byzantine/byzantine_behaviour.rs | 14 +- crates/testing/src/consistency_task.rs | 13 +- crates/testing/src/helpers.rs | 10 +- crates/testing/src/overall_safety_task.rs | 18 +-- crates/testing/src/spinning_task.rs | 11 +- crates/testing/src/test_runner.rs | 6 +- crates/testing/src/view_generator.rs | 63 ++++----- crates/testing/tests/tests_1/message.rs | 31 ++--- .../tests_1/quorum_proposal_recv_task.rs | 6 +- .../tests/tests_1/quorum_proposal_task.rs | 32 ++--- .../testing/tests/tests_1/quorum_vote_task.rs | 6 +- .../tests_1/upgrade_task_with_proposal.rs | 12 +- .../tests/tests_1/upgrade_task_with_vote.rs | 4 +- .../tests/tests_1/vote_dependency_handle.rs | 4 +- crates/types/src/consensus.rs | 51 +++---- crates/types/src/data.rs | 127 +++++++++++++++++- crates/types/src/error.rs | 4 +- crates/types/src/event.rs | 12 +- crates/types/src/message.rs | 31 ++++- crates/types/src/simple_certificate.rs | 53 +++++--- crates/types/src/simple_vote.rs | 38 ++++++ crates/types/src/traits/block_contents.rs | 6 +- .../types/src/traits/node_implementation.rs | 8 +- crates/types/src/traits/states.rs | 4 +- crates/types/src/traits/storage.rs | 4 +- crates/types/src/utils.rs | 4 +- 42 files changed, 574 insertions(+), 365 deletions(-) diff --git a/crates/example-types/src/block_types.rs b/crates/example-types/src/block_types.rs index 7241ac49b1..b3d895e642 100644 --- a/crates/example-types/src/block_types.rs +++ b/crates/example-types/src/block_types.rs @@ -13,7 +13,7 @@ use std::{ use async_trait::async_trait; use committable::{Commitment, Committable, RawCommitmentBuilder}; use hotshot_types::{ - data::{BlockError, Leaf}, + data::{BlockError, Leaf2}, traits::{ block_contents::{BlockHeader, BuilderFee, EncodeBytes, TestableBlock, Transaction}, node_implementation::NodeType, @@ -272,7 +272,7 @@ pub struct TestBlockHeader { impl TestBlockHeader { pub fn new>( - parent_leaf: &Leaf, + parent_leaf: &Leaf2, payload_commitment: VidCommitment, builder_commitment: BuilderCommitment, metadata: TestMetadata, @@ -312,7 +312,7 @@ impl< async fn new_legacy( _parent_state: &TYPES::ValidatedState, instance_state: &>::Instance, - parent_leaf: &Leaf, + parent_leaf: &Leaf2, payload_commitment: VidCommitment, builder_commitment: BuilderCommitment, metadata: >::Metadata, @@ -332,7 +332,7 @@ impl< async fn new_marketplace( _parent_state: &TYPES::ValidatedState, instance_state: &>::Instance, - parent_leaf: &Leaf, + parent_leaf: &Leaf2, payload_commitment: VidCommitment, builder_commitment: BuilderCommitment, metadata: >::Metadata, diff --git a/crates/example-types/src/state_types.rs b/crates/example-types/src/state_types.rs index 50c40fee73..a62f6e5209 100644 --- a/crates/example-types/src/state_types.rs +++ b/crates/example-types/src/state_types.rs @@ -10,7 +10,7 @@ use std::fmt::Debug; use async_trait::async_trait; use committable::{Commitment, Committable}; use hotshot_types::{ - data::{fake_commitment, BlockError, Leaf, ViewNumber}, + data::{fake_commitment, BlockError, Leaf2, ViewNumber}, traits::{ block_contents::BlockHeader, node_implementation::NodeType, @@ -103,7 +103,7 @@ impl ValidatedState for TestValidatedState { async fn validate_and_apply_header( &self, instance: &Self::Instance, - _parent_leaf: &Leaf, + _parent_leaf: &Leaf2, _proposed_header: &TYPES::BlockHeader, _vid_common: VidCommon, _version: Version, diff --git a/crates/example-types/src/storage_types.rs b/crates/example-types/src/storage_types.rs index b20c839c0b..6c928a0be9 100644 --- a/crates/example-types/src/storage_types.rs +++ b/crates/example-types/src/storage_types.rs @@ -17,7 +17,7 @@ use hotshot_types::{ data::{DaProposal, Leaf, Leaf2, QuorumProposal, QuorumProposal2, VidDisperseShare}, event::HotShotAction, message::Proposal, - simple_certificate::{QuorumCertificate, UpgradeCertificate}, + simple_certificate::{QuorumCertificate2, UpgradeCertificate}, traits::{ node_implementation::{ConsensusTime, NodeType}, storage::Storage, @@ -41,6 +41,7 @@ pub struct TestStorageState { proposals: BTreeMap>>, proposals2: BTreeMap>>, high_qc: Option>, + high_qc2: Option>, action: TYPES::View, } @@ -52,6 +53,7 @@ impl Default for TestStorageState { proposals: BTreeMap::new(), proposals2: BTreeMap::new(), high_qc: None, + high_qc2: None, action: TYPES::View::genesis(), } } @@ -90,11 +92,11 @@ impl TestableDelay for TestStorage { impl TestStorage { pub async fn proposals_cloned( &self, - ) -> BTreeMap>> { - self.inner.read().await.proposals.clone() + ) -> BTreeMap>> { + self.inner.read().await.proposals2.clone() } - pub async fn high_qc_cloned(&self) -> Option> { - self.inner.read().await.high_qc.clone() + pub async fn high_qc_cloned(&self) -> Option> { + self.inner.read().await.high_qc2.clone() } pub async fn decided_upgrade_certificate(&self) -> Option> { self.decided_upgrade_certificate.read().await.clone() @@ -198,6 +200,25 @@ impl Storage for TestStorage { } Ok(()) } + + async fn update_high_qc2( + &self, + new_high_qc: hotshot_types::simple_certificate::QuorumCertificate2, + ) -> Result<()> { + if self.should_return_err { + bail!("Failed to update high qc to storage"); + } + Self::run_delay_settings_from_config(&self.delay_config).await; + let mut inner = self.inner.write().await; + if let Some(ref current_high_qc) = inner.high_qc2 { + if new_high_qc.view_number() > current_high_qc.view_number() { + inner.high_qc2 = Some(new_high_qc); + } + } else { + inner.high_qc2 = Some(new_high_qc); + } + Ok(()) + } async fn update_undecided_state( &self, _leafs: CommitmentMap>, diff --git a/crates/hotshot/src/lib.rs b/crates/hotshot/src/lib.rs index 80e409b0f8..a281e13388 100644 --- a/crates/hotshot/src/lib.rs +++ b/crates/hotshot/src/lib.rs @@ -11,6 +11,7 @@ #[cfg(feature = "docs")] pub mod documentation; +use committable::Committable; use futures::future::{select, Either}; use hotshot_types::{ message::UpgradeLock, @@ -51,7 +52,7 @@ use hotshot_types::{ data::{Leaf, Leaf2, QuorumProposal, QuorumProposal2}, event::{EventType, LeafInfo}, message::{convert_proposal, DataMessage, Message, MessageKind, Proposal}, - simple_certificate::{QuorumCertificate, UpgradeCertificate}, + simple_certificate::{QuorumCertificate, QuorumCertificate2, UpgradeCertificate}, traits::{ consensus_api::ConsensusApi, election::Membership, @@ -136,7 +137,7 @@ pub struct SystemContext, V: Versi pub(crate) external_event_stream: (Sender>, InactiveReceiver>), /// Anchored leaf provided by the initializer. - anchored_leaf: Leaf, + anchored_leaf: Leaf2, /// access to the internal event stream, in case we need to, say, shut something down #[allow(clippy::type_complexity)] @@ -302,7 +303,7 @@ impl, V: Versions> SystemContext, V: Versions> SystemContext, V: Versions> SystemContext, V: Versions> SystemContext(&validated_state, self.instance_state.as_ref()) - .await, + .await + .to_qc2(), ); broadcast_event( @@ -543,7 +542,7 @@ impl, V: Versions> SystemContext Leaf { + pub async fn decided_leaf(&self) -> Leaf2 { self.consensus.read().await.decided_leaf() } @@ -554,7 +553,7 @@ impl, V: Versions> SystemContext Option> { + pub fn try_decided_leaf(&self) -> Option> { self.consensus.try_read().map(|guard| guard.decided_leaf()) } @@ -968,7 +967,7 @@ impl, V: Versions> ConsensusApi { /// the leaf specified initialization - inner: Leaf, + inner: Leaf2, /// Instance-level state. instance_state: TYPES::InstanceState, @@ -992,16 +991,16 @@ pub struct HotShotInitializer { /// Highest QC that was seen, for genesis it's the genesis QC. It should be for a view greater /// than `inner`s view number for the non genesis case because we must have seen higher QCs /// to decide on the leaf. - high_qc: QuorumCertificate, + high_qc: QuorumCertificate2, /// Previously decided upgrade certificate; this is necessary if an upgrade has happened and we are not restarting with the new version decided_upgrade_certificate: Option>, /// Undecided leafs that were seen, but not yet decided on. These allow a restarting node /// to vote and propose right away if they didn't miss anything while down. - undecided_leafs: Vec>, + undecided_leafs: Vec>, /// Not yet decided state undecided_state: BTreeMap>, /// Proposals we have sent out to provide to others for catchup - saved_proposals: BTreeMap>>, + saved_proposals: BTreeMap>>, } impl HotShotInitializer { @@ -1012,10 +1011,14 @@ impl HotShotInitializer { instance_state: TYPES::InstanceState, ) -> Result> { let (validated_state, state_delta) = TYPES::ValidatedState::genesis(&instance_state); - let high_qc = QuorumCertificate::genesis::(&validated_state, &instance_state).await; + let high_qc = QuorumCertificate::genesis::(&validated_state, &instance_state) + .await + .to_qc2(); Ok(Self { - inner: Leaf::genesis(&validated_state, &instance_state).await, + inner: Leaf::genesis(&validated_state, &instance_state) + .await + .into(), validated_state: Some(Arc::new(validated_state)), state_delta: Some(Arc::new(state_delta)), start_view: TYPES::View::new(0), @@ -1038,15 +1041,15 @@ impl HotShotInitializer { /// `SystemContext`. #[allow(clippy::too_many_arguments)] pub fn from_reload( - anchor_leaf: Leaf, + anchor_leaf: Leaf2, instance_state: TYPES::InstanceState, validated_state: Option>, start_view: TYPES::View, actioned_view: TYPES::View, - saved_proposals: BTreeMap>>, - high_qc: QuorumCertificate, + saved_proposals: BTreeMap>>, + high_qc: QuorumCertificate2, decided_upgrade_certificate: Option>, - undecided_leafs: Vec>, + undecided_leafs: Vec>, undecided_state: BTreeMap>, ) -> Self { Self { diff --git a/crates/hotshot/src/types/handle.rs b/crates/hotshot/src/types/handle.rs index 2ab7874b8f..2956f39465 100644 --- a/crates/hotshot/src/types/handle.rs +++ b/crates/hotshot/src/types/handle.rs @@ -20,7 +20,7 @@ use hotshot_task::{ use hotshot_task_impls::{events::HotShotEvent, helpers::broadcast_event}; use hotshot_types::{ consensus::Consensus, - data::{Leaf, QuorumProposal}, + data::{Leaf2, QuorumProposal2}, error::HotShotError, message::{Message, MessageKind, Proposal, RecipientList}, request_response::ProposalRequestPayload, @@ -141,8 +141,9 @@ impl + 'static, V: Versions> &self, view: TYPES::View, epoch: TYPES::Epoch, - leaf_commitment: Commitment>, - ) -> Result>>>> { + leaf_commitment: Commitment>, + ) -> Result>>>> + { // We need to be able to sign this request before submitting it to the network. Compute the // payload first. let signed_proposal_request = ProposalRequestPayload { @@ -195,8 +196,8 @@ impl + 'static, V: Versions> tracing::warn!("Invalid Proposal Received after Request. Err {:?}", err); continue; } - let proposed_leaf = Leaf::from_quorum_proposal(&quorum_proposal.data); - let commit = proposed_leaf.commit(&upgrade_lock).await; + let proposed_leaf = Leaf2::from_quorum_proposal(&quorum_proposal.data); + let commit = proposed_leaf.commit(); if commit == leaf_commitment { return Ok(quorum_proposal.clone()); } @@ -255,7 +256,7 @@ impl + 'static, V: Versions> /// /// # Panics /// If the internal consensus is in an inconsistent state. - pub async fn decided_leaf(&self) -> Leaf { + pub async fn decided_leaf(&self) -> Leaf2 { self.hotshot.decided_leaf().await } @@ -265,7 +266,7 @@ impl + 'static, V: Versions> /// # Panics /// Panics if internal consensus is in an inconsistent state. #[must_use] - pub fn try_decided_leaf(&self) -> Option> { + pub fn try_decided_leaf(&self) -> Option> { self.hotshot.try_decided_leaf() } diff --git a/crates/task-impls/src/consensus/handlers.rs b/crates/task-impls/src/consensus/handlers.rs index 41db9a2e13..a5bf1a5ed1 100644 --- a/crates/task-impls/src/consensus/handlers.rs +++ b/crates/task-impls/src/consensus/handlers.rs @@ -10,7 +10,7 @@ use async_broadcast::Sender; use chrono::Utc; use hotshot_types::{ event::{Event, EventType}, - simple_vote::{QuorumVote, TimeoutData, TimeoutVote}, + simple_vote::{QuorumVote2, TimeoutData, TimeoutVote}, traits::{ election::Membership, node_implementation::{ConsensusTime, NodeImplementation, NodeType}, @@ -35,7 +35,7 @@ pub(crate) async fn handle_quorum_vote_recv< I: NodeImplementation, V: Versions, >( - vote: &QuorumVote, + vote: &QuorumVote2, event: Arc>, sender: &Sender>>, task_state: &mut ConsensusTaskState, diff --git a/crates/task-impls/src/consensus/mod.rs b/crates/task-impls/src/consensus/mod.rs index 2de584199a..a5e861b5f9 100644 --- a/crates/task-impls/src/consensus/mod.rs +++ b/crates/task-impls/src/consensus/mod.rs @@ -14,8 +14,8 @@ use hotshot_types::{ consensus::OuterConsensus, event::Event, message::UpgradeLock, - simple_certificate::{QuorumCertificate, TimeoutCertificate}, - simple_vote::{QuorumVote, TimeoutVote}, + simple_certificate::{QuorumCertificate2, TimeoutCertificate}, + simple_vote::{QuorumVote2, TimeoutVote}, traits::{ node_implementation::{NodeImplementation, NodeType, Versions}, signature_key::SignatureKey, @@ -57,7 +57,7 @@ pub struct ConsensusTaskState, V: pub committee_membership: Arc, /// A map of `QuorumVote` collector tasks. - pub vote_collectors: VoteCollectorsMap, QuorumCertificate, V>, + pub vote_collectors: VoteCollectorsMap, QuorumCertificate2, V>, /// A map of `TimeoutVote` collector tasks. pub timeout_vote_collectors: diff --git a/crates/task-impls/src/events.rs b/crates/task-impls/src/events.rs index 3de709dbe9..e2c1459e63 100644 --- a/crates/task-impls/src/events.rs +++ b/crates/task-impls/src/events.rs @@ -11,18 +11,19 @@ use either::Either; use hotshot_task::task::TaskEvent; use hotshot_types::{ data::{ - DaProposal, Leaf, PackedBundle, QuorumProposal, QuorumProposal2, UpgradeProposal, + DaProposal, Leaf2, PackedBundle, QuorumProposal, QuorumProposal2, UpgradeProposal, VidDisperse, VidDisperseShare, }, message::Proposal, request_response::ProposalRequestPayload, simple_certificate::{ - DaCertificate, QuorumCertificate, TimeoutCertificate, UpgradeCertificate, - ViewSyncCommitCertificate2, ViewSyncFinalizeCertificate2, ViewSyncPreCommitCertificate2, + DaCertificate, QuorumCertificate, QuorumCertificate2, TimeoutCertificate, + UpgradeCertificate, ViewSyncCommitCertificate2, ViewSyncFinalizeCertificate2, + ViewSyncPreCommitCertificate2, }, simple_vote::{ - DaVote, QuorumVote, QuorumVote2, TimeoutVote, UpgradeVote, ViewSyncCommitVote, - ViewSyncFinalizeVote, ViewSyncPreCommitVote, + DaVote, QuorumVote2, TimeoutVote, UpgradeVote, ViewSyncCommitVote, ViewSyncFinalizeVote, + ViewSyncPreCommitVote, }, traits::{ block_contents::BuilderFee, network::DataRequest, node_implementation::NodeType, @@ -71,13 +72,9 @@ pub enum HotShotEvent { /// Shutdown the task Shutdown, /// A quorum proposal has been received from the network; handled by the consensus task - QuorumProposalRecv(Proposal>, TYPES::SignatureKey), - /// A quorum proposal has been received from the network; handled by the consensus task - QuorumProposal2Recv(Proposal>, TYPES::SignatureKey), - /// A quorum vote has been received from the network; handled by the consensus task - QuorumVoteRecv(QuorumVote), + QuorumProposalRecv(Proposal>, TYPES::SignatureKey), /// A quorum vote has been received from the network; handled by the consensus task - QuorumVote2Recv(QuorumVote2), + QuorumVoteRecv(QuorumVote2), /// A timeout vote received from the network; handled by consensus task TimeoutVoteRecv(TimeoutVote), /// Send a timeout vote to the network; emitted by consensus task replicas @@ -93,13 +90,9 @@ pub enum HotShotEvent { /// A DAC is validated. DaCertificateValidated(DaCertificate), /// Send a quorum proposal to the network; emitted by the leader in the consensus task - QuorumProposalSend(Proposal>, TYPES::SignatureKey), - /// Send a quorum proposal to the network; emitted by the leader in the consensus task - QuorumProposal2Send(Proposal>, TYPES::SignatureKey), + QuorumProposalSend(Proposal>, TYPES::SignatureKey), /// Send a quorum vote to the next leader; emitted by a replica in the consensus task after seeing a valid quorum proposal - QuorumVoteSend(QuorumVote), - /// Send a quorum vote to the next leader; emitted by a replica in the consensus task after seeing a valid quorum proposal - QuorumVote2Send(QuorumVote2), + QuorumVoteSend(QuorumVote2), /// All dependencies for the quorum vote are validated. QuorumVoteDependenciesValidated(TYPES::View), /// A quorum proposal with the given parent leaf is validated. @@ -108,7 +101,7 @@ pub enum HotShotEvent { /// 2. The proposal has been correctly signed by the leader of the current view /// 3. The justify QC is valid /// 4. The proposal passes either liveness or safety check. - QuorumProposalValidated(Proposal>, Leaf), + QuorumProposalValidated(Proposal>, Leaf2), /// A quorum proposal is missing for a view that we need. QuorumProposalRequestSend( ProposalRequestPayload, @@ -120,19 +113,17 @@ pub enum HotShotEvent { ::PureAssembledSignatureType, ), /// A quorum proposal was missing for a view. As the leader, we send a reply to the recipient with their key. - QuorumProposalResponseSend(TYPES::SignatureKey, Proposal>), - /// A quorum proposal was missing for a view. As the leader, we send a reply to the recipient with their key. - QuorumProposal2ResponseSend(TYPES::SignatureKey, Proposal>), + QuorumProposalResponseSend(TYPES::SignatureKey, Proposal>), /// A quorum proposal was requested by a node for a view. - QuorumProposalResponseRecv(Proposal>), - /// A quorum proposal was requested by a node for a view. - QuorumProposal2ResponseRecv(Proposal>), + QuorumProposalResponseRecv(Proposal>), /// Send a DA proposal to the DA committee; emitted by the DA leader (which is the same node as the leader of view v + 1) in the DA task DaProposalSend(Proposal>, TYPES::SignatureKey), /// Send a DA vote to the DA leader; emitted by DA committee members in the DA task after seeing a valid DA proposal DaVoteSend(DaVote), /// The next leader has collected enough votes to form a QC; emitted by the next leader in the consensus task; an internal event only QcFormed(Either, TimeoutCertificate>), + /// The next leader has collected enough votes to form a QC; emitted by the next leader in the consensus task; an internal event only + Qc2Formed(Either, TimeoutCertificate>), /// The DA leader has collected enough votes to form a DAC; emitted by the DA leader in the DA task; sent to the entire network via the networking task DacSend(DaCertificate, TYPES::SignatureKey), /// The current view has changed; emitted by the replica in the consensus task or replica in the view sync task; received by almost all other tasks @@ -190,7 +181,7 @@ pub enum HotShotEvent { /// Event when the transactions task has a block formed BlockReady(VidDisperse, TYPES::View), /// Event when consensus decided on a leaf - LeafDecided(Vec>), + LeafDecided(Vec>), /// Send VID shares to VID storage nodes; emitted by the DA leader /// /// Like [`HotShotEvent::DaProposalSend`]. @@ -223,17 +214,17 @@ pub enum HotShotEvent { LastDecidedViewUpdated(TYPES::View), /// A new high_qc has been reached by this node. - UpdateHighQc(QuorumCertificate), + UpdateHighQc(QuorumCertificate2), /// A new high_qc has been updated in `Consensus`. - HighQcUpdated(QuorumCertificate), + HighQcUpdated(QuorumCertificate2), /// A quorum proposal has been preliminarily validated. /// The preliminary checks include: /// 1. The proposal is not for an old view /// 2. The proposal has been correctly signed by the leader of the current view /// 3. The justify QC is valid - QuorumProposalPreliminarilyValidated(Proposal>), + QuorumProposalPreliminarilyValidated(Proposal>), /// Send a VID request to the network; emitted to on of the members of DA committee. /// Includes the data request, node's public key and signature as well as public key of DA committee who we want to send to. @@ -278,21 +269,12 @@ impl HotShotEvent { HotShotEvent::QuorumProposalRecv(proposal, _) | HotShotEvent::QuorumProposalSend(proposal, _) | HotShotEvent::QuorumProposalValidated(proposal, _) - | HotShotEvent::QuorumProposalResponseSend(_, proposal) | HotShotEvent::QuorumProposalResponseRecv(proposal) + | HotShotEvent::QuorumProposalResponseSend(_, proposal) | HotShotEvent::QuorumProposalPreliminarilyValidated(proposal) => { Some(proposal.data.view_number()) } - HotShotEvent::QuorumProposal2Recv(proposal, _) - | HotShotEvent::QuorumProposal2Send(proposal, _) - | HotShotEvent::QuorumProposal2ResponseSend(_, proposal) - | HotShotEvent::QuorumProposal2ResponseRecv(proposal) => { - Some(proposal.data.view_number()) - } HotShotEvent::QuorumVoteSend(vote) => Some(vote.view_number()), - HotShotEvent::QuorumVote2Send(vote) | HotShotEvent::QuorumVote2Recv(vote) => { - Some(vote.view_number()) - } HotShotEvent::DaProposalRecv(proposal, _) | HotShotEvent::DaProposalValidated(proposal, _) | HotShotEvent::DaProposalSend(proposal, _) => Some(proposal.data.view_number()), @@ -303,6 +285,10 @@ impl HotShotEvent { either::Left(qc) => Some(qc.view_number()), either::Right(tc) => Some(tc.view_number()), }, + HotShotEvent::Qc2Formed(cert) => match cert { + either::Left(qc) => Some(qc.view_number()), + either::Right(tc) => Some(tc.view_number()), + }, HotShotEvent::ViewSyncCommitVoteSend(vote) | HotShotEvent::ViewSyncCommitVoteRecv(vote) => Some(vote.view_number()), HotShotEvent::ViewSyncPreCommitVoteRecv(vote) @@ -368,17 +354,9 @@ impl Display for HotShotEvent { "QuorumProposalRecv(view_number={:?})", proposal.data.view_number() ), - HotShotEvent::QuorumProposal2Recv(proposal, _) => write!( - f, - "QuorumProposal2Recv(view_number={:?})", - proposal.data.view_number() - ), HotShotEvent::QuorumVoteRecv(v) => { write!(f, "QuorumVoteRecv(view_number={:?})", v.view_number()) } - HotShotEvent::QuorumVote2Recv(v) => { - write!(f, "QuorumVote2Recv(view_number={:?})", v.view_number()) - } HotShotEvent::TimeoutVoteRecv(v) => { write!(f, "TimeoutVoteRecv(view_number={:?})", v.view_number()) } @@ -411,17 +389,9 @@ impl Display for HotShotEvent { "QuorumProposalSend(view_number={:?})", proposal.data.view_number() ), - HotShotEvent::QuorumProposal2Send(proposal, _) => write!( - f, - "QuorumProposal2Send(view_number={:?})", - proposal.data.view_number() - ), HotShotEvent::QuorumVoteSend(vote) => { write!(f, "QuorumVoteSend(view_number={:?})", vote.view_number()) } - HotShotEvent::QuorumVote2Send(vote) => { - write!(f, "QuorumVote2Send(view_number={:?})", vote.view_number()) - } HotShotEvent::QuorumVoteDependenciesValidated(view_number) => { write!( f, @@ -445,6 +415,10 @@ impl Display for HotShotEvent { either::Left(qc) => write!(f, "QcFormed(view_number={:?})", qc.view_number()), either::Right(tc) => write!(f, "QcFormed(view_number={:?})", tc.view_number()), }, + HotShotEvent::Qc2Formed(cert) => match cert { + either::Left(qc) => write!(f, "QcFormed(view_number={:?})", qc.view_number()), + either::Right(tc) => write!(f, "QcFormed(view_number={:?})", tc.view_number()), + }, HotShotEvent::DacSend(cert, _) => { write!(f, "DacSend(view_number={:?})", cert.view_number()) } @@ -546,7 +520,7 @@ impl Display for HotShotEvent { } HotShotEvent::LeafDecided(leaves) => { let view_numbers: Vec<::View> = - leaves.iter().map(Leaf::view_number).collect(); + leaves.iter().map(Leaf2::view_number).collect(); write!(f, "LeafDecided({view_numbers:?})") } HotShotEvent::VidDisperseSend(proposal, _) => write!( @@ -598,13 +572,6 @@ impl Display for HotShotEvent { proposal.data.view_number ) } - HotShotEvent::QuorumProposal2ResponseSend(_, proposal) => { - write!( - f, - "QuorumProposal2ResponseSend(view_number={:?})", - proposal.data.view_number - ) - } HotShotEvent::QuorumProposalResponseRecv(proposal) => { write!( f, @@ -612,13 +579,6 @@ impl Display for HotShotEvent { proposal.data.view_number ) } - HotShotEvent::QuorumProposal2ResponseRecv(proposal) => { - write!( - f, - "QuorumProposal2ResponseRecv(view_number={:?})", - proposal.data.view_number - ) - } HotShotEvent::LockedViewUpdated(view_number) => { write!(f, "LockedViewUpdated(view_number={view_number:?})") } diff --git a/crates/task-impls/src/helpers.rs b/crates/task-impls/src/helpers.rs index 8fbfd6251d..ff648a1472 100644 --- a/crates/task-impls/src/helpers.rs +++ b/crates/task-impls/src/helpers.rs @@ -15,11 +15,11 @@ use committable::{Commitment, Committable}; use hotshot_task::dependency::{Dependency, EventDependency}; use hotshot_types::{ consensus::OuterConsensus, - data::{Leaf, QuorumProposal, ViewChangeEvidence}, + data::{Leaf2, QuorumProposal2, ViewChangeEvidence}, event::{Event, EventType, LeafInfo}, message::{Proposal, UpgradeLock}, request_response::ProposalRequestPayload, - simple_certificate::{QuorumCertificate, UpgradeCertificate}, + simple_certificate::{QuorumCertificate2, UpgradeCertificate}, traits::{ block_contents::BlockHeader, election::Membership, @@ -48,7 +48,7 @@ pub(crate) async fn fetch_proposal( sender_public_key: TYPES::SignatureKey, sender_private_key: ::PrivateKey, upgrade_lock: &UpgradeLock, -) -> Result<(Leaf, View)> { +) -> Result<(Leaf2, View)> { // We need to be able to sign this request before submitting it to the network. Compute the // payload first. let signed_proposal_request = ProposalRequestPayload { @@ -131,7 +131,7 @@ pub(crate) async fn fetch_proposal( bail!("Invalid justify_qc in proposal for view {}", *view_number); } let mut consensus_writer = consensus.write().await; - let leaf = Leaf::from_quorum_proposal(&proposal.data); + let leaf = Leaf2::from_quorum_proposal(&proposal.data); let state = Arc::new( >::from_header(&proposal.data.block_header), ); @@ -144,7 +144,7 @@ pub(crate) async fn fetch_proposal( } let view = View { view_inner: ViewInner::Leaf { - leaf: leaf.commit(upgrade_lock).await, + leaf: leaf.commit(), state, delta: None, }, @@ -162,13 +162,13 @@ pub struct LeafChainTraversalOutcome { pub new_decided_view_number: Option, /// The qc for the decided chain. - pub new_decide_qc: Option>, + pub new_decide_qc: Option>, /// The decided leaves with corresponding validated state and VID info. pub leaf_views: Vec>, /// The decided leaves. - pub leaves_decided: Vec>, + pub leaves_decided: Vec>, /// The transactions in the block payload for each leaf. pub included_txns: Option::Transaction>>>, @@ -223,7 +223,7 @@ impl Default for LeafChainTraversalOutcome { /// Upon receipt then of a proposal for view 9, assuming it is valid, this entire process will repeat, and /// the anchor view will be set to view 6, with the locked view as view 7. pub async fn decide_from_proposal( - proposal: &QuorumProposal, + proposal: &QuorumProposal2, consensus: OuterConsensus, existing_upgrade_cert: Arc>>>, public_key: &TYPES::SignatureKey, @@ -355,7 +355,7 @@ pub(crate) async fn parent_leaf_and_state( consensus: OuterConsensus, upgrade_lock: &UpgradeLock, parent_view_number: TYPES::View, -) -> Result<(Leaf, Arc<::ValidatedState>)> { +) -> Result<(Leaf2, Arc<::ValidatedState>)> { let consensus_reader = consensus.read().await; let cur_epoch = consensus_reader.cur_epoch(); ensure!( @@ -425,18 +425,17 @@ pub async fn validate_proposal_safety_and_liveness< I: NodeImplementation, V: Versions, >( - proposal: Proposal>, - parent_leaf: Leaf, + proposal: Proposal>, + parent_leaf: Leaf2, validation_info: &ValidationInfo, event_stream: Sender>>, sender: TYPES::SignatureKey, ) -> Result<()> { let view_number = proposal.data.view_number(); - let proposed_leaf = Leaf::from_quorum_proposal(&proposal.data); + let proposed_leaf = Leaf2::from_quorum_proposal(&proposal.data); ensure!( - proposed_leaf.parent_commitment() - == parent_leaf.commit(&validation_info.upgrade_lock).await, + proposed_leaf.parent_commitment() == parent_leaf.commit(), "Proposed leaf does not extend the parent leaf." ); @@ -558,7 +557,7 @@ pub(crate) async fn validate_proposal_view_and_certs< I: NodeImplementation, V: Versions, >( - proposal: &Proposal>, + proposal: &Proposal>, validation_info: &ValidationInfo, ) -> Result<()> { let view_number = proposal.data.view_number(); diff --git a/crates/task-impls/src/network.rs b/crates/task-impls/src/network.rs index 4ffab75b68..ca23d7a87c 100644 --- a/crates/task-impls/src/network.rs +++ b/crates/task-impls/src/network.rs @@ -19,8 +19,8 @@ use hotshot_types::{ data::{VidDisperse, VidDisperseShare}, event::{Event, EventType, HotShotAction}, message::{ - DaConsensusMessage, DataMessage, GeneralConsensusMessage, Message, MessageKind, Proposal, - SequencingMessage, UpgradeLock, + convert_proposal, DaConsensusMessage, DataMessage, GeneralConsensusMessage, Message, + MessageKind, Proposal, SequencingMessage, UpgradeLock, }, traits::{ election::Membership, @@ -69,16 +69,16 @@ impl NetworkMessageTaskState { let event = match consensus_message { SequencingMessage::General(general_message) => match general_message { GeneralConsensusMessage::Proposal(proposal) => { - HotShotEvent::QuorumProposalRecv(proposal, sender) + HotShotEvent::QuorumProposalRecv(convert_proposal(proposal), sender) } GeneralConsensusMessage::ProposalRequested(req, sig) => { HotShotEvent::QuorumProposalRequestRecv(req, sig) } GeneralConsensusMessage::LeaderProposalAvailable(proposal) => { - HotShotEvent::QuorumProposalResponseRecv(proposal) + HotShotEvent::QuorumProposalResponseRecv(convert_proposal(proposal)) } GeneralConsensusMessage::Vote(vote) => { - HotShotEvent::QuorumVoteRecv(vote.clone()) + HotShotEvent::QuorumVoteRecv(vote.to_vote2()) } GeneralConsensusMessage::ViewSyncPreCommitVote(view_sync_message) => { HotShotEvent::ViewSyncPreCommitVoteRecv(view_sync_message) @@ -369,7 +369,7 @@ impl< Some(( sender, MessageKind::::from_consensus_message(SequencingMessage::General( - GeneralConsensusMessage::Proposal(proposal), + GeneralConsensusMessage::Proposal(convert_proposal(proposal)), )), TransmitType::Broadcast, )) @@ -394,7 +394,7 @@ impl< Some(( vote.signing_key(), MessageKind::::from_consensus_message(SequencingMessage::General( - GeneralConsensusMessage::Vote(vote.clone()), + GeneralConsensusMessage::Vote(vote.clone().to_vote()), )), TransmitType::Direct(leader), )) @@ -409,7 +409,7 @@ impl< HotShotEvent::QuorumProposalResponseSend(sender_key, proposal) => Some(( sender_key.clone(), MessageKind::::from_consensus_message(SequencingMessage::General( - GeneralConsensusMessage::LeaderProposalAvailable(proposal), + GeneralConsensusMessage::LeaderProposalAvailable(convert_proposal(proposal)), )), TransmitType::Direct(sender_key), )), diff --git a/crates/task-impls/src/quorum_proposal/handlers.rs b/crates/task-impls/src/quorum_proposal/handlers.rs index 1add742a7a..cd703dd3a4 100644 --- a/crates/task-impls/src/quorum_proposal/handlers.rs +++ b/crates/task-impls/src/quorum_proposal/handlers.rs @@ -12,10 +12,11 @@ use std::{marker::PhantomData, sync::Arc}; use anyhow::{ensure, Context, Result}; use async_broadcast::{InactiveReceiver, Sender}; use async_lock::RwLock; +use committable::Committable; use hotshot_task::dependency_task::HandleDepOutput; use hotshot_types::{ consensus::{CommitmentAndMetadata, OuterConsensus}, - data::{Leaf, QuorumProposal, VidDisperse, ViewChangeEvidence}, + data::{Leaf2, QuorumProposal, VidDisperse, ViewChangeEvidence}, message::Proposal, simple_certificate::UpgradeCertificate, traits::{ @@ -217,23 +218,22 @@ impl ProposalDependencyHandle { let proposal = QuorumProposal { block_header, view_number: self.view_number, - justify_qc: high_qc, + justify_qc: high_qc.to_qc(), upgrade_certificate, proposal_certificate, - }; + } + .into(); - let proposed_leaf = Leaf::from_quorum_proposal(&proposal); + let proposed_leaf = Leaf2::from_quorum_proposal(&proposal); ensure!( - proposed_leaf.parent_commitment() == parent_leaf.commit(&self.upgrade_lock).await, + proposed_leaf.parent_commitment() == parent_leaf.commit(), "Proposed leaf parent does not equal high qc" ); - let signature = TYPES::SignatureKey::sign( - &self.private_key, - proposed_leaf.commit(&self.upgrade_lock).await.as_ref(), - ) - .wrap() - .context(error!("Failed to compute proposed_leaf.commit()"))?; + let signature = + TYPES::SignatureKey::sign(&self.private_key, proposed_leaf.commit().as_ref()) + .wrap() + .context(error!("Failed to compute proposed_leaf.commit()"))?; let message = Proposal { data: proposal, diff --git a/crates/task-impls/src/quorum_proposal/mod.rs b/crates/task-impls/src/quorum_proposal/mod.rs index 1cc857f335..149fad34df 100644 --- a/crates/task-impls/src/quorum_proposal/mod.rs +++ b/crates/task-impls/src/quorum_proposal/mod.rs @@ -382,7 +382,7 @@ impl, V: Versions> self.formed_upgrade_certificate = Some(cert.clone()); } } - HotShotEvent::QcFormed(cert) => match cert.clone() { + HotShotEvent::Qc2Formed(cert) => match cert.clone() { either::Right(timeout_cert) => { let view_number = timeout_cert.view_number + 1; let epoch_number = self.consensus.read().await.cur_epoch(); @@ -504,7 +504,7 @@ impl, V: Versions> self.storage .write() .await - .update_high_qc(qc.clone()) + .update_high_qc2(qc.clone()) .await .wrap() .context(error!("Failed to update high QC in storage!"))?; diff --git a/crates/task-impls/src/quorum_proposal_recv/handlers.rs b/crates/task-impls/src/quorum_proposal_recv/handlers.rs index 88b04d149f..852acdccba 100644 --- a/crates/task-impls/src/quorum_proposal_recv/handlers.rs +++ b/crates/task-impls/src/quorum_proposal_recv/handlers.rs @@ -13,7 +13,7 @@ use async_lock::RwLockUpgradableReadGuard; use committable::Committable; use hotshot_types::{ consensus::OuterConsensus, - data::{Leaf, QuorumProposal}, + data::{Leaf2, QuorumProposal, QuorumProposal2}, message::Proposal, simple_certificate::QuorumCertificate, traits::{ @@ -43,12 +43,12 @@ use crate::{ /// Update states in the event that the parent state is not found for a given `proposal`. #[instrument(skip_all)] async fn validate_proposal_liveness, V: Versions>( - proposal: &Proposal>, + proposal: &Proposal>, validation_info: &ValidationInfo, ) -> Result<()> { let mut consensus_writer = validation_info.consensus.write().await; - let leaf = Leaf::from_quorum_proposal(&proposal.data); + let leaf = Leaf2::from_quorum_proposal(&proposal.data); let state = Arc::new( >::from_header(&proposal.data.block_header), @@ -65,7 +65,7 @@ async fn validate_proposal_liveness, V: Versions, >( - proposal: &Proposal>, + proposal: &Proposal>, quorum_proposal_sender_key: &TYPES::SignatureKey, event_sender: &Sender>>, event_receiver: &Receiver>>, @@ -207,7 +207,7 @@ pub(crate) async fn handle_quorum_proposal_recv< .storage .write() .await - .update_high_qc(justify_qc.clone()) + .update_high_qc2(justify_qc.clone()) .await { bail!("Failed to store High QC, not voting; error = {:?}", e); diff --git a/crates/task-impls/src/quorum_vote/handlers.rs b/crates/task-impls/src/quorum_vote/handlers.rs index 4225113440..403db09855 100644 --- a/crates/task-impls/src/quorum_vote/handlers.rs +++ b/crates/task-impls/src/quorum_vote/handlers.rs @@ -9,12 +9,13 @@ use std::sync::Arc; use async_broadcast::{InactiveReceiver, Sender}; use async_lock::RwLock; use chrono::Utc; +use committable::Committable; use hotshot_types::{ consensus::OuterConsensus, - data::{Leaf, QuorumProposal, VidDisperseShare}, + data::{Leaf2, QuorumProposal2, VidDisperseShare}, event::{Event, EventType}, message::{Proposal, UpgradeLock}, - simple_vote::{QuorumData, QuorumVote}, + simple_vote::{QuorumData2, QuorumVote2}, traits::{ election::Membership, node_implementation::{ConsensusTime, NodeImplementation, NodeType}, @@ -41,7 +42,7 @@ pub(crate) async fn handle_quorum_proposal_validated< I: NodeImplementation, V: Versions, >( - proposal: &QuorumProposal, + proposal: &QuorumProposal2, sender: &Sender>>, task_state: &mut QuorumVoteTaskState, ) -> Result<()> { @@ -169,7 +170,7 @@ pub(crate) async fn update_shared_state< view_number: TYPES::View, instance_state: Arc, storage: Arc>, - proposed_leaf: &Leaf, + proposed_leaf: &Leaf2, vid_share: &Proposal>, parent_view_number: Option, ) -> Result<()> { @@ -276,7 +277,7 @@ pub(crate) async fn update_shared_state< storage .write() .await - .update_undecided_state(new_leaves, new_state) + .update_undecided_state2(new_leaves, new_state) .await .wrap() .context(error!("Failed to update undecided state"))?; @@ -296,7 +297,7 @@ pub(crate) async fn submit_vote, V view_number: TYPES::View, epoch_number: TYPES::Epoch, storage: Arc>, - leaf: Leaf, + leaf: Leaf2, vid_share: Proposal>, ) -> Result<()> { ensure!( @@ -308,9 +309,9 @@ pub(crate) async fn submit_vote, V ); // Create and send the vote. - let vote = QuorumVote::::create_signed_vote( - QuorumData { - leaf_commit: leaf.commit(&upgrade_lock).await, + let vote = QuorumVote2::::create_signed_vote( + QuorumData2 { + leaf_commit: leaf.commit(), }, view_number, &public_key, diff --git a/crates/task-impls/src/quorum_vote/mod.rs b/crates/task-impls/src/quorum_vote/mod.rs index 563fa6fa41..a26b8ad747 100644 --- a/crates/task-impls/src/quorum_vote/mod.rs +++ b/crates/task-impls/src/quorum_vote/mod.rs @@ -9,6 +9,7 @@ use std::{collections::BTreeMap, sync::Arc}; use async_broadcast::{InactiveReceiver, Receiver, Sender}; use async_lock::RwLock; use async_trait::async_trait; +use committable::Committable; use hotshot_task::{ dependency::{AndDependency, EventDependency}, dependency_task::{DependencyTask, HandleDepOutput}, @@ -16,7 +17,7 @@ use hotshot_task::{ }; use hotshot_types::{ consensus::OuterConsensus, - data::{Leaf, QuorumProposal}, + data::{Leaf2, QuorumProposal2}, event::Event, message::{Proposal, UpgradeLock}, traits::{ @@ -107,8 +108,8 @@ impl + 'static, V: Versions> Handl } }; let proposal_payload_comm = proposal.data.block_header.payload_commitment(); - let parent_commitment = parent_leaf.commit(&self.upgrade_lock).await; - let proposed_leaf = Leaf::from_quorum_proposal(&proposal.data); + let parent_commitment = parent_leaf.commit(); + let proposed_leaf = Leaf2::from_quorum_proposal(&proposal.data); if version >= V::Epochs::VERSION && self @@ -134,7 +135,7 @@ impl + 'static, V: Versions> Handl } // Update our persistent storage of the proposal. If we cannot store the proposal return // and error so we don't vote - if let Err(e) = self.storage.write().await.append_proposal(proposal).await { + if let Err(e) = self.storage.write().await.append_proposal2(proposal).await { tracing::error!("failed to store proposal, not voting. error = {e:#}"); return; } @@ -599,15 +600,15 @@ impl, V: Versions> QuorumVoteTaskS /// Handles voting for the last block in the epoch to form the Extended QC. async fn handle_eqc_voting( &self, - proposal: &Proposal>, - parent_leaf: &Leaf, + proposal: &Proposal>, + parent_leaf: &Leaf2, event_sender: Sender>>, event_receiver: Receiver>>, epoch_number: TYPES::Epoch, ) { tracing::info!("Reached end of epoch. Justify QC is for the last block in the epoch."); - let proposed_leaf = Leaf::from_quorum_proposal(&proposal.data); - let parent_commitment = parent_leaf.commit(&self.upgrade_lock).await; + let proposed_leaf = Leaf2::from_quorum_proposal(&proposal.data); + let parent_commitment = parent_leaf.commit(); if proposed_leaf.height() != parent_leaf.height() || proposed_leaf.payload_commitment() != parent_leaf.payload_commitment() { @@ -646,7 +647,7 @@ impl, V: Versions> QuorumVoteTaskS } // Update our persistent storage of the proposal. If we cannot store the proposal return // and error so we don't vote - if let Err(e) = self.storage.write().await.append_proposal(proposal).await { + if let Err(e) = self.storage.write().await.append_proposal2(proposal).await { tracing::error!("failed to store proposal, not voting. error = {e:#}"); return; } diff --git a/crates/task-impls/src/vote_collection.rs b/crates/task-impls/src/vote_collection.rs index 4c685ca978..26321cecac 100644 --- a/crates/task-impls/src/vote_collection.rs +++ b/crates/task-impls/src/vote_collection.rs @@ -17,12 +17,13 @@ use either::Either::{self, Left, Right}; use hotshot_types::{ message::UpgradeLock, simple_certificate::{ - DaCertificate, QuorumCertificate, TimeoutCertificate, UpgradeCertificate, - ViewSyncCommitCertificate2, ViewSyncFinalizeCertificate2, ViewSyncPreCommitCertificate2, + DaCertificate, QuorumCertificate, QuorumCertificate2, TimeoutCertificate, + UpgradeCertificate, ViewSyncCommitCertificate2, ViewSyncFinalizeCertificate2, + ViewSyncPreCommitCertificate2, }, simple_vote::{ - DaVote, QuorumVote, TimeoutVote, UpgradeVote, ViewSyncCommitVote, ViewSyncFinalizeVote, - ViewSyncPreCommitVote, + DaVote, QuorumVote, QuorumVote2, TimeoutVote, UpgradeVote, ViewSyncCommitVote, + ViewSyncFinalizeVote, ViewSyncPreCommitVote, }, traits::{ election::Membership, @@ -291,7 +292,7 @@ where /// Alias for Quorum vote accumulator type QuorumVoteState = - VoteCollectionTaskState, QuorumCertificate, V>; + VoteCollectionTaskState, QuorumCertificate2, V>; /// Alias for DA vote accumulator type DaVoteState = VoteCollectionTaskState, DaCertificate, V>; /// Alias for Timeout vote accumulator @@ -336,6 +337,24 @@ impl AggregatableVote, QuorumCertifica } } +impl AggregatableVote, QuorumCertificate2> + for QuorumVote2 +{ + fn leader( + &self, + membership: &TYPES::Membership, + epoch: TYPES::Epoch, + ) -> Result { + membership.leader(self.view_number() + 1, epoch) + } + fn make_cert_event( + certificate: QuorumCertificate2, + _key: &TYPES::SignatureKey, + ) -> HotShotEvent { + HotShotEvent::Qc2Formed(Left(certificate)) + } +} + impl AggregatableVote, UpgradeCertificate> for UpgradeVote { @@ -450,14 +469,14 @@ impl // Handlers for all vote accumulators #[async_trait] impl - HandleVoteEvent, QuorumCertificate> + HandleVoteEvent, QuorumCertificate2> for QuorumVoteState { async fn handle_vote_event( &mut self, event: Arc>, sender: &Sender>>, - ) -> Result>> { + ) -> Result>> { match event.as_ref() { HotShotEvent::QuorumVoteRecv(vote) => self.accumulate_vote(vote, sender).await, _ => Ok(None), diff --git a/crates/testing/src/byzantine/byzantine_behaviour.rs b/crates/testing/src/byzantine/byzantine_behaviour.rs index e21580a3a5..b81284d139 100644 --- a/crates/testing/src/byzantine/byzantine_behaviour.rs +++ b/crates/testing/src/byzantine/byzantine_behaviour.rs @@ -19,9 +19,9 @@ use hotshot_task_impls::{ }; use hotshot_types::{ consensus::Consensus, - data::QuorumProposal, + data::QuorumProposal2, message::{Proposal, UpgradeLock}, - simple_vote::QuorumVote, + simple_vote::QuorumVote2, traits::node_implementation::{ConsensusTime, NodeImplementation, NodeType, Versions}, }; @@ -106,7 +106,7 @@ impl, V: Versions> EventTransforme /// An `EventHandlerState` that modifies justify_qc on `QuorumProposalSend` to that of a previous view to mock dishonest leader pub struct DishonestLeader { /// Store events from previous views - pub validated_proposals: Vec>, + pub validated_proposals: Vec>, /// How many times current node has been elected leader and sent proposal pub total_proposals_from_node: u64, /// Which proposals to be dishonest at @@ -126,7 +126,7 @@ impl DishonestLeader { async fn handle_proposal_send_event( &self, event: &HotShotEvent, - proposal: &Proposal>, + proposal: &Proposal>, sender: &TYPES::SignatureKey, ) -> HotShotEvent { let length = self.validated_proposals.len(); @@ -319,7 +319,7 @@ impl + std::fmt::Debug, V: Version ) -> Vec> { if let HotShotEvent::QuorumVoteSend(vote) = event { let new_view = vote.view_number + self.view_increment; - let spoofed_vote = QuorumVote::::create_signed_vote( + let spoofed_vote = QuorumVote2::::create_signed_vote( vote.data.clone(), new_view, public_key, @@ -373,7 +373,7 @@ impl std::fmt::Debug for DishonestVoting { /// An `EventHandlerState` that will send a vote for a bad proposal pub struct DishonestVoter { /// Collect all votes the node sends - pub votes_sent: Vec>, + pub votes_sent: Vec>, /// Shared state with views numbers that leaders were dishonest at pub dishonest_proposal_view_numbers: Arc>>, } @@ -402,7 +402,7 @@ impl + std::fmt::Debug, V: Version // Create a vote using data from most recent vote and the current event number // We wont update internal consensus state for this Byzantine replica but we are at least // Going to send a vote to the next honest leader - let vote = QuorumVote::::create_signed_vote( + let vote = QuorumVote2::::create_signed_vote( self.votes_sent.last().unwrap().data.clone(), event.view_number().unwrap(), public_key, diff --git a/crates/testing/src/consistency_task.rs b/crates/testing/src/consistency_task.rs index 0d78c4d12c..9d51d50a78 100644 --- a/crates/testing/src/consistency_task.rs +++ b/crates/testing/src/consistency_task.rs @@ -9,9 +9,10 @@ use std::{collections::BTreeMap, marker::PhantomData}; use anyhow::{bail, ensure, Context, Result}; use async_trait::async_trait; +use committable::Committable; use hotshot_example_types::block_types::TestBlockHeader; use hotshot_types::{ - data::Leaf, + data::Leaf2, event::{Event, EventType}, message::UpgradeLock, traits::node_implementation::{ConsensusTime, NodeType, Versions}, @@ -24,10 +25,10 @@ use crate::{ }; /// Map from views to leaves for a single node, allowing multiple leaves for each view (because the node may a priori send us multiple leaves for a given view). -pub type NodeMap = BTreeMap<::View, Vec>>; +pub type NodeMap = BTreeMap<::View, Vec>>; /// A sanitized map from views to leaves for a single node, with only a single leaf per view. -pub type NodeMapSanitized = BTreeMap<::View, Leaf>; +pub type NodeMapSanitized = BTreeMap<::View, Leaf2>; /// Validate that the `NodeMap` only has a single leaf per view. fn sanitize_node_map( @@ -104,7 +105,7 @@ async fn validate_node_map( // We want to make sure the commitment matches, // but allow for the possibility that we may have skipped views in between. if child.justify_qc().view_number == parent.view_number() - && child.justify_qc().data.leaf_commit != parent.commit(&upgrade_lock).await + && child.justify_qc().data.leaf_commit != parent.commit() { bail!("The node has provided leaf:\n\n{child:?}\n\nwhich points to:\n\n{parent:?}\n\nbut the commits do not match."); } @@ -144,7 +145,7 @@ fn sanitize_network_map( Ok(result) } -pub type ViewMap = BTreeMap<::View, BTreeMap>>; +pub type ViewMap = BTreeMap<::View, BTreeMap>>; // Invert the network map by interchanging the roles of the node_id and view number. // @@ -171,7 +172,7 @@ async fn invert_network_map( } /// A view map, sanitized to have exactly one leaf per view. -pub type ViewMapSanitized = BTreeMap<::View, Leaf>; +pub type ViewMapSanitized = BTreeMap<::View, Leaf2>; fn sanitize_view_map( view_map: &ViewMap, diff --git a/crates/testing/src/helpers.rs b/crates/testing/src/helpers.rs index 40760372c6..49f28e2504 100644 --- a/crates/testing/src/helpers.rs +++ b/crates/testing/src/helpers.rs @@ -26,7 +26,7 @@ use hotshot_example_types::{ use hotshot_task_impls::events::HotShotEvent; use hotshot_types::{ consensus::ConsensusMetricsValue, - data::{Leaf, QuorumProposal, VidDisperse, VidDisperseShare}, + data::{Leaf, Leaf2, QuorumProposal, VidDisperse, VidDisperseShare}, message::{GeneralConsensusMessage, Proposal, UpgradeLock}, simple_certificate::DaCertificate, simple_vote::{DaData, DaVote, QuorumData, QuorumVote, SimpleVote, VersionedVoteData}, @@ -437,7 +437,7 @@ where /// This function will create a fake [`View`] from a provided [`Leaf`]. pub async fn build_fake_view_with_leaf( - leaf: Leaf, + leaf: Leaf2, upgrade_lock: &UpgradeLock, ) -> View { build_fake_view_with_leaf_and_state(leaf, TestValidatedState::default(), upgrade_lock).await @@ -445,13 +445,13 @@ pub async fn build_fake_view_with_leaf( /// This function will create a fake [`View`] from a provided [`Leaf`] and `state`. pub async fn build_fake_view_with_leaf_and_state( - leaf: Leaf, + leaf: Leaf2, state: TestValidatedState, - upgrade_lock: &UpgradeLock, + _upgrade_lock: &UpgradeLock, ) -> View { View { view_inner: ViewInner::Leaf { - leaf: leaf.commit(upgrade_lock).await, + leaf: leaf.commit(), state: state.into(), delta: None, }, diff --git a/crates/testing/src/overall_safety_task.rs b/crates/testing/src/overall_safety_task.rs index ba7136c90f..c82315d18d 100644 --- a/crates/testing/src/overall_safety_task.rs +++ b/crates/testing/src/overall_safety_task.rs @@ -15,10 +15,10 @@ use async_lock::RwLock; use async_trait::async_trait; use hotshot::{traits::TestableNodeImplementation, HotShotError}; use hotshot_types::{ - data::Leaf, + data::Leaf2, error::RoundTimedoutState, event::{Event, EventType, LeafChain}, - simple_certificate::QuorumCertificate, + simple_certificate::QuorumCertificate2, traits::node_implementation::{ConsensusTime, NodeType, Versions}, vid::VidCommitment, }; @@ -311,7 +311,7 @@ pub struct RoundResult { /// Nodes that committed this round /// id -> (leaf, qc) - success_nodes: HashMap, QuorumCertificate)>, + success_nodes: HashMap, QuorumCertificate2)>, /// Nodes that failed to commit this round pub failed_nodes: HashMap>>, @@ -322,7 +322,7 @@ pub struct RoundResult { /// NOTE: technically a map is not needed /// left one anyway for ease of viewing /// leaf -> # entries decided on that leaf - pub leaf_map: HashMap, usize>, + pub leaf_map: HashMap, usize>, /// block -> # entries decided on that block pub block_map: HashMap, @@ -403,9 +403,9 @@ impl RoundResult { pub fn insert_into_result( &mut self, idx: usize, - result: (LeafChain, QuorumCertificate), + result: (LeafChain, QuorumCertificate2), maybe_block_size: Option, - ) -> Option> { + ) -> Option> { self.success_nodes.insert(idx as u64, result.clone()); let maybe_leaf = result.0.first(); @@ -458,7 +458,7 @@ impl RoundResult { &mut self, threshold: usize, total_num_nodes: usize, - key: &Leaf, + key: &Leaf2, check_leaf: bool, check_block: bool, transaction_threshold: u64, @@ -530,8 +530,8 @@ impl RoundResult { /// generate leaves #[must_use] - pub fn gen_leaves(&self) -> HashMap, usize> { - let mut leaves = HashMap::, usize>::new(); + pub fn gen_leaves(&self) -> HashMap, usize> { + let mut leaves = HashMap::, usize>::new(); for (leaf_vec, _) in self.success_nodes.values() { let most_recent_leaf = leaf_vec.iter().last(); diff --git a/crates/testing/src/spinning_task.rs b/crates/testing/src/spinning_task.rs index 5593a4336e..87128070fe 100644 --- a/crates/testing/src/spinning_task.rs +++ b/crates/testing/src/spinning_task.rs @@ -26,9 +26,9 @@ use hotshot_example_types::{ }; use hotshot_types::{ constants::EVENT_CHANNEL_SIZE, - data::Leaf, + data::Leaf2, event::Event, - simple_certificate::QuorumCertificate, + simple_certificate::{QuorumCertificate, QuorumCertificate2}, traits::{ network::{AsyncGenerator, ConnectedNetwork}, node_implementation::{ConsensusTime, NodeImplementation, NodeType, Versions}, @@ -62,9 +62,9 @@ pub struct SpinningTask< /// most recent view seen by spinning task pub(crate) latest_view: Option, /// Last decided leaf that can be used as the anchor leaf to initialize the node. - pub(crate) last_decided_leaf: Leaf, + pub(crate) last_decided_leaf: Leaf2, /// Highest qc seen in the test for restarting nodes - pub(crate) high_qc: QuorumCertificate, + pub(crate) high_qc: QuorumCertificate2, /// Add specified delay to async calls pub(crate) async_delay_config: DelayConfig, /// Context stored for nodes to be restarted with @@ -245,7 +245,8 @@ where &TestValidatedState::default(), &TestInstanceState::default(), ) - .await, + .await + .to_qc2(), ), read_storage.decided_upgrade_certificate().await, Vec::new(), diff --git a/crates/testing/src/test_runner.rs b/crates/testing/src/test_runner.rs index 2a08fda5e4..8631badc9d 100644 --- a/crates/testing/src/test_runner.rs +++ b/crates/testing/src/test_runner.rs @@ -183,12 +183,14 @@ where &TestValidatedState::default(), &TestInstanceState::default(), ) - .await, + .await + .into(), high_qc: QuorumCertificate::genesis::( &TestValidatedState::default(), &TestInstanceState::default(), ) - .await, + .await + .to_qc2(), async_delay_config: launcher.metadata.async_delay_config, restart_contexts: HashMap::new(), channel_generator: launcher.resource_generator.channel_generator, diff --git a/crates/testing/src/view_generator.rs b/crates/testing/src/view_generator.rs index 87a688b74b..afa5a8e229 100644 --- a/crates/testing/src/view_generator.rs +++ b/crates/testing/src/view_generator.rs @@ -12,6 +12,7 @@ use std::{ task::{Context, Poll}, }; +use committable::Committable; use futures::{FutureExt, Stream}; use hotshot::types::{BLSPubKey, SignatureKey, SystemContextHandle}; use hotshot_example_types::{ @@ -21,16 +22,16 @@ use hotshot_example_types::{ }; use hotshot_types::{ data::{ - DaProposal, EpochNumber, Leaf, QuorumProposal, VidDisperse, VidDisperseShare, + DaProposal, EpochNumber, Leaf, Leaf2, QuorumProposal2, VidDisperse, VidDisperseShare, ViewChangeEvidence, ViewNumber, }, message::{Proposal, UpgradeLock}, simple_certificate::{ - DaCertificate, QuorumCertificate, TimeoutCertificate, UpgradeCertificate, - ViewSyncFinalizeCertificate2, + DaCertificate, QuorumCertificate, QuorumCertificate2, TimeoutCertificate, + UpgradeCertificate, ViewSyncFinalizeCertificate2, }, simple_vote::{ - DaData, DaVote, QuorumData, QuorumVote, TimeoutData, TimeoutVote, UpgradeProposalData, + DaData, DaVote, QuorumData2, QuorumVote2, TimeoutData, TimeoutVote, UpgradeProposalData, UpgradeVote, ViewSyncFinalizeData, ViewSyncFinalizeVote, }, traits::{ @@ -49,8 +50,8 @@ use crate::helpers::{ #[derive(Clone)] pub struct TestView { pub da_proposal: Proposal>, - pub quorum_proposal: Proposal>, - pub leaf: Leaf, + pub quorum_proposal: Proposal>, + pub leaf: Leaf2, pub view_number: ViewNumber, pub epoch_number: EpochNumber, pub quorum_membership: ::Membership, @@ -130,22 +131,26 @@ impl TestView { &TestValidatedState::default(), &TestInstanceState::default(), ) - .await, + .await + .into(), payload_commitment, builder_commitment, metadata, ); - let quorum_proposal_inner = QuorumProposal:: { + let quorum_proposal_inner = QuorumProposal2:: { block_header: block_header.clone(), view_number: genesis_view, justify_qc: QuorumCertificate::genesis::( &TestValidatedState::default(), &TestInstanceState::default(), ) - .await, + .await + .to_qc2(), upgrade_certificate: None, proposal_certificate: None, + drb_result: [0; 32], + drb_seed: [0; 96], }; let encoded_transactions = Arc::from(TestTransaction::encode(&transactions)); @@ -166,16 +171,13 @@ impl TestView { _pd: PhantomData, }; - let mut leaf = Leaf::from_quorum_proposal(&quorum_proposal_inner); + let mut leaf = Leaf2::from_quorum_proposal(&quorum_proposal_inner); leaf.fill_block_payload_unchecked(TestBlockPayload { transactions: transactions.clone(), }); - let signature = ::sign( - &private_key, - leaf.commit(&upgrade_lock).await.as_ref(), - ) - .expect("Failed to sign leaf commitment!"); + let signature = ::sign(&private_key, leaf.commit().as_ref()) + .expect("Failed to sign leaf commitment!"); let quorum_proposal = Proposal { data: quorum_proposal_inner, @@ -222,8 +224,8 @@ impl TestView { let transactions = &self.transactions; - let quorum_data = QuorumData { - leaf_commit: old.leaf.commit(&self.upgrade_lock).await, + let quorum_data = QuorumData2 { + leaf_commit: old.leaf.commit(), }; let (old_private_key, old_public_key) = key_pair_for_id::(*old_view); @@ -274,9 +276,9 @@ impl TestView { let quorum_certificate = build_cert::< TestTypes, TestVersions, - QuorumData, - QuorumVote, - QuorumCertificate, + QuorumData2, + QuorumVote2, + QuorumCertificate2, >( quorum_data, quorum_membership, @@ -374,24 +376,23 @@ impl TestView { random, }; - let proposal = QuorumProposal:: { + let proposal = QuorumProposal2:: { block_header: block_header.clone(), view_number: next_view, justify_qc: quorum_certificate.clone(), upgrade_certificate: upgrade_certificate.clone(), proposal_certificate, + drb_result: [0; 32], + drb_seed: [0; 96], }; - let mut leaf = Leaf::from_quorum_proposal(&proposal); + let mut leaf = Leaf2::from_quorum_proposal(&proposal); leaf.fill_block_payload_unchecked(TestBlockPayload { transactions: transactions.clone(), }); - let signature = ::sign( - &private_key, - leaf.commit(&self.upgrade_lock).await.as_ref(), - ) - .expect("Failed to sign leaf commitment."); + let signature = ::sign(&private_key, leaf.commit().as_ref()) + .expect("Failed to sign leaf commitment."); let quorum_proposal = Proposal { data: proposal, @@ -451,10 +452,10 @@ impl TestView { pub async fn create_quorum_vote( &self, handle: &SystemContextHandle, - ) -> QuorumVote { - QuorumVote::::create_signed_vote( - QuorumData { - leaf_commit: self.leaf.commit(&handle.hotshot.upgrade_lock).await, + ) -> QuorumVote2 { + QuorumVote2::::create_signed_vote( + QuorumData2 { + leaf_commit: self.leaf.commit(), }, self.view_number, &handle.public_key(), diff --git a/crates/testing/tests/tests_1/message.rs b/crates/testing/tests/tests_1/message.rs index 68e28c4e59..5e0ff49eaa 100644 --- a/crates/testing/tests/tests_1/message.rs +++ b/crates/testing/tests/tests_1/message.rs @@ -63,14 +63,10 @@ fn version_number_at_start_of_serialization() { async fn test_certificate2_validity() { use futures::StreamExt; use hotshot_example_types::node_types::{MemoryImpl, TestTypes, TestVersions}; - use hotshot_testing::{ - helpers::{build_payload_commitment, build_system_handle}, - view_generator::TestViewGenerator, - }; + use hotshot_testing::{helpers::build_system_handle, view_generator::TestViewGenerator}; use hotshot_types::{ - data::{EpochNumber, Leaf, Leaf2, ViewNumber}, - simple_certificate::convert_quorum_certificate, - traits::{election::Membership, node_implementation::ConsensusTime}, + data::{EpochNumber, Leaf, Leaf2}, + traits::node_implementation::ConsensusTime, vote::Certificate, }; @@ -83,12 +79,6 @@ async fn test_certificate2_validity() { let quorum_membership = handle.hotshot.memberships.quorum_membership.clone(); let da_membership = handle.hotshot.memberships.da_membership.clone(); - let payload_commitment = build_payload_commitment::( - &quorum_membership, - ViewNumber::new(node_id), - EpochNumber::new(1), - ); - let mut generator = TestViewGenerator::generate(quorum_membership.clone(), da_membership); let mut proposals = Vec::new(); @@ -96,8 +86,7 @@ async fn test_certificate2_validity() { let mut leaves = Vec::new(); let mut vids = Vec::new(); let mut vid_dispersals = Vec::new(); - let consensus = handle.hotshot.consensus(); - let mut consensus_writer = consensus.write().await; + for view in (&mut generator).take(4).collect::>().await { proposals.push(view.quorum_proposal.clone()); leaders.push(view.leader_public_key); @@ -110,8 +99,8 @@ async fn test_certificate2_validity() { let parent_proposal = proposals[2].clone(); // ensure that we don't break certificate validation - let qc = proposal.data.justify_qc.clone(); - let qc2 = convert_quorum_certificate(qc); + let qc2 = proposal.data.justify_qc.clone(); + let qc = qc2.clone().to_qc(); assert!( qc.is_valid_cert( @@ -132,11 +121,11 @@ async fn test_certificate2_validity() { ); // ensure that we don't break the leaf commitment chain - let leaf = Leaf::from_quorum_proposal(&proposal.data); - let parent_leaf = Leaf::from_quorum_proposal(&parent_proposal.data); + let leaf2 = Leaf2::from_quorum_proposal(&proposal.data); + let parent_leaf2 = Leaf2::from_quorum_proposal(&parent_proposal.data); - let leaf2 = Leaf2::from_quorum_proposal(&proposal.data.into()); - let parent_leaf2 = Leaf2::from_quorum_proposal(&parent_proposal.data.into()); + let leaf = Leaf::from_quorum_proposal(&proposal.data.into()); + let parent_leaf = Leaf::from_quorum_proposal(&parent_proposal.data.into()); assert!(leaf.parent_commitment() == parent_leaf.commit(&handle.hotshot.upgrade_lock).await); diff --git a/crates/testing/tests/tests_1/quorum_proposal_recv_task.rs b/crates/testing/tests/tests_1/quorum_proposal_recv_task.rs index cb216376d7..ffafb4cffe 100644 --- a/crates/testing/tests/tests_1/quorum_proposal_recv_task.rs +++ b/crates/testing/tests/tests_1/quorum_proposal_recv_task.rs @@ -28,7 +28,7 @@ use hotshot_testing::{ view_generator::TestViewGenerator, }; use hotshot_types::{ - data::{Leaf, ViewNumber}, + data::{Leaf2, ViewNumber}, request_response::ProposalRequestPayload, traits::{ consensus_api::ConsensusApi, @@ -77,7 +77,7 @@ async fn test_quorum_proposal_recv_task() { // to that, we'll just put them in here. consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, &handle.hotshot.upgrade_lock, @@ -125,7 +125,7 @@ async fn test_quorum_proposal_recv_task_liveness_check() { helpers::{build_fake_view_with_leaf, build_fake_view_with_leaf_and_state}, script::{Expectations, TaskScript}, }; - use hotshot_types::{data::Leaf, vote::HasViewNumber}; + use hotshot_types::{data::Leaf2, vote::HasViewNumber}; hotshot::helpers::initialize_logging(); diff --git a/crates/testing/tests/tests_1/quorum_proposal_task.rs b/crates/testing/tests/tests_1/quorum_proposal_task.rs index 26fe07ab61..5e08ef1e8d 100644 --- a/crates/testing/tests/tests_1/quorum_proposal_task.rs +++ b/crates/testing/tests/tests_1/quorum_proposal_task.rs @@ -25,7 +25,7 @@ use hotshot_testing::{ view_generator::TestViewGenerator, }; use hotshot_types::{ - data::{null_block, EpochNumber, Leaf, ViewChangeEvidence, ViewNumber}, + data::{null_block, EpochNumber, Leaf2, ViewChangeEvidence, ViewNumber}, simple_vote::{TimeoutData, ViewSyncFinalizeData}, traits::{ election::Membership, @@ -79,7 +79,7 @@ async fn test_quorum_proposal_task_quorum_proposal_view_1() { // to make sure they show up during tests. consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, &handle.hotshot.upgrade_lock, @@ -104,7 +104,7 @@ async fn test_quorum_proposal_task_quorum_proposal_view_1() { handle.public_key() )], random![ - QcFormed(either::Left(genesis_cert.clone())), + Qc2Formed(either::Left(genesis_cert.clone())), SendPayloadCommitmentAndMetadata( payload_commitment, builder_commitment, @@ -172,7 +172,7 @@ async fn test_quorum_proposal_task_quorum_proposal_view_gt_1() { // to make sure they show up during tests. consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, &handle.hotshot.upgrade_lock, @@ -196,7 +196,7 @@ async fn test_quorum_proposal_task_quorum_proposal_view_gt_1() { let inputs = vec![ random![ - QcFormed(either::Left(genesis_cert.clone())), + Qc2Formed(either::Left(genesis_cert.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -215,7 +215,7 @@ async fn test_quorum_proposal_task_quorum_proposal_view_gt_1() { ], random![ QuorumProposalPreliminarilyValidated(proposals[0].clone()), - QcFormed(either::Left(proposals[1].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[1].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -232,7 +232,7 @@ async fn test_quorum_proposal_task_quorum_proposal_view_gt_1() { ], random![ QuorumProposalPreliminarilyValidated(proposals[1].clone()), - QcFormed(either::Left(proposals[2].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[2].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -249,7 +249,7 @@ async fn test_quorum_proposal_task_quorum_proposal_view_gt_1() { ], random![ QuorumProposalPreliminarilyValidated(proposals[2].clone()), - QcFormed(either::Left(proposals[3].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[3].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -266,7 +266,7 @@ async fn test_quorum_proposal_task_quorum_proposal_view_gt_1() { ], random![ QuorumProposalPreliminarilyValidated(proposals[3].clone()), - QcFormed(either::Left(proposals[4].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[4].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -373,7 +373,7 @@ async fn test_quorum_proposal_task_qc_timeout() { }; let inputs = vec![random![ - QcFormed(either::Right(cert.clone())), + Qc2Formed(either::Right(cert.clone())), SendPayloadCommitmentAndMetadata( payload_commitment, builder_commitment, @@ -527,7 +527,7 @@ async fn test_quorum_proposal_task_liveness_check() { // to make sure they show up during tests. consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, &handle.hotshot.upgrade_lock, @@ -550,7 +550,7 @@ async fn test_quorum_proposal_task_liveness_check() { let inputs = vec![ random![ - QcFormed(either::Left(genesis_cert.clone())), + Qc2Formed(either::Left(genesis_cert.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -569,7 +569,7 @@ async fn test_quorum_proposal_task_liveness_check() { ], random![ QuorumProposalPreliminarilyValidated(proposals[0].clone()), - QcFormed(either::Left(proposals[1].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[1].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -586,7 +586,7 @@ async fn test_quorum_proposal_task_liveness_check() { ], random![ QuorumProposalPreliminarilyValidated(proposals[1].clone()), - QcFormed(either::Left(proposals[2].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[2].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -603,7 +603,7 @@ async fn test_quorum_proposal_task_liveness_check() { ], random![ QuorumProposalPreliminarilyValidated(proposals[2].clone()), - QcFormed(either::Left(proposals[3].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[3].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -620,7 +620,7 @@ async fn test_quorum_proposal_task_liveness_check() { ], random![ QuorumProposalPreliminarilyValidated(proposals[3].clone()), - QcFormed(either::Left(proposals[4].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[4].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, diff --git a/crates/testing/tests/tests_1/quorum_vote_task.rs b/crates/testing/tests/tests_1/quorum_vote_task.rs index 9735d07b74..3243b75409 100644 --- a/crates/testing/tests/tests_1/quorum_vote_task.rs +++ b/crates/testing/tests/tests_1/quorum_vote_task.rs @@ -23,7 +23,7 @@ use hotshot_testing::{ script::{Expectations, InputOrder, TaskScript}, }; use hotshot_types::{ - data::{Leaf, ViewNumber}, + data::{Leaf2, ViewNumber}, traits::node_implementation::ConsensusTime, }; @@ -64,7 +64,7 @@ async fn test_quorum_vote_task_success() { vids.push(view.vid_proposal.clone()); consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, &handle.hotshot.upgrade_lock, @@ -137,7 +137,7 @@ async fn test_quorum_vote_task_miss_dependency() { consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, &handle.hotshot.upgrade_lock, diff --git a/crates/testing/tests/tests_1/upgrade_task_with_proposal.rs b/crates/testing/tests/tests_1/upgrade_task_with_proposal.rs index 828b823ab0..1d67eefbdc 100644 --- a/crates/testing/tests/tests_1/upgrade_task_with_proposal.rs +++ b/crates/testing/tests/tests_1/upgrade_task_with_proposal.rs @@ -30,7 +30,7 @@ use hotshot_testing::{ view_generator::TestViewGenerator, }; use hotshot_types::{ - data::{null_block, EpochNumber, Leaf, ViewNumber}, + data::{null_block, EpochNumber, Leaf2, ViewNumber}, simple_vote::UpgradeProposalData, traits::{ election::Membership, @@ -96,7 +96,7 @@ async fn test_upgrade_task_with_proposal() { views.push(view.clone()); consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, &handle.hotshot.upgrade_lock, @@ -117,7 +117,7 @@ async fn test_upgrade_task_with_proposal() { views.push(view.clone()); consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, &handle.hotshot.upgrade_lock, @@ -154,7 +154,7 @@ async fn test_upgrade_task_with_proposal() { let inputs = vec![ random![ - QcFormed(either::Left(genesis_cert.clone())), + Qc2Formed(either::Left(genesis_cert.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -173,7 +173,7 @@ async fn test_upgrade_task_with_proposal() { ], random![ QuorumProposalPreliminarilyValidated(proposals[0].clone()), - QcFormed(either::Left(proposals[1].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[1].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -191,7 +191,7 @@ async fn test_upgrade_task_with_proposal() { InputOrder::Random(upgrade_vote_recvs), random![ QuorumProposalPreliminarilyValidated(proposals[1].clone()), - QcFormed(either::Left(proposals[2].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[2].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, diff --git a/crates/testing/tests/tests_1/upgrade_task_with_vote.rs b/crates/testing/tests/tests_1/upgrade_task_with_vote.rs index eaaa70fc94..9c935163ae 100644 --- a/crates/testing/tests/tests_1/upgrade_task_with_vote.rs +++ b/crates/testing/tests/tests_1/upgrade_task_with_vote.rs @@ -29,7 +29,7 @@ use hotshot_testing::{ view_generator::TestViewGenerator, }; use hotshot_types::{ - data::{null_block, Leaf, ViewNumber}, + data::{null_block, Leaf2, ViewNumber}, simple_vote::UpgradeProposalData, traits::{election::Membership, node_implementation::ConsensusTime}, vote::HasViewNumber, @@ -82,7 +82,7 @@ async fn test_upgrade_task_with_vote() { leaves.push(view.leaf.clone()); consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, &handle.hotshot.upgrade_lock, diff --git a/crates/testing/tests/tests_1/vote_dependency_handle.rs b/crates/testing/tests/tests_1/vote_dependency_handle.rs index 35fd7200af..614693b4e8 100644 --- a/crates/testing/tests/tests_1/vote_dependency_handle.rs +++ b/crates/testing/tests/tests_1/vote_dependency_handle.rs @@ -15,7 +15,7 @@ use hotshot_testing::{ }; use hotshot_types::{ consensus::OuterConsensus, - data::{EpochNumber, Leaf, ViewNumber}, + data::{EpochNumber, Leaf2, ViewNumber}, traits::{consensus_api::ConsensusApi, node_implementation::ConsensusTime}, }; use itertools::Itertools; @@ -55,7 +55,7 @@ async fn test_vote_dependency_handle() { vids.push(view.vid_proposal.clone()); consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, &handle.hotshot.upgrade_lock, diff --git a/crates/types/src/consensus.rs b/crates/types/src/consensus.rs index 72964d3705..cddd13def7 100644 --- a/crates/types/src/consensus.rs +++ b/crates/types/src/consensus.rs @@ -14,18 +14,18 @@ use std::{ }; use async_lock::{RwLock, RwLockReadGuard, RwLockUpgradableReadGuard, RwLockWriteGuard}; -use committable::Commitment; +use committable::{Commitment, Committable}; use tracing::instrument; use utils::anytrace::*; use vec1::Vec1; pub use crate::utils::{View, ViewInner}; use crate::{ - data::{Leaf, QuorumProposal, VidDisperse, VidDisperseShare}, + data::{Leaf2, QuorumProposal2, VidDisperse, VidDisperseShare}, error::HotShotError, event::HotShotAction, message::{Proposal, UpgradeLock}, - simple_certificate::{DaCertificate, QuorumCertificate}, + simple_certificate::{DaCertificate, QuorumCertificate2}, traits::{ block_contents::{BlockHeader, BuilderFee}, metrics::{Counter, Gauge, Histogram, Metrics, NoMetrics}, @@ -289,7 +289,7 @@ pub struct Consensus { /// Last proposals we sent out, None if we haven't proposed yet. /// Prevents duplicate proposals, and can be served to those trying to catchup - last_proposals: BTreeMap>>, + last_proposals: BTreeMap>>, /// last view had a successful decide event last_decided_view: TYPES::View, @@ -300,7 +300,7 @@ pub struct Consensus { /// Map of leaf hash -> leaf /// - contains undecided leaves /// - includes the MOST RECENT decided leaf - saved_leaves: CommitmentMap>, + saved_leaves: CommitmentMap>, /// Bundle of views which we performed the most recent action /// visibible to the network. Actions are votes and proposals @@ -313,7 +313,7 @@ pub struct Consensus { saved_payloads: BTreeMap>, /// the highqc per spec - high_qc: QuorumCertificate, + high_qc: QuorumCertificate2, /// A reference to the metrics trait pub metrics: Arc, @@ -403,10 +403,10 @@ impl Consensus { locked_view: TYPES::View, last_decided_view: TYPES::View, last_actioned_view: TYPES::View, - last_proposals: BTreeMap>>, - saved_leaves: CommitmentMap>, + last_proposals: BTreeMap>>, + saved_leaves: CommitmentMap>, saved_payloads: BTreeMap>, - high_qc: QuorumCertificate, + high_qc: QuorumCertificate2, metrics: Arc, epoch_height: u64, ) -> Self { @@ -449,7 +449,7 @@ impl Consensus { } /// Get the high QC. - pub fn high_qc(&self) -> &QuorumCertificate { + pub fn high_qc(&self) -> &QuorumCertificate2 { &self.high_qc } @@ -459,7 +459,7 @@ impl Consensus { } /// Get the saved leaves. - pub fn saved_leaves(&self) -> &CommitmentMap> { + pub fn saved_leaves(&self) -> &CommitmentMap> { &self.saved_leaves } @@ -479,7 +479,9 @@ impl Consensus { } /// Get the map of our recent proposals - pub fn last_proposals(&self) -> &BTreeMap>> { + pub fn last_proposals( + &self, + ) -> &BTreeMap>> { &self.last_proposals } @@ -545,7 +547,7 @@ impl Consensus { /// Can return an error when the new view_number is not higher than the existing proposed view number. pub fn update_proposed_view( &mut self, - proposal: Proposal>, + proposal: Proposal>, ) -> Result<()> { ensure!( proposal.data.view_number() @@ -609,7 +611,7 @@ impl Consensus { /// with the same view number. pub async fn update_leaf( &mut self, - leaf: Leaf, + leaf: Leaf2, state: Arc, delta: Option>::Delta>>, upgrade_lock: &UpgradeLock, @@ -617,7 +619,7 @@ impl Consensus { let view_number = leaf.view_number(); let view = View { view_inner: ViewInner::Leaf { - leaf: leaf.commit(upgrade_lock).await, + leaf: leaf.commit(), state, delta, }, @@ -664,11 +666,10 @@ impl Consensus { /// Update the saved leaves with a new leaf. async fn update_saved_leaves( &mut self, - leaf: Leaf, - upgrade_lock: &UpgradeLock, + leaf: Leaf2, + _upgrade_lock: &UpgradeLock, ) { - self.saved_leaves - .insert(leaf.commit(upgrade_lock).await, leaf); + self.saved_leaves.insert(leaf.commit(), leaf); } /// Update the saved payloads with a new encoded transaction. @@ -691,7 +692,7 @@ impl Consensus { /// Update the high QC if given a newer one. /// # Errors /// Can return an error when the provided high_qc is not newer than the existing entry. - pub fn update_high_qc(&mut self, high_qc: QuorumCertificate) -> Result<()> { + pub fn update_high_qc(&mut self, high_qc: QuorumCertificate2) -> Result<()> { ensure!( high_qc.view_number > self.high_qc.view_number || high_qc == self.high_qc, debug!("High QC with an equal or higher view exists.") @@ -731,7 +732,7 @@ impl Consensus { ) -> std::result::Result<(), HotShotError> where F: FnMut( - &Leaf, + &Leaf2, Arc<::ValidatedState>, Option::ValidatedState as ValidatedState>::Delta>>, ) -> bool, @@ -817,7 +818,7 @@ impl Consensus { /// if the last decided view's leaf does not exist in the state map or saved leaves, which /// should never happen. #[must_use] - pub fn decided_leaf(&self) -> Leaf { + pub fn decided_leaf(&self) -> Leaf2 { let decided_view_num = self.last_decided_view; let view = self.validated_state_map.get(&decided_view_num).unwrap(); let leaf = view @@ -889,7 +890,7 @@ impl Consensus { } /// Returns true if the given qc is for the last block in the epoch - pub fn is_qc_for_last_block(&self, cert: &QuorumCertificate) -> bool { + pub fn is_qc_for_last_block(&self, cert: &QuorumCertificate2) -> bool { let Some(leaf) = self.saved_leaves.get(&cert.data().leaf_commit) else { return false; }; @@ -916,7 +917,7 @@ impl Consensus { /// Returns true if the given qc is an extended Quorum Certificate /// The Extended Quorum Certificate (eQC) is the third Quorum Certificate formed in three /// consecutive views for the last block in the epoch. - pub fn is_qc_extended(&self, cert: &QuorumCertificate) -> bool { + pub fn is_qc_extended(&self, cert: &QuorumCertificate2) -> bool { if !self.is_qc_for_last_block(cert) { tracing::debug!("High QC is not for the last block in the epoch."); return false; @@ -968,7 +969,7 @@ impl Consensus { /// Return true if the given Quorum Certificate takes part in forming an eQC, i.e. /// it is one of the 3-chain certificates but not the eQC itself - pub fn is_qc_forming_eqc(&self, cert: &QuorumCertificate) -> bool { + pub fn is_qc_forming_eqc(&self, cert: &QuorumCertificate2) -> bool { self.is_qc_for_last_block(cert) && !self.is_qc_extended(cert) } diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index f8ac0f54bd..b9e278db1e 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -33,8 +33,8 @@ use vec1::Vec1; use crate::{ message::{Proposal, UpgradeLock}, simple_certificate::{ - convert_quorum_certificate, QuorumCertificate, QuorumCertificate2, TimeoutCertificate, - UpgradeCertificate, ViewSyncFinalizeCertificate2, + QuorumCertificate, QuorumCertificate2, TimeoutCertificate, UpgradeCertificate, + ViewSyncFinalizeCertificate2, }, simple_vote::{QuorumData, UpgradeProposalData, VersionedVoteData}, traits::{ @@ -406,7 +406,7 @@ impl From> for QuorumProposal2 { Self { block_header: quorum_proposal.block_header, view_number: quorum_proposal.view_number, - justify_qc: convert_quorum_certificate(quorum_proposal.justify_qc), + justify_qc: quorum_proposal.justify_qc.to_qc2(), upgrade_certificate: quorum_proposal.upgrade_certificate, proposal_certificate: quorum_proposal.proposal_certificate, drb_seed: [0; 96], @@ -415,13 +415,25 @@ impl From> for QuorumProposal2 { } } +impl From> for QuorumProposal { + fn from(quorum_proposal: QuorumProposal2) -> Self { + Self { + block_header: quorum_proposal.block_header, + view_number: quorum_proposal.view_number, + justify_qc: quorum_proposal.justify_qc.to_qc(), + upgrade_certificate: quorum_proposal.upgrade_certificate, + proposal_certificate: quorum_proposal.proposal_certificate, + } + } +} + impl From> for Leaf2 { fn from(leaf: Leaf) -> Self { let bytes: [u8; 32] = leaf.parent_commitment.into(); Self { view_number: leaf.view_number, - justify_qc: convert_quorum_certificate(leaf.justify_qc), + justify_qc: leaf.justify_qc.to_qc2(), parent_commitment: Commitment::from_raw(bytes), block_header: leaf.block_header, upgrade_certificate: leaf.upgrade_certificate, @@ -578,6 +590,87 @@ impl Leaf2 { pub fn block_header_mut(&mut self) -> &mut ::BlockHeader { &mut self.block_header } + /// Fill this leaf with the block payload. + /// + /// # Errors + /// + /// Fails if the payload commitment doesn't match `self.block_header.payload_commitment()` + /// or if the transactions are of invalid length + pub fn fill_block_payload( + &mut self, + block_payload: TYPES::BlockPayload, + num_storage_nodes: usize, + ) -> std::result::Result<(), BlockError> { + let encoded_txns = block_payload.encode(); + let commitment = vid_commitment(&encoded_txns, num_storage_nodes); + if commitment != self.block_header.payload_commitment() { + return Err(BlockError::InconsistentPayloadCommitment); + } + self.block_payload = Some(block_payload); + Ok(()) + } + + /// Take the block payload from the leaf and return it if it is present + pub fn unfill_block_payload(&mut self) -> Option { + self.block_payload.take() + } + + /// Fill this leaf with the block payload, without checking + /// header and payload consistency + pub fn fill_block_payload_unchecked(&mut self, block_payload: TYPES::BlockPayload) { + self.block_payload = Some(block_payload); + } + + /// Optional block payload. + pub fn block_payload(&self) -> Option { + self.block_payload.clone() + } + + /// A commitment to the block payload contained in this leaf. + pub fn payload_commitment(&self) -> VidCommitment { + self.block_header().payload_commitment() + } + + /// Validate that a leaf has the right upgrade certificate to be the immediate child of another leaf + /// + /// This may not be a complete function. Please double-check that it performs the checks you expect before subtituting validation logic with it. + /// + /// # Errors + /// Returns an error if the certificates are not identical, or that when we no longer see a + /// cert, it's for the right reason. + pub async fn extends_upgrade( + &self, + parent: &Self, + decided_upgrade_certificate: &Arc>>>, + ) -> Result<()> { + match (self.upgrade_certificate(), parent.upgrade_certificate()) { + // Easiest cases are: + // - no upgrade certificate on either: this is the most common case, and is always fine. + // - if the parent didn't have a certificate, but we see one now, it just means that we have begun an upgrade: again, this is always fine. + (None | Some(_), None) => {} + // If we no longer see a cert, we have to make sure that we either: + // - no longer care because we have passed new_version_first_view, or + // - no longer care because we have passed `decide_by` without deciding the certificate. + (None, Some(parent_cert)) => { + let decided_upgrade_certificate_read = decided_upgrade_certificate.read().await; + ensure!(self.view_number() > parent_cert.data.new_version_first_view + || (self.view_number() > parent_cert.data.decide_by && decided_upgrade_certificate_read.is_none()), + "The new leaf is missing an upgrade certificate that was present in its parent, and should still be live." + ); + } + // If we both have a certificate, they should be identical. + // Technically, this prevents us from initiating a new upgrade in the view immediately following an upgrade. + // I think this is a fairly lax restriction. + (Some(cert), Some(parent_cert)) => { + ensure!(cert == parent_cert, "The new leaf does not extend the parent leaf, because it has attached a different upgrade certificate."); + } + } + + // This check should be added once we sort out the genesis leaf/justify_qc issue. + // ensure!(self.parent_commitment() == parent_leaf.commit(), "The commitment of the parent leaf does not match the specified parent commitment."); + + Ok(()) + } } impl Committable for Leaf2 { @@ -622,6 +715,16 @@ impl Hash for Leaf { } } +impl Hash for Leaf2 { + fn hash(&self, state: &mut H) { + self.commit().hash(state); + self.view_number.hash(state); + self.justify_qc.hash(state); + self.parent_commitment.hash(state); + self.block_header.hash(state); + } +} + impl Display for Leaf { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( @@ -849,6 +952,22 @@ where TYPES::ValidatedState::create_random_transaction(None, rng, padding) } } +impl TestableLeaf for Leaf2 +where + TYPES::ValidatedState: TestableState, + TYPES::BlockPayload: TestableBlock, +{ + type NodeType = TYPES; + + fn create_random_transaction( + &self, + rng: &mut dyn rand::RngCore, + padding: u64, + ) -> <::BlockPayload as BlockPayload>::Transaction + { + TYPES::ValidatedState::create_random_transaction(None, rng, padding) + } +} /// Fake the thing a genesis block points to. Needed to avoid infinite recursion #[must_use] pub fn fake_commitment() -> Commitment { diff --git a/crates/types/src/error.rs b/crates/types/src/error.rs index f6c25e376f..6c4e29611e 100644 --- a/crates/types/src/error.rs +++ b/crates/types/src/error.rs @@ -13,7 +13,7 @@ use committable::Commitment; use serde::{Deserialize, Serialize}; use thiserror::Error; -use crate::{data::Leaf, traits::node_implementation::NodeType}; +use crate::{data::Leaf2, traits::node_implementation::NodeType}; /// Error type for `HotShot` #[derive(Debug, Error)] @@ -25,7 +25,7 @@ pub enum HotShotError { /// Leaf was not present in storage #[error("Missing leaf with commitment: {0}")] - MissingLeaf(Commitment>), + MissingLeaf(Commitment>), /// Failed to serialize data #[error("Failed to serialize: {0}")] diff --git a/crates/types/src/event.rs b/crates/types/src/event.rs index f9d48779e8..da8e71ab85 100644 --- a/crates/types/src/event.rs +++ b/crates/types/src/event.rs @@ -11,10 +11,10 @@ use std::sync::Arc; use serde::{Deserialize, Serialize}; use crate::{ - data::{DaProposal, Leaf, QuorumProposal, UpgradeProposal, VidDisperseShare}, + data::{DaProposal, Leaf2, QuorumProposal2, UpgradeProposal, VidDisperseShare}, error::HotShotError, message::Proposal, - simple_certificate::QuorumCertificate, + simple_certificate::QuorumCertificate2, traits::{node_implementation::NodeType, ValidatedState}, }; /// A status event emitted by a `HotShot` instance @@ -35,7 +35,7 @@ pub struct Event { #[serde(bound(deserialize = "TYPES: NodeType"))] pub struct LeafInfo { /// Decided leaf. - pub leaf: Leaf, + pub leaf: Leaf2, /// Validated state. pub state: Arc<::ValidatedState>, /// Optional application-specific state delta. @@ -47,7 +47,7 @@ pub struct LeafInfo { impl LeafInfo { /// Constructor. pub fn new( - leaf: Leaf, + leaf: Leaf2, state: Arc<::ValidatedState>, delta: Option::ValidatedState as ValidatedState>::Delta>>, vid_share: Option>, @@ -121,7 +121,7 @@ pub enum EventType { /// /// Note that the QC for each additional leaf in the chain can be obtained from the leaf /// before it using - qc: Arc>, + qc: Arc>, /// Optional information of the number of transactions in the block, for logging purposes. block_size: Option, }, @@ -158,7 +158,7 @@ pub enum EventType { /// or submitted to the network by us QuorumProposal { /// Contents of the proposal - proposal: Proposal>, + proposal: Proposal>, /// Public key of the leader submitting the proposal sender: TYPES::SignatureKey, }, diff --git a/crates/types/src/message.rs b/crates/types/src/message.rs index 7cf53c69d4..4b6ee6a03a 100644 --- a/crates/types/src/message.rs +++ b/crates/types/src/message.rs @@ -17,6 +17,7 @@ use std::{ use async_lock::RwLock; use cdn_proto::util::mnemonic; +use committable::Committable; use derivative::Derivative; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use utils::anytrace::*; @@ -26,7 +27,9 @@ use vbs::{ }; use crate::{ - data::{DaProposal, Leaf, QuorumProposal, UpgradeProposal, VidDisperseShare}, + data::{ + DaProposal, Leaf, Leaf2, QuorumProposal, QuorumProposal2, UpgradeProposal, VidDisperseShare, + }, request_response::ProposalRequestPayload, simple_certificate::{ DaCertificate, UpgradeCertificate, ViewSyncCommitCertificate2, @@ -408,6 +411,32 @@ where } } +impl Proposal> +where + TYPES: NodeType, +{ + /// Checks that the signature of the quorum proposal is valid. + /// # Errors + /// Returns an error when the proposal signature is invalid. + pub async fn validate_signature( + &self, + quorum_membership: &TYPES::Membership, + epoch: TYPES::Epoch, + _upgrade_lock: &UpgradeLock, + ) -> Result<()> { + let view_number = self.data.view_number(); + let view_leader_key = quorum_membership.leader(view_number, epoch)?; + let proposed_leaf = Leaf2::from_quorum_proposal(&self.data); + + ensure!( + view_leader_key.validate(&self.signature, proposed_leaf.commit().as_ref()), + "Proposal signature is invalid." + ); + + Ok(()) + } +} + #[derive(Clone, Debug)] /// A lock for an upgrade certificate decided by HotShot, which doubles as `PhantomData` for an instance of the `Versions` trait. pub struct UpgradeLock { diff --git a/crates/types/src/simple_certificate.rs b/crates/types/src/simple_certificate.rs index 46684a8d77..9c75b294a3 100644 --- a/crates/types/src/simple_certificate.rs +++ b/crates/types/src/simple_certificate.rs @@ -249,24 +249,45 @@ impl UpgradeCertificate { } } -/// Convert a QC into a QC2 -pub fn convert_quorum_certificate( - quorum_certificate: QuorumCertificate, -) -> QuorumCertificate2 { - let bytes: [u8; 32] = quorum_certificate.data.leaf_commit.into(); - let data = QuorumData2 { - leaf_commit: Commitment::from_raw(bytes), - }; +impl QuorumCertificate { + /// Convert a `QuorumCertificate` into a `QuorumCertificate2` + pub fn to_qc2(self) -> QuorumCertificate2 { + let bytes: [u8; 32] = self.data.leaf_commit.into(); + let data = QuorumData2 { + leaf_commit: Commitment::from_raw(bytes), + }; + + let bytes: [u8; 32] = self.vote_commitment.into(); + let vote_commitment = Commitment::from_raw(bytes); - let bytes: [u8; 32] = quorum_certificate.vote_commitment.into(); - let vote_commitment = Commitment::from_raw(bytes); + SimpleCertificate { + data, + vote_commitment, + view_number: self.view_number, + signatures: self.signatures.clone(), + _pd: PhantomData, + } + } +} - SimpleCertificate { - data, - vote_commitment, - view_number: quorum_certificate.view_number, - signatures: quorum_certificate.signatures.clone(), - _pd: PhantomData, +impl QuorumCertificate2 { + /// Convert a `QuorumCertificate2` into a `QuorumCertificate` + pub fn to_qc(self) -> QuorumCertificate { + let bytes: [u8; 32] = self.data.leaf_commit.into(); + let data = QuorumData { + leaf_commit: Commitment::from_raw(bytes), + }; + + let bytes: [u8; 32] = self.vote_commitment.into(); + let vote_commitment = Commitment::from_raw(bytes); + + SimpleCertificate { + data, + vote_commitment, + view_number: self.view_number, + signatures: self.signatures.clone(), + _pd: PhantomData, + } } } diff --git a/crates/types/src/simple_vote.rs b/crates/types/src/simple_vote.rs index bcb7e0baab..013653a959 100644 --- a/crates/types/src/simple_vote.rs +++ b/crates/types/src/simple_vote.rs @@ -346,6 +346,44 @@ impl QuorumVote { + /// Convert a `QuorumVote` to a `QuorumVote2` + pub fn to_vote2(self) -> QuorumVote2 { + let bytes: [u8; 32] = self.data.leaf_commit.into(); + + let signature = self.signature; + let data = QuorumData2 { + leaf_commit: Commitment::from_raw(bytes), + }; + let view_number = self.view_number; + + SimpleVote { + signature, + data, + view_number, + } + } +} + +impl QuorumVote2 { + /// Convert a `QuorumVote2` to a `QuorumVote` + pub fn to_vote(self) -> QuorumVote { + let bytes: [u8; 32] = self.data.leaf_commit.into(); + + let signature = self.signature; + let data = QuorumData { + leaf_commit: Commitment::from_raw(bytes), + }; + let view_number = self.view_number; + + SimpleVote { + signature, + data, + view_number, + } + } +} + // Type aliases for simple use of all the main votes. We should never see `SimpleVote` outside this file /// Quorum vote Alias pub type QuorumVote = SimpleVote>; diff --git a/crates/types/src/traits/block_contents.rs b/crates/types/src/traits/block_contents.rs index de9968488c..3ce73eade7 100644 --- a/crates/types/src/traits/block_contents.rs +++ b/crates/types/src/traits/block_contents.rs @@ -25,7 +25,7 @@ use vbs::version::Version; use super::signature_key::BuilderSignatureKey; use crate::{ - data::Leaf, + data::Leaf2, traits::{node_implementation::NodeType, states::InstanceState, ValidatedState}, utils::BuilderCommitment, vid::{vid_scheme, VidCommitment, VidCommon, VidSchemeType}, @@ -199,7 +199,7 @@ pub trait BlockHeader: fn new_legacy( parent_state: &TYPES::ValidatedState, instance_state: &>::Instance, - parent_leaf: &Leaf, + parent_leaf: &Leaf2, payload_commitment: VidCommitment, builder_commitment: BuilderCommitment, metadata: >::Metadata, @@ -215,7 +215,7 @@ pub trait BlockHeader: fn new_marketplace( parent_state: &TYPES::ValidatedState, instance_state: &>::Instance, - parent_leaf: &Leaf, + parent_leaf: &Leaf2, payload_commitment: VidCommitment, builder_commitment: BuilderCommitment, metadata: >::Metadata, diff --git a/crates/types/src/traits/node_implementation.rs b/crates/types/src/traits/node_implementation.rs index d05652b783..32093b8714 100644 --- a/crates/types/src/traits/node_implementation.rs +++ b/crates/types/src/traits/node_implementation.rs @@ -35,7 +35,7 @@ use super::{ ValidatedState, }; use crate::{ - data::{Leaf, TestableLeaf}, + data::{Leaf2, TestableLeaf}, traits::{ election::Membership, signature_key::SignatureKey, states::InstanceState, BlockPayload, }, @@ -87,7 +87,7 @@ pub trait TestableNodeImplementation: NodeImplementation /// otherwise panics /// `padding` is the bytes of padding to add to the transaction fn leaf_create_random_transaction( - leaf: &Leaf, + leaf: &Leaf2, rng: &mut dyn rand::RngCore, padding: u64, ) -> >::Transaction; @@ -126,11 +126,11 @@ where } fn leaf_create_random_transaction( - leaf: &Leaf, + leaf: &Leaf2, rng: &mut dyn rand::RngCore, padding: u64, ) -> >::Transaction { - Leaf::create_random_transaction(leaf, rng, padding) + Leaf2::create_random_transaction(leaf, rng, padding) } fn block_genesis() -> TYPES::BlockPayload { diff --git a/crates/types/src/traits/states.rs b/crates/types/src/traits/states.rs index 9c951bb8d0..a19bf1f453 100644 --- a/crates/types/src/traits/states.rs +++ b/crates/types/src/traits/states.rs @@ -17,7 +17,7 @@ use vbs::version::Version; use super::block_contents::TestableBlock; use crate::{ - data::Leaf, + data::Leaf2, traits::{ node_implementation::{ConsensusTime, NodeType}, BlockPayload, @@ -69,7 +69,7 @@ pub trait ValidatedState: fn validate_and_apply_header( &self, instance: &Self::Instance, - parent_leaf: &Leaf, + parent_leaf: &Leaf2, proposed_header: &TYPES::BlockHeader, vid_common: VidCommon, version: Version, diff --git a/crates/types/src/traits/storage.rs b/crates/types/src/traits/storage.rs index 6c03e50bde..4ce9fdeac4 100644 --- a/crates/types/src/traits/storage.rs +++ b/crates/types/src/traits/storage.rs @@ -21,7 +21,7 @@ use crate::{ data::{DaProposal, Leaf, Leaf2, QuorumProposal, QuorumProposal2, VidDisperseShare}, event::HotShotAction, message::Proposal, - simple_certificate::{QuorumCertificate, UpgradeCertificate}, + simple_certificate::{QuorumCertificate, QuorumCertificate2, UpgradeCertificate}, vid::VidSchemeType, }; @@ -50,6 +50,8 @@ pub trait Storage: Send + Sync + Clone { async fn record_action(&self, view: TYPES::View, action: HotShotAction) -> Result<()>; /// Update the current high QC in storage. async fn update_high_qc(&self, high_qc: QuorumCertificate) -> Result<()>; + /// Update the current high QC in storage. + async fn update_high_qc2(&self, high_qc: QuorumCertificate2) -> Result<()>; /// Update the currently undecided state of consensus. This includes the undecided leaf chain, /// and the undecided state. async fn update_undecided_state( diff --git a/crates/types/src/utils.rs b/crates/types/src/utils.rs index e3d19a8286..e5e5fd1be0 100644 --- a/crates/types/src/utils.rs +++ b/crates/types/src/utils.rs @@ -24,7 +24,7 @@ use tagged_base64::tagged; use typenum::Unsigned; use crate::{ - data::Leaf, + data::Leaf2, traits::{node_implementation::NodeType, ValidatedState}, vid::VidCommitment, }; @@ -70,7 +70,7 @@ impl Clone for ViewInner { } } /// The hash of a leaf. -type LeafCommitment = Commitment>; +type LeafCommitment = Commitment>; /// Optional validated state and state delta. pub type StateAndDelta = ( From 922d07394b7fcc6210f1cbb57b27465b33642cb3 Mon Sep 17 00:00:00 2001 From: ss-es <155648797+ss-es@users.noreply.github.com> Date: Wed, 13 Nov 2024 13:59:16 -0500 Subject: [PATCH 11/15] fix? --- crates/hotshot/src/lib.rs | 7 ++--- crates/hotshot/src/types/handle.rs | 6 +--- crates/task-impls/src/helpers.rs | 28 +++++-------------- .../src/quorum_proposal/handlers.rs | 6 ++-- crates/task-impls/src/quorum_proposal/mod.rs | 8 +++--- .../src/quorum_proposal_recv/handlers.rs | 5 +--- crates/task-impls/src/quorum_vote/handlers.rs | 14 ++++------ crates/testing/src/test_runner.rs | 1 - .../tests_1/quorum_proposal_recv_task.rs | 2 -- .../tests/tests_1/quorum_proposal_task.rs | 6 ---- .../testing/tests/tests_1/quorum_vote_task.rs | 4 --- .../tests_1/upgrade_task_with_proposal.rs | 4 --- .../tests/tests_1/upgrade_task_with_vote.rs | 2 -- .../tests/tests_1/vote_dependency_handle.rs | 2 -- crates/types/src/consensus.rs | 15 ++++------ crates/types/src/event.rs | 1 + crates/types/src/message.rs | 3 +- 17 files changed, 31 insertions(+), 83 deletions(-) diff --git a/crates/hotshot/src/lib.rs b/crates/hotshot/src/lib.rs index a53a054835..9f6d8591e5 100644 --- a/crates/hotshot/src/lib.rs +++ b/crates/hotshot/src/lib.rs @@ -245,7 +245,6 @@ impl, V: Versions> SystemContext`] with the given configuration options. @@ -256,7 +255,7 @@ impl, V: Versions> SystemContext::PrivateKey, nonce: u64, @@ -382,7 +381,7 @@ impl, V: Versions> SystemContext, V: Versions> SystemContext + 'static, V: Versions> )?; let mem = self.memberships.quorum_membership.clone(); - let upgrade_lock = self.hotshot.upgrade_lock.clone(); let receiver = self.internal_event_stream.1.activate_cloned(); let sender = self.internal_event_stream.0.clone(); Ok(async move { @@ -189,10 +188,7 @@ impl + 'static, V: Versions> if let HotShotEvent::QuorumProposalResponseRecv(quorum_proposal) = hs_event.as_ref() { // Make sure that the quorum_proposal is valid - if let Err(err) = quorum_proposal - .validate_signature(&mem, epoch, &upgrade_lock) - .await - { + if let Err(err) = quorum_proposal.validate_signature(&mem, epoch) { tracing::warn!("Invalid Proposal Received after Request. Err {:?}", err); continue; } diff --git a/crates/task-impls/src/helpers.rs b/crates/task-impls/src/helpers.rs index 2392f50490..e456ee6032 100644 --- a/crates/task-impls/src/helpers.rs +++ b/crates/task-impls/src/helpers.rs @@ -104,7 +104,7 @@ pub(crate) async fn fetch_proposal( hs_event.as_ref() { // Make sure that the quorum_proposal is valid - if quorum_proposal.validate_signature(&mem, cur_epoch, upgrade_lock).await.is_ok() { + if quorum_proposal.validate_signature(&mem, cur_epoch).is_ok() { proposal = Some(quorum_proposal.clone()); } @@ -136,10 +136,7 @@ pub(crate) async fn fetch_proposal( >::from_header(&proposal.data.block_header), ); - if let Err(e) = consensus_writer - .update_leaf(leaf.clone(), Arc::clone(&state), None, upgrade_lock) - .await - { + if let Err(e) = consensus_writer.update_leaf(leaf.clone(), Arc::clone(&state), None) { tracing::trace!("{e:?}"); } let view = View { @@ -440,15 +437,7 @@ pub async fn validate_proposal_safety_and_liveness< { let mut consensus_writer = validation_info.consensus.write().await; - if let Err(e) = consensus_writer - .update_leaf( - proposed_leaf.clone(), - state, - None, - &validation_info.upgrade_lock, - ) - .await - { + if let Err(e) = consensus_writer.update_leaf(proposed_leaf.clone(), state, None) { tracing::trace!("{e:?}"); } @@ -563,13 +552,10 @@ pub(crate) async fn validate_proposal_view_and_certs< ); // Validate the proposal's signature. This should also catch if the leaf_commitment does not equal our calculated parent commitment - proposal - .validate_signature( - &validation_info.quorum_membership, - validation_info.cur_epoch, - &validation_info.upgrade_lock, - ) - .await?; + proposal.validate_signature( + &validation_info.quorum_membership, + validation_info.cur_epoch, + )?; // Verify a timeout certificate OR a view sync certificate exists and is valid. if proposal.data.justify_qc.view_number() != view_number - 1 { diff --git a/crates/task-impls/src/quorum_proposal/handlers.rs b/crates/task-impls/src/quorum_proposal/handlers.rs index 385a8f0475..5eca34fec8 100644 --- a/crates/task-impls/src/quorum_proposal/handlers.rs +++ b/crates/task-impls/src/quorum_proposal/handlers.rs @@ -40,13 +40,13 @@ pub(crate) enum ProposalDependency { /// For the `SendPayloadCommitmentAndMetadata` event. PayloadAndMetadata, - /// For the `QcFormed` event. + /// For the `Qc2Formed` event. Qc, /// For the `ViewSyncFinalizeCertificate2Recv` event. ViewSyncCert, - /// For the `QcFormed` event timeout branch. + /// For the `Qc2Formed` event timeout branch. TimeoutCert, /// For the `QuroumProposalRecv` event. @@ -288,7 +288,7 @@ impl HandleDepOutput for ProposalDependencyHandle< auction_result: auction_result.clone(), }); } - HotShotEvent::QcFormed(cert) => match cert { + HotShotEvent::Qc2Formed(cert) => match cert { either::Right(timeout) => { timeout_certificate = Some(timeout.clone()); } diff --git a/crates/task-impls/src/quorum_proposal/mod.rs b/crates/task-impls/src/quorum_proposal/mod.rs index e5c114556e..b1ee986193 100644 --- a/crates/task-impls/src/quorum_proposal/mod.rs +++ b/crates/task-impls/src/quorum_proposal/mod.rs @@ -110,14 +110,14 @@ impl, V: Versions> let event = event.as_ref(); let event_view = match dependency_type { ProposalDependency::Qc => { - if let HotShotEvent::QcFormed(either::Left(qc)) = event { + if let HotShotEvent::Qc2Formed(either::Left(qc)) = event { qc.view_number() + 1 } else { return false; } } ProposalDependency::TimeoutCert => { - if let HotShotEvent::QcFormed(either::Right(timeout)) = event { + if let HotShotEvent::Qc2Formed(either::Right(timeout)) = event { timeout.view_number() + 1 } else { return false; @@ -224,7 +224,7 @@ impl, V: Versions> HotShotEvent::QuorumProposalPreliminarilyValidated(..) => { proposal_dependency.mark_as_completed(event); } - HotShotEvent::QcFormed(quorum_certificate) => match quorum_certificate { + HotShotEvent::Qc2Formed(quorum_certificate) => match quorum_certificate { Either::Right(_) => { timeout_dependency.mark_as_completed(event); } @@ -248,7 +248,7 @@ impl, V: Versions> // 2. A view sync cert was received. AndDependency::from_deps(vec![view_sync_dependency]), ]; - // 3. A `QcFormed`` event (and `QuorumProposalRecv` event) + // 3. A `Qc2Formed`` event (and `QuorumProposalRecv` event) if *view_number > 1 { secondary_deps.push(AndDependency::from_deps(vec![ qc_dependency, diff --git a/crates/task-impls/src/quorum_proposal_recv/handlers.rs b/crates/task-impls/src/quorum_proposal_recv/handlers.rs index 9a1fb8106e..49edb38deb 100644 --- a/crates/task-impls/src/quorum_proposal_recv/handlers.rs +++ b/crates/task-impls/src/quorum_proposal_recv/handlers.rs @@ -55,10 +55,7 @@ async fn validate_proposal_liveness>::from_header(&proposal.data.block_header), ); - if let Err(e) = consensus_writer - .update_leaf(leaf.clone(), state, None, &validation_info.upgrade_lock) - .await - { + if let Err(e) = consensus_writer.update_leaf(leaf.clone(), state, None) { tracing::trace!("{e:?}"); } diff --git a/crates/task-impls/src/quorum_vote/handlers.rs b/crates/task-impls/src/quorum_vote/handlers.rs index 9abe7ab4f8..85aaec0899 100644 --- a/crates/task-impls/src/quorum_vote/handlers.rs +++ b/crates/task-impls/src/quorum_vote/handlers.rs @@ -241,15 +241,11 @@ pub(crate) async fn update_shared_state< // Now that we've rounded everyone up, we need to update the shared state let mut consensus_writer = consensus.write().await; - if let Err(e) = consensus_writer - .update_leaf( - proposed_leaf.clone(), - Arc::clone(&state), - Some(Arc::clone(&delta)), - &upgrade_lock, - ) - .await - { + if let Err(e) = consensus_writer.update_leaf( + proposed_leaf.clone(), + Arc::clone(&state), + Some(Arc::clone(&delta)), + ) { tracing::trace!("{e:?}"); } diff --git a/crates/testing/src/test_runner.rs b/crates/testing/src/test_runner.rs index 8631badc9d..3460795db0 100644 --- a/crates/testing/src/test_runner.rs +++ b/crates/testing/src/test_runner.rs @@ -647,7 +647,6 @@ where internal_channel, external_channel, ) - .await } } diff --git a/crates/testing/tests/tests_1/quorum_proposal_recv_task.rs b/crates/testing/tests/tests_1/quorum_proposal_recv_task.rs index c55e6921f1..7a1636d9c6 100644 --- a/crates/testing/tests/tests_1/quorum_proposal_recv_task.rs +++ b/crates/testing/tests/tests_1/quorum_proposal_recv_task.rs @@ -80,9 +80,7 @@ async fn test_quorum_proposal_recv_task() { Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } drop(consensus_writer); diff --git a/crates/testing/tests/tests_1/quorum_proposal_task.rs b/crates/testing/tests/tests_1/quorum_proposal_task.rs index 89502896c4..c3b7d2b6cd 100644 --- a/crates/testing/tests/tests_1/quorum_proposal_task.rs +++ b/crates/testing/tests/tests_1/quorum_proposal_task.rs @@ -82,9 +82,7 @@ async fn test_quorum_proposal_task_quorum_proposal_view_1() { Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } @@ -172,9 +170,7 @@ async fn test_quorum_proposal_task_quorum_proposal_view_gt_1() { Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } @@ -514,9 +510,7 @@ async fn test_quorum_proposal_task_liveness_check() { Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } drop(consensus_writer); diff --git a/crates/testing/tests/tests_1/quorum_vote_task.rs b/crates/testing/tests/tests_1/quorum_vote_task.rs index 1eb80cb218..f5c5940b20 100644 --- a/crates/testing/tests/tests_1/quorum_vote_task.rs +++ b/crates/testing/tests/tests_1/quorum_vote_task.rs @@ -67,9 +67,7 @@ async fn test_quorum_vote_task_success() { Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } drop(consensus_writer); @@ -139,9 +137,7 @@ async fn test_quorum_vote_task_miss_dependency() { Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } drop(consensus_writer); diff --git a/crates/testing/tests/tests_1/upgrade_task_with_proposal.rs b/crates/testing/tests/tests_1/upgrade_task_with_proposal.rs index 1e4f123941..0cef44ab81 100644 --- a/crates/testing/tests/tests_1/upgrade_task_with_proposal.rs +++ b/crates/testing/tests/tests_1/upgrade_task_with_proposal.rs @@ -99,9 +99,7 @@ async fn test_upgrade_task_with_proposal() { Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } @@ -120,9 +118,7 @@ async fn test_upgrade_task_with_proposal() { Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } drop(consensus_writer); diff --git a/crates/testing/tests/tests_1/upgrade_task_with_vote.rs b/crates/testing/tests/tests_1/upgrade_task_with_vote.rs index f12c4a9912..cc942451f1 100644 --- a/crates/testing/tests/tests_1/upgrade_task_with_vote.rs +++ b/crates/testing/tests/tests_1/upgrade_task_with_vote.rs @@ -85,9 +85,7 @@ async fn test_upgrade_task_with_vote() { Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } drop(consensus_writer); diff --git a/crates/testing/tests/tests_1/vote_dependency_handle.rs b/crates/testing/tests/tests_1/vote_dependency_handle.rs index df89abf2ab..f22d658d34 100644 --- a/crates/testing/tests/tests_1/vote_dependency_handle.rs +++ b/crates/testing/tests/tests_1/vote_dependency_handle.rs @@ -58,9 +58,7 @@ async fn test_vote_dependency_handle() { Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } drop(consensus_writer); diff --git a/crates/types/src/consensus.rs b/crates/types/src/consensus.rs index c339a2840e..10645526d1 100644 --- a/crates/types/src/consensus.rs +++ b/crates/types/src/consensus.rs @@ -24,12 +24,12 @@ use crate::{ data::{Leaf2, QuorumProposal2, VidDisperse, VidDisperseShare}, error::HotShotError, event::HotShotAction, - message::{Proposal, UpgradeLock}, + message::Proposal, simple_certificate::{DaCertificate, QuorumCertificate2}, traits::{ block_contents::BuilderFee, metrics::{Counter, Gauge, Histogram, Metrics, NoMetrics}, - node_implementation::{ConsensusTime, NodeType, Versions}, + node_implementation::{ConsensusTime, NodeType}, signature_key::SignatureKey, BlockPayload, ValidatedState, }, @@ -610,12 +610,11 @@ impl Consensus { /// # Errors /// Can return an error when the new view contains less information than the exisiting view /// with the same view number. - pub async fn update_leaf( + pub fn update_leaf( &mut self, leaf: Leaf2, state: Arc, delta: Option>::Delta>>, - upgrade_lock: &UpgradeLock, ) -> Result<()> { let view_number = leaf.view_number(); let view = View { @@ -626,7 +625,7 @@ impl Consensus { }, }; self.update_validated_state_map(view_number, view)?; - self.update_saved_leaves(leaf, upgrade_lock).await; + self.update_saved_leaves(leaf); Ok(()) } @@ -665,11 +664,7 @@ impl Consensus { } /// Update the saved leaves with a new leaf. - async fn update_saved_leaves( - &mut self, - leaf: Leaf2, - _upgrade_lock: &UpgradeLock, - ) { + fn update_saved_leaves(&mut self, leaf: Leaf2) { self.saved_leaves.insert(leaf.commit(), leaf); } diff --git a/crates/types/src/event.rs b/crates/types/src/event.rs index da8e71ab85..02d7f07ac2 100644 --- a/crates/types/src/event.rs +++ b/crates/types/src/event.rs @@ -100,6 +100,7 @@ pub mod error_adaptor { #[non_exhaustive] #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(bound(deserialize = "TYPES: NodeType"))] +#[allow(clippy::large_enum_variant)] pub enum EventType { /// A view encountered an error and was interrupted Error { diff --git a/crates/types/src/message.rs b/crates/types/src/message.rs index 4b6ee6a03a..df55d3dc97 100644 --- a/crates/types/src/message.rs +++ b/crates/types/src/message.rs @@ -418,11 +418,10 @@ where /// Checks that the signature of the quorum proposal is valid. /// # Errors /// Returns an error when the proposal signature is invalid. - pub async fn validate_signature( + pub fn validate_signature( &self, quorum_membership: &TYPES::Membership, epoch: TYPES::Epoch, - _upgrade_lock: &UpgradeLock, ) -> Result<()> { let view_number = self.data.view_number(); let view_leader_key = quorum_membership.leader(view_number, epoch)?; From abb7d780ca295dafede49904dfc35ce0c121e13e Mon Sep 17 00:00:00 2001 From: ss-es <155648797+ss-es@users.noreply.github.com> Date: Wed, 13 Nov 2024 14:03:53 -0500 Subject: [PATCH 12/15] rename proposal_certificate --- crates/task-impls/src/helpers.rs | 2 +- crates/testing/src/view_generator.rs | 6 +++--- crates/testing/tests/tests_1/quorum_proposal_task.rs | 4 ++-- crates/types/src/data.rs | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/task-impls/src/helpers.rs b/crates/task-impls/src/helpers.rs index e456ee6032..468dedf14f 100644 --- a/crates/task-impls/src/helpers.rs +++ b/crates/task-impls/src/helpers.rs @@ -560,7 +560,7 @@ pub(crate) async fn validate_proposal_view_and_certs< // Verify a timeout certificate OR a view sync certificate exists and is valid. if proposal.data.justify_qc.view_number() != view_number - 1 { let received_proposal_cert = - proposal.data.proposal_certificate.clone().context(debug!( + proposal.data.view_change_evidence.clone().context(debug!( "Quorum proposal for view {} needed a timeout or view sync certificate, but did not have one", *view_number ))?; diff --git a/crates/testing/src/view_generator.rs b/crates/testing/src/view_generator.rs index afa5a8e229..53a0d74b7e 100644 --- a/crates/testing/src/view_generator.rs +++ b/crates/testing/src/view_generator.rs @@ -148,7 +148,7 @@ impl TestView { .await .to_qc2(), upgrade_certificate: None, - proposal_certificate: None, + view_change_evidence: None, drb_result: [0; 32], drb_seed: [0; 96], }; @@ -359,7 +359,7 @@ impl TestView { None }; - let proposal_certificate = if let Some(tc) = timeout_certificate { + let view_change_evidence = if let Some(tc) = timeout_certificate { Some(ViewChangeEvidence::Timeout(tc)) } else { view_sync_certificate.map(ViewChangeEvidence::ViewSync) @@ -381,7 +381,7 @@ impl TestView { view_number: next_view, justify_qc: quorum_certificate.clone(), upgrade_certificate: upgrade_certificate.clone(), - proposal_certificate, + view_change_evidence, drb_result: [0; 32], drb_seed: [0; 96], }; diff --git a/crates/testing/tests/tests_1/quorum_proposal_task.rs b/crates/testing/tests/tests_1/quorum_proposal_task.rs index c3b7d2b6cd..f522d28f7c 100644 --- a/crates/testing/tests/tests_1/quorum_proposal_task.rs +++ b/crates/testing/tests/tests_1/quorum_proposal_task.rs @@ -345,7 +345,7 @@ async fn test_quorum_proposal_task_qc_timeout() { } // Get the proposal cert out for the view sync input - let cert = match proposals[1].data.proposal_certificate.clone().unwrap() { + let cert = match proposals[1].data.view_change_evidence.clone().unwrap() { ViewChangeEvidence::Timeout(tc) => tc, _ => panic!("Found a View Sync Cert when there should have been a Timeout cert"), }; @@ -435,7 +435,7 @@ async fn test_quorum_proposal_task_view_sync() { } // Get the proposal cert out for the view sync input - let cert = match proposals[1].data.proposal_certificate.clone().unwrap() { + let cert = match proposals[1].data.view_change_evidence.clone().unwrap() { ViewChangeEvidence::ViewSync(vsc) => vsc, _ => panic!("Found a TC when there should have been a view sync cert"), }; diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index 28ce26c455..4fcb48f478 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -390,7 +390,7 @@ pub struct QuorumProposal2 { pub upgrade_certificate: Option>, /// Possible timeout or view sync certificate. If the `justify_qc` is not for a proposal in the immediately preceding view, then either a timeout or view sync certificate must be attached. - pub proposal_certificate: Option>, + pub view_change_evidence: Option>, /// the DRB seed currently being calculated #[serde(with = "serde_bytes")] @@ -408,7 +408,7 @@ impl From> for QuorumProposal2 { view_number: quorum_proposal.view_number, justify_qc: quorum_proposal.justify_qc.to_qc2(), upgrade_certificate: quorum_proposal.upgrade_certificate, - proposal_certificate: quorum_proposal.proposal_certificate, + view_change_evidence: quorum_proposal.proposal_certificate, drb_seed: [0; 96], drb_result: [0; 32], } @@ -422,7 +422,7 @@ impl From> for QuorumProposal { view_number: quorum_proposal.view_number, justify_qc: quorum_proposal.justify_qc.to_qc(), upgrade_certificate: quorum_proposal.upgrade_certificate, - proposal_certificate: quorum_proposal.proposal_certificate, + proposal_certificate: quorum_proposal.view_change_evidence, } } } @@ -1029,7 +1029,7 @@ impl Leaf2 { justify_qc, block_header, upgrade_certificate, - proposal_certificate: _, + view_change_evidence: _, drb_seed: _, drb_result: _, } = quorum_proposal; From 863b169f47cad061d8bed1b16f0e0dee94c0e60c Mon Sep 17 00:00:00 2001 From: ss-es <155648797+ss-es@users.noreply.github.com> Date: Thu, 14 Nov 2024 10:53:33 -0500 Subject: [PATCH 13/15] fix --- crates/types/src/data.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index 4fcb48f478..f257919ce8 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -533,7 +533,7 @@ pub struct Leaf { /// This is the consensus-internal analogous concept to a block, and it contains the block proper, /// as well as the hash of its parent `Leaf`. -#[derive(Serialize, Deserialize, Clone, Debug, Derivative, PartialEq, Eq)] +#[derive(Serialize, Deserialize, Clone, Debug, Derivative, Eq)] #[serde(bound(deserialize = ""))] pub struct Leaf2 { /// CurView from leader when proposing leaf @@ -706,6 +706,25 @@ impl PartialEq for Leaf { } } +impl PartialEq for Leaf2 { + fn eq(&self, other: &Self) -> bool { + let Leaf2 { + view_number, + justify_qc, + parent_commitment, + block_header, + upgrade_certificate, + block_payload: _, + } = self; + + *view_number == other.view_number + && *justify_qc == other.justify_qc + && *parent_commitment == other.parent_commitment + && *block_header == other.block_header + && *upgrade_certificate == other.upgrade_certificate + } +} + impl Hash for Leaf { fn hash(&self, state: &mut H) { self.view_number.hash(state); From 5d189d85a7ac897a481c917fcc03519122158690 Mon Sep 17 00:00:00 2001 From: ss-es <155648797+ss-es@users.noreply.github.com> Date: Thu, 14 Nov 2024 13:41:21 -0500 Subject: [PATCH 14/15] fix --- crates/task-impls/src/events.rs | 6 +++--- crates/task-impls/src/vote_collection.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/task-impls/src/events.rs b/crates/task-impls/src/events.rs index 3af3e55a2e..47aa69d8fb 100644 --- a/crates/task-impls/src/events.rs +++ b/crates/task-impls/src/events.rs @@ -11,8 +11,8 @@ use either::Either; use hotshot_task::task::TaskEvent; use hotshot_types::{ data::{ - DaProposal, Leaf2, PackedBundle, QuorumProposal, QuorumProposal2, UpgradeProposal, - VidDisperse, VidDisperseShare, + DaProposal, Leaf2, PackedBundle, QuorumProposal2, UpgradeProposal, VidDisperse, + VidDisperseShare, }, message::Proposal, request_response::ProposalRequestPayload, @@ -50,7 +50,7 @@ pub struct ProposalMissing { /// View of missing proposal pub view: TYPES::View, /// Channel to send the response back to - pub response_chan: Sender>>>, + pub response_chan: Sender>>>, } impl PartialEq for ProposalMissing { diff --git a/crates/task-impls/src/vote_collection.rs b/crates/task-impls/src/vote_collection.rs index aeac95af98..6023f5cdfb 100644 --- a/crates/task-impls/src/vote_collection.rs +++ b/crates/task-impls/src/vote_collection.rs @@ -414,7 +414,7 @@ impl AggregatableVote, TimeoutCertifi certificate: TimeoutCertificate, _key: &TYPES::SignatureKey, ) -> HotShotEvent { - HotShotEvent::QcFormed(Right(certificate)) + HotShotEvent::Qc2Formed(Right(certificate)) } } From 72b733eb755fe84386a130a4402679500025ec54 Mon Sep 17 00:00:00 2001 From: ss-es <155648797+ss-es@users.noreply.github.com> Date: Fri, 15 Nov 2024 02:20:57 -0500 Subject: [PATCH 15/15] add to leaf2 --- crates/types/src/data.rs | 55 ++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index f257919ce8..b97c84a62c 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -438,6 +438,9 @@ impl From> for Leaf2 { block_header: leaf.block_header, upgrade_certificate: leaf.upgrade_certificate, block_payload: leaf.block_payload, + view_change_evidence: None, + drb_seed: [0; 96], + drb_result: [0; 32], } } } @@ -556,6 +559,17 @@ pub struct Leaf2 { /// /// It may be empty for nodes not in the DA committee. block_payload: Option, + + /// Possible timeout or view sync certificate. If the `justify_qc` is not for a proposal in the immediately preceding view, then either a timeout or view sync certificate must be attached. + pub view_change_evidence: Option>, + + /// the DRB seed currently being calculated + #[serde(with = "serde_bytes")] + pub drb_seed: [u8; 96], + + /// the result of the DRB calculation + #[serde(with = "serde_bytes")] + pub drb_result: [u8; 32], } impl Leaf2 { @@ -675,13 +689,25 @@ impl Leaf2 { impl Committable for Leaf2 { fn commit(&self) -> committable::Commitment { - RawCommitmentBuilder::new("leaf commitment") - .u64_field("view number", *self.view_number) - .field("parent leaf commitment", self.parent_commitment) - .field("block header", self.block_header.commit()) - .field("justify qc", self.justify_qc.commit()) - .optional("upgrade certificate", &self.upgrade_certificate) - .finalize() + if self.drb_seed == [0; 96] && self.drb_result == [0; 32] { + RawCommitmentBuilder::new("leaf commitment") + .u64_field("view number", *self.view_number) + .field("parent leaf commitment", self.parent_commitment) + .field("block header", self.block_header.commit()) + .field("justify qc", self.justify_qc.commit()) + .optional("upgrade certificate", &self.upgrade_certificate) + .finalize() + } else { + RawCommitmentBuilder::new("leaf commitment") + .u64_field("view number", *self.view_number) + .field("parent leaf commitment", self.parent_commitment) + .field("block header", self.block_header.commit()) + .field("justify qc", self.justify_qc.commit()) + .optional("upgrade certificate", &self.upgrade_certificate) + .fixed_size_bytes(&self.drb_seed) + .fixed_size_bytes(&self.drb_result) + .finalize() + } } } @@ -715,6 +741,9 @@ impl PartialEq for Leaf2 { block_header, upgrade_certificate, block_payload: _, + view_change_evidence, + drb_seed, + drb_result, } = self; *view_number == other.view_number @@ -722,6 +751,9 @@ impl PartialEq for Leaf2 { && *parent_commitment == other.parent_commitment && *block_header == other.block_header && *upgrade_certificate == other.upgrade_certificate + && *view_change_evidence == other.view_change_evidence + && *drb_seed == other.drb_seed + && *drb_result == other.drb_result } } @@ -1048,9 +1080,9 @@ impl Leaf2 { justify_qc, block_header, upgrade_certificate, - view_change_evidence: _, - drb_seed: _, - drb_result: _, + view_change_evidence, + drb_seed, + drb_result, } = quorum_proposal; Self { @@ -1060,6 +1092,9 @@ impl Leaf2 { block_header: block_header.clone(), upgrade_certificate: upgrade_certificate.clone(), block_payload: None, + view_change_evidence: view_change_evidence.clone(), + drb_seed: *drb_seed, + drb_result: *drb_result, } } }