From 23b06aa51ee2ab9c13a90c7d5e92f8e085b3eafc Mon Sep 17 00:00:00 2001 From: int88 Date: Thu, 29 Jun 2023 09:39:15 +0000 Subject: [PATCH 01/51] avoid relocking head during builder health check (#4323) ## Issue Addressed #4314 ## Proposed Changes avoid relocking head during builder health check ## Additional Info NA --- beacon_node/beacon_chain/src/beacon_chain.rs | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index ceda7222e69..4aea7bc55fc 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -5699,13 +5699,9 @@ impl BeaconChain { /// Since we are likely calling this during the slot we are going to propose in, don't take into /// account the current slot when accounting for skips. pub fn is_healthy(&self, parent_root: &Hash256) -> Result { + let cached_head = self.canonical_head.cached_head(); // Check if the merge has been finalized. - if let Some(finalized_hash) = self - .canonical_head - .cached_head() - .forkchoice_update_parameters() - .finalized_hash - { + if let Some(finalized_hash) = cached_head.forkchoice_update_parameters().finalized_hash { if ExecutionBlockHash::zero() == finalized_hash { return Ok(ChainHealth::PreMerge); } @@ -5732,17 +5728,13 @@ impl BeaconChain { // Check slots at the head of the chain. let prev_slot = current_slot.saturating_sub(Slot::new(1)); - let head_skips = prev_slot.saturating_sub(self.canonical_head.cached_head().head_slot()); + let head_skips = prev_slot.saturating_sub(cached_head.head_slot()); let head_skips_check = head_skips.as_usize() <= self.config.builder_fallback_skips; // Check if finalization is advancing. let current_epoch = current_slot.epoch(T::EthSpec::slots_per_epoch()); - let epochs_since_finalization = current_epoch.saturating_sub( - self.canonical_head - .cached_head() - .finalized_checkpoint() - .epoch, - ); + let epochs_since_finalization = + current_epoch.saturating_sub(cached_head.finalized_checkpoint().epoch); let finalization_check = epochs_since_finalization.as_usize() <= self.config.builder_fallback_epochs_since_finalization; From 1aff082eeaf7ccb66e06153a6db24bb530a4e2af Mon Sep 17 00:00:00 2001 From: Jack McPherson Date: Thu, 29 Jun 2023 12:02:38 +0000 Subject: [PATCH 02/51] Add broadcast validation routes to Beacon Node HTTP API (#4316) ## Issue Addressed - #4293 - #4264 ## Proposed Changes *Changes largely follow those suggested in the main issue*. - Add new routes to HTTP API - `post_beacon_blocks_v2` - `post_blinded_beacon_blocks_v2` - Add new routes to `BeaconNodeHttpClient` - `post_beacon_blocks_v2` - `post_blinded_beacon_blocks_v2` - Define new Eth2 common types - `BroadcastValidation`, enum representing the level of validation to apply to blocks prior to broadcast - `BroadcastValidationQuery`, the corresponding HTTP query string type for the above type - ~~Define `_checked` variants of both `publish_block` and `publish_blinded_block` that enforce a validation level at a type level~~ - Add interactive tests to the `bn_http_api_tests` test target covering each validation level (to their own test module, `broadcast_validation_tests`) - `beacon/blocks` - `broadcast_validation=gossip` - Invalid (400) - Full Pass (200) - Partial Pass (202) - `broadcast_validation=consensus` - Invalid (400) - Only gossip (400) - Only consensus pass (i.e., equivocates) (200) - Full pass (200) - `broadcast_validation=consensus_and_equivocation` - Invalid (400) - Invalid due to early equivocation (400) - Only gossip (400) - Only consensus (400) - Pass (200) - `beacon/blinded_blocks` - `broadcast_validation=gossip` - Invalid (400) - Full Pass (200) - Partial Pass (202) - `broadcast_validation=consensus` - Invalid (400) - Only gossip (400) - ~~Only consensus pass (i.e., equivocates) (200)~~ - Full pass (200) - `broadcast_validation=consensus_and_equivocation` - Invalid (400) - Invalid due to early equivocation (400) - Only gossip (400) - Only consensus (400) - Pass (200) - Add a new trait, `IntoGossipVerifiedBlock`, which allows type-level guarantees to be made as to gossip validity - Modify the structure of the `ObservedBlockProducers` cache from a `(slot, validator_index)` mapping to a `((slot, validator_index), block_root)` mapping - Modify `ObservedBlockProducers::proposer_has_been_observed` to return a `SeenBlock` rather than a boolean on success - Punish gossip peer (low) for submitting equivocating blocks - Rename `BlockError::SlashablePublish` to `BlockError::SlashableProposal` ## Additional Info This PR contains changes that directly modify how blocks are verified within the client. For more context, consult [comments in-thread](https://github.com/sigp/lighthouse/pull/4316#discussion_r1234724202). Co-authored-by: Michael Sproul --- beacon_node/beacon_chain/src/beacon_chain.rs | 5 +- .../beacon_chain/src/block_verification.rs | 81 +- beacon_node/beacon_chain/src/builder.rs | 1 - beacon_node/beacon_chain/src/errors.rs | 1 + beacon_node/beacon_chain/src/lib.rs | 2 +- .../src/observed_block_producers.rs | 225 ++- beacon_node/beacon_chain/src/test_utils.rs | 17 +- .../beacon_chain/tests/block_verification.rs | 19 +- .../tests/payload_invalidation.rs | 14 +- beacon_node/beacon_chain/tests/store_tests.rs | 1 + beacon_node/beacon_chain/tests/tests.rs | 1 + beacon_node/http_api/src/lib.rs | 101 +- beacon_node/http_api/src/publish_blocks.rs | 201 ++- .../tests/broadcast_validation_tests.rs | 1270 +++++++++++++++++ beacon_node/http_api/tests/main.rs | 1 + beacon_node/http_api/tests/tests.rs | 23 +- .../beacon_processor/worker/gossip_methods.rs | 22 +- .../beacon_processor/worker/sync_methods.rs | 24 +- common/eth2/src/lib.rs | 90 ++ common/eth2/src/types.rs | 46 +- consensus/types/src/beacon_block_body.rs | 2 +- testing/ef_tests/src/cases/fork_choice.rs | 1 + 22 files changed, 1965 insertions(+), 183 deletions(-) create mode 100644 beacon_node/http_api/tests/broadcast_validation_tests.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 4aea7bc55fc..772e4c1529e 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2578,6 +2578,7 @@ impl BeaconChain { signature_verified_block.block_root(), signature_verified_block, notify_execution_layer, + || Ok(()), ) .await { @@ -2666,6 +2667,7 @@ impl BeaconChain { block_root: Hash256, unverified_block: B, notify_execution_layer: NotifyExecutionLayer, + publish_fn: impl FnOnce() -> Result<(), BlockError> + Send + 'static, ) -> Result> { // Start the Prometheus timer. let _full_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_TIMES); @@ -2684,6 +2686,7 @@ impl BeaconChain { &chain, notify_execution_layer, )?; + publish_fn()?; chain .import_execution_pending_block(execution_pending) .await @@ -2725,7 +2728,7 @@ impl BeaconChain { } // The block failed verification. Err(other) => { - trace!( + debug!( self.log, "Beacon block rejected"; "reason" => other.to_string(), diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 3cb8fbdb524..492f492521e 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -52,6 +52,7 @@ use crate::execution_payload::{ is_optimistic_candidate_block, validate_execution_payload_for_gossip, validate_merge_block, AllowOptimisticImport, NotifyExecutionLayer, PayloadNotifier, }; +use crate::observed_block_producers::SeenBlock; use crate::snapshot_cache::PreProcessingSnapshot; use crate::validator_monitor::HISTORIC_EPOCHS as VALIDATOR_MONITOR_HISTORIC_EPOCHS; use crate::validator_pubkey_cache::ValidatorPubkeyCache; @@ -181,13 +182,6 @@ pub enum BlockError { /// /// The block is valid and we have already imported a block with this hash. BlockIsAlreadyKnown, - /// A block for this proposer and slot has already been observed. - /// - /// ## Peer scoring - /// - /// The `proposer` has already proposed a block at this slot. The existing block may or may not - /// be equal to the given block. - RepeatProposal { proposer: u64, slot: Slot }, /// The block slot exceeds the MAXIMUM_BLOCK_SLOT_NUMBER. /// /// ## Peer scoring @@ -283,6 +277,13 @@ pub enum BlockError { /// problems to worry about than losing peers, and we're doing the network a favour by /// disconnecting. ParentExecutionPayloadInvalid { parent_root: Hash256 }, + /// The block is a slashable equivocation from the proposer. + /// + /// ## Peer scoring + /// + /// Honest peers shouldn't forward more than 1 equivocating block from the same proposer, so + /// we penalise them with a mid-tolerance error. + Slashable, } /// Returned when block validation failed due to some issue verifying @@ -631,6 +632,40 @@ pub struct ExecutionPendingBlock { pub payload_verification_handle: PayloadVerificationHandle, } +pub trait IntoGossipVerifiedBlock: Sized { + fn into_gossip_verified_block( + self, + chain: &BeaconChain, + ) -> Result, BlockError>; + fn inner(&self) -> Arc>; +} + +impl IntoGossipVerifiedBlock for GossipVerifiedBlock { + fn into_gossip_verified_block( + self, + _chain: &BeaconChain, + ) -> Result, BlockError> { + Ok(self) + } + + fn inner(&self) -> Arc> { + self.block.clone() + } +} + +impl IntoGossipVerifiedBlock for Arc> { + fn into_gossip_verified_block( + self, + chain: &BeaconChain, + ) -> Result, BlockError> { + GossipVerifiedBlock::new(self, chain) + } + + fn inner(&self) -> Arc> { + self.clone() + } +} + /// Implemented on types that can be converted into a `ExecutionPendingBlock`. /// /// Used to allow functions to accept blocks at various stages of verification. @@ -727,19 +762,6 @@ impl GossipVerifiedBlock { return Err(BlockError::BlockIsAlreadyKnown); } - // Check that we have not already received a block with a valid signature for this slot. - if chain - .observed_block_producers - .read() - .proposer_has_been_observed(block.message()) - .map_err(|e| BlockError::BeaconChainError(e.into()))? - { - return Err(BlockError::RepeatProposal { - proposer: block.message().proposer_index(), - slot: block.slot(), - }); - } - // Do not process a block that doesn't descend from the finalized root. // // We check this *before* we load the parent so that we can return a more detailed error. @@ -855,17 +877,16 @@ impl GossipVerifiedBlock { // // It's important to double-check that the proposer still hasn't been observed so we don't // have a race-condition when verifying two blocks simultaneously. - if chain + match chain .observed_block_producers .write() - .observe_proposer(block.message()) + .observe_proposal(block_root, block.message()) .map_err(|e| BlockError::BeaconChainError(e.into()))? { - return Err(BlockError::RepeatProposal { - proposer: block.message().proposer_index(), - slot: block.slot(), - }); - } + SeenBlock::Slashable => return Err(BlockError::Slashable), + SeenBlock::Duplicate => return Err(BlockError::BlockIsAlreadyKnown), + SeenBlock::UniqueNonSlashable => {} + }; if block.message().proposer_index() != expected_proposer as u64 { return Err(BlockError::IncorrectBlockProposer { @@ -1101,6 +1122,12 @@ impl ExecutionPendingBlock { chain: &Arc>, notify_execution_layer: NotifyExecutionLayer, ) -> Result> { + chain + .observed_block_producers + .write() + .observe_proposal(block_root, block.message()) + .map_err(|e| BlockError::BeaconChainError(e.into()))?; + if let Some(parent) = chain .canonical_head .fork_choice_read_lock() diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index 84148fbfb18..9bb39396326 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -826,7 +826,6 @@ where observed_sync_aggregators: <_>::default(), // TODO: allow for persisting and loading the pool from disk. observed_block_producers: <_>::default(), - // TODO: allow for persisting and loading the pool from disk. observed_voluntary_exits: <_>::default(), observed_proposer_slashings: <_>::default(), observed_attester_slashings: <_>::default(), diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index e789b54a21b..50bcf426536 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -213,6 +213,7 @@ pub enum BeaconChainError { BlsToExecutionConflictsWithPool, InconsistentFork(InconsistentFork), ProposerHeadForkChoiceError(fork_choice::Error), + UnableToPublish, } easy_from_to!(SlotProcessingError, BeaconChainError); diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index d672c168288..c5cf74e179c 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -64,7 +64,7 @@ pub use attestation_verification::Error as AttestationError; pub use beacon_fork_choice_store::{BeaconForkChoiceStore, Error as ForkChoiceStoreError}; pub use block_verification::{ get_block_root, BlockError, ExecutionPayloadError, GossipVerifiedBlock, - IntoExecutionPendingBlock, + IntoExecutionPendingBlock, IntoGossipVerifiedBlock, }; pub use canonical_head::{CachedHead, CanonicalHead, CanonicalHeadRwLock}; pub use eth1_chain::{Eth1Chain, Eth1ChainBackend}; diff --git a/beacon_node/beacon_chain/src/observed_block_producers.rs b/beacon_node/beacon_chain/src/observed_block_producers.rs index b5995121b99..f76fc537967 100644 --- a/beacon_node/beacon_chain/src/observed_block_producers.rs +++ b/beacon_node/beacon_chain/src/observed_block_producers.rs @@ -1,9 +1,10 @@ //! Provides the `ObservedBlockProducers` struct which allows for rejecting gossip blocks from //! validators that have already produced a block. +use std::collections::hash_map::Entry; use std::collections::{HashMap, HashSet}; use std::marker::PhantomData; -use types::{BeaconBlockRef, Epoch, EthSpec, Slot, Unsigned}; +use types::{BeaconBlockRef, Epoch, EthSpec, Hash256, Slot, Unsigned}; #[derive(Debug, PartialEq)] pub enum Error { @@ -14,6 +15,12 @@ pub enum Error { ValidatorIndexTooHigh(u64), } +#[derive(Eq, Hash, PartialEq, Debug, Default)] +struct ProposalKey { + slot: Slot, + proposer: u64, +} + /// Maintains a cache of observed `(block.slot, block.proposer)`. /// /// The cache supports pruning based upon the finalized epoch. It does not automatically prune, you @@ -27,7 +34,7 @@ pub enum Error { /// known_distinct_shufflings` which is much smaller. pub struct ObservedBlockProducers { finalized_slot: Slot, - items: HashMap>, + items: HashMap>, _phantom: PhantomData, } @@ -42,6 +49,24 @@ impl Default for ObservedBlockProducers { } } +pub enum SeenBlock { + Duplicate, + Slashable, + UniqueNonSlashable, +} + +impl SeenBlock { + pub fn proposer_previously_observed(self) -> bool { + match self { + Self::Duplicate | Self::Slashable => true, + Self::UniqueNonSlashable => false, + } + } + pub fn is_slashable(&self) -> bool { + matches!(self, Self::Slashable) + } +} + impl ObservedBlockProducers { /// Observe that the `block` was produced by `block.proposer_index` at `block.slot`. This will /// update `self` so future calls to it indicate that this block is known. @@ -52,16 +77,44 @@ impl ObservedBlockProducers { /// /// - `block.proposer_index` is greater than `VALIDATOR_REGISTRY_LIMIT`. /// - `block.slot` is equal to or less than the latest pruned `finalized_slot`. - pub fn observe_proposer(&mut self, block: BeaconBlockRef<'_, E>) -> Result { + pub fn observe_proposal( + &mut self, + block_root: Hash256, + block: BeaconBlockRef<'_, E>, + ) -> Result { self.sanitize_block(block)?; - let did_not_exist = self - .items - .entry(block.slot()) - .or_insert_with(|| HashSet::with_capacity(E::SlotsPerEpoch::to_usize())) - .insert(block.proposer_index()); - - Ok(!did_not_exist) + let key = ProposalKey { + slot: block.slot(), + proposer: block.proposer_index(), + }; + + let entry = self.items.entry(key); + + let slashable_proposal = match entry { + Entry::Occupied(mut occupied_entry) => { + let block_roots = occupied_entry.get_mut(); + let newly_inserted = block_roots.insert(block_root); + + let is_equivocation = block_roots.len() > 1; + + if is_equivocation { + SeenBlock::Slashable + } else if !newly_inserted { + SeenBlock::Duplicate + } else { + SeenBlock::UniqueNonSlashable + } + } + Entry::Vacant(vacant_entry) => { + let block_roots = HashSet::from([block_root]); + vacant_entry.insert(block_roots); + + SeenBlock::UniqueNonSlashable + } + }; + + Ok(slashable_proposal) } /// Returns `Ok(true)` if the `block` has been observed before, `Ok(false)` if not. Does not @@ -72,15 +125,33 @@ impl ObservedBlockProducers { /// /// - `block.proposer_index` is greater than `VALIDATOR_REGISTRY_LIMIT`. /// - `block.slot` is equal to or less than the latest pruned `finalized_slot`. - pub fn proposer_has_been_observed(&self, block: BeaconBlockRef<'_, E>) -> Result { + pub fn proposer_has_been_observed( + &self, + block: BeaconBlockRef<'_, E>, + block_root: Hash256, + ) -> Result { self.sanitize_block(block)?; - let exists = self - .items - .get(&block.slot()) - .map_or(false, |set| set.contains(&block.proposer_index())); - - Ok(exists) + let key = ProposalKey { + slot: block.slot(), + proposer: block.proposer_index(), + }; + + if let Some(block_roots) = self.items.get(&key) { + let block_already_known = block_roots.contains(&block_root); + let no_prev_known_blocks = + block_roots.difference(&HashSet::from([block_root])).count() == 0; + + if !no_prev_known_blocks { + Ok(SeenBlock::Slashable) + } else if block_already_known { + Ok(SeenBlock::Duplicate) + } else { + Ok(SeenBlock::UniqueNonSlashable) + } + } else { + Ok(SeenBlock::UniqueNonSlashable) + } } /// Returns `Ok(())` if the given `block` is sane. @@ -112,15 +183,15 @@ impl ObservedBlockProducers { } self.finalized_slot = finalized_slot; - self.items.retain(|slot, _set| *slot > finalized_slot); + self.items.retain(|key, _| key.slot > finalized_slot); } /// Returns `true` if the given `validator_index` has been stored in `self` at `epoch`. /// /// This is useful for doppelganger detection. pub fn index_seen_at_epoch(&self, validator_index: u64, epoch: Epoch) -> bool { - self.items.iter().any(|(slot, producers)| { - slot.epoch(E::slots_per_epoch()) == epoch && producers.contains(&validator_index) + self.items.iter().any(|(key, _)| { + key.slot.epoch(E::slots_per_epoch()) == epoch && key.proposer == validator_index }) } } @@ -148,9 +219,12 @@ mod tests { // Slot 0, proposer 0 let block_a = get_block(0, 0); + let block_root = block_a.canonical_root(); assert_eq!( - cache.observe_proposer(block_a.to_ref()), + cache + .observe_proposal(block_root, block_a.to_ref()) + .map(SeenBlock::proposer_previously_observed), Ok(false), "can observe proposer, indicates proposer unobserved" ); @@ -164,7 +238,10 @@ mod tests { assert_eq!( cache .items - .get(&Slot::new(0)) + .get(&ProposalKey { + slot: Slot::new(0), + proposer: 0 + }) .expect("slot zero should be present") .len(), 1, @@ -182,7 +259,10 @@ mod tests { assert_eq!( cache .items - .get(&Slot::new(0)) + .get(&ProposalKey { + slot: Slot::new(0), + proposer: 0 + }) .expect("slot zero should be present") .len(), 1, @@ -207,9 +287,12 @@ mod tests { // First slot of finalized epoch, proposer 0 let block_b = get_block(E::slots_per_epoch(), 0); + let block_root_b = block_b.canonical_root(); assert_eq!( - cache.observe_proposer(block_b.to_ref()), + cache + .observe_proposal(block_root_b, block_b.to_ref()) + .map(SeenBlock::proposer_previously_observed), Err(Error::FinalizedBlock { slot: E::slots_per_epoch().into(), finalized_slot: E::slots_per_epoch().into(), @@ -229,7 +312,9 @@ mod tests { let block_b = get_block(three_epochs, 0); assert_eq!( - cache.observe_proposer(block_b.to_ref()), + cache + .observe_proposal(block_root_b, block_b.to_ref()) + .map(SeenBlock::proposer_previously_observed), Ok(false), "can insert non-finalized block" ); @@ -238,7 +323,10 @@ mod tests { assert_eq!( cache .items - .get(&Slot::new(three_epochs)) + .get(&ProposalKey { + slot: Slot::new(three_epochs), + proposer: 0 + }) .expect("the three epochs slot should be present") .len(), 1, @@ -262,7 +350,10 @@ mod tests { assert_eq!( cache .items - .get(&Slot::new(three_epochs)) + .get(&ProposalKey { + slot: Slot::new(three_epochs), + proposer: 0 + }) .expect("the three epochs slot should be present") .len(), 1, @@ -276,24 +367,33 @@ mod tests { // Slot 0, proposer 0 let block_a = get_block(0, 0); + let block_root_a = block_a.canonical_root(); assert_eq!( - cache.proposer_has_been_observed(block_a.to_ref()), + cache + .proposer_has_been_observed(block_a.to_ref(), block_a.canonical_root()) + .map(|x| x.proposer_previously_observed()), Ok(false), "no observation in empty cache" ); assert_eq!( - cache.observe_proposer(block_a.to_ref()), + cache + .observe_proposal(block_root_a, block_a.to_ref()) + .map(SeenBlock::proposer_previously_observed), Ok(false), "can observe proposer, indicates proposer unobserved" ); assert_eq!( - cache.proposer_has_been_observed(block_a.to_ref()), + cache + .proposer_has_been_observed(block_a.to_ref(), block_a.canonical_root()) + .map(|x| x.proposer_previously_observed()), Ok(true), "observed block is indicated as true" ); assert_eq!( - cache.observe_proposer(block_a.to_ref()), + cache + .observe_proposal(block_root_a, block_a.to_ref()) + .map(SeenBlock::proposer_previously_observed), Ok(true), "observing again indicates true" ); @@ -303,7 +403,10 @@ mod tests { assert_eq!( cache .items - .get(&Slot::new(0)) + .get(&ProposalKey { + slot: Slot::new(0), + proposer: 0 + }) .expect("slot zero should be present") .len(), 1, @@ -312,24 +415,33 @@ mod tests { // Slot 1, proposer 0 let block_b = get_block(1, 0); + let block_root_b = block_b.canonical_root(); assert_eq!( - cache.proposer_has_been_observed(block_b.to_ref()), + cache + .proposer_has_been_observed(block_b.to_ref(), block_b.canonical_root()) + .map(|x| x.proposer_previously_observed()), Ok(false), "no observation for new slot" ); assert_eq!( - cache.observe_proposer(block_b.to_ref()), + cache + .observe_proposal(block_root_b, block_b.to_ref()) + .map(SeenBlock::proposer_previously_observed), Ok(false), "can observe proposer for new slot, indicates proposer unobserved" ); assert_eq!( - cache.proposer_has_been_observed(block_b.to_ref()), + cache + .proposer_has_been_observed(block_b.to_ref(), block_b.canonical_root()) + .map(|x| x.proposer_previously_observed()), Ok(true), "observed block in slot 1 is indicated as true" ); assert_eq!( - cache.observe_proposer(block_b.to_ref()), + cache + .observe_proposal(block_root_b, block_b.to_ref()) + .map(SeenBlock::proposer_previously_observed), Ok(true), "observing slot 1 again indicates true" ); @@ -339,7 +451,10 @@ mod tests { assert_eq!( cache .items - .get(&Slot::new(0)) + .get(&ProposalKey { + slot: Slot::new(0), + proposer: 0 + }) .expect("slot zero should be present") .len(), 1, @@ -348,7 +463,10 @@ mod tests { assert_eq!( cache .items - .get(&Slot::new(1)) + .get(&ProposalKey { + slot: Slot::new(1), + proposer: 0 + }) .expect("slot zero should be present") .len(), 1, @@ -357,45 +475,54 @@ mod tests { // Slot 0, proposer 1 let block_c = get_block(0, 1); + let block_root_c = block_c.canonical_root(); assert_eq!( - cache.proposer_has_been_observed(block_c.to_ref()), + cache + .proposer_has_been_observed(block_c.to_ref(), block_c.canonical_root()) + .map(|x| x.proposer_previously_observed()), Ok(false), "no observation for new proposer" ); assert_eq!( - cache.observe_proposer(block_c.to_ref()), + cache + .observe_proposal(block_root_c, block_c.to_ref()) + .map(SeenBlock::proposer_previously_observed), Ok(false), "can observe new proposer, indicates proposer unobserved" ); assert_eq!( - cache.proposer_has_been_observed(block_c.to_ref()), + cache + .proposer_has_been_observed(block_c.to_ref(), block_c.canonical_root()) + .map(|x| x.proposer_previously_observed()), Ok(true), "observed new proposer block is indicated as true" ); assert_eq!( - cache.observe_proposer(block_c.to_ref()), + cache + .observe_proposal(block_root_c, block_c.to_ref()) + .map(SeenBlock::proposer_previously_observed), Ok(true), "observing new proposer again indicates true" ); assert_eq!(cache.finalized_slot, 0, "finalized slot is zero"); - assert_eq!(cache.items.len(), 2, "two slots should be present"); + assert_eq!(cache.items.len(), 3, "three slots should be present"); assert_eq!( cache .items - .get(&Slot::new(0)) - .expect("slot zero should be present") - .len(), + .iter() + .filter(|(k, _)| k.slot == cache.finalized_slot) + .count(), 2, "two proposers should be present in slot 0" ); assert_eq!( cache .items - .get(&Slot::new(1)) - .expect("slot zero should be present") - .len(), + .iter() + .filter(|(k, _)| k.slot == Slot::new(1)) + .count(), 1, "only one proposer should be present in slot 1" ); diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 55ea016fbda..21f7248cee5 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -733,6 +733,15 @@ where state.get_block_root(slot).unwrap() == state.get_block_root(slot - 1).unwrap() } + pub async fn make_blinded_block( + &self, + state: BeaconState, + slot: Slot, + ) -> (SignedBlindedBeaconBlock, BeaconState) { + let (unblinded, new_state) = self.make_block(state, slot).await; + (unblinded.into(), new_state) + } + /// Returns a newly created block, signed by the proposer for the given slot. pub async fn make_block( &self, @@ -1692,7 +1701,12 @@ where self.set_current_slot(slot); let block_hash: SignedBeaconBlockHash = self .chain - .process_block(block_root, Arc::new(block), NotifyExecutionLayer::Yes) + .process_block( + block_root, + Arc::new(block), + NotifyExecutionLayer::Yes, + || Ok(()), + ) .await? .into(); self.chain.recompute_head_at_current_slot().await; @@ -1709,6 +1723,7 @@ where block.canonical_root(), Arc::new(block), NotifyExecutionLayer::Yes, + || Ok(()), ) .await? .into(); diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index a88931367f7..75b00b2b442 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -351,6 +351,7 @@ async fn assert_invalid_signature( snapshots[block_index].beacon_block.canonical_root(), snapshots[block_index].beacon_block.clone(), NotifyExecutionLayer::Yes, + || Ok(()), ) .await; assert!( @@ -415,6 +416,7 @@ async fn invalid_signature_gossip_block() { signed_block.canonical_root(), Arc::new(signed_block), NotifyExecutionLayer::Yes, + || Ok(()), ) .await, Err(BlockError::InvalidSignature) @@ -727,6 +729,7 @@ async fn block_gossip_verification() { gossip_verified.block_root, gossip_verified, NotifyExecutionLayer::Yes, + || Ok(()), ) .await .expect("should import valid gossip verified block"); @@ -923,11 +926,7 @@ async fn block_gossip_verification() { assert!( matches!( unwrap_err(harness.chain.verify_block_for_gossip(Arc::new(block.clone())).await), - BlockError::RepeatProposal { - proposer, - slot, - } - if proposer == other_proposer && slot == block.message().slot() + BlockError::BlockIsAlreadyKnown, ), "should register any valid signature against the proposer, even if the block failed later verification" ); @@ -956,11 +955,7 @@ async fn block_gossip_verification() { .await .err() .expect("should error when processing known block"), - BlockError::RepeatProposal { - proposer, - slot, - } - if proposer == block.message().proposer_index() && slot == block.message().slot() + BlockError::BlockIsAlreadyKnown ), "the second proposal by this validator should be rejected" ); @@ -998,6 +993,7 @@ async fn verify_block_for_gossip_slashing_detection() { verified_block.block_root, verified_block, NotifyExecutionLayer::Yes, + || Ok(()), ) .await .unwrap(); @@ -1037,6 +1033,7 @@ async fn verify_block_for_gossip_doppelganger_detection() { verified_block.block_root, verified_block, NotifyExecutionLayer::Yes, + || Ok(()), ) .await .unwrap(); @@ -1184,6 +1181,7 @@ async fn add_base_block_to_altair_chain() { base_block.canonical_root(), Arc::new(base_block.clone()), NotifyExecutionLayer::Yes, + || Ok(()), ) .await .err() @@ -1318,6 +1316,7 @@ async fn add_altair_block_to_base_chain() { altair_block.canonical_root(), Arc::new(altair_block.clone()), NotifyExecutionLayer::Yes, + || Ok(()), ) .await .err() diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index c39bdeaf366..018defd2f00 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -697,6 +697,7 @@ async fn invalidates_all_descendants() { fork_block.canonical_root(), Arc::new(fork_block), NotifyExecutionLayer::Yes, + || Ok(()), ) .await .unwrap(); @@ -793,6 +794,7 @@ async fn switches_heads() { fork_block.canonical_root(), Arc::new(fork_block), NotifyExecutionLayer::Yes, + || Ok(()), ) .await .unwrap(); @@ -1046,7 +1048,9 @@ async fn invalid_parent() { // Ensure the block built atop an invalid payload is invalid for import. assert!(matches!( - rig.harness.chain.process_block(block.canonical_root(), block.clone(), NotifyExecutionLayer::Yes).await, + rig.harness.chain.process_block(block.canonical_root(), block.clone(), NotifyExecutionLayer::Yes, + || Ok(()), + ).await, Err(BlockError::ParentExecutionPayloadInvalid { parent_root: invalid_root }) if invalid_root == parent_root )); @@ -1332,7 +1336,12 @@ async fn build_optimistic_chain( for block in blocks { rig.harness .chain - .process_block(block.canonical_root(), block, NotifyExecutionLayer::Yes) + .process_block( + block.canonical_root(), + block, + NotifyExecutionLayer::Yes, + || Ok(()), + ) .await .unwrap(); } @@ -1892,6 +1901,7 @@ async fn recover_from_invalid_head_by_importing_blocks() { fork_block.canonical_root(), fork_block.clone(), NotifyExecutionLayer::Yes, + || Ok(()), ) .await .unwrap(); diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index 0bc7798a7ff..29027748259 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -2151,6 +2151,7 @@ async fn weak_subjectivity_sync() { full_block.canonical_root(), Arc::new(full_block), NotifyExecutionLayer::Yes, + || Ok(()), ) .await .unwrap(); diff --git a/beacon_node/beacon_chain/tests/tests.rs b/beacon_node/beacon_chain/tests/tests.rs index f97f7069dce..c5b2892cbdb 100644 --- a/beacon_node/beacon_chain/tests/tests.rs +++ b/beacon_node/beacon_chain/tests/tests.rs @@ -687,6 +687,7 @@ async fn run_skip_slot_test(skip_slots: u64) { harness_a.chain.head_snapshot().beacon_block_root, harness_a.chain.head_snapshot().beacon_block.clone(), NotifyExecutionLayer::Yes, + || Ok(()) ) .await .unwrap(), diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 025b54f326e..93bfe524bc2 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -31,8 +31,8 @@ use beacon_chain::{ pub use block_id::BlockId; use directory::DEFAULT_ROOT_DIR; use eth2::types::{ - self as api_types, EndpointVersion, ForkChoice, ForkChoiceNode, SkipRandaoVerification, - ValidatorId, ValidatorStatus, + self as api_types, BroadcastValidation, EndpointVersion, ForkChoice, ForkChoiceNode, + SkipRandaoVerification, ValidatorId, ValidatorStatus, }; use lighthouse_network::{types::SyncState, EnrExt, NetworkGlobals, PeerId, PubsubMessage}; use lighthouse_version::version_with_platform; @@ -40,7 +40,9 @@ use logging::SSELoggingComponents; use network::{NetworkMessage, NetworkSenders, ValidatorSubscriptionMessage}; use operation_pool::ReceivedPreCapella; use parking_lot::RwLock; -use publish_blocks::ProvenancedBlock; +pub use publish_blocks::{ + publish_blinded_block, publish_block, reconstruct_block, ProvenancedBlock, +}; use serde::{Deserialize, Serialize}; use slog::{crit, debug, error, info, warn, Logger}; use slot_clock::SlotClock; @@ -324,6 +326,7 @@ pub fn serve( }; let eth_v1 = single_version(V1); + let eth_v2 = single_version(V2); // Create a `warp` filter that provides access to the network globals. let inner_network_globals = ctx.network_globals.clone(); @@ -1222,16 +1225,55 @@ pub fn serve( log: Logger| async move { publish_blocks::publish_block( None, - ProvenancedBlock::Local(block), + ProvenancedBlock::local(block), chain, &network_tx, log, + BroadcastValidation::default(), ) .await .map(|()| warp::reply().into_response()) }, ); + let post_beacon_blocks_v2 = eth_v2 + .and(warp::path("beacon")) + .and(warp::path("blocks")) + .and(warp::query::()) + .and(warp::path::end()) + .and(warp::body::json()) + .and(chain_filter.clone()) + .and(network_tx_filter.clone()) + .and(log_filter.clone()) + .then( + |validation_level: api_types::BroadcastValidationQuery, + block: Arc>, + chain: Arc>, + network_tx: UnboundedSender>, + log: Logger| async move { + match publish_blocks::publish_block( + None, + ProvenancedBlock::local(block), + chain, + &network_tx, + log, + validation_level.broadcast_validation, + ) + .await + { + Ok(()) => warp::reply().into_response(), + Err(e) => match warp_utils::reject::handle_rejection(e).await { + Ok(reply) => reply.into_response(), + Err(_) => warp::reply::with_status( + StatusCode::INTERNAL_SERVER_ERROR, + eth2::StatusCode::INTERNAL_SERVER_ERROR, + ) + .into_response(), + }, + } + }, + ); + /* * beacon/blocks */ @@ -1250,9 +1292,52 @@ pub fn serve( chain: Arc>, network_tx: UnboundedSender>, log: Logger| async move { - publish_blocks::publish_blinded_block(block, chain, &network_tx, log) - .await - .map(|()| warp::reply().into_response()) + publish_blocks::publish_blinded_block( + block, + chain, + &network_tx, + log, + BroadcastValidation::default(), + ) + .await + .map(|()| warp::reply().into_response()) + }, + ); + + let post_beacon_blinded_blocks_v2 = eth_v2 + .and(warp::path("beacon")) + .and(warp::path("blinded_blocks")) + .and(warp::query::()) + .and(warp::path::end()) + .and(warp::body::json()) + .and(chain_filter.clone()) + .and(network_tx_filter.clone()) + .and(log_filter.clone()) + .then( + |validation_level: api_types::BroadcastValidationQuery, + block: SignedBeaconBlock>, + chain: Arc>, + network_tx: UnboundedSender>, + log: Logger| async move { + match publish_blocks::publish_blinded_block( + block, + chain, + &network_tx, + log, + validation_level.broadcast_validation, + ) + .await + { + Ok(()) => warp::reply().into_response(), + Err(e) => match warp_utils::reject::handle_rejection(e).await { + Ok(reply) => reply.into_response(), + Err(_) => warp::reply::with_status( + StatusCode::INTERNAL_SERVER_ERROR, + eth2::StatusCode::INTERNAL_SERVER_ERROR, + ) + .into_response(), + }, + } }, ); @@ -3847,6 +3932,8 @@ pub fn serve( warp::post().and( post_beacon_blocks .uor(post_beacon_blinded_blocks) + .uor(post_beacon_blocks_v2) + .uor(post_beacon_blinded_blocks_v2) .uor(post_beacon_pool_attestations) .uor(post_beacon_pool_attester_slashings) .uor(post_beacon_pool_proposer_slashings) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 8bcad6ba401..0f2f7b361c7 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -1,11 +1,16 @@ use crate::metrics; use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now}; -use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError, NotifyExecutionLayer}; +use beacon_chain::{ + BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, IntoGossipVerifiedBlock, + NotifyExecutionLayer, +}; +use eth2::types::BroadcastValidation; use execution_layer::ProvenancedPayload; use lighthouse_network::PubsubMessage; use network::NetworkMessage; use slog::{debug, error, info, warn, Logger}; use slot_clock::SlotClock; +use std::marker::PhantomData; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; @@ -16,45 +21,115 @@ use types::{ }; use warp::Rejection; -pub enum ProvenancedBlock { +pub enum ProvenancedBlock> { /// The payload was built using a local EE. - Local(Arc>>), + Local(B, PhantomData), /// The payload was build using a remote builder (e.g., via a mev-boost /// compatible relay). - Builder(Arc>>), + Builder(B, PhantomData), +} + +impl> ProvenancedBlock { + pub fn local(block: B) -> Self { + Self::Local(block, PhantomData) + } + + pub fn builder(block: B) -> Self { + Self::Builder(block, PhantomData) + } } /// Handles a request from the HTTP API for full blocks. -pub async fn publish_block( +pub async fn publish_block>( block_root: Option, - provenanced_block: ProvenancedBlock, + provenanced_block: ProvenancedBlock, chain: Arc>, network_tx: &UnboundedSender>, log: Logger, + validation_level: BroadcastValidation, ) -> Result<(), Rejection> { let seen_timestamp = timestamp_now(); let (block, is_locally_built_block) = match provenanced_block { - ProvenancedBlock::Local(block) => (block, true), - ProvenancedBlock::Builder(block) => (block, false), + ProvenancedBlock::Local(block, _) => (block, true), + ProvenancedBlock::Builder(block, _) => (block, false), }; - let delay = get_block_delay_ms(seen_timestamp, block.message(), &chain.slot_clock); + let beacon_block = block.inner(); + let delay = get_block_delay_ms(seen_timestamp, beacon_block.message(), &chain.slot_clock); + debug!(log, "Signed block received in HTTP API"; "slot" => beacon_block.slot()); - debug!( - log, - "Signed block published to HTTP API"; - "slot" => block.slot() - ); + /* actually publish a block */ + let publish_block = move |block: Arc>, + sender, + log, + seen_timestamp| { + let publish_timestamp = timestamp_now(); + let publish_delay = publish_timestamp + .checked_sub(seen_timestamp) + .unwrap_or_else(|| Duration::from_secs(0)); - // Send the block, regardless of whether or not it is valid. The API - // specification is very clear that this is the desired behaviour. + info!(log, "Signed block published to network via HTTP API"; "slot" => block.slot(), "publish_delay" => ?publish_delay); - let message = PubsubMessage::BeaconBlock(block.clone()); - crate::publish_pubsub_message(network_tx, message)?; + let message = PubsubMessage::BeaconBlock(block); + crate::publish_pubsub_message(&sender, message) + .map_err(|_| BeaconChainError::UnableToPublish.into()) + }; - let block_root = block_root.unwrap_or_else(|| block.canonical_root()); + /* if we can form a `GossipVerifiedBlock`, we've passed our basic gossip checks */ + let gossip_verified_block = block.into_gossip_verified_block(&chain).map_err(|e| { + warn!(log, "Not publishing block, not gossip verified"; "slot" => beacon_block.slot(), "error" => ?e); + warp_utils::reject::custom_bad_request(e.to_string()) + })?; + + let block_root = block_root.unwrap_or(gossip_verified_block.block_root); + + if let BroadcastValidation::Gossip = validation_level { + publish_block( + beacon_block.clone(), + network_tx.clone(), + log.clone(), + seen_timestamp, + ) + .map_err(|_| warp_utils::reject::custom_server_error("unable to publish".into()))?; + } + + /* only publish if gossip- and consensus-valid and equivocation-free */ + let chain_clone = chain.clone(); + let block_clone = beacon_block.clone(); + let log_clone = log.clone(); + let sender_clone = network_tx.clone(); + + let publish_fn = move || match validation_level { + BroadcastValidation::Gossip => Ok(()), + BroadcastValidation::Consensus => { + publish_block(block_clone, sender_clone, log_clone, seen_timestamp) + } + BroadcastValidation::ConsensusAndEquivocation => { + if chain_clone + .observed_block_producers + .read() + .proposer_has_been_observed(block_clone.message(), block_root) + .map_err(|e| BlockError::BeaconChainError(e.into()))? + .is_slashable() + { + warn!( + log_clone, + "Not publishing equivocating block"; + "slot" => block_clone.slot() + ); + Err(BlockError::Slashable) + } else { + publish_block(block_clone, sender_clone, log_clone, seen_timestamp) + } + } + }; match chain - .process_block(block_root, block.clone(), NotifyExecutionLayer::Yes) + .process_block( + block_root, + gossip_verified_block, + NotifyExecutionLayer::Yes, + publish_fn, + ) .await { Ok(root) => { @@ -63,14 +138,14 @@ pub async fn publish_block( "Valid block from HTTP API"; "block_delay" => ?delay, "root" => format!("{}", root), - "proposer_index" => block.message().proposer_index(), - "slot" => block.slot(), + "proposer_index" => beacon_block.message().proposer_index(), + "slot" => beacon_block.slot(), ); // Notify the validator monitor. chain.validator_monitor.read().register_api_block( seen_timestamp, - block.message(), + beacon_block.message(), root, &chain.slot_clock, ); @@ -83,40 +158,44 @@ pub async fn publish_block( // blocks built with builders we consider the broadcast time to be // when the blinded block is published to the builder. if is_locally_built_block { - late_block_logging(&chain, seen_timestamp, block.message(), root, "local", &log) + late_block_logging( + &chain, + seen_timestamp, + beacon_block.message(), + root, + "local", + &log, + ) } Ok(()) } - Err(BlockError::BlockIsAlreadyKnown) => { - info!( - log, - "Block from HTTP API already known"; - "block" => ?block.canonical_root(), - "slot" => block.slot(), - ); - Ok(()) + Err(BlockError::BeaconChainError(BeaconChainError::UnableToPublish)) => { + Err(warp_utils::reject::custom_server_error( + "unable to publish to network channel".to_string(), + )) } - Err(BlockError::RepeatProposal { proposer, slot }) => { - warn!( - log, - "Block ignored due to repeat proposal"; - "msg" => "this can happen when a VC uses fallback BNs. \ - whilst this is not necessarily an error, it can indicate issues with a BN \ - or between the VC and BN.", - "slot" => slot, - "proposer" => proposer, - ); + Err(BlockError::Slashable) => Err(warp_utils::reject::custom_bad_request( + "proposal for this slot and proposer has already been seen".to_string(), + )), + Err(BlockError::BlockIsAlreadyKnown) => { + info!(log, "Block from HTTP API already known"; "block" => ?block_root); Ok(()) } Err(e) => { - let msg = format!("{:?}", e); - error!( - log, - "Invalid block provided to HTTP API"; - "reason" => &msg - ); - Err(warp_utils::reject::broadcast_without_import(msg)) + if let BroadcastValidation::Gossip = validation_level { + Err(warp_utils::reject::broadcast_without_import(format!("{e}"))) + } else { + let msg = format!("{:?}", e); + error!( + log, + "Invalid block provided to HTTP API"; + "reason" => &msg + ); + Err(warp_utils::reject::custom_bad_request(format!( + "Invalid block: {e}" + ))) + } } } } @@ -128,21 +207,31 @@ pub async fn publish_blinded_block( chain: Arc>, network_tx: &UnboundedSender>, log: Logger, + validation_level: BroadcastValidation, ) -> Result<(), Rejection> { let block_root = block.canonical_root(); - let full_block = reconstruct_block(chain.clone(), block_root, block, log.clone()).await?; - publish_block::(Some(block_root), full_block, chain, network_tx, log).await + let full_block: ProvenancedBlock>> = + reconstruct_block(chain.clone(), block_root, block, log.clone()).await?; + publish_block::( + Some(block_root), + full_block, + chain, + network_tx, + log, + validation_level, + ) + .await } /// Deconstruct the given blinded block, and construct a full block. This attempts to use the /// execution layer's payload cache, and if that misses, attempts a blind block proposal to retrieve /// the full payload. -async fn reconstruct_block( +pub async fn reconstruct_block( chain: Arc>, block_root: Hash256, block: SignedBeaconBlock>, log: Logger, -) -> Result, Rejection> { +) -> Result>>, Rejection> { let full_payload_opt = if let Ok(payload_header) = block.message().body().execution_payload() { let el = chain.execution_layer.as_ref().ok_or_else(|| { warp_utils::reject::custom_server_error("Missing execution layer".to_string()) @@ -208,15 +297,15 @@ async fn reconstruct_block( None => block .try_into_full_block(None) .map(Arc::new) - .map(ProvenancedBlock::Local), + .map(ProvenancedBlock::local), Some(ProvenancedPayload::Local(full_payload)) => block .try_into_full_block(Some(full_payload)) .map(Arc::new) - .map(ProvenancedBlock::Local), + .map(ProvenancedBlock::local), Some(ProvenancedPayload::Builder(full_payload)) => block .try_into_full_block(Some(full_payload)) .map(Arc::new) - .map(ProvenancedBlock::Builder), + .map(ProvenancedBlock::builder), } .ok_or_else(|| { warp_utils::reject::custom_server_error("Unable to add payload to block".to_string()) diff --git a/beacon_node/http_api/tests/broadcast_validation_tests.rs b/beacon_node/http_api/tests/broadcast_validation_tests.rs new file mode 100644 index 00000000000..4819dd99e7a --- /dev/null +++ b/beacon_node/http_api/tests/broadcast_validation_tests.rs @@ -0,0 +1,1270 @@ +use beacon_chain::{ + test_utils::{AttestationStrategy, BlockStrategy}, + GossipVerifiedBlock, +}; +use eth2::types::{BroadcastValidation, SignedBeaconBlock, SignedBlindedBeaconBlock}; +use http_api::test_utils::InteractiveTester; +use http_api::{publish_blinded_block, publish_block, reconstruct_block, ProvenancedBlock}; +use tree_hash::TreeHash; +use types::{Hash256, MainnetEthSpec, Slot}; +use warp::Rejection; +use warp_utils::reject::CustomBadRequest; + +use eth2::reqwest::StatusCode; + +type E = MainnetEthSpec; + +/* + * We have the following test cases, which are duplicated for the blinded variant of the route: + * + * - `broadcast_validation=gossip` + * - Invalid (400) + * - Full Pass (200) + * - Partial Pass (202) + * - `broadcast_validation=consensus` + * - Invalid (400) + * - Only gossip (400) + * - Only consensus pass (i.e., equivocates) (200) + * - Full pass (200) + * - `broadcast_validation=consensus_and_equivocation` + * - Invalid (400) + * - Invalid due to early equivocation (400) + * - Only gossip (400) + * - Only consensus (400) + * - Pass (200) + * + */ + +/// This test checks that a block that is **invalid** from a gossip perspective gets rejected when using `broadcast_validation=gossip`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn gossip_invalid() { + /* this test targets gossip-level validation */ + let validation_level: Option = Some(BroadcastValidation::Gossip); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + + let chain_state_before = tester.harness.get_current_state(); + let slot = chain_state_before.slot() + 1; + + tester.harness.advance_slot(); + + let (block, _): (SignedBeaconBlock, _) = tester + .harness + .make_block_with_modifier(chain_state_before, slot, |b| { + *b.state_root_mut() = Hash256::zero(); + *b.parent_root_mut() = Hash256::zero(); + }) + .await; + + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blocks_v2(&block, validation_level) + .await; + assert!(response.is_err()); + + let error_response: eth2::Error = response.err().unwrap(); + + /* mandated by Beacon API spec */ + assert_eq!(error_response.status(), Some(StatusCode::BAD_REQUEST)); + + assert!( + matches!(error_response, eth2::Error::ServerMessage(err) if err.message == "BAD_REQUEST: NotFinalizedDescendant { block_parent_root: 0x0000000000000000000000000000000000000000000000000000000000000000 }".to_string()) + ); +} + +/// This test checks that a block that is valid from a gossip perspective is accepted when using `broadcast_validation=gossip`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn gossip_partial_pass() { + /* this test targets gossip-level validation */ + let validation_level: Option = Some(BroadcastValidation::Gossip); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + + let chain_state_before = tester.harness.get_current_state(); + let slot = chain_state_before.slot() + 1; + + tester.harness.advance_slot(); + + let (block, _): (SignedBeaconBlock, _) = tester + .harness + .make_block_with_modifier(chain_state_before, slot, |b| { + *b.state_root_mut() = Hash256::random() + }) + .await; + + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blocks_v2(&block, validation_level) + .await; + assert!(response.is_err()); + + let error_response = response.unwrap_err(); + + assert_eq!(error_response.status(), Some(StatusCode::ACCEPTED)); +} + +// This test checks that a block that is valid from both a gossip and consensus perspective is accepted when using `broadcast_validation=gossip`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn gossip_full_pass() { + /* this test targets gossip-level validation */ + let validation_level: Option = Some(BroadcastValidation::Gossip); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + tester.harness.advance_slot(); + + let slot_a = Slot::new(num_initial); + let slot_b = slot_a + 1; + + let state_a = tester.harness.get_current_state(); + let (block, _): (SignedBeaconBlock, _) = tester.harness.make_block(state_a, slot_b).await; + + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blocks_v2(&block, validation_level) + .await; + + assert!(response.is_ok()); + assert!(tester + .harness + .chain + .block_is_known_to_fork_choice(&block.canonical_root())); +} + +/// This test checks that a block that is **invalid** from a gossip perspective gets rejected when using `broadcast_validation=consensus`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn consensus_invalid() { + let validation_level: Option = Some(BroadcastValidation::Consensus); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + + let chain_state_before = tester.harness.get_current_state(); + let slot = chain_state_before.slot() + 1; + + tester.harness.advance_slot(); + + let (block, _): (SignedBeaconBlock, _) = tester + .harness + .make_block_with_modifier(chain_state_before, slot, |b| { + *b.state_root_mut() = Hash256::zero(); + *b.parent_root_mut() = Hash256::zero(); + }) + .await; + + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blocks_v2(&block, validation_level) + .await; + assert!(response.is_err()); + + let error_response: eth2::Error = response.err().unwrap(); + + /* mandated by Beacon API spec */ + assert_eq!(error_response.status(), Some(StatusCode::BAD_REQUEST)); + + assert!( + matches!(error_response, eth2::Error::ServerMessage(err) if err.message == "BAD_REQUEST: NotFinalizedDescendant { block_parent_root: 0x0000000000000000000000000000000000000000000000000000000000000000 }".to_string()) + ); +} + +/// This test checks that a block that is only valid from a gossip perspective is rejected when using `broadcast_validation=consensus`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn consensus_gossip() { + /* this test targets gossip-level validation */ + let validation_level: Option = Some(BroadcastValidation::Consensus); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + tester.harness.advance_slot(); + + let slot_a = Slot::new(num_initial); + let slot_b = slot_a + 1; + + let state_a = tester.harness.get_current_state(); + let (block, _): (SignedBeaconBlock, _) = tester + .harness + .make_block_with_modifier(state_a, slot_b, |b| *b.state_root_mut() = Hash256::zero()) + .await; + + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blocks_v2(&block, validation_level) + .await; + assert!(response.is_err()); + + let error_response: eth2::Error = response.err().unwrap(); + + /* mandated by Beacon API spec */ + assert_eq!(error_response.status(), Some(StatusCode::BAD_REQUEST)); + + assert!( + matches!(error_response, eth2::Error::ServerMessage(err) if err.message == "BAD_REQUEST: Invalid block: StateRootMismatch { block: 0x0000000000000000000000000000000000000000000000000000000000000000, local: 0xfc675d642ff7a06458eb33c7d7b62a5813e34d1b2bb1aee3e395100b579da026 }".to_string()) + ); +} + +/// This test checks that a block that is valid from both a gossip and consensus perspective, but nonetheless equivocates, is accepted when using `broadcast_validation=consensus`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn consensus_partial_pass_only_consensus() { + /* this test targets gossip-level validation */ + let validation_level: Option = Some(BroadcastValidation::Consensus); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + let test_logger = tester.harness.logger().clone(); + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + tester.harness.advance_slot(); + + let slot_a = Slot::new(num_initial); + let slot_b = slot_a + 1; + + let state_a = tester.harness.get_current_state(); + let (block_a, state_after_a): (SignedBeaconBlock, _) = + tester.harness.make_block(state_a.clone(), slot_b).await; + let (block_b, state_after_b): (SignedBeaconBlock, _) = + tester.harness.make_block(state_a, slot_b).await; + + /* check for `make_block` curios */ + assert_eq!(block_a.state_root(), state_after_a.tree_hash_root()); + assert_eq!(block_b.state_root(), state_after_b.tree_hash_root()); + assert_ne!(block_a.state_root(), block_b.state_root()); + + let gossip_block_b = GossipVerifiedBlock::new(block_b.clone().into(), &tester.harness.chain); + assert!(gossip_block_b.is_ok()); + let gossip_block_a = GossipVerifiedBlock::new(block_a.clone().into(), &tester.harness.chain); + assert!(gossip_block_a.is_err()); + + /* submit `block_b` which should induce equivocation */ + let channel = tokio::sync::mpsc::unbounded_channel(); + + let publication_result: Result<(), Rejection> = publish_block( + None, + ProvenancedBlock::local(gossip_block_b.unwrap()), + tester.harness.chain.clone(), + &channel.0, + test_logger, + validation_level.unwrap(), + ) + .await; + + assert!(publication_result.is_ok()); + assert!(tester + .harness + .chain + .block_is_known_to_fork_choice(&block_b.canonical_root())); +} + +/// This test checks that a block that is valid from both a gossip and consensus perspective is accepted when using `broadcast_validation=consensus`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn consensus_full_pass() { + /* this test targets gossip-level validation */ + let validation_level: Option = Some(BroadcastValidation::Consensus); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + tester.harness.advance_slot(); + + let slot_a = Slot::new(num_initial); + let slot_b = slot_a + 1; + + let state_a = tester.harness.get_current_state(); + let (block, _): (SignedBeaconBlock, _) = tester.harness.make_block(state_a, slot_b).await; + + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blocks_v2(&block, validation_level) + .await; + + assert!(response.is_ok()); + assert!(tester + .harness + .chain + .block_is_known_to_fork_choice(&block.canonical_root())); +} + +/// This test checks that a block that is **invalid** from a gossip perspective gets rejected when using `broadcast_validation=consensus_and_equivocation`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn equivocation_invalid() { + /* this test targets gossip-level validation */ + let validation_level: Option = + Some(BroadcastValidation::ConsensusAndEquivocation); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + + let chain_state_before = tester.harness.get_current_state(); + let slot = chain_state_before.slot() + 1; + + tester.harness.advance_slot(); + + let (block, _): (SignedBeaconBlock, _) = tester + .harness + .make_block_with_modifier(chain_state_before, slot, |b| { + *b.state_root_mut() = Hash256::zero(); + *b.parent_root_mut() = Hash256::zero(); + }) + .await; + + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blocks_v2(&block, validation_level) + .await; + assert!(response.is_err()); + + let error_response: eth2::Error = response.err().unwrap(); + + /* mandated by Beacon API spec */ + assert_eq!(error_response.status(), Some(StatusCode::BAD_REQUEST)); + + assert!( + matches!(error_response, eth2::Error::ServerMessage(err) if err.message == "BAD_REQUEST: NotFinalizedDescendant { block_parent_root: 0x0000000000000000000000000000000000000000000000000000000000000000 }".to_string()) + ); +} + +/// This test checks that a block that is valid from both a gossip and consensus perspective is rejected when using `broadcast_validation=consensus_and_equivocation`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn equivocation_consensus_early_equivocation() { + /* this test targets gossip-level validation */ + let validation_level: Option = + Some(BroadcastValidation::ConsensusAndEquivocation); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + tester.harness.advance_slot(); + + let slot_a = Slot::new(num_initial); + let slot_b = slot_a + 1; + + let state_a = tester.harness.get_current_state(); + let (block_a, state_after_a): (SignedBeaconBlock, _) = + tester.harness.make_block(state_a.clone(), slot_b).await; + let (block_b, state_after_b): (SignedBeaconBlock, _) = + tester.harness.make_block(state_a, slot_b).await; + + /* check for `make_block` curios */ + assert_eq!(block_a.state_root(), state_after_a.tree_hash_root()); + assert_eq!(block_b.state_root(), state_after_b.tree_hash_root()); + assert_ne!(block_a.state_root(), block_b.state_root()); + + /* submit `block_a` as valid */ + assert!(tester + .client + .post_beacon_blocks_v2(&block_a, validation_level) + .await + .is_ok()); + assert!(tester + .harness + .chain + .block_is_known_to_fork_choice(&block_a.canonical_root())); + + /* submit `block_b` which should induce equivocation */ + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blocks_v2(&block_b, validation_level) + .await; + assert!(response.is_err()); + + let error_response: eth2::Error = response.err().unwrap(); + + assert_eq!(error_response.status(), Some(StatusCode::BAD_REQUEST)); + + assert!( + matches!(error_response, eth2::Error::ServerMessage(err) if err.message == "BAD_REQUEST: Slashable".to_string()) + ); +} + +/// This test checks that a block that is only valid from a gossip perspective is rejected when using `broadcast_validation=consensus_and_equivocation`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn equivocation_gossip() { + /* this test targets gossip-level validation */ + let validation_level: Option = + Some(BroadcastValidation::ConsensusAndEquivocation); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + tester.harness.advance_slot(); + + let slot_a = Slot::new(num_initial); + let slot_b = slot_a + 1; + + let state_a = tester.harness.get_current_state(); + let (block, _): (SignedBeaconBlock, _) = tester + .harness + .make_block_with_modifier(state_a, slot_b, |b| *b.state_root_mut() = Hash256::zero()) + .await; + + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blocks_v2(&block, validation_level) + .await; + assert!(response.is_err()); + + let error_response: eth2::Error = response.err().unwrap(); + + /* mandated by Beacon API spec */ + assert_eq!(error_response.status(), Some(StatusCode::BAD_REQUEST)); + + assert!( + matches!(error_response, eth2::Error::ServerMessage(err) if err.message == "BAD_REQUEST: Invalid block: StateRootMismatch { block: 0x0000000000000000000000000000000000000000000000000000000000000000, local: 0xfc675d642ff7a06458eb33c7d7b62a5813e34d1b2bb1aee3e395100b579da026 }".to_string()) + ); +} + +/// This test checks that a block that is valid from both a gossip and consensus perspective but that equivocates **late** is rejected when using `broadcast_validation=consensus_and_equivocation`. +/// +/// This test is unique in that we can't actually test the HTTP API directly, but instead have to hook into the `publish_blocks` code manually. This is in order to handle the late equivocation case. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn equivocation_consensus_late_equivocation() { + /* this test targets gossip-level validation */ + let validation_level: Option = + Some(BroadcastValidation::ConsensusAndEquivocation); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + let test_logger = tester.harness.logger().clone(); + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + tester.harness.advance_slot(); + + let slot_a = Slot::new(num_initial); + let slot_b = slot_a + 1; + + let state_a = tester.harness.get_current_state(); + let (block_a, state_after_a): (SignedBeaconBlock, _) = + tester.harness.make_block(state_a.clone(), slot_b).await; + let (block_b, state_after_b): (SignedBeaconBlock, _) = + tester.harness.make_block(state_a, slot_b).await; + + /* check for `make_block` curios */ + assert_eq!(block_a.state_root(), state_after_a.tree_hash_root()); + assert_eq!(block_b.state_root(), state_after_b.tree_hash_root()); + assert_ne!(block_a.state_root(), block_b.state_root()); + + let gossip_block_b = GossipVerifiedBlock::new(block_b.clone().into(), &tester.harness.chain); + assert!(gossip_block_b.is_ok()); + let gossip_block_a = GossipVerifiedBlock::new(block_a.clone().into(), &tester.harness.chain); + assert!(gossip_block_a.is_err()); + + let channel = tokio::sync::mpsc::unbounded_channel(); + + let publication_result: Result<(), Rejection> = publish_block( + None, + ProvenancedBlock::local(gossip_block_b.unwrap()), + tester.harness.chain, + &channel.0, + test_logger, + validation_level.unwrap(), + ) + .await; + + assert!(publication_result.is_err()); + + let publication_error = publication_result.unwrap_err(); + + assert!(publication_error.find::().is_some()); + + assert_eq!( + *publication_error.find::().unwrap().0, + "proposal for this slot and proposer has already been seen".to_string() + ); +} + +/// This test checks that a block that is valid from both a gossip and consensus perspective (and does not equivocate) is accepted when using `broadcast_validation=consensus_and_equivocation`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn equivocation_full_pass() { + /* this test targets gossip-level validation */ + let validation_level: Option = + Some(BroadcastValidation::ConsensusAndEquivocation); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + tester.harness.advance_slot(); + + let slot_a = Slot::new(num_initial); + let slot_b = slot_a + 1; + + let state_a = tester.harness.get_current_state(); + let (block, _): (SignedBeaconBlock, _) = tester.harness.make_block(state_a, slot_b).await; + + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blocks_v2(&block, validation_level) + .await; + + assert!(response.is_ok()); + assert!(tester + .harness + .chain + .block_is_known_to_fork_choice(&block.canonical_root())); +} + +/// This test checks that a block that is **invalid** from a gossip perspective gets rejected when using `broadcast_validation=gossip`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn blinded_gossip_invalid() { + /* this test targets gossip-level validation */ + let validation_level: Option = Some(BroadcastValidation::Gossip); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + + let chain_state_before = tester.harness.get_current_state(); + let slot = chain_state_before.slot() + 1; + + tester.harness.advance_slot(); + + let (block, _): (SignedBeaconBlock, _) = tester + .harness + .make_block_with_modifier(chain_state_before, slot, |b| { + *b.state_root_mut() = Hash256::zero(); + *b.parent_root_mut() = Hash256::zero(); + }) + .await; + + let blinded_block: SignedBlindedBeaconBlock = block.into(); + + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blinded_blocks_v2(&blinded_block, validation_level) + .await; + assert!(response.is_err()); + + let error_response: eth2::Error = response.err().unwrap(); + + /* mandated by Beacon API spec */ + assert_eq!(error_response.status(), Some(StatusCode::BAD_REQUEST)); + + assert!( + matches!(error_response, eth2::Error::ServerMessage(err) if err.message == "BAD_REQUEST: NotFinalizedDescendant { block_parent_root: 0x0000000000000000000000000000000000000000000000000000000000000000 }".to_string()) + ); +} + +/// This test checks that a block that is valid from a gossip perspective is accepted when using `broadcast_validation=gossip`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn blinded_gossip_partial_pass() { + /* this test targets gossip-level validation */ + let validation_level: Option = Some(BroadcastValidation::Gossip); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + + let chain_state_before = tester.harness.get_current_state(); + let slot = chain_state_before.slot() + 1; + + tester.harness.advance_slot(); + + let (block, _): (SignedBeaconBlock, _) = tester + .harness + .make_block_with_modifier(chain_state_before, slot, |b| { + *b.state_root_mut() = Hash256::zero() + }) + .await; + + let blinded_block: SignedBlindedBeaconBlock = block.into(); + + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blinded_blocks_v2(&blinded_block, validation_level) + .await; + assert!(response.is_err()); + + let error_response = response.unwrap_err(); + + assert_eq!(error_response.status(), Some(StatusCode::ACCEPTED)); +} + +// This test checks that a block that is valid from both a gossip and consensus perspective is accepted when using `broadcast_validation=gossip`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn blinded_gossip_full_pass() { + /* this test targets gossip-level validation */ + let validation_level: Option = Some(BroadcastValidation::Gossip); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + tester.harness.advance_slot(); + + let slot_a = Slot::new(num_initial); + let slot_b = slot_a + 1; + + let state_a = tester.harness.get_current_state(); + let (block, _): (SignedBlindedBeaconBlock, _) = + tester.harness.make_blinded_block(state_a, slot_b).await; + + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blinded_blocks_v2(&block, validation_level) + .await; + + assert!(response.is_ok()); + assert!(tester + .harness + .chain + .block_is_known_to_fork_choice(&block.canonical_root())); +} + +/// This test checks that a block that is **invalid** from a gossip perspective gets rejected when using `broadcast_validation=consensus`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn blinded_consensus_invalid() { + /* this test targets gossip-level validation */ + let validation_level: Option = Some(BroadcastValidation::Consensus); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + + let chain_state_before = tester.harness.get_current_state(); + let slot = chain_state_before.slot() + 1; + + tester.harness.advance_slot(); + + let (block, _): (SignedBeaconBlock, _) = tester + .harness + .make_block_with_modifier(chain_state_before, slot, |b| { + *b.state_root_mut() = Hash256::zero(); + *b.parent_root_mut() = Hash256::zero(); + }) + .await; + + let blinded_block: SignedBlindedBeaconBlock = block.into(); + + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blinded_blocks_v2(&blinded_block, validation_level) + .await; + assert!(response.is_err()); + + let error_response: eth2::Error = response.err().unwrap(); + + /* mandated by Beacon API spec */ + assert_eq!(error_response.status(), Some(StatusCode::BAD_REQUEST)); + + assert!( + matches!(error_response, eth2::Error::ServerMessage(err) if err.message == "BAD_REQUEST: NotFinalizedDescendant { block_parent_root: 0x0000000000000000000000000000000000000000000000000000000000000000 }".to_string()) + ); +} + +/// This test checks that a block that is only valid from a gossip perspective is rejected when using `broadcast_validation=consensus`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn blinded_consensus_gossip() { + /* this test targets gossip-level validation */ + let validation_level: Option = Some(BroadcastValidation::Consensus); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + tester.harness.advance_slot(); + + let slot_a = Slot::new(num_initial); + let slot_b = slot_a + 1; + + let state_a = tester.harness.get_current_state(); + let (block, _): (SignedBeaconBlock, _) = tester + .harness + .make_block_with_modifier(state_a, slot_b, |b| *b.state_root_mut() = Hash256::zero()) + .await; + + let blinded_block: SignedBlindedBeaconBlock = block.into(); + + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blinded_blocks_v2(&blinded_block, validation_level) + .await; + assert!(response.is_err()); + + let error_response: eth2::Error = response.err().unwrap(); + + /* mandated by Beacon API spec */ + assert_eq!(error_response.status(), Some(StatusCode::BAD_REQUEST)); + + assert!( + matches!(error_response, eth2::Error::ServerMessage(err) if err.message == "BAD_REQUEST: Invalid block: StateRootMismatch { block: 0x0000000000000000000000000000000000000000000000000000000000000000, local: 0xfc675d642ff7a06458eb33c7d7b62a5813e34d1b2bb1aee3e395100b579da026 }".to_string()) + ); +} + +/// This test checks that a block that is valid from both a gossip and consensus perspective is accepted when using `broadcast_validation=consensus`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn blinded_consensus_full_pass() { + /* this test targets gossip-level validation */ + let validation_level: Option = Some(BroadcastValidation::Consensus); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + tester.harness.advance_slot(); + + let slot_a = Slot::new(num_initial); + let slot_b = slot_a + 1; + + let state_a = tester.harness.get_current_state(); + let (block, _): (SignedBlindedBeaconBlock, _) = + tester.harness.make_blinded_block(state_a, slot_b).await; + + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blinded_blocks_v2(&block, validation_level) + .await; + + assert!(response.is_ok()); + assert!(tester + .harness + .chain + .block_is_known_to_fork_choice(&block.canonical_root())); +} + +/// This test checks that a block that is **invalid** from a gossip perspective gets rejected when using `broadcast_validation=consensus_and_equivocation`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn blinded_equivocation_invalid() { + /* this test targets gossip-level validation */ + let validation_level: Option = + Some(BroadcastValidation::ConsensusAndEquivocation); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + + let chain_state_before = tester.harness.get_current_state(); + let slot = chain_state_before.slot() + 1; + + tester.harness.advance_slot(); + + let (block, _): (SignedBeaconBlock, _) = tester + .harness + .make_block_with_modifier(chain_state_before, slot, |b| { + *b.state_root_mut() = Hash256::zero(); + *b.parent_root_mut() = Hash256::zero(); + }) + .await; + + let blinded_block: SignedBlindedBeaconBlock = block.into(); + + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blinded_blocks_v2(&blinded_block, validation_level) + .await; + assert!(response.is_err()); + + let error_response: eth2::Error = response.err().unwrap(); + + /* mandated by Beacon API spec */ + assert_eq!(error_response.status(), Some(StatusCode::BAD_REQUEST)); + + assert!( + matches!(error_response, eth2::Error::ServerMessage(err) if err.message == "BAD_REQUEST: NotFinalizedDescendant { block_parent_root: 0x0000000000000000000000000000000000000000000000000000000000000000 }".to_string()) + ); +} + +/// This test checks that a block that is valid from both a gossip and consensus perspective is rejected when using `broadcast_validation=consensus_and_equivocation`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn blinded_equivocation_consensus_early_equivocation() { + /* this test targets gossip-level validation */ + let validation_level: Option = + Some(BroadcastValidation::ConsensusAndEquivocation); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + tester.harness.advance_slot(); + + let slot_a = Slot::new(num_initial); + let slot_b = slot_a + 1; + + let state_a = tester.harness.get_current_state(); + let (block_a, state_after_a): (SignedBlindedBeaconBlock, _) = tester + .harness + .make_blinded_block(state_a.clone(), slot_b) + .await; + let (block_b, state_after_b): (SignedBlindedBeaconBlock, _) = + tester.harness.make_blinded_block(state_a, slot_b).await; + + /* check for `make_blinded_block` curios */ + assert_eq!(block_a.state_root(), state_after_a.tree_hash_root()); + assert_eq!(block_b.state_root(), state_after_b.tree_hash_root()); + assert_ne!(block_a.state_root(), block_b.state_root()); + + /* submit `block_a` as valid */ + assert!(tester + .client + .post_beacon_blinded_blocks_v2(&block_a, validation_level) + .await + .is_ok()); + assert!(tester + .harness + .chain + .block_is_known_to_fork_choice(&block_a.canonical_root())); + + /* submit `block_b` which should induce equivocation */ + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blinded_blocks_v2(&block_b, validation_level) + .await; + assert!(response.is_err()); + + let error_response: eth2::Error = response.err().unwrap(); + + assert_eq!(error_response.status(), Some(StatusCode::BAD_REQUEST)); + + assert!( + matches!(error_response, eth2::Error::ServerMessage(err) if err.message == "BAD_REQUEST: Slashable".to_string()) + ); +} + +/// This test checks that a block that is only valid from a gossip perspective is rejected when using `broadcast_validation=consensus_and_equivocation`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn blinded_equivocation_gossip() { + /* this test targets gossip-level validation */ + let validation_level: Option = + Some(BroadcastValidation::ConsensusAndEquivocation); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + tester.harness.advance_slot(); + + let slot_a = Slot::new(num_initial); + let slot_b = slot_a + 1; + + let state_a = tester.harness.get_current_state(); + let (block, _): (SignedBeaconBlock, _) = tester + .harness + .make_block_with_modifier(state_a, slot_b, |b| *b.state_root_mut() = Hash256::zero()) + .await; + + let blinded_block: SignedBlindedBeaconBlock = block.into(); + + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blinded_blocks_v2(&blinded_block, validation_level) + .await; + assert!(response.is_err()); + + let error_response: eth2::Error = response.err().unwrap(); + + /* mandated by Beacon API spec */ + assert_eq!(error_response.status(), Some(StatusCode::BAD_REQUEST)); + + assert!( + matches!(error_response, eth2::Error::ServerMessage(err) if err.message == "BAD_REQUEST: Invalid block: StateRootMismatch { block: 0x0000000000000000000000000000000000000000000000000000000000000000, local: 0xfc675d642ff7a06458eb33c7d7b62a5813e34d1b2bb1aee3e395100b579da026 }".to_string()) + ); +} + +/// This test checks that a block that is valid from both a gossip and consensus perspective but that equivocates **late** is rejected when using `broadcast_validation=consensus_and_equivocation`. +/// +/// This test is unique in that we can't actually test the HTTP API directly, but instead have to hook into the `publish_blocks` code manually. This is in order to handle the late equivocation case. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn blinded_equivocation_consensus_late_equivocation() { + /* this test targets gossip-level validation */ + let validation_level: Option = + Some(BroadcastValidation::ConsensusAndEquivocation); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + let test_logger = tester.harness.logger().clone(); + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + tester.harness.advance_slot(); + + let slot_a = Slot::new(num_initial); + let slot_b = slot_a + 1; + + let state_a = tester.harness.get_current_state(); + let (block_a, state_after_a): (SignedBlindedBeaconBlock, _) = tester + .harness + .make_blinded_block(state_a.clone(), slot_b) + .await; + let (block_b, state_after_b): (SignedBlindedBeaconBlock, _) = + tester.harness.make_blinded_block(state_a, slot_b).await; + + /* check for `make_blinded_block` curios */ + assert_eq!(block_a.state_root(), state_after_a.tree_hash_root()); + assert_eq!(block_b.state_root(), state_after_b.tree_hash_root()); + assert_ne!(block_a.state_root(), block_b.state_root()); + + let unblinded_block_a = reconstruct_block( + tester.harness.chain.clone(), + block_a.state_root(), + block_a, + test_logger.clone(), + ) + .await + .unwrap(); + let unblinded_block_b = reconstruct_block( + tester.harness.chain.clone(), + block_b.clone().state_root(), + block_b.clone(), + test_logger.clone(), + ) + .await + .unwrap(); + + let inner_block_a = match unblinded_block_a { + ProvenancedBlock::Local(a, _) => a, + ProvenancedBlock::Builder(a, _) => a, + }; + let inner_block_b = match unblinded_block_b { + ProvenancedBlock::Local(b, _) => b, + ProvenancedBlock::Builder(b, _) => b, + }; + + let gossip_block_b = GossipVerifiedBlock::new(inner_block_b, &tester.harness.chain); + assert!(gossip_block_b.is_ok()); + let gossip_block_a = GossipVerifiedBlock::new(inner_block_a, &tester.harness.chain); + assert!(gossip_block_a.is_err()); + + let channel = tokio::sync::mpsc::unbounded_channel(); + + let publication_result: Result<(), Rejection> = publish_blinded_block( + block_b, + tester.harness.chain, + &channel.0, + test_logger, + validation_level.unwrap(), + ) + .await; + + assert!(publication_result.is_err()); + + let publication_error: Rejection = publication_result.unwrap_err(); + + assert!(publication_error.find::().is_some()); +} + +/// This test checks that a block that is valid from both a gossip and consensus perspective (and does not equivocate) is accepted when using `broadcast_validation=consensus_and_equivocation`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn blinded_equivocation_full_pass() { + /* this test targets gossip-level validation */ + let validation_level: Option = + Some(BroadcastValidation::ConsensusAndEquivocation); + + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 64; + let num_initial: u64 = 31; + let tester = InteractiveTester::::new(None, validator_count).await; + + // Create some chain depth. + tester.harness.advance_slot(); + tester + .harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + tester.harness.advance_slot(); + + let slot_a = Slot::new(num_initial); + let slot_b = slot_a + 1; + + let state_a = tester.harness.get_current_state(); + let (block, _): (SignedBlindedBeaconBlock, _) = + tester.harness.make_blinded_block(state_a, slot_b).await; + + let response: Result<(), eth2::Error> = tester + .client + .post_beacon_blocks_v2(&block, validation_level) + .await; + + assert!(response.is_ok()); + assert!(tester + .harness + .chain + .block_is_known_to_fork_choice(&block.canonical_root())); +} diff --git a/beacon_node/http_api/tests/main.rs b/beacon_node/http_api/tests/main.rs index f5916d8506a..e0636424e48 100644 --- a/beacon_node/http_api/tests/main.rs +++ b/beacon_node/http_api/tests/main.rs @@ -1,5 +1,6 @@ #![cfg(not(debug_assertions))] // Tests are too slow in debug. +pub mod broadcast_validation_tests; pub mod fork_tests; pub mod interactive_tests; pub mod status_tests; diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index a6c49ddaeef..bcd192c699e 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -159,7 +159,7 @@ impl ApiTester { // `make_block` adds random graffiti, so this will produce an alternate block let (reorg_block, _reorg_state) = harness - .make_block(head.beacon_state.clone(), harness.chain.slot().unwrap()) + .make_block(head.beacon_state.clone(), harness.chain.slot().unwrap() + 1) .await; let head_state_root = head.beacon_state_root(); @@ -1248,14 +1248,23 @@ impl ApiTester { } pub async fn test_post_beacon_blocks_invalid(mut self) -> Self { - let mut next_block = self.next_block.clone(); - *next_block.message_mut().proposer_index_mut() += 1; + let block = self + .harness + .make_block_with_modifier( + self.harness.get_current_state(), + self.harness.get_current_slot(), + |b| { + *b.state_root_mut() = Hash256::zero(); + }, + ) + .await + .0; - assert!(self.client.post_beacon_blocks(&next_block).await.is_err()); + assert!(self.client.post_beacon_blocks(&block).await.is_err()); assert!( self.network_rx.network_recv.recv().await.is_some(), - "invalid blocks should be sent to network" + "gossip valid blocks should be sent to network" ); self @@ -4126,7 +4135,7 @@ impl ApiTester { .unwrap(); let expected_reorg = EventKind::ChainReorg(SseChainReorg { - slot: self.next_block.slot(), + slot: self.reorg_block.slot(), depth: 1, old_head_block: self.next_block.canonical_root(), old_head_state: self.next_block.state_root(), @@ -4136,6 +4145,8 @@ impl ApiTester { execution_optimistic: false, }); + self.harness.advance_slot(); + self.client .post_beacon_blocks(&self.reorg_block) .await diff --git a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs index 185634c3081..91ec81b18d3 100644 --- a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs @@ -785,6 +785,20 @@ impl Worker { verified_block } + Err(e @ BlockError::Slashable) => { + warn!( + self.log, + "Received equivocating block from peer"; + "error" => ?e + ); + /* punish peer for submitting an equivocation, but not too harshly as honest peers may conceivably forward equivocating blocks to us from time to time */ + self.gossip_penalize_peer( + peer_id, + PeerAction::MidToleranceError, + "gossip_block_mid", + ); + return None; + } Err(BlockError::ParentUnknown(block)) => { debug!( self.log, @@ -806,7 +820,6 @@ impl Worker { Err(e @ BlockError::FutureSlot { .. }) | Err(e @ BlockError::WouldRevertFinalizedSlot { .. }) | Err(e @ BlockError::BlockIsAlreadyKnown) - | Err(e @ BlockError::RepeatProposal { .. }) | Err(e @ BlockError::NotFinalizedDescendant { .. }) => { debug!(self.log, "Could not verify block for gossip. Ignoring the block"; "error" => %e); @@ -948,7 +961,12 @@ impl Worker { let result = self .chain - .process_block(block_root, verified_block, NotifyExecutionLayer::Yes) + .process_block( + block_root, + verified_block, + NotifyExecutionLayer::Yes, + || Ok(()), + ) .await; match &result { diff --git a/beacon_node/network/src/beacon_processor/worker/sync_methods.rs b/beacon_node/network/src/beacon_processor/worker/sync_methods.rs index 7e8fce35632..ac59b1daa93 100644 --- a/beacon_node/network/src/beacon_processor/worker/sync_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/sync_methods.rs @@ -98,33 +98,21 @@ impl Worker { }); // Checks if a block from this proposer is already known. - let proposal_already_known = || { + let block_equivocates = || { match self .chain .observed_block_producers .read() - .proposer_has_been_observed(block.message()) + .proposer_has_been_observed(block.message(), block.canonical_root()) { - Ok(is_observed) => is_observed, - // Both of these blocks will be rejected, so reject them now rather + Ok(seen_status) => seen_status.is_slashable(), + //Both of these blocks will be rejected, so reject them now rather // than re-queuing them. Err(ObserveError::FinalizedBlock { .. }) | Err(ObserveError::ValidatorIndexTooHigh { .. }) => false, } }; - // Returns `true` if the block is already known to fork choice. Notably, - // this will return `false` for blocks that we've already imported but - // ancestors of the finalized checkpoint. That should not be an issue - // for our use here since finalized blocks will always be late and won't - // be requeued anyway. - let block_is_already_known = || { - self.chain - .canonical_head - .fork_choice_read_lock() - .contains_block(&block_root) - }; - // If we've already seen a block from this proposer *and* the block // arrived before the attestation deadline, requeue it to ensure it is // imported late enough that it won't receive a proposer boost. @@ -132,7 +120,7 @@ impl Worker { // Don't requeue blocks if they're already known to fork choice, just // push them through to block processing so they can be handled through // the normal channels. - if !block_is_late && proposal_already_known() && !block_is_already_known() { + if !block_is_late && block_equivocates() { debug!( self.log, "Delaying processing of duplicate RPC block"; @@ -165,7 +153,7 @@ impl Worker { let parent_root = block.message().parent_root(); let result = self .chain - .process_block(block_root, block, NotifyExecutionLayer::Yes) + .process_block(block_root, block, NotifyExecutionLayer::Yes, || Ok(())) .await; metrics::inc_counter(&metrics::BEACON_PROCESSOR_RPC_BLOCK_IMPORTED_TOTAL); diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 217d3569689..e34916bebab 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -322,6 +322,26 @@ impl BeaconNodeHttpClient { ok_or_error(response).await } + /// Generic POST function supporting arbitrary responses and timeouts. + async fn post_generic_with_consensus_version( + &self, + url: U, + body: &T, + timeout: Option, + fork: ForkName, + ) -> Result { + let mut builder = self.client.post(url); + if let Some(timeout) = timeout { + builder = builder.timeout(timeout); + } + let response = builder + .header(CONSENSUS_VERSION_HEADER, fork.to_string()) + .json(body) + .send() + .await?; + ok_or_error(response).await + } + /// `GET beacon/genesis` /// /// ## Errors @@ -654,6 +674,76 @@ impl BeaconNodeHttpClient { Ok(()) } + pub fn post_beacon_blocks_v2_path( + &self, + validation_level: Option, + ) -> Result { + let mut path = self.eth_path(V2)?; + path.path_segments_mut() + .map_err(|_| Error::InvalidUrl(self.server.clone()))? + .extend(&["beacon", "blocks"]); + + path.set_query( + validation_level + .map(|v| format!("broadcast_validation={}", v)) + .as_deref(), + ); + + Ok(path) + } + + pub fn post_beacon_blinded_blocks_v2_path( + &self, + validation_level: Option, + ) -> Result { + let mut path = self.eth_path(V2)?; + path.path_segments_mut() + .map_err(|_| Error::InvalidUrl(self.server.clone()))? + .extend(&["beacon", "blinded_blocks"]); + + path.set_query( + validation_level + .map(|v| format!("broadcast_validation={}", v)) + .as_deref(), + ); + + Ok(path) + } + + /// `POST v2/beacon/blocks` + pub async fn post_beacon_blocks_v2>( + &self, + block: &SignedBeaconBlock, + validation_level: Option, + ) -> Result<(), Error> { + self.post_generic_with_consensus_version( + self.post_beacon_blocks_v2_path(validation_level)?, + block, + Some(self.timeouts.proposal), + block.message().body().fork_name(), + ) + .await?; + + Ok(()) + } + + /// `POST v2/beacon/blinded_blocks` + pub async fn post_beacon_blinded_blocks_v2( + &self, + block: &SignedBlindedBeaconBlock, + validation_level: Option, + ) -> Result<(), Error> { + self.post_generic_with_consensus_version( + self.post_beacon_blinded_blocks_v2_path(validation_level)?, + block, + Some(self.timeouts.proposal), + block.message().body().fork_name(), + ) + .await?; + + Ok(()) + } + /// Path for `v2/beacon/blocks` pub fn get_beacon_blocks_path(&self, block_id: BlockId) -> Result { let mut path = self.eth_path(V2)?; diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 55759a2e158..5f2e1ada7be 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -6,7 +6,7 @@ use lighthouse_network::{ConnectionDirection, Enr, Multiaddr, PeerConnectionStat use mediatype::{names, MediaType, MediaTypeList}; use serde::{Deserialize, Serialize}; use std::convert::TryFrom; -use std::fmt; +use std::fmt::{self, Display}; use std::str::{from_utf8, FromStr}; use std::time::Duration; pub use types::*; @@ -1260,6 +1260,50 @@ pub struct ForkChoiceNode { pub execution_block_hash: Option, } +#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum BroadcastValidation { + Gossip, + Consensus, + ConsensusAndEquivocation, +} + +impl Default for BroadcastValidation { + fn default() -> Self { + Self::Gossip + } +} + +impl Display for BroadcastValidation { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Gossip => write!(f, "gossip"), + Self::Consensus => write!(f, "consensus"), + Self::ConsensusAndEquivocation => write!(f, "consensus_and_equivocation"), + } + } +} + +impl FromStr for BroadcastValidation { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + match s { + "gossip" => Ok(Self::Gossip), + "consensus" => Ok(Self::Consensus), + "consensus_and_equivocation" => Ok(Self::ConsensusAndEquivocation), + _ => Err("Invalid broadcast validation level"), + } + } +} + +#[derive(Default, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub struct BroadcastValidationQuery { + #[serde(default)] + pub broadcast_validation: BroadcastValidation, +} + #[cfg(test)] mod tests { use super::*; diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index c0ba8694100..dce1be742ff 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -89,7 +89,7 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, T, } } -impl<'a, T: EthSpec> BeaconBlockBodyRef<'a, T> { +impl<'a, T: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, T, Payload> { /// Get the fork_name of this object pub fn fork_name(self) -> ForkName { match self { diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index e0f4043ac21..65528de1757 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -382,6 +382,7 @@ impl Tester { block_root, block.clone(), NotifyExecutionLayer::Yes, + || Ok(()), ))?; if result.is_ok() != valid { return Err(Error::DidntFail(format!( From edd093293a34b68e5fc32382f7fa0c374f8543ed Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Fri, 30 Jun 2023 01:13:03 +0000 Subject: [PATCH 03/51] added debounce to log (#4269) ## Issue Addressed [#4259](https://github.com/sigp/lighthouse/issues/4259) ## Proposed Changes debounce spammy `Unable to send message to the beacon processor` log messages ## Additional Info We could potentially debounce other logs that have the potential to be "spammy". After some feedback we decided to additionally add the following change: create a newtype wrapper around `mpsc::Sender>`. When there is an error on the try_send method on the wrapper, we increase a counter metric with one label per work type. --- .../network/src/beacon_processor/mod.rs | 18 +++++++++++++++ beacon_node/network/src/metrics.rs | 6 +++++ beacon_node/network/src/router.rs | 22 +++++++++++++------ .../network/src/sync/block_lookups/tests.rs | 3 ++- beacon_node/network/src/sync/manager.rs | 4 ++-- .../network/src/sync/network_context.rs | 10 ++++----- .../network/src/sync/range_sync/range.rs | 4 ++-- 7 files changed, 50 insertions(+), 17 deletions(-) diff --git a/beacon_node/network/src/beacon_processor/mod.rs b/beacon_node/network/src/beacon_processor/mod.rs index 26d2c19b519..84d8e1b07a8 100644 --- a/beacon_node/network/src/beacon_processor/mod.rs +++ b/beacon_node/network/src/beacon_processor/mod.rs @@ -750,6 +750,24 @@ impl std::convert::From> for WorkEvent { } } +pub struct BeaconProcessorSend(pub mpsc::Sender>); + +impl BeaconProcessorSend { + pub fn try_send(&self, message: WorkEvent) -> Result<(), Box>>> { + let work_type = message.work_type(); + match self.0.try_send(message) { + Ok(res) => Ok(res), + Err(e) => { + metrics::inc_counter_vec( + &metrics::BEACON_PROCESSOR_SEND_ERROR_PER_WORK_TYPE, + &[work_type], + ); + Err(Box::new(e)) + } + } + } +} + /// A consensus message (or multiple) from the network that requires processing. #[derive(Derivative)] #[derivative(Debug(bound = "T: BeaconChainTypes"))] diff --git a/beacon_node/network/src/metrics.rs b/beacon_node/network/src/metrics.rs index 09caaaa11e3..27d7dc9625d 100644 --- a/beacon_node/network/src/metrics.rs +++ b/beacon_node/network/src/metrics.rs @@ -279,6 +279,12 @@ lazy_static! { "Gossipsub light_client_optimistic_update errors per error type", &["type"] ); + pub static ref BEACON_PROCESSOR_SEND_ERROR_PER_WORK_TYPE: Result = + try_create_int_counter_vec( + "beacon_processor_send_error_per_work_type", + "Total number of beacon processor send error per work type", + &["type"] + ); /* diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 1b0f1fb41e1..7a91f2d0b1a 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -6,7 +6,8 @@ #![allow(clippy::unit_arg)] use crate::beacon_processor::{ - BeaconProcessor, InvalidBlockStorage, WorkEvent as BeaconWorkEvent, MAX_WORK_EVENT_QUEUE_LEN, + BeaconProcessor, BeaconProcessorSend, InvalidBlockStorage, WorkEvent as BeaconWorkEvent, + MAX_WORK_EVENT_QUEUE_LEN, }; use crate::error; use crate::service::{NetworkMessage, RequestId}; @@ -19,6 +20,7 @@ use lighthouse_network::rpc::*; use lighthouse_network::{ MessageId, NetworkGlobals, PeerId, PeerRequestId, PubsubMessage, Request, Response, }; +use logging::TimeLatch; use slog::{debug, o, trace}; use slog::{error, warn}; use std::cmp; @@ -39,9 +41,11 @@ pub struct Router { /// A network context to return and handle RPC requests. network: HandlerNetworkContext, /// A multi-threaded, non-blocking processor for applying messages to the beacon chain. - beacon_processor_send: mpsc::Sender>, + beacon_processor_send: BeaconProcessorSend, /// The `Router` logger. log: slog::Logger, + /// Provides de-bounce functionality for logging. + logger_debounce: TimeLatch, } /// Types of messages the router can receive. @@ -100,7 +104,7 @@ impl Router { beacon_chain.clone(), network_globals.clone(), network_send.clone(), - beacon_processor_send.clone(), + BeaconProcessorSend(beacon_processor_send.clone()), sync_logger, ); @@ -124,8 +128,9 @@ impl Router { chain: beacon_chain, sync_send, network: HandlerNetworkContext::new(network_send, log.clone()), - beacon_processor_send, + beacon_processor_send: BeaconProcessorSend(beacon_processor_send), log: message_handler_log, + logger_debounce: TimeLatch::default(), }; // spawn handler task and move the message handler instance into the spawned thread @@ -479,12 +484,15 @@ impl Router { self.beacon_processor_send .try_send(work) .unwrap_or_else(|e| { - let work_type = match &e { + let work_type = match &*e { mpsc::error::TrySendError::Closed(work) | mpsc::error::TrySendError::Full(work) => work.work_type(), }; - error!(&self.log, "Unable to send message to the beacon processor"; - "error" => %e, "type" => work_type) + + if self.logger_debounce.elapsed() { + error!(&self.log, "Unable to send message to the beacon processor"; + "error" => %e, "type" => work_type) + } }) } } diff --git a/beacon_node/network/src/sync/block_lookups/tests.rs b/beacon_node/network/src/sync/block_lookups/tests.rs index 5a70944f6cb..82334db0f8e 100644 --- a/beacon_node/network/src/sync/block_lookups/tests.rs +++ b/beacon_node/network/src/sync/block_lookups/tests.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +use crate::beacon_processor::BeaconProcessorSend; use crate::service::RequestId; use crate::sync::manager::RequestId as SyncId; use crate::NetworkMessage; @@ -54,7 +55,7 @@ impl TestRig { SyncNetworkContext::new( network_tx, globals, - beacon_processor_tx, + BeaconProcessorSend(beacon_processor_tx), log.new(slog::o!("component" => "network_context")), ) }; diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 37b63cdba78..c24d4c192b1 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -38,7 +38,7 @@ use super::block_lookups::BlockLookups; use super::network_context::SyncNetworkContext; use super::peer_sync_info::{remote_sync_type, PeerSyncType}; use super::range_sync::{RangeSync, RangeSyncType, EPOCHS_PER_BATCH}; -use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent as BeaconWorkEvent}; +use crate::beacon_processor::{BeaconProcessorSend, ChainSegmentProcessId}; use crate::service::NetworkMessage; use crate::status::ToStatusMessage; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError, EngineState}; @@ -188,7 +188,7 @@ pub fn spawn( beacon_chain: Arc>, network_globals: Arc>, network_send: mpsc::UnboundedSender>, - beacon_processor_send: mpsc::Sender>, + beacon_processor_send: BeaconProcessorSend, log: slog::Logger, ) -> mpsc::UnboundedSender> { assert!( diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 23d42002f47..03c466eecea 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -3,7 +3,7 @@ use super::manager::{Id, RequestId as SyncRequestId}; use super::range_sync::{BatchId, ChainId}; -use crate::beacon_processor::WorkEvent; +use crate::beacon_processor::BeaconProcessorSend; use crate::service::{NetworkMessage, RequestId}; use crate::status::ToStatusMessage; use beacon_chain::{BeaconChainTypes, EngineState}; @@ -37,7 +37,7 @@ pub struct SyncNetworkContext { execution_engine_state: EngineState, /// Channel to send work to the beacon processor. - beacon_processor_send: mpsc::Sender>, + beacon_processor_send: BeaconProcessorSend, /// Logger for the `SyncNetworkContext`. log: slog::Logger, @@ -47,7 +47,7 @@ impl SyncNetworkContext { pub fn new( network_send: mpsc::UnboundedSender>, network_globals: Arc>, - beacon_processor_send: mpsc::Sender>, + beacon_processor_send: BeaconProcessorSend, log: slog::Logger, ) -> Self { Self { @@ -278,12 +278,12 @@ impl SyncNetworkContext { }) } - pub fn processor_channel_if_enabled(&self) -> Option<&mpsc::Sender>> { + pub fn processor_channel_if_enabled(&self) -> Option<&BeaconProcessorSend> { self.is_execution_engine_online() .then_some(&self.beacon_processor_send) } - pub fn processor_channel(&self) -> &mpsc::Sender> { + pub fn processor_channel(&self) -> &BeaconProcessorSend { &self.beacon_processor_send } diff --git a/beacon_node/network/src/sync/range_sync/range.rs b/beacon_node/network/src/sync/range_sync/range.rs index 0f1c00e509f..2c35c57d9e4 100644 --- a/beacon_node/network/src/sync/range_sync/range.rs +++ b/beacon_node/network/src/sync/range_sync/range.rs @@ -375,7 +375,7 @@ mod tests { use crate::NetworkMessage; use super::*; - use crate::beacon_processor::WorkEvent as BeaconWorkEvent; + use crate::beacon_processor::{BeaconProcessorSend, WorkEvent as BeaconWorkEvent}; use beacon_chain::builder::Witness; use beacon_chain::eth1_chain::CachingEth1Backend; use beacon_chain::parking_lot::RwLock; @@ -603,7 +603,7 @@ mod tests { let cx = SyncNetworkContext::new( network_tx, globals.clone(), - beacon_processor_tx, + BeaconProcessorSend(beacon_processor_tx), log.new(o!("component" => "network_context")), ); let test_rig = TestRig { From 826e090f50da93a0cca297bc4e57a4300f0db9f2 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Fri, 30 Jun 2023 01:13:04 +0000 Subject: [PATCH 04/51] Update node health endpoint (#4310) ## Issue Addressed [#4292](https://github.com/sigp/lighthouse/issues/4292) ## Proposed Changes Updated the node health endpoint will return a 200 status code if `!syncing && !el_offline && !optimistic` wil return a 206 if `(syncing || optimistic) && !el_offline` will return a 503 if `el_offline` ## Additional Info --- beacon_node/http_api/src/lib.rs | 53 +++++++++----- beacon_node/http_api/tests/status_tests.rs | 80 ++++++++++++++++++++++ beacon_node/http_api/tests/tests.rs | 14 ++-- 3 files changed, 125 insertions(+), 22 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 93bfe524bc2..27bcc4d8a13 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -2418,24 +2418,41 @@ pub fn serve( .and(warp::path("health")) .and(warp::path::end()) .and(network_globals.clone()) - .and_then(|network_globals: Arc>| { - blocking_response_task(move || match *network_globals.sync_state.read() { - SyncState::SyncingFinalized { .. } - | SyncState::SyncingHead { .. } - | SyncState::SyncTransition - | SyncState::BackFillSyncing { .. } => Ok(warp::reply::with_status( - warp::reply(), - warp::http::StatusCode::PARTIAL_CONTENT, - )), - SyncState::Synced => Ok(warp::reply::with_status( - warp::reply(), - warp::http::StatusCode::OK, - )), - SyncState::Stalled => Err(warp_utils::reject::not_synced( - "sync stalled, beacon chain may not yet be initialized.".to_string(), - )), - }) - }); + .and(chain_filter.clone()) + .and_then( + |network_globals: Arc>, chain: Arc>| { + async move { + let el_offline = if let Some(el) = &chain.execution_layer { + el.is_offline_or_erroring().await + } else { + true + }; + + blocking_response_task(move || { + let is_optimistic = chain + .is_optimistic_or_invalid_head() + .map_err(warp_utils::reject::beacon_chain_error)?; + + let is_syncing = !network_globals.sync_state.read().is_synced(); + + if el_offline { + Err(warp_utils::reject::not_synced("execution layer is offline".to_string())) + } else if is_syncing || is_optimistic { + Ok(warp::reply::with_status( + warp::reply(), + warp::http::StatusCode::PARTIAL_CONTENT, + )) + } else { + Ok(warp::reply::with_status( + warp::reply(), + warp::http::StatusCode::OK, + )) + } + }) + .await + } + }, + ); // GET node/peers/{peer_id} let get_node_peers_by_id = eth_v1 diff --git a/beacon_node/http_api/tests/status_tests.rs b/beacon_node/http_api/tests/status_tests.rs index ce725b75a9a..95f885faa56 100644 --- a/beacon_node/http_api/tests/status_tests.rs +++ b/beacon_node/http_api/tests/status_tests.rs @@ -3,6 +3,7 @@ use beacon_chain::{ test_utils::{AttestationStrategy, BlockStrategy, SyncCommitteeStrategy}, BlockError, }; +use eth2::StatusCode; use execution_layer::{PayloadStatusV1, PayloadStatusV1Status}; use http_api::test_utils::InteractiveTester; use types::{EthSpec, ExecPayload, ForkName, MinimalEthSpec, Slot}; @@ -143,3 +144,82 @@ async fn el_error_on_new_payload() { assert_eq!(api_response.is_optimistic, Some(false)); assert_eq!(api_response.is_syncing, false); } + +/// Check `node health` endpoint when the EL is offline. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn node_health_el_offline() { + let num_blocks = E::slots_per_epoch() / 2; + let num_validators = E::slots_per_epoch(); + let tester = post_merge_tester(num_blocks, num_validators).await; + let harness = &tester.harness; + let mock_el = harness.mock_execution_layer.as_ref().unwrap(); + + // EL offline + mock_el.server.set_syncing_response(Err("offline".into())); + mock_el.el.upcheck().await; + + let status = tester.client.get_node_health().await; + match status { + Ok(_) => { + panic!("should return 503 error status code"); + } + Err(e) => { + assert_eq!(e.status().unwrap(), 503); + } + } +} + +/// Check `node health` endpoint when the EL is online and synced. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn node_health_el_online_and_synced() { + let num_blocks = E::slots_per_epoch() / 2; + let num_validators = E::slots_per_epoch(); + let tester = post_merge_tester(num_blocks, num_validators).await; + let harness = &tester.harness; + let mock_el = harness.mock_execution_layer.as_ref().unwrap(); + + // EL synced + mock_el.server.set_syncing_response(Ok(false)); + mock_el.el.upcheck().await; + + let status = tester.client.get_node_health().await; + match status { + Ok(response) => { + assert_eq!(response, StatusCode::OK); + } + Err(_) => { + panic!("should return 200 status code"); + } + } +} + +/// Check `node health` endpoint when the EL is online but not synced. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn node_health_el_online_and_not_synced() { + let num_blocks = E::slots_per_epoch() / 2; + let num_validators = E::slots_per_epoch(); + let tester = post_merge_tester(num_blocks, num_validators).await; + let harness = &tester.harness; + let mock_el = harness.mock_execution_layer.as_ref().unwrap(); + + // EL not synced + harness.advance_slot(); + mock_el.server.all_payloads_syncing(true); + harness + .extend_chain( + 1, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + + let status = tester.client.get_node_health().await; + match status { + Ok(response) => { + assert_eq!(response, StatusCode::PARTIAL_CONTENT); + } + Err(_) => { + panic!("should return 206 status code"); + } + } +} diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index bcd192c699e..741ee1ffc06 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -8,7 +8,7 @@ use eth2::{ mixin::{RequestAccept, ResponseForkName, ResponseOptional}, reqwest::RequestBuilder, types::{BlockId as CoreBlockId, ForkChoiceNode, StateId as CoreStateId, *}, - BeaconNodeHttpClient, Error, StatusCode, Timeouts, + BeaconNodeHttpClient, Error, Timeouts, }; use execution_layer::test_utils::TestingBuilder; use execution_layer::test_utils::DEFAULT_BUILDER_THRESHOLD_WEI; @@ -1762,9 +1762,15 @@ impl ApiTester { } pub async fn test_get_node_health(self) -> Self { - let status = self.client.get_node_health().await.unwrap(); - assert_eq!(status, StatusCode::OK); - + let status = self.client.get_node_health().await; + match status { + Ok(_) => { + panic!("should return 503 error status code"); + } + Err(e) => { + assert_eq!(e.status().unwrap(), 503); + } + } self } From 46be05f7280c3afdfbd614dbc8f6a9b3985ab821 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Fri, 30 Jun 2023 01:13:06 +0000 Subject: [PATCH 05/51] Cache target attester balances for unrealized FFG progression calculation (#4362) ## Issue Addressed #4118 ## Proposed Changes This PR introduces a "progressive balances" cache on the `BeaconState`, which keeps track of the accumulated target attestation balance for the current & previous epochs. The cached values are utilised by fork choice to calculate unrealized justification and finalization (instead of converting epoch participation arrays to balances for each block we receive). This optimization will be rolled out gradually to allow for more testing. A new `--progressive-balances disabled|checked|strict|fast` flag is introduced to support this: - `checked`: enabled with checks against participation cache, and falls back to the existing epoch processing calculation if there is a total target attester balance mismatch. There is no performance gain from this as the participation cache still needs to be computed. **This is the default mode for now.** - `strict`: enabled with checks against participation cache, returns error if there is a mismatch. **Used for testing only**. - `fast`: enabled with no comparative checks and without computing the participation cache. This mode gives us the performance gains from the optimization. This is still experimental and not currently recommended for production usage, but will become the default mode in a future release. - `disabled`: disable the usage of progressive cache, and use the existing method for FFG progression calculation. This mode may be useful if we find a bug and want to stop the frequent error logs. ### Tasks - [x] Initial cache implementation in `BeaconState` - [x] Perform checks in fork choice to compare the progressive balances cache against results from `ParticipationCache` - [x] Add CLI flag, and disable the optimization by default - [x] Testing on Goerli & Benchmarking - [x] Move caching logic from state processing to the `ProgressiveBalancesCache` (see [this comment](https://github.com/sigp/lighthouse/pull/4362#discussion_r1230877001)) - [x] Add attesting balance metrics Co-authored-by: Jimmy Chen --- Cargo.lock | 1 + beacon_node/beacon_chain/src/beacon_chain.rs | 2 + beacon_node/beacon_chain/src/builder.rs | 8 +- beacon_node/beacon_chain/src/chain_config.rs | 5 +- beacon_node/beacon_chain/src/fork_revert.rs | 9 +- beacon_node/beacon_chain/src/test_utils.rs | 38 +++- beacon_node/beacon_chain/tests/capella.rs | 18 +- .../tests/payload_invalidation.rs | 3 +- beacon_node/http_api/src/block_rewards.rs | 2 +- beacon_node/src/cli.rs | 14 ++ beacon_node/src/config.rs | 6 + beacon_node/store/src/partial_beacon_state.rs | 1 + beacon_node/store/src/reconstruct.rs | 2 +- consensus/fork_choice/src/fork_choice.rs | 209 +++++++++++++++--- consensus/fork_choice/tests/tests.rs | 106 ++++++++- consensus/state_processing/src/common/mod.rs | 1 + .../src/common/slash_validator.rs | 3 + .../update_progressive_balances_cache.rs | 142 ++++++++++++ consensus/state_processing/src/genesis.rs | 4 +- consensus/state_processing/src/metrics.rs | 11 + .../src/per_block_processing.rs | 9 + .../src/per_block_processing/errors.rs | 9 + .../process_operations.rs | 11 + .../src/per_epoch_processing/altair.rs | 8 +- .../altair/participation_cache.rs | 54 ++--- .../src/per_epoch_processing/base.rs | 2 +- .../src/per_epoch_processing/capella.rs | 8 +- .../effective_balance_updates.rs | 41 +++- .../src/per_epoch_processing/slashings.rs | 2 +- .../state_processing/src/upgrade/altair.rs | 4 + .../state_processing/src/upgrade/capella.rs | 1 + .../state_processing/src/upgrade/merge.rs | 1 + consensus/types/Cargo.toml | 1 + consensus/types/benches/benches.rs | 2 +- consensus/types/src/beacon_state.rs | 55 ++++- consensus/types/src/beacon_state/balance.rs | 33 +++ .../types/src/beacon_state/clone_config.rs | 2 + .../progressive_balances_cache.rs | 184 +++++++++++++++ consensus/types/src/beacon_state/tests.rs | 5 +- lcli/src/new_testnet.rs | 2 +- lcli/src/skip_slots.rs | 2 +- lcli/src/transition_blocks.rs | 6 +- lighthouse/tests/beacon_node.rs | 30 ++- .../ef_tests/src/cases/epoch_processing.rs | 4 +- testing/ef_tests/src/cases/fork_choice.rs | 5 +- testing/ef_tests/src/cases/operations.rs | 4 + testing/ef_tests/src/cases/sanity_blocks.rs | 2 +- testing/ef_tests/src/cases/sanity_slots.rs | 2 +- 48 files changed, 953 insertions(+), 121 deletions(-) create mode 100644 consensus/state_processing/src/common/update_progressive_balances_cache.rs create mode 100644 consensus/types/src/beacon_state/balance.rs create mode 100644 consensus/types/src/beacon_state/progressive_balances_cache.rs diff --git a/Cargo.lock b/Cargo.lock index 02922b2d7e4..efc6a5d6abe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8935,6 +8935,7 @@ dependencies = [ "smallvec", "ssz_types", "state_processing", + "strum", "superstruct 0.6.0", "swap_or_not_shuffle", "tempfile", diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 772e4c1529e..01343ff3b15 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2898,7 +2898,9 @@ impl BeaconChain { block_delay, &state, payload_verification_status, + self.config.progressive_balances_mode, &self.spec, + &self.log, ) .map_err(|e| BlockError::BeaconChainError(e.into()))?; } diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index 9bb39396326..044391c415e 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -338,7 +338,7 @@ where let beacon_block = genesis_block(&mut beacon_state, &self.spec)?; beacon_state - .build_all_caches(&self.spec) + .build_caches(&self.spec) .map_err(|e| format!("Failed to build genesis state caches: {:?}", e))?; let beacon_state_root = beacon_block.message().state_root(); @@ -437,7 +437,7 @@ where // Prime all caches before storing the state in the database and computing the tree hash // root. weak_subj_state - .build_all_caches(&self.spec) + .build_caches(&self.spec) .map_err(|e| format!("Error building caches on checkpoint state: {e:?}"))?; let computed_state_root = weak_subj_state @@ -687,6 +687,8 @@ where store.clone(), Some(current_slot), &self.spec, + self.chain_config.progressive_balances_mode, + &log, )?; } @@ -700,7 +702,7 @@ where head_snapshot .beacon_state - .build_all_caches(&self.spec) + .build_caches(&self.spec) .map_err(|e| format!("Failed to build state caches: {:?}", e))?; // Perform a check to ensure that the finalization points of the head and fork choice are diff --git a/beacon_node/beacon_chain/src/chain_config.rs b/beacon_node/beacon_chain/src/chain_config.rs index 34a5c9a4ec9..cc7a957ecc8 100644 --- a/beacon_node/beacon_chain/src/chain_config.rs +++ b/beacon_node/beacon_chain/src/chain_config.rs @@ -1,7 +1,7 @@ pub use proto_array::{DisallowedReOrgOffsets, ReOrgThreshold}; use serde_derive::{Deserialize, Serialize}; use std::time::Duration; -use types::{Checkpoint, Epoch}; +use types::{Checkpoint, Epoch, ProgressiveBalancesMode}; pub const DEFAULT_RE_ORG_THRESHOLD: ReOrgThreshold = ReOrgThreshold(20); pub const DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION: Epoch = Epoch::new(2); @@ -81,6 +81,8 @@ pub struct ChainConfig { pub always_prepare_payload: bool, /// Whether backfill sync processing should be rate-limited. pub enable_backfill_rate_limiting: bool, + /// Whether to use `ProgressiveBalancesCache` in unrealized FFG progression calculation. + pub progressive_balances_mode: ProgressiveBalancesMode, } impl Default for ChainConfig { @@ -111,6 +113,7 @@ impl Default for ChainConfig { genesis_backfill: false, always_prepare_payload: false, enable_backfill_rate_limiting: true, + progressive_balances_mode: ProgressiveBalancesMode::Checked, } } } diff --git a/beacon_node/beacon_chain/src/fork_revert.rs b/beacon_node/beacon_chain/src/fork_revert.rs index 084ae95e096..dc0e34277c9 100644 --- a/beacon_node/beacon_chain/src/fork_revert.rs +++ b/beacon_node/beacon_chain/src/fork_revert.rs @@ -10,7 +10,10 @@ use state_processing::{ use std::sync::Arc; use std::time::Duration; use store::{iter::ParentRootBlockIterator, HotColdDB, ItemStore}; -use types::{BeaconState, ChainSpec, EthSpec, ForkName, Hash256, SignedBeaconBlock, Slot}; +use types::{ + BeaconState, ChainSpec, EthSpec, ForkName, Hash256, ProgressiveBalancesMode, SignedBeaconBlock, + Slot, +}; const CORRUPT_DB_MESSAGE: &str = "The database could be corrupt. Check its file permissions or \ consider deleting it by running with the --purge-db flag."; @@ -100,6 +103,8 @@ pub fn reset_fork_choice_to_finalization, Cold: It store: Arc>, current_slot: Option, spec: &ChainSpec, + progressive_balances_mode: ProgressiveBalancesMode, + log: &Logger, ) -> Result, E>, String> { // Fetch finalized block. let finalized_checkpoint = head_state.finalized_checkpoint(); @@ -197,7 +202,9 @@ pub fn reset_fork_choice_to_finalization, Cold: It Duration::from_secs(0), &state, payload_verification_status, + progressive_balances_mode, spec, + log, ) .map_err(|e| format!("Error applying replayed block to fork choice: {:?}", e))?; } diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 21f7248cee5..6520c9ba9c8 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -754,9 +754,7 @@ where complete_state_advance(&mut state, None, slot, &self.spec) .expect("should be able to advance state to slot"); - state - .build_all_caches(&self.spec) - .expect("should build caches"); + state.build_caches(&self.spec).expect("should build caches"); let proposer_index = state.get_beacon_proposer_index(slot, &self.spec).unwrap(); @@ -803,9 +801,7 @@ where complete_state_advance(&mut state, None, slot, &self.spec) .expect("should be able to advance state to slot"); - state - .build_all_caches(&self.spec) - .expect("should build caches"); + state.build_caches(&self.spec).expect("should build caches"); let proposer_index = state.get_beacon_proposer_index(slot, &self.spec).unwrap(); @@ -1523,6 +1519,36 @@ where .sign(sk, &fork, genesis_validators_root, &self.chain.spec) } + pub fn add_proposer_slashing(&self, validator_index: u64) -> Result<(), String> { + let propposer_slashing = self.make_proposer_slashing(validator_index); + if let ObservationOutcome::New(verified_proposer_slashing) = self + .chain + .verify_proposer_slashing_for_gossip(propposer_slashing) + .expect("should verify proposer slashing for gossip") + { + self.chain + .import_proposer_slashing(verified_proposer_slashing); + Ok(()) + } else { + Err("should observe new proposer slashing".to_string()) + } + } + + pub fn add_attester_slashing(&self, validator_indices: Vec) -> Result<(), String> { + let attester_slashing = self.make_attester_slashing(validator_indices); + if let ObservationOutcome::New(verified_attester_slashing) = self + .chain + .verify_attester_slashing_for_gossip(attester_slashing) + .expect("should verify attester slashing for gossip") + { + self.chain + .import_attester_slashing(verified_attester_slashing); + Ok(()) + } else { + Err("should observe new attester slashing".to_string()) + } + } + pub fn add_bls_to_execution_change( &self, validator_index: u64, diff --git a/beacon_node/beacon_chain/tests/capella.rs b/beacon_node/beacon_chain/tests/capella.rs index e910e8134f1..f0b799ec9f7 100644 --- a/beacon_node/beacon_chain/tests/capella.rs +++ b/beacon_node/beacon_chain/tests/capella.rs @@ -133,13 +133,8 @@ async fn base_altair_merge_capella() { for _ in (merge_fork_slot.as_u64() + 3)..capella_fork_slot.as_u64() { harness.extend_slots(1).await; let block = &harness.chain.head_snapshot().beacon_block; - let full_payload: FullPayload = block - .message() - .body() - .execution_payload() - .unwrap() - .clone() - .into(); + let full_payload: FullPayload = + block.message().body().execution_payload().unwrap().into(); // pre-capella shouldn't have withdrawals assert!(full_payload.withdrawals_root().is_err()); execution_payloads.push(full_payload); @@ -151,13 +146,8 @@ async fn base_altair_merge_capella() { for _ in 0..16 { harness.extend_slots(1).await; let block = &harness.chain.head_snapshot().beacon_block; - let full_payload: FullPayload = block - .message() - .body() - .execution_payload() - .unwrap() - .clone() - .into(); + let full_payload: FullPayload = + block.message().body().execution_payload().unwrap().into(); // post-capella should have withdrawals assert!(full_payload.withdrawals_root().is_ok()); execution_payloads.push(full_payload); diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index 018defd2f00..9a8c324d09f 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -1064,8 +1064,9 @@ async fn invalid_parent() { Duration::from_secs(0), &state, PayloadVerificationStatus::Optimistic, + rig.harness.chain.config.progressive_balances_mode, &rig.harness.chain.spec, - + rig.harness.logger() ), Err(ForkChoiceError::ProtoArrayStringError(message)) if message.contains(&format!( diff --git a/beacon_node/http_api/src/block_rewards.rs b/beacon_node/http_api/src/block_rewards.rs index 828be8e5760..299bc019c40 100644 --- a/beacon_node/http_api/src/block_rewards.rs +++ b/beacon_node/http_api/src/block_rewards.rs @@ -49,7 +49,7 @@ pub fn get_block_rewards( .map_err(beacon_chain_error)?; state - .build_all_caches(&chain.spec) + .build_caches(&chain.spec) .map_err(beacon_state_error)?; let mut reward_cache = Default::default(); diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 206cd3c72f3..646356b6cb4 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -1,5 +1,6 @@ use clap::{App, Arg}; use strum::VariantNames; +use types::ProgressiveBalancesMode; pub fn cli_app<'a, 'b>() -> App<'a, 'b> { App::new("beacon_node") @@ -1117,4 +1118,17 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { developers. This directory is not pruned, users should be careful to avoid \ filling up their disks.") ) + .arg( + Arg::with_name("progressive-balances") + .long("progressive-balances") + .value_name("MODE") + .help("Options to enable or disable the progressive balances cache for \ + unrealized FFG progression calculation. The default `checked` mode compares \ + the progressive balances from the cache against results from the existing \ + method. If there is a mismatch, it falls back to the existing method. The \ + optimized mode (`fast`) is faster but is still experimental, and is \ + not recommended for mainnet usage at this time.") + .takes_value(true) + .possible_values(ProgressiveBalancesMode::VARIANTS) + ) } diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index c59b297c1b2..948c70dd41f 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -800,6 +800,12 @@ pub fn get_config( client_config.network.invalid_block_storage = Some(path); } + if let Some(progressive_balances_mode) = + clap_utils::parse_optional(cli_args, "progressive-balances")? + { + client_config.chain.progressive_balances_mode = progressive_balances_mode; + } + Ok(client_config) } diff --git a/beacon_node/store/src/partial_beacon_state.rs b/beacon_node/store/src/partial_beacon_state.rs index cd923da40dc..9f2532d0a75 100644 --- a/beacon_node/store/src/partial_beacon_state.rs +++ b/beacon_node/store/src/partial_beacon_state.rs @@ -373,6 +373,7 @@ macro_rules! impl_try_into_beacon_state { // Caching total_active_balance: <_>::default(), + progressive_balances_cache: <_>::default(), committee_caches: <_>::default(), pubkey_cache: <_>::default(), exit_cache: <_>::default(), diff --git a/beacon_node/store/src/reconstruct.rs b/beacon_node/store/src/reconstruct.rs index cd50babdb0c..bac5d3cc823 100644 --- a/beacon_node/store/src/reconstruct.rs +++ b/beacon_node/store/src/reconstruct.rs @@ -63,7 +63,7 @@ where .load_cold_state_by_slot(lower_limit_slot)? .ok_or(HotColdDBError::MissingLowerLimitState(lower_limit_slot))?; - state.build_all_caches(&self.spec)?; + state.build_caches(&self.spec)?; process_results(block_root_iter, |iter| -> Result<(), Error> { let mut io_batch = vec![]; diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index 5d86f99f1ab..e60774fc86e 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -1,10 +1,15 @@ use crate::{ForkChoiceStore, InvalidationOperation}; +use per_epoch_processing::altair::participation_cache::Error as ParticipationCacheError; use proto_array::{ Block as ProtoBlock, DisallowedReOrgOffsets, ExecutionStatus, ProposerHeadError, ProposerHeadInfo, ProtoArrayForkChoice, ReOrgThreshold, }; -use slog::{crit, debug, warn, Logger}; +use slog::{crit, debug, error, warn, Logger}; use ssz_derive::{Decode, Encode}; +use state_processing::per_epoch_processing::altair::ParticipationCache; +use state_processing::per_epoch_processing::{ + weigh_justification_and_finalization, JustificationAndFinalizationState, +}; use state_processing::{ per_block_processing::errors::AttesterSlashingValidationError, per_epoch_processing, }; @@ -18,6 +23,7 @@ use types::{ EthSpec, ExecPayload, ExecutionBlockHash, Hash256, IndexedAttestation, RelativeEpoch, SignedBeaconBlock, Slot, }; +use types::{ProgressiveBalancesCache, ProgressiveBalancesMode}; #[derive(Debug)] pub enum Error { @@ -72,7 +78,9 @@ pub enum Error { }, UnrealizedVoteProcessing(state_processing::EpochProcessingError), ParticipationCacheBuild(BeaconStateError), + ParticipationCacheError(ParticipationCacheError), ValidatorStatuses(BeaconStateError), + ProgressiveBalancesCacheCheckFailed(String), } impl From for Error { @@ -93,6 +101,18 @@ impl From for Error { } } +impl From for Error { + fn from(e: BeaconStateError) -> Self { + Error::BeaconStateError(e) + } +} + +impl From for Error { + fn from(e: ParticipationCacheError) -> Self { + Error::ParticipationCacheError(e) + } +} + #[derive(Debug, Clone, Copy)] /// Controls how fork choice should behave when restoring from a persisted fork choice. pub enum ResetPayloadStatuses { @@ -643,7 +663,9 @@ where block_delay: Duration, state: &BeaconState, payload_verification_status: PayloadVerificationStatus, + progressive_balances_mode: ProgressiveBalancesMode, spec: &ChainSpec, + log: &Logger, ) -> Result<(), Error> { // If this block has already been processed we do not need to reprocess it. // We check this immediately in case re-processing the block mutates some property of the @@ -737,43 +759,84 @@ where parent_justified.epoch == block_epoch && parent_finalized.epoch + 1 >= block_epoch }); - let (unrealized_justified_checkpoint, unrealized_finalized_checkpoint) = - if let Some((parent_justified, parent_finalized)) = parent_checkpoints { - (parent_justified, parent_finalized) - } else { - let justification_and_finalization_state = match block { - BeaconBlockRef::Capella(_) - | BeaconBlockRef::Merge(_) - | BeaconBlockRef::Altair(_) => { - let participation_cache = - per_epoch_processing::altair::ParticipationCache::new(state, spec) - .map_err(Error::ParticipationCacheBuild)?; + let (unrealized_justified_checkpoint, unrealized_finalized_checkpoint) = if let Some(( + parent_justified, + parent_finalized, + )) = + parent_checkpoints + { + (parent_justified, parent_finalized) + } else { + let justification_and_finalization_state = match block { + BeaconBlockRef::Capella(_) + | BeaconBlockRef::Merge(_) + | BeaconBlockRef::Altair(_) => match progressive_balances_mode { + ProgressiveBalancesMode::Disabled => { + let participation_cache = ParticipationCache::new(state, spec) + .map_err(Error::ParticipationCacheBuild)?; per_epoch_processing::altair::process_justification_and_finalization( state, &participation_cache, )? } - BeaconBlockRef::Base(_) => { - let mut validator_statuses = - per_epoch_processing::base::ValidatorStatuses::new(state, spec) - .map_err(Error::ValidatorStatuses)?; - validator_statuses - .process_attestations(state) - .map_err(Error::ValidatorStatuses)?; - per_epoch_processing::base::process_justification_and_finalization( - state, - &validator_statuses.total_balances, - spec, - )? + ProgressiveBalancesMode::Fast + | ProgressiveBalancesMode::Checked + | ProgressiveBalancesMode::Strict => { + let maybe_participation_cache = progressive_balances_mode + .perform_comparative_checks() + .then(|| { + ParticipationCache::new(state, spec) + .map_err(Error::ParticipationCacheBuild) + }) + .transpose()?; + + process_justification_and_finalization_from_progressive_cache::( + state, + maybe_participation_cache.as_ref(), + ) + .or_else(|e| { + if progressive_balances_mode != ProgressiveBalancesMode::Strict { + error!( + log, + "Processing with progressive balances cache failed"; + "info" => "falling back to the non-optimized processing method", + "error" => ?e, + ); + let participation_cache = maybe_participation_cache + .map(Ok) + .unwrap_or_else(|| ParticipationCache::new(state, spec)) + .map_err(Error::ParticipationCacheBuild)?; + per_epoch_processing::altair::process_justification_and_finalization( + state, + &participation_cache, + ).map_err(Error::from) + } else { + Err(e) + } + })? } - }; - - ( - justification_and_finalization_state.current_justified_checkpoint(), - justification_and_finalization_state.finalized_checkpoint(), - ) + }, + BeaconBlockRef::Base(_) => { + let mut validator_statuses = + per_epoch_processing::base::ValidatorStatuses::new(state, spec) + .map_err(Error::ValidatorStatuses)?; + validator_statuses + .process_attestations(state) + .map_err(Error::ValidatorStatuses)?; + per_epoch_processing::base::process_justification_and_finalization( + state, + &validator_statuses.total_balances, + spec, + )? + } }; + ( + justification_and_finalization_state.current_justified_checkpoint(), + justification_and_finalization_state.finalized_checkpoint(), + ) + }; + // Update best known unrealized justified & finalized checkpoints if unrealized_justified_checkpoint.epoch > self.fc_store.unrealized_justified_checkpoint().epoch @@ -1499,6 +1562,92 @@ where } } +/// Process justification and finalization using progressive cache. Also performs a comparative +/// check against the `ParticipationCache` if it is supplied. +/// +/// Returns an error if the cache is not initialized or if there is a mismatch on the comparative check. +fn process_justification_and_finalization_from_progressive_cache( + state: &BeaconState, + maybe_participation_cache: Option<&ParticipationCache>, +) -> Result, Error> +where + E: EthSpec, + T: ForkChoiceStore, +{ + let justification_and_finalization_state = JustificationAndFinalizationState::new(state); + if state.current_epoch() <= E::genesis_epoch() + 1 { + return Ok(justification_and_finalization_state); + } + + // Load cached balances + let progressive_balances_cache: &ProgressiveBalancesCache = state.progressive_balances_cache(); + let previous_target_balance = + progressive_balances_cache.previous_epoch_target_attesting_balance()?; + let current_target_balance = + progressive_balances_cache.current_epoch_target_attesting_balance()?; + let total_active_balance = state.get_total_active_balance()?; + + if let Some(participation_cache) = maybe_participation_cache { + check_progressive_balances::( + state, + participation_cache, + previous_target_balance, + current_target_balance, + total_active_balance, + )?; + } + + weigh_justification_and_finalization( + justification_and_finalization_state, + total_active_balance, + previous_target_balance, + current_target_balance, + ) + .map_err(Error::from) +} + +/// Perform comparative checks against `ParticipationCache`, will return error if there's a mismatch. +fn check_progressive_balances( + state: &BeaconState, + participation_cache: &ParticipationCache, + cached_previous_target_balance: u64, + cached_current_target_balance: u64, + cached_total_active_balance: u64, +) -> Result<(), Error> +where + E: EthSpec, + T: ForkChoiceStore, +{ + let slot = state.slot(); + let epoch = state.current_epoch(); + + // Check previous epoch target balances + let previous_target_balance = participation_cache.previous_epoch_target_attesting_balance()?; + if previous_target_balance != cached_previous_target_balance { + return Err(Error::ProgressiveBalancesCacheCheckFailed( + format!("Previous epoch target attesting balance mismatch, slot: {}, epoch: {}, actual: {}, cached: {}", slot, epoch, previous_target_balance, cached_previous_target_balance) + )); + } + + // Check current epoch target balances + let current_target_balance = participation_cache.current_epoch_target_attesting_balance()?; + if current_target_balance != cached_current_target_balance { + return Err(Error::ProgressiveBalancesCacheCheckFailed( + format!("Current epoch target attesting balance mismatch, slot: {}, epoch: {}, actual: {}, cached: {}", slot, epoch, current_target_balance, cached_current_target_balance) + )); + } + + // Check current epoch total balances + let total_active_balance = participation_cache.current_epoch_total_active_balance(); + if total_active_balance != cached_total_active_balance { + return Err(Error::ProgressiveBalancesCacheCheckFailed( + format!("Current epoch total active balance mismatch, slot: {}, epoch: {}, actual: {}, cached: {}", slot, epoch, total_active_balance, cached_total_active_balance) + )); + } + + Ok(()) +} + /// Helper struct that is used to encode/decode the state of the `ForkChoice` as SSZ bytes. /// /// This is used when persisting the state of the fork choice to disk. diff --git a/consensus/fork_choice/tests/tests.rs b/consensus/fork_choice/tests/tests.rs index ef262b58c00..d28210aa1bd 100644 --- a/consensus/fork_choice/tests/tests.rs +++ b/consensus/fork_choice/tests/tests.rs @@ -17,12 +17,13 @@ use fork_choice::{ use store::MemoryStore; use types::{ test_utils::generate_deterministic_keypair, BeaconBlockRef, BeaconState, ChainSpec, Checkpoint, - Epoch, EthSpec, Hash256, IndexedAttestation, MainnetEthSpec, SignedBeaconBlock, Slot, SubnetId, + Epoch, EthSpec, ForkName, Hash256, IndexedAttestation, MainnetEthSpec, ProgressiveBalancesMode, + RelativeEpoch, SignedBeaconBlock, Slot, SubnetId, }; pub type E = MainnetEthSpec; -pub const VALIDATOR_COUNT: usize = 32; +pub const VALIDATOR_COUNT: usize = 64; /// Defines some delay between when an attestation is created and when it is mutated. pub enum MutationDelay { @@ -68,6 +69,24 @@ impl ForkChoiceTest { Self { harness } } + /// Creates a new tester with the specified `ProgressiveBalancesMode` and genesis from latest fork. + fn new_with_progressive_balances_mode(mode: ProgressiveBalancesMode) -> ForkChoiceTest { + // genesis with latest fork (at least altair required to test the cache) + let spec = ForkName::latest().make_genesis_spec(ChainSpec::default()); + let harness = BeaconChainHarness::builder(MainnetEthSpec) + .spec(spec) + .chain_config(ChainConfig { + progressive_balances_mode: mode, + ..ChainConfig::default() + }) + .deterministic_keypairs(VALIDATOR_COUNT) + .fresh_ephemeral_store() + .mock_execution_layer() + .build(); + + Self { harness } + } + /// Get a value from the `ForkChoice` instantiation. fn get(&self, func: T) -> U where @@ -212,6 +231,39 @@ impl ForkChoiceTest { self } + /// Slash a validator from the previous epoch committee. + pub async fn add_previous_epoch_attester_slashing(self) -> Self { + let state = self.harness.get_current_state(); + let previous_epoch_shuffling = state.get_shuffling(RelativeEpoch::Previous).unwrap(); + let validator_indices = previous_epoch_shuffling + .iter() + .map(|idx| *idx as u64) + .take(1) + .collect(); + + self.harness + .add_attester_slashing(validator_indices) + .unwrap(); + + self + } + + /// Slash the proposer of a block in the previous epoch. + pub async fn add_previous_epoch_proposer_slashing(self, slots_per_epoch: u64) -> Self { + let previous_epoch_slot = self.harness.get_current_slot() - slots_per_epoch; + let previous_epoch_block = self + .harness + .chain + .block_at_slot(previous_epoch_slot, WhenSlotSkipped::None) + .unwrap() + .unwrap(); + let proposer_index: u64 = previous_epoch_block.message().proposer_index(); + + self.harness.add_proposer_slashing(proposer_index).unwrap(); + + self + } + /// Apply `count` blocks to the chain (without attestations). pub async fn apply_blocks_without_new_attestations(self, count: usize) -> Self { self.harness.advance_slot(); @@ -286,7 +338,9 @@ impl ForkChoiceTest { Duration::from_secs(0), &state, PayloadVerificationStatus::Verified, + self.harness.chain.config.progressive_balances_mode, &self.harness.chain.spec, + self.harness.logger(), ) .unwrap(); self @@ -328,7 +382,9 @@ impl ForkChoiceTest { Duration::from_secs(0), &state, PayloadVerificationStatus::Verified, + self.harness.chain.config.progressive_balances_mode, &self.harness.chain.spec, + self.harness.logger(), ) .err() .expect("on_block did not return an error"); @@ -1287,3 +1343,49 @@ async fn weak_subjectivity_check_epoch_boundary_is_skip_slot_failure() { .assert_finalized_epoch_is_less_than(checkpoint.epoch) .assert_shutdown_signal_sent(); } + +/// Checks that `ProgressiveBalancesCache` is updated correctly after an attester slashing event, +/// where the slashed validator is a target attester in previous / current epoch. +#[tokio::test] +async fn progressive_balances_cache_attester_slashing() { + ForkChoiceTest::new_with_progressive_balances_mode(ProgressiveBalancesMode::Strict) + // first two epochs + .apply_blocks_while(|_, state| state.finalized_checkpoint().epoch == 0) + .await + .unwrap() + .add_previous_epoch_attester_slashing() + .await + // expect fork choice to import blocks successfully after a previous epoch attester is + // slashed, i.e. the slashed attester's balance is correctly excluded from + // the previous epoch total balance in `ProgressiveBalancesCache`. + .apply_blocks(1) + .await + // expect fork choice to import another epoch of blocks successfully - the slashed + // attester's balance should be excluded from the current epoch total balance in + // `ProgressiveBalancesCache` as well. + .apply_blocks(MainnetEthSpec::slots_per_epoch() as usize) + .await; +} + +/// Checks that `ProgressiveBalancesCache` is updated correctly after a proposer slashing event, +/// where the slashed validator is a target attester in previous / current epoch. +#[tokio::test] +async fn progressive_balances_cache_proposer_slashing() { + ForkChoiceTest::new_with_progressive_balances_mode(ProgressiveBalancesMode::Strict) + // first two epochs + .apply_blocks_while(|_, state| state.finalized_checkpoint().epoch == 0) + .await + .unwrap() + .add_previous_epoch_proposer_slashing(MainnetEthSpec::slots_per_epoch()) + .await + // expect fork choice to import blocks successfully after a previous epoch proposer is + // slashed, i.e. the slashed proposer's balance is correctly excluded from + // the previous epoch total balance in `ProgressiveBalancesCache`. + .apply_blocks(1) + .await + // expect fork choice to import another epoch of blocks successfully - the slashed + // proposer's balance should be excluded from the current epoch total balance in + // `ProgressiveBalancesCache` as well. + .apply_blocks(MainnetEthSpec::slots_per_epoch() as usize) + .await; +} diff --git a/consensus/state_processing/src/common/mod.rs b/consensus/state_processing/src/common/mod.rs index 8a2e2439bb6..ffe8be3a041 100644 --- a/consensus/state_processing/src/common/mod.rs +++ b/consensus/state_processing/src/common/mod.rs @@ -7,6 +7,7 @@ mod slash_validator; pub mod altair; pub mod base; +pub mod update_progressive_balances_cache; pub use deposit_data_tree::DepositDataTree; pub use get_attestation_participation::get_attestation_participation_flag_indices; diff --git a/consensus/state_processing/src/common/slash_validator.rs b/consensus/state_processing/src/common/slash_validator.rs index d4675f5ef5d..d54da43a04b 100644 --- a/consensus/state_processing/src/common/slash_validator.rs +++ b/consensus/state_processing/src/common/slash_validator.rs @@ -1,3 +1,4 @@ +use crate::common::update_progressive_balances_cache::update_progressive_balances_on_slashing; use crate::{ common::{decrease_balance, increase_balance, initiate_validator_exit}, per_block_processing::errors::BlockProcessingError, @@ -43,6 +44,8 @@ pub fn slash_validator( .safe_div(spec.min_slashing_penalty_quotient_for_state(state))?, )?; + update_progressive_balances_on_slashing(state, slashed_index)?; + // Apply proposer and whistleblower rewards let proposer_index = ctxt.get_proposer_index(state, spec)? as usize; let whistleblower_index = opt_whistleblower_index.unwrap_or(proposer_index); diff --git a/consensus/state_processing/src/common/update_progressive_balances_cache.rs b/consensus/state_processing/src/common/update_progressive_balances_cache.rs new file mode 100644 index 00000000000..45b5d657a6e --- /dev/null +++ b/consensus/state_processing/src/common/update_progressive_balances_cache.rs @@ -0,0 +1,142 @@ +/// A collection of all functions that mutates the `ProgressiveBalancesCache`. +use crate::metrics::{ + PARTICIPATION_CURR_EPOCH_TARGET_ATTESTING_GWEI_PROGRESSIVE_TOTAL, + PARTICIPATION_PREV_EPOCH_TARGET_ATTESTING_GWEI_PROGRESSIVE_TOTAL, +}; +use crate::per_epoch_processing::altair::ParticipationCache; +use crate::{BlockProcessingError, EpochProcessingError}; +use lighthouse_metrics::set_gauge; +use ssz_types::VariableList; +use std::borrow::Cow; +use types::consts::altair::TIMELY_TARGET_FLAG_INDEX; +use types::{ + is_progressive_balances_enabled, BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, + ParticipationFlags, ProgressiveBalancesCache, +}; + +/// Initializes the `ProgressiveBalancesCache` cache using balance values from the +/// `ParticipationCache`. If the optional `&ParticipationCache` is not supplied, it will be computed +/// from the `BeaconState`. +pub fn initialize_progressive_balances_cache( + state: &mut BeaconState, + maybe_participation_cache: Option<&ParticipationCache>, + spec: &ChainSpec, +) -> Result<(), BeaconStateError> { + if !is_progressive_balances_enabled(state) + || state.progressive_balances_cache().is_initialized() + { + return Ok(()); + } + + let participation_cache = match maybe_participation_cache { + Some(cache) => Cow::Borrowed(cache), + None => Cow::Owned(ParticipationCache::new(state, spec)?), + }; + + let previous_epoch_target_attesting_balance = participation_cache + .previous_epoch_target_attesting_balance_raw() + .map_err(|e| BeaconStateError::ParticipationCacheError(format!("{e:?}")))?; + + let current_epoch_target_attesting_balance = participation_cache + .current_epoch_target_attesting_balance_raw() + .map_err(|e| BeaconStateError::ParticipationCacheError(format!("{e:?}")))?; + + let current_epoch = state.current_epoch(); + state.progressive_balances_cache_mut().initialize( + current_epoch, + previous_epoch_target_attesting_balance, + current_epoch_target_attesting_balance, + ); + + update_progressive_balances_metrics(state.progressive_balances_cache())?; + + Ok(()) +} + +/// Updates the `ProgressiveBalancesCache` when a new target attestation has been processed. +pub fn update_progressive_balances_on_attestation( + state: &mut BeaconState, + epoch: Epoch, + validator_index: usize, +) -> Result<(), BlockProcessingError> { + if is_progressive_balances_enabled(state) { + let validator = state.get_validator(validator_index)?; + if !validator.slashed { + let validator_effective_balance = validator.effective_balance; + state + .progressive_balances_cache_mut() + .on_new_target_attestation(epoch, validator_effective_balance)?; + } + } + Ok(()) +} + +/// Updates the `ProgressiveBalancesCache` when a target attester has been slashed. +pub fn update_progressive_balances_on_slashing( + state: &mut BeaconState, + validator_index: usize, +) -> Result<(), BlockProcessingError> { + if is_progressive_balances_enabled(state) { + let previous_epoch_participation = state.previous_epoch_participation()?; + let is_previous_epoch_target_attester = + is_target_attester_in_epoch::(previous_epoch_participation, validator_index)?; + + let current_epoch_participation = state.current_epoch_participation()?; + let is_current_epoch_target_attester = + is_target_attester_in_epoch::(current_epoch_participation, validator_index)?; + + let validator_effective_balance = state.get_effective_balance(validator_index)?; + + state.progressive_balances_cache_mut().on_slashing( + is_previous_epoch_target_attester, + is_current_epoch_target_attester, + validator_effective_balance, + )?; + } + + Ok(()) +} + +/// Updates the `ProgressiveBalancesCache` on epoch transition. +pub fn update_progressive_balances_on_epoch_transition( + state: &mut BeaconState, + spec: &ChainSpec, +) -> Result<(), EpochProcessingError> { + if is_progressive_balances_enabled(state) { + state + .progressive_balances_cache_mut() + .on_epoch_transition(spec)?; + + update_progressive_balances_metrics(state.progressive_balances_cache())?; + } + + Ok(()) +} + +pub fn update_progressive_balances_metrics( + cache: &ProgressiveBalancesCache, +) -> Result<(), BeaconStateError> { + set_gauge( + &PARTICIPATION_PREV_EPOCH_TARGET_ATTESTING_GWEI_PROGRESSIVE_TOTAL, + cache.previous_epoch_target_attesting_balance()? as i64, + ); + + set_gauge( + &PARTICIPATION_CURR_EPOCH_TARGET_ATTESTING_GWEI_PROGRESSIVE_TOTAL, + cache.current_epoch_target_attesting_balance()? as i64, + ); + + Ok(()) +} + +fn is_target_attester_in_epoch( + epoch_participation: &VariableList, + validator_index: usize, +) -> Result { + let participation_flags = epoch_participation + .get(validator_index) + .ok_or(BeaconStateError::UnknownValidator(validator_index))?; + participation_flags + .has_flag(TIMELY_TARGET_FLAG_INDEX) + .map_err(|e| e.into()) +} diff --git a/consensus/state_processing/src/genesis.rs b/consensus/state_processing/src/genesis.rs index 68f04b554e3..ebbc8f9f31e 100644 --- a/consensus/state_processing/src/genesis.rs +++ b/consensus/state_processing/src/genesis.rs @@ -92,7 +92,7 @@ pub fn initialize_beacon_state_from_eth1( } // Now that we have our validators, initialize the caches (including the committees) - state.build_all_caches(spec)?; + state.build_caches(spec)?; // Set genesis validators root for domain separation and chain versioning *state.genesis_validators_root_mut() = state.update_validators_tree_hash_cache()?; @@ -115,7 +115,7 @@ pub fn process_activations( state: &mut BeaconState, spec: &ChainSpec, ) -> Result<(), Error> { - let (validators, balances) = state.validators_and_balances_mut(); + let (validators, balances, _) = state.validators_and_balances_and_progressive_balances_mut(); for (index, validator) in validators.iter_mut().enumerate() { let balance = balances .get(index) diff --git a/consensus/state_processing/src/metrics.rs b/consensus/state_processing/src/metrics.rs index ddfaae56403..360b007678a 100644 --- a/consensus/state_processing/src/metrics.rs +++ b/consensus/state_processing/src/metrics.rs @@ -23,4 +23,15 @@ lazy_static! { "beacon_participation_prev_epoch_active_gwei_total", "Total effective balance (gwei) of validators active in the previous epoch" ); + /* + * Participation Metrics (progressive balances) + */ + pub static ref PARTICIPATION_PREV_EPOCH_TARGET_ATTESTING_GWEI_PROGRESSIVE_TOTAL: Result = try_create_int_gauge( + "beacon_participation_prev_epoch_target_attesting_gwei_progressive_total", + "Progressive total effective balance (gwei) of validators who attested to the target in the previous epoch" + ); + pub static ref PARTICIPATION_CURR_EPOCH_TARGET_ATTESTING_GWEI_PROGRESSIVE_TOTAL: Result = try_create_int_gauge( + "beacon_participation_curr_epoch_target_attesting_gwei_progressive_total", + "Progressive total effective balance (gwei) of validators who attested to the target in the current epoch" + ); } diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index 124fdf6500b..b8b76a499d3 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -41,6 +41,9 @@ mod verify_proposer_slashing; use crate::common::decrease_balance; use crate::StateProcessingStrategy; +use crate::common::update_progressive_balances_cache::{ + initialize_progressive_balances_cache, update_progressive_balances_metrics, +}; #[cfg(feature = "arbitrary-fuzz")] use arbitrary::Arbitrary; @@ -114,6 +117,8 @@ pub fn per_block_processing>( .fork_name(spec) .map_err(BlockProcessingError::InconsistentStateFork)?; + initialize_progressive_balances_cache(state, None, spec)?; + let verify_signatures = match block_signature_strategy { BlockSignatureStrategy::VerifyBulk => { // Verify all signatures in the block at once. @@ -182,6 +187,10 @@ pub fn per_block_processing>( )?; } + if is_progressive_balances_enabled(state) { + update_progressive_balances_metrics(state.progressive_balances_cache())?; + } + Ok(()) } diff --git a/consensus/state_processing/src/per_block_processing/errors.rs b/consensus/state_processing/src/per_block_processing/errors.rs index 1aaf298d690..0aba1d83faf 100644 --- a/consensus/state_processing/src/per_block_processing/errors.rs +++ b/consensus/state_processing/src/per_block_processing/errors.rs @@ -1,6 +1,8 @@ use super::signature_sets::Error as SignatureSetError; +use crate::per_epoch_processing::altair::participation_cache; use crate::ContextError; use merkle_proof::MerkleTreeError; +use participation_cache::Error as ParticipationCacheError; use safe_arith::ArithError; use ssz::DecodeError; use types::*; @@ -83,6 +85,7 @@ pub enum BlockProcessingError { found: Hash256, }, WithdrawalCredentialsInvalid, + ParticipationCacheError(ParticipationCacheError), } impl From for BlockProcessingError { @@ -140,6 +143,12 @@ impl From> for BlockProcessingError { } } +impl From for BlockProcessingError { + fn from(e: ParticipationCacheError) -> Self { + BlockProcessingError::ParticipationCacheError(e) + } +} + /// A conversion that consumes `self` and adds an `index` variable to resulting struct. /// /// Used here to allow converting an error into an upstream error that points to the object that diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 4bee596615a..1dbcb7fb8fe 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -97,6 +97,8 @@ pub mod base { pub mod altair { use super::*; + use crate::common::update_progressive_balances_cache::update_progressive_balances_on_attestation; + use types::consts::altair::TIMELY_TARGET_FLAG_INDEX; pub fn process_attestations( state: &mut BeaconState, @@ -163,6 +165,14 @@ pub mod altair { get_base_reward(state, index, base_reward_per_increment, spec)? .safe_mul(weight)?, )?; + + if flag_index == TIMELY_TARGET_FLAG_INDEX { + update_progressive_balances_on_attestation( + state, + data.target.epoch, + index, + )?; + } } } } @@ -235,6 +245,7 @@ pub fn process_attester_slashings( Ok(()) } + /// Wrapper function to handle calling the correct version of `process_attestations` based on /// the fork. pub fn process_attestations>( diff --git a/consensus/state_processing/src/per_epoch_processing/altair.rs b/consensus/state_processing/src/per_epoch_processing/altair.rs index d5df2fc975d..0abbd16a989 100644 --- a/consensus/state_processing/src/per_epoch_processing/altair.rs +++ b/consensus/state_processing/src/per_epoch_processing/altair.rs @@ -1,4 +1,7 @@ use super::{process_registry_updates, process_slashings, EpochProcessingSummary, Error}; +use crate::common::update_progressive_balances_cache::{ + initialize_progressive_balances_cache, update_progressive_balances_on_epoch_transition, +}; use crate::per_epoch_processing::{ effective_balance_updates::process_effective_balance_updates, historical_roots_update::process_historical_roots_update, @@ -31,6 +34,7 @@ pub fn process_epoch( // Pre-compute participating indices and total balances. let participation_cache = ParticipationCache::new(state, spec)?; let sync_committee = state.current_sync_committee()?.clone(); + initialize_progressive_balances_cache::(state, Some(&participation_cache), spec)?; // Justification and finalization. let justification_and_finalization_state = @@ -56,7 +60,7 @@ pub fn process_epoch( process_eth1_data_reset(state)?; // Update effective balances with hysteresis (lag). - process_effective_balance_updates(state, spec)?; + process_effective_balance_updates(state, Some(&participation_cache), spec)?; // Reset slashings process_slashings_reset(state)?; @@ -75,6 +79,8 @@ pub fn process_epoch( // Rotate the epoch caches to suit the epoch transition. state.advance_caches(spec)?; + update_progressive_balances_on_epoch_transition(state, spec)?; + Ok(EpochProcessingSummary::Altair { participation_cache, sync_committee, diff --git a/consensus/state_processing/src/per_epoch_processing/altair/participation_cache.rs b/consensus/state_processing/src/per_epoch_processing/altair/participation_cache.rs index 004726923e9..a5caddd0455 100644 --- a/consensus/state_processing/src/per_epoch_processing/altair/participation_cache.rs +++ b/consensus/state_processing/src/per_epoch_processing/altair/participation_cache.rs @@ -11,49 +11,23 @@ //! Additionally, this cache is returned from the `altair::process_epoch` function and can be used //! to get useful summaries about the validator participation in an epoch. -use safe_arith::{ArithError, SafeArith}; use types::{ consts::altair::{ NUM_FLAG_INDICES, TIMELY_HEAD_FLAG_INDEX, TIMELY_SOURCE_FLAG_INDEX, TIMELY_TARGET_FLAG_INDEX, }, - BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, ParticipationFlags, RelativeEpoch, + Balance, BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, ParticipationFlags, + RelativeEpoch, }; -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] pub enum Error { InvalidFlagIndex(usize), InvalidValidatorIndex(usize), } -/// A balance which will never be below the specified `minimum`. -/// -/// This is an effort to ensure the `EFFECTIVE_BALANCE_INCREMENT` minimum is always respected. -#[derive(PartialEq, Debug, Clone, Copy)] -struct Balance { - raw: u64, - minimum: u64, -} - -impl Balance { - /// Initialize the balance to `0`, or the given `minimum`. - pub fn zero(minimum: u64) -> Self { - Self { raw: 0, minimum } - } - - /// Returns the balance with respect to the initialization `minimum`. - pub fn get(&self) -> u64 { - std::cmp::max(self.raw, self.minimum) - } - - /// Add-assign to the balance. - pub fn safe_add_assign(&mut self, other: u64) -> Result<(), ArithError> { - self.raw.safe_add_assign(other) - } -} - /// Caches the participation values for one epoch (either the previous or current). -#[derive(PartialEq, Debug)] +#[derive(PartialEq, Debug, Clone)] struct SingleEpochParticipationCache { /// Maps an active validator index to their participation flags. /// @@ -95,6 +69,14 @@ impl SingleEpochParticipationCache { .ok_or(Error::InvalidFlagIndex(flag_index)) } + /// Returns the raw total balance of attesters who have `flag_index` set. + fn total_flag_balance_raw(&self, flag_index: usize) -> Result { + self.total_flag_balances + .get(flag_index) + .copied() + .ok_or(Error::InvalidFlagIndex(flag_index)) + } + /// Returns `true` if `val_index` is active, unslashed and has `flag_index` set. /// /// ## Errors @@ -173,7 +155,7 @@ impl SingleEpochParticipationCache { } /// Maintains a cache to be used during `altair::process_epoch`. -#[derive(PartialEq, Debug)] +#[derive(PartialEq, Debug, Clone)] pub struct ParticipationCache { current_epoch: Epoch, /// Caches information about active validators pertaining to `self.current_epoch`. @@ -291,6 +273,11 @@ impl ParticipationCache { .total_flag_balance(TIMELY_TARGET_FLAG_INDEX) } + pub fn current_epoch_target_attesting_balance_raw(&self) -> Result { + self.current_epoch_participation + .total_flag_balance_raw(TIMELY_TARGET_FLAG_INDEX) + } + pub fn previous_epoch_total_active_balance(&self) -> u64 { self.previous_epoch_participation.total_active_balance.get() } @@ -300,6 +287,11 @@ impl ParticipationCache { .total_flag_balance(TIMELY_TARGET_FLAG_INDEX) } + pub fn previous_epoch_target_attesting_balance_raw(&self) -> Result { + self.previous_epoch_participation + .total_flag_balance_raw(TIMELY_TARGET_FLAG_INDEX) + } + pub fn previous_epoch_source_attesting_balance(&self) -> Result { self.previous_epoch_participation .total_flag_balance(TIMELY_SOURCE_FLAG_INDEX) diff --git a/consensus/state_processing/src/per_epoch_processing/base.rs b/consensus/state_processing/src/per_epoch_processing/base.rs index cb7e7d4b300..680563ce74c 100644 --- a/consensus/state_processing/src/per_epoch_processing/base.rs +++ b/consensus/state_processing/src/per_epoch_processing/base.rs @@ -52,7 +52,7 @@ pub fn process_epoch( process_eth1_data_reset(state)?; // Update effective balances with hysteresis (lag). - process_effective_balance_updates(state, spec)?; + process_effective_balance_updates(state, None, spec)?; // Reset slashings process_slashings_reset(state)?; diff --git a/consensus/state_processing/src/per_epoch_processing/capella.rs b/consensus/state_processing/src/per_epoch_processing/capella.rs index aaf301f29ec..911510ed0ce 100644 --- a/consensus/state_processing/src/per_epoch_processing/capella.rs +++ b/consensus/state_processing/src/per_epoch_processing/capella.rs @@ -11,6 +11,9 @@ use crate::per_epoch_processing::{ }; use types::{BeaconState, ChainSpec, EthSpec, RelativeEpoch}; +use crate::common::update_progressive_balances_cache::{ + initialize_progressive_balances_cache, update_progressive_balances_on_epoch_transition, +}; pub use historical_summaries_update::process_historical_summaries_update; mod historical_summaries_update; @@ -27,6 +30,7 @@ pub fn process_epoch( // Pre-compute participating indices and total balances. let participation_cache = ParticipationCache::new(state, spec)?; let sync_committee = state.current_sync_committee()?.clone(); + initialize_progressive_balances_cache(state, Some(&participation_cache), spec)?; // Justification and finalization. let justification_and_finalization_state = @@ -52,7 +56,7 @@ pub fn process_epoch( process_eth1_data_reset(state)?; // Update effective balances with hysteresis (lag). - process_effective_balance_updates(state, spec)?; + process_effective_balance_updates(state, Some(&participation_cache), spec)?; // Reset slashings process_slashings_reset(state)?; @@ -71,6 +75,8 @@ pub fn process_epoch( // Rotate the epoch caches to suit the epoch transition. state.advance_caches(spec)?; + update_progressive_balances_on_epoch_transition(state, spec)?; + Ok(EpochProcessingSummary::Altair { participation_cache, sync_committee, diff --git a/consensus/state_processing/src/per_epoch_processing/effective_balance_updates.rs b/consensus/state_processing/src/per_epoch_processing/effective_balance_updates.rs index c166667b5a9..1759f7e1402 100644 --- a/consensus/state_processing/src/per_epoch_processing/effective_balance_updates.rs +++ b/consensus/state_processing/src/per_epoch_processing/effective_balance_updates.rs @@ -1,11 +1,13 @@ use super::errors::EpochProcessingError; +use crate::per_epoch_processing::altair::ParticipationCache; use safe_arith::SafeArith; use types::beacon_state::BeaconState; use types::chain_spec::ChainSpec; -use types::{BeaconStateError, EthSpec}; +use types::{BeaconStateError, EthSpec, ProgressiveBalancesCache}; pub fn process_effective_balance_updates( state: &mut BeaconState, + maybe_participation_cache: Option<&ParticipationCache>, spec: &ChainSpec, ) -> Result<(), EpochProcessingError> { let hysteresis_increment = spec @@ -13,7 +15,8 @@ pub fn process_effective_balance_updates( .safe_div(spec.hysteresis_quotient)?; let downward_threshold = hysteresis_increment.safe_mul(spec.hysteresis_downward_multiplier)?; let upward_threshold = hysteresis_increment.safe_mul(spec.hysteresis_upward_multiplier)?; - let (validators, balances) = state.validators_and_balances_mut(); + let (validators, balances, progressive_balances_cache) = + state.validators_and_balances_and_progressive_balances_mut(); for (index, validator) in validators.iter_mut().enumerate() { let balance = balances .get(index) @@ -23,11 +26,43 @@ pub fn process_effective_balance_updates( if balance.safe_add(downward_threshold)? < validator.effective_balance || validator.effective_balance.safe_add(upward_threshold)? < balance { - validator.effective_balance = std::cmp::min( + let old_effective_balance = validator.effective_balance; + let new_effective_balance = std::cmp::min( balance.safe_sub(balance.safe_rem(spec.effective_balance_increment)?)?, spec.max_effective_balance, ); + + if let Some(participation_cache) = maybe_participation_cache { + update_progressive_balances( + participation_cache, + progressive_balances_cache, + index, + old_effective_balance, + new_effective_balance, + )?; + } + + validator.effective_balance = new_effective_balance; } } Ok(()) } + +fn update_progressive_balances( + participation_cache: &ParticipationCache, + progressive_balances_cache: &mut ProgressiveBalancesCache, + index: usize, + old_effective_balance: u64, + new_effective_balance: u64, +) -> Result<(), EpochProcessingError> { + if old_effective_balance != new_effective_balance { + let is_current_epoch_target_attester = + participation_cache.is_current_epoch_timely_target_attester(index)?; + progressive_balances_cache.on_effective_balance_change( + is_current_epoch_target_attester, + old_effective_balance, + new_effective_balance, + )?; + } + Ok(()) +} diff --git a/consensus/state_processing/src/per_epoch_processing/slashings.rs b/consensus/state_processing/src/per_epoch_processing/slashings.rs index 6d5342cd363..2d595491c11 100644 --- a/consensus/state_processing/src/per_epoch_processing/slashings.rs +++ b/consensus/state_processing/src/per_epoch_processing/slashings.rs @@ -16,7 +16,7 @@ pub fn process_slashings( total_balance, ); - let (validators, balances) = state.validators_and_balances_mut(); + let (validators, balances, _) = state.validators_and_balances_and_progressive_balances_mut(); for (index, validator) in validators.iter().enumerate() { if validator.slashed && epoch.safe_add(T::EpochsPerSlashingsVector::to_u64().safe_div(2)?)? diff --git a/consensus/state_processing/src/upgrade/altair.rs b/consensus/state_processing/src/upgrade/altair.rs index 176f1af15c6..26b1192bc16 100644 --- a/consensus/state_processing/src/upgrade/altair.rs +++ b/consensus/state_processing/src/upgrade/altair.rs @@ -1,3 +1,4 @@ +use crate::common::update_progressive_balances_cache::initialize_progressive_balances_cache; use crate::common::{get_attestation_participation_flag_indices, get_attesting_indices}; use std::mem; use std::sync::Arc; @@ -101,6 +102,7 @@ pub fn upgrade_to_altair( next_sync_committee: temp_sync_committee, // not read // Caches total_active_balance: pre.total_active_balance, + progressive_balances_cache: mem::take(&mut pre.progressive_balances_cache), committee_caches: mem::take(&mut pre.committee_caches), pubkey_cache: mem::take(&mut pre.pubkey_cache), exit_cache: mem::take(&mut pre.exit_cache), @@ -110,6 +112,8 @@ pub fn upgrade_to_altair( // Fill in previous epoch participation from the pre state's pending attestations. translate_participation(&mut post, &pre.previous_epoch_attestations, spec)?; + initialize_progressive_balances_cache(&mut post, None, spec)?; + // Fill in sync committees // Note: A duplicate committee is assigned for the current and next committee at the fork // boundary diff --git a/consensus/state_processing/src/upgrade/capella.rs b/consensus/state_processing/src/upgrade/capella.rs index 3b933fac37a..5153e35f447 100644 --- a/consensus/state_processing/src/upgrade/capella.rs +++ b/consensus/state_processing/src/upgrade/capella.rs @@ -62,6 +62,7 @@ pub fn upgrade_to_capella( historical_summaries: VariableList::default(), // Caches total_active_balance: pre.total_active_balance, + progressive_balances_cache: mem::take(&mut pre.progressive_balances_cache), committee_caches: mem::take(&mut pre.committee_caches), pubkey_cache: mem::take(&mut pre.pubkey_cache), exit_cache: mem::take(&mut pre.exit_cache), diff --git a/consensus/state_processing/src/upgrade/merge.rs b/consensus/state_processing/src/upgrade/merge.rs index c172466248a..eb744501072 100644 --- a/consensus/state_processing/src/upgrade/merge.rs +++ b/consensus/state_processing/src/upgrade/merge.rs @@ -60,6 +60,7 @@ pub fn upgrade_to_bellatrix( latest_execution_payload_header: >::default(), // Caches total_active_balance: pre.total_active_balance, + progressive_balances_cache: mem::take(&mut pre.progressive_balances_cache), committee_caches: mem::take(&mut pre.committee_caches), pubkey_cache: mem::take(&mut pre.pubkey_cache), exit_cache: mem::take(&mut pre.exit_cache), diff --git a/consensus/types/Cargo.toml b/consensus/types/Cargo.toml index 583b940d5f4..ba15f6d4885 100644 --- a/consensus/types/Cargo.toml +++ b/consensus/types/Cargo.toml @@ -52,6 +52,7 @@ serde_json = "1.0.74" smallvec = "1.8.0" serde_with = "1.13.0" maplit = "1.0.2" +strum = { version = "0.24.0", features = ["derive"] } [dev-dependencies] criterion = "0.3.3" diff --git a/consensus/types/benches/benches.rs b/consensus/types/benches/benches.rs index 28f57e70804..bb2b527109f 100644 --- a/consensus/types/benches/benches.rs +++ b/consensus/types/benches/benches.rs @@ -51,7 +51,7 @@ fn all_benches(c: &mut Criterion) { let spec = Arc::new(MainnetEthSpec::default_spec()); let mut state = get_state::(validator_count); - state.build_all_caches(&spec).expect("should build caches"); + state.build_caches(&spec).expect("should build caches"); let state_bytes = state.as_ssz_bytes(); let inner_state = state.clone(); diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 4a9da364047..1fa4dee3a0e 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -26,6 +26,8 @@ pub use self::committee_cache::{ compute_committee_index_in_epoch, compute_committee_range_in_epoch, epoch_committee_count, CommitteeCache, }; +pub use crate::beacon_state::balance::Balance; +pub use crate::beacon_state::progressive_balances_cache::*; use crate::historical_summary::HistoricalSummary; pub use clone_config::CloneConfig; pub use eth_spec::*; @@ -34,9 +36,11 @@ pub use tree_hash_cache::BeaconTreeHashCache; #[macro_use] mod committee_cache; +mod balance; mod clone_config; mod exit_cache; mod iter; +mod progressive_balances_cache; mod pubkey_cache; mod tests; mod tree_hash_cache; @@ -101,6 +105,9 @@ pub enum Error { SszTypesError(ssz_types::Error), TreeHashCacheNotInitialized, NonLinearTreeHashCacheHistory, + ParticipationCacheError(String), + ProgressiveBalancesCacheNotInitialized, + ProgressiveBalancesCacheInconsistent, TreeHashCacheSkippedSlot { cache: Slot, state: Slot, @@ -317,6 +324,12 @@ where #[tree_hash(skip_hashing)] #[test_random(default)] #[derivative(Clone(clone_with = "clone_default"))] + pub progressive_balances_cache: ProgressiveBalancesCache, + #[serde(skip_serializing, skip_deserializing)] + #[ssz(skip_serializing, skip_deserializing)] + #[tree_hash(skip_hashing)] + #[test_random(default)] + #[derivative(Clone(clone_with = "clone_default"))] pub committee_caches: [CommitteeCache; CACHED_EPOCHS], #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] @@ -393,6 +406,7 @@ impl BeaconState { // Caching (not in spec) total_active_balance: None, + progressive_balances_cache: <_>::default(), committee_caches: [ CommitteeCache::default(), CommitteeCache::default(), @@ -757,7 +771,7 @@ impl BeaconState { Ok(signature_hash_int.safe_rem(modulo)? == 0) } - /// Returns the beacon proposer index for the `slot` in the given `relative_epoch`. + /// Returns the beacon proposer index for the `slot` in `self.current_epoch()`. /// /// Spec v0.12.1 pub fn get_beacon_proposer_index(&self, slot: Slot, spec: &ChainSpec) -> Result { @@ -1150,12 +1164,30 @@ impl BeaconState { } /// Convenience accessor for validators and balances simultaneously. - pub fn validators_and_balances_mut(&mut self) -> (&mut [Validator], &mut [u64]) { + pub fn validators_and_balances_and_progressive_balances_mut( + &mut self, + ) -> (&mut [Validator], &mut [u64], &mut ProgressiveBalancesCache) { match self { - BeaconState::Base(state) => (&mut state.validators, &mut state.balances), - BeaconState::Altair(state) => (&mut state.validators, &mut state.balances), - BeaconState::Merge(state) => (&mut state.validators, &mut state.balances), - BeaconState::Capella(state) => (&mut state.validators, &mut state.balances), + BeaconState::Base(state) => ( + &mut state.validators, + &mut state.balances, + &mut state.progressive_balances_cache, + ), + BeaconState::Altair(state) => ( + &mut state.validators, + &mut state.balances, + &mut state.progressive_balances_cache, + ), + BeaconState::Merge(state) => ( + &mut state.validators, + &mut state.balances, + &mut state.progressive_balances_cache, + ), + BeaconState::Capella(state) => ( + &mut state.validators, + &mut state.balances, + &mut state.progressive_balances_cache, + ), } } @@ -1380,7 +1412,7 @@ impl BeaconState { } /// Build all caches (except the tree hash cache), if they need to be built. - pub fn build_all_caches(&mut self, spec: &ChainSpec) -> Result<(), Error> { + pub fn build_caches(&mut self, spec: &ChainSpec) -> Result<(), Error> { self.build_all_committee_caches(spec)?; self.update_pubkey_cache()?; self.build_exit_cache(spec)?; @@ -1412,6 +1444,7 @@ impl BeaconState { self.drop_committee_cache(RelativeEpoch::Next)?; self.drop_pubkey_cache(); self.drop_tree_hash_cache(); + self.drop_progressive_balances_cache(); *self.exit_cache_mut() = ExitCache::default(); Ok(()) } @@ -1608,6 +1641,11 @@ impl BeaconState { *self.pubkey_cache_mut() = PubkeyCache::default() } + /// Completely drops the `progressive_balances_cache` cache, replacing it with a new, empty cache. + fn drop_progressive_balances_cache(&mut self) { + *self.progressive_balances_cache_mut() = ProgressiveBalancesCache::default(); + } + /// Initialize but don't fill the tree hash cache, if it isn't already initialized. pub fn initialize_tree_hash_cache(&mut self) { if !self.tree_hash_cache().is_initialized() { @@ -1679,6 +1717,9 @@ impl BeaconState { if config.tree_hash_cache { *res.tree_hash_cache_mut() = self.tree_hash_cache().clone(); } + if config.progressive_balances_cache { + *res.progressive_balances_cache_mut() = self.progressive_balances_cache().clone(); + } res } diff --git a/consensus/types/src/beacon_state/balance.rs b/consensus/types/src/beacon_state/balance.rs new file mode 100644 index 00000000000..e537a5b9842 --- /dev/null +++ b/consensus/types/src/beacon_state/balance.rs @@ -0,0 +1,33 @@ +use arbitrary::Arbitrary; +use safe_arith::{ArithError, SafeArith}; + +/// A balance which will never be below the specified `minimum`. +/// +/// This is an effort to ensure the `EFFECTIVE_BALANCE_INCREMENT` minimum is always respected. +#[derive(PartialEq, Debug, Clone, Copy, Arbitrary)] +pub struct Balance { + raw: u64, + minimum: u64, +} + +impl Balance { + /// Initialize the balance to `0`, or the given `minimum`. + pub fn zero(minimum: u64) -> Self { + Self { raw: 0, minimum } + } + + /// Returns the balance with respect to the initialization `minimum`. + pub fn get(&self) -> u64 { + std::cmp::max(self.raw, self.minimum) + } + + /// Add-assign to the balance. + pub fn safe_add_assign(&mut self, other: u64) -> Result<(), ArithError> { + self.raw.safe_add_assign(other) + } + + /// Sub-assign to the balance. + pub fn safe_sub_assign(&mut self, other: u64) -> Result<(), ArithError> { + self.raw.safe_sub_assign(other) + } +} diff --git a/consensus/types/src/beacon_state/clone_config.rs b/consensus/types/src/beacon_state/clone_config.rs index e5f050aee69..c6e7f47421f 100644 --- a/consensus/types/src/beacon_state/clone_config.rs +++ b/consensus/types/src/beacon_state/clone_config.rs @@ -5,6 +5,7 @@ pub struct CloneConfig { pub pubkey_cache: bool, pub exit_cache: bool, pub tree_hash_cache: bool, + pub progressive_balances_cache: bool, } impl CloneConfig { @@ -14,6 +15,7 @@ impl CloneConfig { pubkey_cache: true, exit_cache: true, tree_hash_cache: true, + progressive_balances_cache: true, } } diff --git a/consensus/types/src/beacon_state/progressive_balances_cache.rs b/consensus/types/src/beacon_state/progressive_balances_cache.rs new file mode 100644 index 00000000000..9f5c223d573 --- /dev/null +++ b/consensus/types/src/beacon_state/progressive_balances_cache.rs @@ -0,0 +1,184 @@ +use crate::beacon_state::balance::Balance; +use crate::{BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec}; +use arbitrary::Arbitrary; +use safe_arith::SafeArith; +use serde_derive::{Deserialize, Serialize}; +use strum::{Display, EnumString, EnumVariantNames}; + +/// This cache keeps track of the accumulated target attestation balance for the current & previous +/// epochs. The cached values can be utilised by fork choice to calculate unrealized justification +/// and finalization instead of converting epoch participation arrays to balances for each block we +/// process. +#[derive(Default, Debug, PartialEq, Arbitrary, Clone)] +pub struct ProgressiveBalancesCache { + inner: Option, +} + +#[derive(Debug, PartialEq, Arbitrary, Clone)] +struct Inner { + pub current_epoch: Epoch, + pub previous_epoch_target_attesting_balance: Balance, + pub current_epoch_target_attesting_balance: Balance, +} + +impl ProgressiveBalancesCache { + pub fn initialize( + &mut self, + current_epoch: Epoch, + previous_epoch_target_attesting_balance: Balance, + current_epoch_target_attesting_balance: Balance, + ) { + self.inner = Some(Inner { + current_epoch, + previous_epoch_target_attesting_balance, + current_epoch_target_attesting_balance, + }); + } + + pub fn is_initialized(&self) -> bool { + self.inner.is_some() + } + + /// When a new target attestation has been processed, we update the cached + /// `current_epoch_target_attesting_balance` to include the validator effective balance. + /// If the epoch is neither the current epoch nor the previous epoch, an error is returned. + pub fn on_new_target_attestation( + &mut self, + epoch: Epoch, + validator_effective_balance: u64, + ) -> Result<(), BeaconStateError> { + let cache = self.get_inner_mut()?; + + if epoch == cache.current_epoch { + cache + .current_epoch_target_attesting_balance + .safe_add_assign(validator_effective_balance)?; + } else if epoch.safe_add(1)? == cache.current_epoch { + cache + .previous_epoch_target_attesting_balance + .safe_add_assign(validator_effective_balance)?; + } else { + return Err(BeaconStateError::ProgressiveBalancesCacheInconsistent); + } + + Ok(()) + } + + /// When a validator is slashed, we reduce the `current_epoch_target_attesting_balance` by the + /// validator's effective balance to exclude the validator weight. + pub fn on_slashing( + &mut self, + is_previous_epoch_target_attester: bool, + is_current_epoch_target_attester: bool, + effective_balance: u64, + ) -> Result<(), BeaconStateError> { + let cache = self.get_inner_mut()?; + if is_previous_epoch_target_attester { + cache + .previous_epoch_target_attesting_balance + .safe_sub_assign(effective_balance)?; + } + if is_current_epoch_target_attester { + cache + .current_epoch_target_attesting_balance + .safe_sub_assign(effective_balance)?; + } + Ok(()) + } + + /// When a current epoch target attester has its effective balance changed, we adjust the + /// its share of the target attesting balance in the cache. + pub fn on_effective_balance_change( + &mut self, + is_current_epoch_target_attester: bool, + old_effective_balance: u64, + new_effective_balance: u64, + ) -> Result<(), BeaconStateError> { + let cache = self.get_inner_mut()?; + if is_current_epoch_target_attester { + if new_effective_balance > old_effective_balance { + cache + .current_epoch_target_attesting_balance + .safe_add_assign(new_effective_balance.safe_sub(old_effective_balance)?)?; + } else { + cache + .current_epoch_target_attesting_balance + .safe_sub_assign(old_effective_balance.safe_sub(new_effective_balance)?)?; + } + } + Ok(()) + } + + /// On epoch transition, the balance from current epoch is shifted to previous epoch, and the + /// current epoch balance is reset to 0. + pub fn on_epoch_transition(&mut self, spec: &ChainSpec) -> Result<(), BeaconStateError> { + let cache = self.get_inner_mut()?; + cache.current_epoch.safe_add_assign(1)?; + cache.previous_epoch_target_attesting_balance = + cache.current_epoch_target_attesting_balance; + cache.current_epoch_target_attesting_balance = + Balance::zero(spec.effective_balance_increment); + Ok(()) + } + + pub fn previous_epoch_target_attesting_balance(&self) -> Result { + Ok(self + .get_inner()? + .previous_epoch_target_attesting_balance + .get()) + } + + pub fn current_epoch_target_attesting_balance(&self) -> Result { + Ok(self + .get_inner()? + .current_epoch_target_attesting_balance + .get()) + } + + fn get_inner_mut(&mut self) -> Result<&mut Inner, BeaconStateError> { + self.inner + .as_mut() + .ok_or(BeaconStateError::ProgressiveBalancesCacheNotInitialized) + } + + fn get_inner(&self) -> Result<&Inner, BeaconStateError> { + self.inner + .as_ref() + .ok_or(BeaconStateError::ProgressiveBalancesCacheNotInitialized) + } +} + +#[derive( + Debug, PartialEq, Eq, Clone, Copy, Deserialize, Serialize, Display, EnumString, EnumVariantNames, +)] +#[strum(serialize_all = "lowercase")] +pub enum ProgressiveBalancesMode { + /// Disable the usage of progressive cache, and use the existing `ParticipationCache` calculation. + Disabled, + /// Enable the usage of progressive cache, with checks against the `ParticipationCache` and falls + /// back to the existing calculation if there is a balance mismatch. + Checked, + /// Enable the usage of progressive cache, with checks against the `ParticipationCache`. Errors + /// if there is a balance mismatch. Used in testing only. + Strict, + /// Enable the usage of progressive cache, with no comparative checks against the + /// `ParticipationCache`. This is fast but an experimental mode, use with caution. + Fast, +} + +impl ProgressiveBalancesMode { + pub fn perform_comparative_checks(&self) -> bool { + match self { + Self::Disabled | Self::Fast => false, + Self::Checked | Self::Strict => true, + } + } +} + +/// `ProgressiveBalancesCache` is only enabled from `Altair` as it requires `ParticipationCache`. +pub fn is_progressive_balances_enabled(state: &BeaconState) -> bool { + match state { + BeaconState::Base(_) => false, + BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) => true, + } +} diff --git a/consensus/types/src/beacon_state/tests.rs b/consensus/types/src/beacon_state/tests.rs index d63eaafc4b9..6cd9c1dbf88 100644 --- a/consensus/types/src/beacon_state/tests.rs +++ b/consensus/types/src/beacon_state/tests.rs @@ -219,17 +219,18 @@ async fn clone_config() { let mut state = build_state::(16).await; - state.build_all_caches(&spec).unwrap(); + state.build_caches(&spec).unwrap(); state .update_tree_hash_cache() .expect("should update tree hash cache"); - let num_caches = 4; + let num_caches = 5; let all_configs = (0..2u8.pow(num_caches)).map(|i| CloneConfig { committee_caches: (i & 1) != 0, pubkey_cache: ((i >> 1) & 1) != 0, exit_cache: ((i >> 2) & 1) != 0, tree_hash_cache: ((i >> 3) & 1) != 0, + progressive_balances_cache: ((i >> 4) & 1) != 0, }); for config in all_configs { diff --git a/lcli/src/new_testnet.rs b/lcli/src/new_testnet.rs index aa5f52eef8c..01a44cabef7 100644 --- a/lcli/src/new_testnet.rs +++ b/lcli/src/new_testnet.rs @@ -303,7 +303,7 @@ fn initialize_state_with_validators( } // Now that we have our validators, initialize the caches (including the committees) - state.build_all_caches(spec).unwrap(); + state.build_caches(spec).unwrap(); // Set genesis validators root for domain separation and chain versioning *state.genesis_validators_root_mut() = state.update_validators_tree_hash_cache().unwrap(); diff --git a/lcli/src/skip_slots.rs b/lcli/src/skip_slots.rs index 49d1dd424da..e3b2a5acbfd 100644 --- a/lcli/src/skip_slots.rs +++ b/lcli/src/skip_slots.rs @@ -109,7 +109,7 @@ pub fn run(env: Environment, matches: &ArgMatches) -> Result<(), let target_slot = initial_slot + slots; state - .build_all_caches(spec) + .build_caches(spec) .map_err(|e| format!("Unable to build caches: {:?}", e))?; let state_root = if let Some(root) = cli_state_root.or(state_root) { diff --git a/lcli/src/transition_blocks.rs b/lcli/src/transition_blocks.rs index cf971c69f00..34a45607610 100644 --- a/lcli/src/transition_blocks.rs +++ b/lcli/src/transition_blocks.rs @@ -205,7 +205,7 @@ pub fn run(env: Environment, matches: &ArgMatches) -> Result<(), if config.exclude_cache_builds { pre_state - .build_all_caches(spec) + .build_caches(spec) .map_err(|e| format!("Unable to build caches: {:?}", e))?; let state_root = pre_state .update_tree_hash_cache() @@ -303,7 +303,7 @@ fn do_transition( if !config.exclude_cache_builds { let t = Instant::now(); pre_state - .build_all_caches(spec) + .build_caches(spec) .map_err(|e| format!("Unable to build caches: {:?}", e))?; debug!("Build caches: {:?}", t.elapsed()); @@ -335,7 +335,7 @@ fn do_transition( let t = Instant::now(); pre_state - .build_all_caches(spec) + .build_caches(spec) .map_err(|e| format!("Unable to build caches: {:?}", e))?; debug!("Build all caches (again): {:?}", t.elapsed()); diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 65d7bd08b28..ac0780015f6 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -16,7 +16,10 @@ use std::str::FromStr; use std::string::ToString; use std::time::Duration; use tempfile::TempDir; -use types::{Address, Checkpoint, Epoch, ExecutionBlockHash, ForkName, Hash256, MainnetEthSpec}; +use types::{ + Address, Checkpoint, Epoch, ExecutionBlockHash, ForkName, Hash256, MainnetEthSpec, + ProgressiveBalancesMode, +}; use unused_port::{unused_tcp4_port, unused_tcp6_port, unused_udp4_port, unused_udp6_port}; const DEFAULT_ETH1_ENDPOINT: &str = "http://localhost:8545/"; @@ -2284,3 +2287,28 @@ fn invalid_gossip_verified_blocks_path() { ) }); } + +#[test] +fn progressive_balances_default() { + CommandLineTest::new() + .run_with_zero_port() + .with_config(|config| { + assert_eq!( + config.chain.progressive_balances_mode, + ProgressiveBalancesMode::Checked + ) + }); +} + +#[test] +fn progressive_balances_fast() { + CommandLineTest::new() + .flag("progressive-balances", Some("fast")) + .run_with_zero_port() + .with_config(|config| { + assert_eq!( + config.chain.progressive_balances_mode, + ProgressiveBalancesMode::Fast + ) + }); +} diff --git a/testing/ef_tests/src/cases/epoch_processing.rs b/testing/ef_tests/src/cases/epoch_processing.rs index 6095e1be6b1..31542ba4476 100644 --- a/testing/ef_tests/src/cases/epoch_processing.rs +++ b/testing/ef_tests/src/cases/epoch_processing.rs @@ -6,9 +6,9 @@ use crate::type_name; use crate::type_name::TypeName; use serde_derive::Deserialize; use state_processing::per_epoch_processing::capella::process_historical_summaries_update; +use state_processing::per_epoch_processing::effective_balance_updates::process_effective_balance_updates; use state_processing::per_epoch_processing::{ altair, base, - effective_balance_updates::process_effective_balance_updates, historical_roots_update::process_historical_roots_update, process_registry_updates, process_slashings, resets::{process_eth1_data_reset, process_randao_mixes_reset, process_slashings_reset}, @@ -173,7 +173,7 @@ impl EpochTransition for Eth1DataReset { impl EpochTransition for EffectiveBalanceUpdates { fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { - process_effective_balance_updates(state, spec) + process_effective_balance_updates(state, None, spec) } } diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index 65528de1757..9627d2cde03 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -18,7 +18,8 @@ use std::sync::Arc; use std::time::Duration; use types::{ Attestation, AttesterSlashing, BeaconBlock, BeaconState, Checkpoint, EthSpec, - ExecutionBlockHash, ForkName, Hash256, IndexedAttestation, SignedBeaconBlock, Slot, Uint256, + ExecutionBlockHash, ForkName, Hash256, IndexedAttestation, ProgressiveBalancesMode, + SignedBeaconBlock, Slot, Uint256, }; #[derive(Default, Debug, PartialEq, Clone, Deserialize, Decode)] @@ -440,7 +441,9 @@ impl Tester { block_delay, &state, PayloadVerificationStatus::Irrelevant, + ProgressiveBalancesMode::Strict, &self.harness.chain.spec, + self.harness.logger(), ); if result.is_ok() { diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index 5fd00285aaa..21a56dcf2a7 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -4,6 +4,7 @@ use crate::case_result::compare_beacon_state_results_without_caches; use crate::decode::{ssz_decode_file, ssz_decode_file_with, ssz_decode_state, yaml_decode_file}; use crate::testing_spec; use serde_derive::Deserialize; +use state_processing::common::update_progressive_balances_cache::initialize_progressive_balances_cache; use state_processing::{ per_block_processing::{ errors::BlockProcessingError, @@ -96,6 +97,7 @@ impl Operation for Attestation { spec, ), BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) => { + initialize_progressive_balances_cache(state, None, spec)?; altair::process_attestation(state, self, 0, &mut ctxt, VerifySignatures::True, spec) } } @@ -118,6 +120,7 @@ impl Operation for AttesterSlashing { _: &Operations, ) -> Result<(), BlockProcessingError> { let mut ctxt = ConsensusContext::new(state.slot()); + initialize_progressive_balances_cache(state, None, spec)?; process_attester_slashings( state, &[self.clone()], @@ -168,6 +171,7 @@ impl Operation for ProposerSlashing { _: &Operations, ) -> Result<(), BlockProcessingError> { let mut ctxt = ConsensusContext::new(state.slot()); + initialize_progressive_balances_cache(state, None, spec)?; process_proposer_slashings( state, &[self.clone()], diff --git a/testing/ef_tests/src/cases/sanity_blocks.rs b/testing/ef_tests/src/cases/sanity_blocks.rs index e51fed1907f..191b45c33a1 100644 --- a/testing/ef_tests/src/cases/sanity_blocks.rs +++ b/testing/ef_tests/src/cases/sanity_blocks.rs @@ -67,7 +67,7 @@ impl Case for SanityBlocks { let spec = &testing_spec::(fork_name); // Processing requires the epoch cache. - bulk_state.build_all_caches(spec).unwrap(); + bulk_state.build_caches(spec).unwrap(); // Spawning a second state to call the VerifyIndiviual strategy to avoid bitrot. // See https://github.com/sigp/lighthouse/issues/742. diff --git a/testing/ef_tests/src/cases/sanity_slots.rs b/testing/ef_tests/src/cases/sanity_slots.rs index a38a8930a0e..dd385d13f4e 100644 --- a/testing/ef_tests/src/cases/sanity_slots.rs +++ b/testing/ef_tests/src/cases/sanity_slots.rs @@ -61,7 +61,7 @@ impl Case for SanitySlots { let spec = &testing_spec::(fork_name); // Processing requires the epoch cache. - state.build_all_caches(spec).unwrap(); + state.build_caches(spec).unwrap(); let mut result = (0..self.slots) .try_for_each(|_| per_slot_processing(&mut state, None, spec).map(|_| ())) From 8e65419455577bb4d4f3fdeb045085a58874d24d Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 3 Jul 2023 03:20:21 +0000 Subject: [PATCH 06/51] Ipv6 bootnodes update (#4394) We now officially have ipv6 support. The mainnet bootnodes have been updated to support ipv6. This PR updates lighthouse's internal bootnodes for mainnet to avoid fetching them on initial load. --- .../mainnet/boot_enr.yaml | 8 ++-- lighthouse/tests/beacon_node.rs | 39 +------------------ 2 files changed, 7 insertions(+), 40 deletions(-) diff --git a/common/eth2_network_config/built_in_network_configs/mainnet/boot_enr.yaml b/common/eth2_network_config/built_in_network_configs/mainnet/boot_enr.yaml index 196629cb8d5..428a082cc0f 100644 --- a/common/eth2_network_config/built_in_network_configs/mainnet/boot_enr.yaml +++ b/common/eth2_network_config/built_in_network_configs/mainnet/boot_enr.yaml @@ -1,6 +1,8 @@ # Lighthouse Team (Sigma Prime) -- enr:-Jq4QItoFUuug_n_qbYbU0OY04-np2wT8rUCauOOXNi0H3BWbDj-zbfZb7otA7jZ6flbBpx1LNZK2TDebZ9dEKx84LYBhGV0aDKQtTA_KgEAAAD__________4JpZIJ2NIJpcISsaa0ZiXNlY3AyNTZrMaEDHAD2JKYevx89W0CcFJFiskdcEzkH_Wdv9iW42qLK79ODdWRwgiMo -- enr:-Jq4QN_YBsUOqQsty1OGvYv48PMaiEt1AzGD1NkYQHaxZoTyVGqMYXg0K9c0LPNWC9pkXmggApp8nygYLsQwScwAgfgBhGV0aDKQtTA_KgEAAAD__________4JpZIJ2NIJpcISLosQxiXNlY3AyNTZrMaEDBJj7_dLFACaxBfaI8KZTh_SSJUjhyAyfshimvSqo22WDdWRwgiMo +- enr:-Le4QPUXJS2BTORXxyx2Ia-9ae4YqA_JWX3ssj4E_J-3z1A-HmFGrU8BpvpqhNabayXeOZ2Nq_sbeDgtzMJpLLnXFgAChGV0aDKQtTA_KgEAAAAAIgEAAAAAAIJpZIJ2NIJpcISsaa0Zg2lwNpAkAIkHAAAAAPA8kv_-awoTiXNlY3AyNTZrMaEDHAD2JKYevx89W0CcFJFiskdcEzkH_Wdv9iW42qLK79ODdWRwgiMohHVkcDaCI4I +- enr:-Le4QLHZDSvkLfqgEo8IWGG96h6mxwe_PsggC20CL3neLBjfXLGAQFOPSltZ7oP6ol54OvaNqO02Rnvb8YmDR274uq8ChGV0aDKQtTA_KgEAAAAAIgEAAAAAAIJpZIJ2NIJpcISLosQxg2lwNpAqAX4AAAAAAPA8kv_-ax65iXNlY3AyNTZrMaEDBJj7_dLFACaxBfaI8KZTh_SSJUjhyAyfshimvSqo22WDdWRwgiMohHVkcDaCI4I +- enr:-Le4QH6LQrusDbAHPjU_HcKOuMeXfdEB5NJyXgHWFadfHgiySqeDyusQMvfphdYWOzuSZO9Uq2AMRJR5O4ip7OvVma8BhGV0aDKQtTA_KgEAAAAAIgEAAAAAAIJpZIJ2NIJpcISLY9ncg2lwNpAkAh8AgQIBAAAAAAAAAAmXiXNlY3AyNTZrMaECDYCZTZEksF-kmgPholqgVt8IXr-8L7Nu7YrZ7HUpgxmDdWRwgiMohHVkcDaCI4I +- enr:-Le4QIqLuWybHNONr933Lk0dcMmAB5WgvGKRyDihy1wHDIVlNuuztX62W51voT4I8qD34GcTEOTmag1bcdZ_8aaT4NUBhGV0aDKQtTA_KgEAAAAAIgEAAAAAAIJpZIJ2NIJpcISLY04ng2lwNpAkAh8AgAIBAAAAAAAAAA-fiXNlY3AyNTZrMaEDscnRV6n1m-D9ID5UsURk0jsoKNXt1TIrj8uKOGW6iluDdWRwgiMohHVkcDaCI4I # EF Team - enr:-Ku4QHqVeJ8PPICcWk1vSn_XcSkjOkNiTg6Fmii5j6vUQgvzMc9L1goFnLKgXqBJspJjIsB91LTOleFmyWWrFVATGngBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhAMRHkWJc2VjcDI1NmsxoQKLVXFOhp2uX6jeT0DvvDpPcU8FWMjQdR4wMuORMhpX24N1ZHCCIyg - enr:-Ku4QG-2_Md3sZIAUebGYT6g0SMskIml77l6yR-M_JXc-UdNHCmHQeOiMLbylPejyJsdAPsTHJyjJB2sYGDLe0dn8uYBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhBLY-NyJc2VjcDI1NmsxoQORcM6e19T1T9gi7jxEZjk_sjVLGFscUNqAY9obgZaxbIN1ZHCCIyg @@ -15,4 +17,4 @@ - enr:-Ku4QPp9z1W4tAO8Ber_NQierYaOStqhDqQdOPY3bB3jDgkjcbk6YrEnVYIiCBbTxuar3CzS528d2iE7TdJsrL-dEKoBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpD1pf1CAAAAAP__________gmlkgnY0gmlwhBLf22SJc2VjcDI1NmsxoQMw5fqqkw2hHC4F5HZZDPsNmPdB1Gi8JPQK7pRc9XHh-oN1ZHCCKvg # Nimbus team - enr:-LK4QA8FfhaAjlb_BXsXxSfiysR7R52Nhi9JBt4F8SPssu8hdE1BXQQEtVDC3qStCW60LSO7hEsVHv5zm8_6Vnjhcn0Bh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhAN4aBKJc2VjcDI1NmsxoQJerDhsJ-KxZ8sHySMOCmTO6sHM3iCFQ6VMvLTe948MyYN0Y3CCI4yDdWRwgiOM -- enr:-LK4QKWrXTpV9T78hNG6s8AM6IO4XH9kFT91uZtFg1GcsJ6dKovDOr1jtAAFPnS2lvNltkOGA9k29BUN7lFh_sjuc9QBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhANAdd-Jc2VjcDI1NmsxoQLQa6ai7y9PMN5hpLe5HmiJSlYzMuzP7ZhwRiwHvqNXdoN0Y3CCI4yDdWRwgiOM +- enr:-LK4QKWrXTpV9T78hNG6s8AM6IO4XH9kFT91uZtFg1GcsJ6dKovDOr1jtAAFPnS2lvNltkOGA9k29BUN7lFh_sjuc9QBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhANAdd-Jc2VjcDI1NmsxoQLQa6ai7y9PMN5hpLe5HmiJSlYzMuzP7ZhwRiwHvqNXdoN0Y3CCI4yDdWRwgiOM \ No newline at end of file diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index ac0780015f6..9b6d23ddcf5 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -1128,48 +1128,13 @@ fn default_backfill_rate_limiting_flag() { } #[test] fn default_boot_nodes() { - let mainnet = vec![ - // Lighthouse Team (Sigma Prime) - "enr:-Jq4QItoFUuug_n_qbYbU0OY04-np2wT8rUCauOOXNi0H3BWbDj-zbfZb7otA7jZ6flbBpx1LNZK2TDebZ9dEKx84LYBhGV0aDKQtTA_KgEAAAD__________4JpZIJ2NIJpcISsaa0ZiXNlY3AyNTZrMaEDHAD2JKYevx89W0CcFJFiskdcEzkH_Wdv9iW42qLK79ODdWRwgiMo", - "enr:-Jq4QN_YBsUOqQsty1OGvYv48PMaiEt1AzGD1NkYQHaxZoTyVGqMYXg0K9c0LPNWC9pkXmggApp8nygYLsQwScwAgfgBhGV0aDKQtTA_KgEAAAD__________4JpZIJ2NIJpcISLosQxiXNlY3AyNTZrMaEDBJj7_dLFACaxBfaI8KZTh_SSJUjhyAyfshimvSqo22WDdWRwgiMo", - // EF Team - "enr:-Ku4QHqVeJ8PPICcWk1vSn_XcSkjOkNiTg6Fmii5j6vUQgvzMc9L1goFnLKgXqBJspJjIsB91LTOleFmyWWrFVATGngBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhAMRHkWJc2VjcDI1NmsxoQKLVXFOhp2uX6jeT0DvvDpPcU8FWMjQdR4wMuORMhpX24N1ZHCCIyg", - "enr:-Ku4QG-2_Md3sZIAUebGYT6g0SMskIml77l6yR-M_JXc-UdNHCmHQeOiMLbylPejyJsdAPsTHJyjJB2sYGDLe0dn8uYBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhBLY-NyJc2VjcDI1NmsxoQORcM6e19T1T9gi7jxEZjk_sjVLGFscUNqAY9obgZaxbIN1ZHCCIyg", - "enr:-Ku4QPn5eVhcoF1opaFEvg1b6JNFD2rqVkHQ8HApOKK61OIcIXD127bKWgAtbwI7pnxx6cDyk_nI88TrZKQaGMZj0q0Bh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhDayLMaJc2VjcDI1NmsxoQK2sBOLGcUb4AwuYzFuAVCaNHA-dy24UuEKkeFNgCVCsIN1ZHCCIyg", - "enr:-Ku4QEWzdnVtXc2Q0ZVigfCGggOVB2Vc1ZCPEc6j21NIFLODSJbvNaef1g4PxhPwl_3kax86YPheFUSLXPRs98vvYsoBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhDZBrP2Jc2VjcDI1NmsxoQM6jr8Rb1ktLEsVcKAPa08wCsKUmvoQ8khiOl_SLozf9IN1ZHCCIyg", - // Teku team (Consensys) - "enr:-KG4QOtcP9X1FbIMOe17QNMKqDxCpm14jcX5tiOE4_TyMrFqbmhPZHK_ZPG2Gxb1GE2xdtodOfx9-cgvNtxnRyHEmC0ghGV0aDKQ9aX9QgAAAAD__________4JpZIJ2NIJpcIQDE8KdiXNlY3AyNTZrMaEDhpehBDbZjM_L9ek699Y7vhUJ-eAdMyQW_Fil522Y0fODdGNwgiMog3VkcIIjKA", - "enr:-KG4QDyytgmE4f7AnvW-ZaUOIi9i79qX4JwjRAiXBZCU65wOfBu-3Nb5I7b_Rmg3KCOcZM_C3y5pg7EBU5XGrcLTduQEhGV0aDKQ9aX9QgAAAAD__________4JpZIJ2NIJpcIQ2_DUbiXNlY3AyNTZrMaEDKnz_-ps3UUOfHWVYaskI5kWYO_vtYMGYCQRAR3gHDouDdGNwgiMog3VkcIIjKA", - // Prysm team (Prysmatic Labs) - "enr:-Ku4QImhMc1z8yCiNJ1TyUxdcfNucje3BGwEHzodEZUan8PherEo4sF7pPHPSIB1NNuSg5fZy7qFsjmUKs2ea1Whi0EBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpD1pf1CAAAAAP__________gmlkgnY0gmlwhBLf22SJc2VjcDI1NmsxoQOVphkDqal4QzPMksc5wnpuC3gvSC8AfbFOnZY_On34wIN1ZHCCIyg", - "enr:-Ku4QP2xDnEtUXIjzJ_DhlCRN9SN99RYQPJL92TMlSv7U5C1YnYLjwOQHgZIUXw6c-BvRg2Yc2QsZxxoS_pPRVe0yK8Bh2F0dG5ldHOIAAAAAAAAAACEZXRoMpD1pf1CAAAAAP__________gmlkgnY0gmlwhBLf22SJc2VjcDI1NmsxoQMeFF5GrS7UZpAH2Ly84aLK-TyvH-dRo0JM1i8yygH50YN1ZHCCJxA", - "enr:-Ku4QPp9z1W4tAO8Ber_NQierYaOStqhDqQdOPY3bB3jDgkjcbk6YrEnVYIiCBbTxuar3CzS528d2iE7TdJsrL-dEKoBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpD1pf1CAAAAAP__________gmlkgnY0gmlwhBLf22SJc2VjcDI1NmsxoQMw5fqqkw2hHC4F5HZZDPsNmPdB1Gi8JPQK7pRc9XHh-oN1ZHCCKvg", - // Nimbus team - "enr:-LK4QA8FfhaAjlb_BXsXxSfiysR7R52Nhi9JBt4F8SPssu8hdE1BXQQEtVDC3qStCW60LSO7hEsVHv5zm8_6Vnjhcn0Bh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhAN4aBKJc2VjcDI1NmsxoQJerDhsJ-KxZ8sHySMOCmTO6sHM3iCFQ6VMvLTe948MyYN0Y3CCI4yDdWRwgiOM", - "enr:-LK4QKWrXTpV9T78hNG6s8AM6IO4XH9kFT91uZtFg1GcsJ6dKovDOr1jtAAFPnS2lvNltkOGA9k29BUN7lFh_sjuc9QBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhANAdd-Jc2VjcDI1NmsxoQLQa6ai7y9PMN5hpLe5HmiJSlYzMuzP7ZhwRiwHvqNXdoN0Y3CCI4yDdWRwgiOM" - ]; + let number_of_boot_nodes = 15; CommandLineTest::new() .run_with_zero_port() .with_config(|config| { // Lighthouse Team (Sigma Prime) - assert_eq!(config.network.boot_nodes_enr[0].to_base64(), mainnet[0]); - assert_eq!(config.network.boot_nodes_enr[1].to_base64(), mainnet[1]); - // EF Team - assert_eq!(config.network.boot_nodes_enr[2].to_base64(), mainnet[2]); - assert_eq!(config.network.boot_nodes_enr[3].to_base64(), mainnet[3]); - assert_eq!(config.network.boot_nodes_enr[4].to_base64(), mainnet[4]); - assert_eq!(config.network.boot_nodes_enr[5].to_base64(), mainnet[5]); - // Teku team (Consensys) - assert_eq!(config.network.boot_nodes_enr[6].to_base64(), mainnet[6]); - assert_eq!(config.network.boot_nodes_enr[7].to_base64(), mainnet[7]); - // Prysm team (Prysmatic Labs) - assert_eq!(config.network.boot_nodes_enr[8].to_base64(), mainnet[8]); - assert_eq!(config.network.boot_nodes_enr[9].to_base64(), mainnet[9]); - assert_eq!(config.network.boot_nodes_enr[10].to_base64(), mainnet[10]); - // Nimbus team - assert_eq!(config.network.boot_nodes_enr[11].to_base64(), mainnet[11]); - assert_eq!(config.network.boot_nodes_enr[12].to_base64(), mainnet[12]); + assert_eq!(config.network.boot_nodes_enr.len(), number_of_boot_nodes); }); } #[test] From dfcb3363c757671eb19d5f8e519b4b94ac74677a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 4 Jul 2023 13:29:55 +0000 Subject: [PATCH 07/51] Release v4.3.0 (#4452) ## Issue Addressed NA ## Proposed Changes Bump versions ## Additional Info NA --- Cargo.lock | 8 ++++---- beacon_node/Cargo.toml | 2 +- boot_node/Cargo.toml | 2 +- common/lighthouse_version/src/lib.rs | 4 ++-- lcli/Cargo.toml | 2 +- lighthouse/Cargo.toml | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index efc6a5d6abe..e360bdd62e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -668,7 +668,7 @@ dependencies = [ [[package]] name = "beacon_node" -version = "4.2.0" +version = "4.3.0" dependencies = [ "beacon_chain", "clap", @@ -847,7 +847,7 @@ dependencies = [ [[package]] name = "boot_node" -version = "4.2.0" +version = "4.3.0" dependencies = [ "beacon_node", "clap", @@ -4022,7 +4022,7 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "lcli" -version = "4.2.0" +version = "4.3.0" dependencies = [ "account_utils", "beacon_chain", @@ -4674,7 +4674,7 @@ dependencies = [ [[package]] name = "lighthouse" -version = "4.2.0" +version = "4.3.0" dependencies = [ "account_manager", "account_utils", diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 67bb2e5e1d3..7c74365418b 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "beacon_node" -version = "4.2.0" +version = "4.3.0" authors = ["Paul Hauner ", "Age Manning "] edition = "2021" diff --git a/common/lighthouse_version/src/lib.rs b/common/lighthouse_version/src/lib.rs index 3f2745bf90c..e874432fbca 100644 --- a/common/lighthouse_version/src/lib.rs +++ b/common/lighthouse_version/src/lib.rs @@ -17,8 +17,8 @@ pub const VERSION: &str = git_version!( // NOTE: using --match instead of --exclude for compatibility with old Git "--match=thiswillnevermatchlol" ], - prefix = "Lighthouse/v4.2.0-", - fallback = "Lighthouse/v4.2.0" + prefix = "Lighthouse/v4.3.0-", + fallback = "Lighthouse/v4.3.0" ); /// Returns `VERSION`, but with platform information appended to the end. diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index b4d1baba4ab..f9d0a6a31c7 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lcli" description = "Lighthouse CLI (modeled after zcli)" -version = "4.2.0" +version = "4.3.0" authors = ["Paul Hauner "] edition = "2021" diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index bbde006efc5..e7746a2db98 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lighthouse" -version = "4.2.0" +version = "4.3.0" authors = ["Sigma Prime "] edition = "2021" autotests = false From d9254b7ded3c249cfab32842b370548019463431 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 5 Jul 2023 12:04:12 -0400 Subject: [PATCH 08/51] update get blobs endpoint name from blobs to blob_sidecars (#4467) * changed name * Fix sidecars * update get blobs endpoint name from blobs to blob_sidecars --------- Co-authored-by: Rahul Dogra --- beacon_node/http_api/src/lib.rs | 4 ++-- common/eth2/src/lib.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index a4fea8f5f9e..1a2a14611d0 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -1394,10 +1394,10 @@ pub fn serve( * beacon/blobs */ - // GET beacon/blobs/{block_id} + // GET beacon/blob_sidecars/{block_id} let get_blobs = eth_v1 .and(warp::path("beacon")) - .and(warp::path("blobs")) + .and(warp::path("blob_sidecars")) .and(block_id_or_err) .and(warp::path::end()) .and(chain_filter.clone()) diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 98eb1eba9d1..32d1bc2be1f 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -665,13 +665,13 @@ impl BeaconNodeHttpClient { Ok(path) } - /// Path for `v1/beacon/blobs/{block_id}` + /// Path for `v1/beacon/blob_sidecars/{block_id}` pub fn get_blobs_path(&self, block_id: BlockId) -> Result { let mut path = self.eth_path(V1)?; path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? .push("beacon") - .push("blobs") + .push("blob_sidecars") .push(&block_id.to_string()); Ok(path) } From d41193c31895b165519dd40b266212a21d31fe88 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 5 Jul 2023 15:52:59 -0400 Subject: [PATCH 09/51] fix failing network tests (#4472) --- beacon_node/network/src/beacon_processor/tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/network/src/beacon_processor/tests.rs b/beacon_node/network/src/beacon_processor/tests.rs index c1e1f764626..1b1b7c7dca8 100644 --- a/beacon_node/network/src/beacon_processor/tests.rs +++ b/beacon_node/network/src/beacon_processor/tests.rs @@ -707,8 +707,8 @@ async fn attestation_to_unknown_block_processed(import_method: BlockImportMethod BlockImportMethod::Rpc => { rig.enqueue_rpc_block(); events.push(RPC_BLOCK); - rig.enqueue_single_lookup_rpc_blobs(); if num_blobs > 0 { + rig.enqueue_single_lookup_rpc_blobs(); events.push(RPC_BLOB); } } @@ -790,8 +790,8 @@ async fn aggregate_attestation_to_unknown_block(import_method: BlockImportMethod BlockImportMethod::Rpc => { rig.enqueue_rpc_block(); events.push(RPC_BLOCK); - rig.enqueue_single_lookup_rpc_blobs(); if num_blobs > 0 { + rig.enqueue_single_lookup_rpc_blobs(); events.push(RPC_BLOB); } } From af4a66846ef6101e7f00f2e1b33ecb5de8d575e8 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 5 Jul 2023 15:53:22 -0400 Subject: [PATCH 10/51] remove clang from Dockerfile (#4471) --- Dockerfile | 2 +- lcli/Dockerfile | 2 +- testing/antithesis/Dockerfile.libvoidstar | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index cc024111644..be01ad7c572 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM rust:1.68.2-bullseye AS builder -RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake clang libclang-dev protobuf-compiler +RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev protobuf-compiler COPY . lighthouse ARG FEATURES ARG PROFILE=release diff --git a/lcli/Dockerfile b/lcli/Dockerfile index 95b34cf3434..98f33f21536 100644 --- a/lcli/Dockerfile +++ b/lcli/Dockerfile @@ -2,7 +2,7 @@ # - from the `lighthouse` dir with the command: `docker build -f ./lcli/Dockerflie .` # - from the current directory with the command: `docker build -f ./Dockerfile ../` FROM rust:1.68.2-bullseye AS builder -RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake clang libclang-dev protobuf-compiler +RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev protobuf-compiler COPY . lighthouse ARG PORTABLE ENV PORTABLE $PORTABLE diff --git a/testing/antithesis/Dockerfile.libvoidstar b/testing/antithesis/Dockerfile.libvoidstar index 0a3f5c6b195..ddc49e13cd7 100644 --- a/testing/antithesis/Dockerfile.libvoidstar +++ b/testing/antithesis/Dockerfile.libvoidstar @@ -1,5 +1,5 @@ FROM rust:1.68.2-bullseye AS builder -RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake clang libclang-dev protobuf-compiler +RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev protobuf-compiler COPY . lighthouse # Build lighthouse directly with a cargo build command, bypassing the Makefile. From ba6581297244c4fc17105ab6602a561227185cce Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 5 Jul 2023 15:53:35 -0400 Subject: [PATCH 11/51] remove patched dependencies (#4470) --- Cargo.lock | 15 +++++++++------ Cargo.toml | 3 --- beacon_node/beacon_chain/Cargo.toml | 6 +++--- beacon_node/eth1/Cargo.toml | 4 ++-- beacon_node/execution_layer/Cargo.toml | 4 ++-- beacon_node/genesis/Cargo.toml | 2 +- beacon_node/http_api/Cargo.toml | 2 +- beacon_node/lighthouse_network/Cargo.toml | 6 +++--- beacon_node/network/Cargo.toml | 2 +- beacon_node/operation_pool/Cargo.toml | 2 +- beacon_node/store/Cargo.toml | 2 +- common/deposit_contract/Cargo.toml | 2 +- common/eth2/Cargo.toml | 2 +- common/validator_dir/Cargo.toml | 2 +- consensus/cached_tree_hash/Cargo.toml | 6 +++--- consensus/fork_choice/Cargo.toml | 2 +- consensus/proto_array/Cargo.toml | 2 +- consensus/state_processing/Cargo.toml | 6 +++--- consensus/types/Cargo.toml | 6 +++--- crypto/bls/Cargo.toml | 2 +- crypto/kzg/Cargo.toml | 4 ++-- lcli/Cargo.toml | 2 +- slasher/Cargo.toml | 4 ++-- testing/ef_tests/Cargo.toml | 4 ++-- validator_client/Cargo.toml | 2 +- 25 files changed, 47 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 751efba0fac..05bb5e0e56a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2667,8 +2667,9 @@ dependencies = [ [[package]] name = "ethereum_ssz_derive" -version = "0.5.2" -source = "git+https://github.com/jimmygchen/ethereum_ssz?rev=231aa8c840262da694e024235dbc638a2980c545#231aa8c840262da694e024235dbc638a2980c545" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6085d7fd3cf84bd2b8fec150d54c8467fb491d8db9c460607c5534f653a0ee38" dependencies = [ "darling 0.13.4", "proc-macro2", @@ -7997,8 +7998,9 @@ dependencies = [ [[package]] name = "ssz_types" -version = "0.5.3" -source = "git+https://github.com/sigp/ssz_types?rev=63a80d04286c8561d5c211230a21bf1299d66059#63a80d04286c8561d5c211230a21bf1299d66059" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "382939886cb24ee8ac885d09116a60f6262d827c7a9e36012b4f6d3d0116d0b3" dependencies = [ "arbitrary", "derivative", @@ -8907,8 +8909,9 @@ dependencies = [ [[package]] name = "tree_hash" -version = "0.5.1" -source = "git+https://github.com/sigp/tree_hash?rev=a2471f3b240f407a0ec7436cff11f03e5ec8c706#a2471f3b240f407a0ec7436cff11f03e5ec8c706" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c998ac5fe2b07c025444bdd522e6258110b63861c6698eedc610c071980238d" dependencies = [ "ethereum-types 0.14.1", "ethereum_hashing", diff --git a/Cargo.toml b/Cargo.toml index bd2561351b3..cb7811ada77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,9 +91,6 @@ resolver = "2" [patch.crates-io] warp = { git = "https://github.com/macladson/warp", rev="7e75acc368229a46a236a8c991bf251fe7fe50ef" } arbitrary = { git = "https://github.com/michaelsproul/arbitrary", rev="f002b99989b561ddce62e4cf2887b0f8860ae991" } -tree_hash = { git = "https://github.com/sigp/tree_hash", rev="a2471f3b240f407a0ec7436cff11f03e5ec8c706" } -ssz_types = { git = "https://github.com/sigp/ssz_types", rev="63a80d04286c8561d5c211230a21bf1299d66059" } -ethereum_ssz_derive = { git = "https://github.com/jimmygchen/ethereum_ssz", rev="231aa8c840262da694e024235dbc638a2980c545"} [patch."https://github.com/ralexstokes/mev-rs"] mev-rs = { git = "https://github.com/ralexstokes//mev-rs", rev = "7813d4a4a564e0754e9aaab2d95520ba437c3889" } diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index e00b44301cb..77bb514cd8a 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -35,11 +35,11 @@ sloggers = { version = "2.1.1", features = ["json"] } slot_clock = { path = "../../common/slot_clock" } ethereum_hashing = "1.0.0-beta.2" ethereum_ssz = "0.5.0" -ssz_types = "0.5.3" -ethereum_ssz_derive = "0.5.0" +ssz_types = "0.5.4" +ethereum_ssz_derive = "0.5.3" state_processing = { path = "../../consensus/state_processing" } tree_hash_derive = "0.5.0" -tree_hash = "0.5.0" +tree_hash = "0.5.2" types = { path = "../../consensus/types" } tokio = "1.14.0" tokio-stream = "0.1.3" diff --git a/beacon_node/eth1/Cargo.toml b/beacon_node/eth1/Cargo.toml index cc982aee089..9ec144a401b 100644 --- a/beacon_node/eth1/Cargo.toml +++ b/beacon_node/eth1/Cargo.toml @@ -20,8 +20,8 @@ hex = "0.4.2" types = { path = "../../consensus/types"} merkle_proof = { path = "../../consensus/merkle_proof"} ethereum_ssz = "0.5.0" -ethereum_ssz_derive = "0.5.0" -tree_hash = "0.5.0" +ethereum_ssz_derive = "0.5.3" +tree_hash = "0.5.2" parking_lot = "0.12.0" slog = "2.5.2" superstruct = "0.5.0" diff --git a/beacon_node/execution_layer/Cargo.toml b/beacon_node/execution_layer/Cargo.toml index c27ed4c65c3..fd776b9abfc 100644 --- a/beacon_node/execution_layer/Cargo.toml +++ b/beacon_node/execution_layer/Cargo.toml @@ -23,14 +23,14 @@ bytes = "1.1.0" task_executor = { path = "../../common/task_executor" } hex = "0.4.2" ethereum_ssz = "0.5.0" -ssz_types = "0.5.3" +ssz_types = "0.5.4" eth2 = { path = "../../common/eth2" } kzg = { path = "../../crypto/kzg" } state_processing = { path = "../../consensus/state_processing" } superstruct = "0.6.0" lru = "0.7.1" exit-future = "0.2.0" -tree_hash = "0.5.0" +tree_hash = "0.5.2" tree_hash_derive = "0.5.0" parking_lot = "0.12.0" slot_clock = { path = "../../common/slot_clock" } diff --git a/beacon_node/genesis/Cargo.toml b/beacon_node/genesis/Cargo.toml index 8a7d224963e..8b465fa6005 100644 --- a/beacon_node/genesis/Cargo.toml +++ b/beacon_node/genesis/Cargo.toml @@ -18,7 +18,7 @@ state_processing = { path = "../../consensus/state_processing" } merkle_proof = { path = "../../consensus/merkle_proof" } ethereum_ssz = "0.5.0" ethereum_hashing = "1.0.0-beta.2" -tree_hash = "0.5.0" +tree_hash = "0.5.2" tokio = { version = "1.14.0", features = ["full"] } slog = "2.5.2" int_to_bytes = { path = "../../consensus/int_to_bytes" } diff --git a/beacon_node/http_api/Cargo.toml b/beacon_node/http_api/Cargo.toml index 2b117b26cef..bbf305717ea 100644 --- a/beacon_node/http_api/Cargo.toml +++ b/beacon_node/http_api/Cargo.toml @@ -32,7 +32,7 @@ parking_lot = "0.12.0" safe_arith = {path = "../../consensus/safe_arith"} task_executor = { path = "../../common/task_executor" } lru = "0.7.7" -tree_hash = "0.5.0" +tree_hash = "0.5.2" sysinfo = "0.26.5" system_health = { path = "../../common/system_health" } directory = { path = "../../common/directory" } diff --git a/beacon_node/lighthouse_network/Cargo.toml b/beacon_node/lighthouse_network/Cargo.toml index 6d056d83505..8a5ff330e8a 100644 --- a/beacon_node/lighthouse_network/Cargo.toml +++ b/beacon_node/lighthouse_network/Cargo.toml @@ -8,12 +8,12 @@ edition = "2021" discv5 = { version = "0.3.0", features = ["libp2p"]} unsigned-varint = { version = "0.6.0", features = ["codec"] } types = { path = "../../consensus/types" } -ssz_types = "0.5.3" +ssz_types = "0.5.4" serde = { version = "1.0.116", features = ["derive"] } serde_derive = "1.0.116" ethereum_ssz = "0.5.0" -ethereum_ssz_derive = "0.5.0" -tree_hash = "0.5.0" +ethereum_ssz_derive = "0.5.3" +tree_hash = "0.5.2" tree_hash_derive = "0.5.0" slog = { version = "2.5.2", features = ["max_level_trace"] } lighthouse_version = { path = "../../common/lighthouse_version" } diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index cb6d1116c26..3420e646843 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -22,7 +22,7 @@ slot_clock = { path = "../../common/slot_clock" } slog = { version = "2.5.2", features = ["max_level_trace", "nested-values"] } hex = "0.4.2" ethereum_ssz = "0.5.0" -ssz_types = "0.5.3" +ssz_types = "0.5.4" futures = "0.3.7" error-chain = "0.12.4" tokio = { version = "1.14.0", features = ["full"] } diff --git a/beacon_node/operation_pool/Cargo.toml b/beacon_node/operation_pool/Cargo.toml index fdbecb656f4..a4ebc7f5ce9 100644 --- a/beacon_node/operation_pool/Cargo.toml +++ b/beacon_node/operation_pool/Cargo.toml @@ -13,7 +13,7 @@ parking_lot = "0.12.0" types = { path = "../../consensus/types" } state_processing = { path = "../../consensus/state_processing" } ethereum_ssz = "0.5.0" -ethereum_ssz_derive = "0.5.0" +ethereum_ssz_derive = "0.5.3" rayon = "1.5.0" serde = "1.0.116" serde_derive = "1.0.116" diff --git a/beacon_node/store/Cargo.toml b/beacon_node/store/Cargo.toml index a952f1b2ffb..e39d46db67a 100644 --- a/beacon_node/store/Cargo.toml +++ b/beacon_node/store/Cargo.toml @@ -14,7 +14,7 @@ leveldb = { version = "0.8.6" } parking_lot = "0.12.0" itertools = "0.10.0" ethereum_ssz = "0.5.0" -ethereum_ssz_derive = "0.5.0" +ethereum_ssz_derive = "0.5.3" types = { path = "../../consensus/types" } state_processing = { path = "../../consensus/state_processing" } slog = "2.5.2" diff --git a/common/deposit_contract/Cargo.toml b/common/deposit_contract/Cargo.toml index aabc07fc524..e5d0025200d 100644 --- a/common/deposit_contract/Cargo.toml +++ b/common/deposit_contract/Cargo.toml @@ -15,5 +15,5 @@ hex = "0.4.2" [dependencies] types = { path = "../../consensus/types"} ethereum_ssz = "0.5.0" -tree_hash = "0.5.0" +tree_hash = "0.5.2" ethabi = "16.0.0" diff --git a/common/eth2/Cargo.toml b/common/eth2/Cargo.toml index d8e1a375fd5..d15c56d9b05 100644 --- a/common/eth2/Cargo.toml +++ b/common/eth2/Cargo.toml @@ -21,7 +21,7 @@ bytes = "1.0.1" account_utils = { path = "../../common/account_utils" } sensitive_url = { path = "../../common/sensitive_url" } ethereum_ssz = "0.5.0" -ethereum_ssz_derive = "0.5.0" +ethereum_ssz_derive = "0.5.3" futures-util = "0.3.8" futures = "0.3.8" store = { path = "../../beacon_node/store", optional = true } diff --git a/common/validator_dir/Cargo.toml b/common/validator_dir/Cargo.toml index 39a14e28377..3c545e04e88 100644 --- a/common/validator_dir/Cargo.toml +++ b/common/validator_dir/Cargo.toml @@ -16,7 +16,7 @@ filesystem = { path = "../filesystem" } types = { path = "../../consensus/types" } rand = "0.8.5" deposit_contract = { path = "../deposit_contract" } -tree_hash = "0.5.0" +tree_hash = "0.5.2" hex = "0.4.2" derivative = "2.1.1" lockfile = { path = "../lockfile" } diff --git a/consensus/cached_tree_hash/Cargo.toml b/consensus/cached_tree_hash/Cargo.toml index 0f43c8890f1..e6c75deae46 100644 --- a/consensus/cached_tree_hash/Cargo.toml +++ b/consensus/cached_tree_hash/Cargo.toml @@ -6,11 +6,11 @@ edition = "2021" [dependencies] ethereum-types = "0.14.1" -ssz_types = "0.5.3" +ssz_types = "0.5.4" ethereum_hashing = "1.0.0-beta.2" -ethereum_ssz_derive = "0.5.0" +ethereum_ssz_derive = "0.5.3" ethereum_ssz = "0.5.0" -tree_hash = "0.5.0" +tree_hash = "0.5.2" smallvec = "1.6.1" [dev-dependencies] diff --git a/consensus/fork_choice/Cargo.toml b/consensus/fork_choice/Cargo.toml index 3864d52d47c..bd0b9888e61 100644 --- a/consensus/fork_choice/Cargo.toml +++ b/consensus/fork_choice/Cargo.toml @@ -11,7 +11,7 @@ types = { path = "../types" } state_processing = { path = "../state_processing" } proto_array = { path = "../proto_array" } ethereum_ssz = "0.5.0" -ethereum_ssz_derive = "0.5.0" +ethereum_ssz_derive = "0.5.3" slog = { version = "2.5.2", features = ["max_level_trace", "release_max_level_trace"] } [dev-dependencies] diff --git a/consensus/proto_array/Cargo.toml b/consensus/proto_array/Cargo.toml index 81a535e34a1..f6e189d9cf9 100644 --- a/consensus/proto_array/Cargo.toml +++ b/consensus/proto_array/Cargo.toml @@ -11,7 +11,7 @@ path = "src/bin.rs" [dependencies] types = { path = "../types" } ethereum_ssz = "0.5.0" -ethereum_ssz_derive = "0.5.0" +ethereum_ssz_derive = "0.5.3" serde = "1.0.116" serde_derive = "1.0.116" serde_yaml = "0.8.13" diff --git a/consensus/state_processing/Cargo.toml b/consensus/state_processing/Cargo.toml index f19cd1d29df..765ba2307c2 100644 --- a/consensus/state_processing/Cargo.toml +++ b/consensus/state_processing/Cargo.toml @@ -14,11 +14,11 @@ bls = { path = "../../crypto/bls" } integer-sqrt = "0.1.5" itertools = "0.10.0" ethereum_ssz = "0.5.0" -ethereum_ssz_derive = "0.5.0" -ssz_types = "0.5.3" +ethereum_ssz_derive = "0.5.3" +ssz_types = "0.5.4" merkle_proof = { path = "../merkle_proof" } safe_arith = { path = "../safe_arith" } -tree_hash = "0.5.0" +tree_hash = "0.5.2" types = { path = "../types", default-features = false } rayon = "1.4.1" ethereum_hashing = "1.0.0-beta.2" diff --git a/consensus/types/Cargo.toml b/consensus/types/Cargo.toml index 8fdb21a06f2..8db70dc85a1 100644 --- a/consensus/types/Cargo.toml +++ b/consensus/types/Cargo.toml @@ -27,11 +27,11 @@ serde = {version = "1.0.116" , features = ["rc"] } serde_derive = "1.0.116" slog = "2.5.2" ethereum_ssz = { version = "0.5.0", features = ["arbitrary"] } -ethereum_ssz_derive = "0.5.0" -ssz_types = { version = "0.5.3", features = ["arbitrary"] } +ethereum_ssz_derive = "0.5.3" +ssz_types = { version = "0.5.4", features = ["arbitrary"] } swap_or_not_shuffle = { path = "../swap_or_not_shuffle", features = ["arbitrary"] } test_random_derive = { path = "../../common/test_random_derive" } -tree_hash = { version = "0.5.0", features = ["arbitrary"] } +tree_hash = { version = "0.5.2", features = ["arbitrary"] } tree_hash_derive = "0.5.0" rand_xorshift = "0.3.0" cached_tree_hash = { path = "../cached_tree_hash" } diff --git a/crypto/bls/Cargo.toml b/crypto/bls/Cargo.toml index a610f257cdb..dc59fd85efb 100644 --- a/crypto/bls/Cargo.toml +++ b/crypto/bls/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" [dependencies] ethereum_ssz = "0.5.0" -tree_hash = "0.5.0" +tree_hash = "0.5.2" milagro_bls = { git = "https://github.com/sigp/milagro_bls", tag = "v1.4.2", optional = true } rand = "0.7.3" serde = "1.0.116" diff --git a/crypto/kzg/Cargo.toml b/crypto/kzg/Cargo.toml index 31a2a665f48..87c4809a067 100644 --- a/crypto/kzg/Cargo.toml +++ b/crypto/kzg/Cargo.toml @@ -8,8 +8,8 @@ edition = "2021" [dependencies] ethereum_ssz = "0.5.0" -ethereum_ssz_derive = "0.5.0" -tree_hash = "0.5.0" +ethereum_ssz_derive = "0.5.3" +tree_hash = "0.5.2" derivative = "2.1.1" serde = "1.0.116" serde_derive = "1.0.116" diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index 077d81eafb8..185da02d0ec 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -28,7 +28,7 @@ environment = { path = "../lighthouse/environment" } eth2_network_config = { path = "../common/eth2_network_config" } genesis = { path = "../beacon_node/genesis" } deposit_contract = { path = "../common/deposit_contract" } -tree_hash = "0.5.0" +tree_hash = "0.5.2" clap_utils = { path = "../common/clap_utils" } lighthouse_network = { path = "../beacon_node/lighthouse_network" } validator_dir = { path = "../common/validator_dir", features = ["insecure_keys"] } diff --git a/slasher/Cargo.toml b/slasher/Cargo.toml index bfa7b5f64c5..230a758d868 100644 --- a/slasher/Cargo.toml +++ b/slasher/Cargo.toml @@ -13,7 +13,7 @@ lmdb = ["lmdb-rkv", "lmdb-rkv-sys"] bincode = "1.3.1" byteorder = "1.3.4" ethereum_ssz = "0.5.0" -ethereum_ssz_derive = "0.5.0" +ethereum_ssz_derive = "0.5.3" flate2 = { version = "1.0.14", features = ["zlib"], default-features = false } lazy_static = "1.4.0" lighthouse_metrics = { path = "../common/lighthouse_metrics" } @@ -26,7 +26,7 @@ serde = "1.0" serde_derive = "1.0" slog = "2.5.2" sloggers = { version = "2.1.1", features = ["json"] } -tree_hash = "0.5.0" +tree_hash = "0.5.2" tree_hash_derive = "0.5.0" types = { path = "../consensus/types" } strum = { version = "0.24.1", features = ["derive"] } diff --git a/testing/ef_tests/Cargo.toml b/testing/ef_tests/Cargo.toml index c10ea7a47f5..9a1815a7025 100644 --- a/testing/ef_tests/Cargo.toml +++ b/testing/ef_tests/Cargo.toml @@ -27,8 +27,8 @@ serde_yaml = "0.8.13" eth2_network_config = { path = "../../common/eth2_network_config" } ethereum_serde_utils = "0.5.0" ethereum_ssz = "0.5.0" -ethereum_ssz_derive = "0.5.0" -tree_hash = "0.5.0" +ethereum_ssz_derive = "0.5.3" +tree_hash = "0.5.2" tree_hash_derive = "0.5.0" cached_tree_hash = { path = "../../consensus/cached_tree_hash" } state_processing = { path = "../../consensus/state_processing" } diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index 494ebcb3dfc..6ccd0f076a5 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -13,7 +13,7 @@ tokio = { version = "1.14.0", features = ["time", "rt-multi-thread", "macros"] } logging = { path = "../common/logging" } [dependencies] -tree_hash = "0.5.0" +tree_hash = "0.5.2" clap = "2.33.3" slashing_protection = { path = "./slashing_protection" } slot_clock = { path = "../common/slot_clock" } From c3ef84bda29d9d6137685ac2640bbed629e6a0c6 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 5 Jul 2023 16:54:08 -0400 Subject: [PATCH 12/51] remove uninlined arg lint suppression (#4469) --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index fc93a1bb8ae..ad125f32442 100644 --- a/Makefile +++ b/Makefile @@ -180,7 +180,6 @@ lint: cargo clippy --workspace --tests $(EXTRA_CLIPPY_OPTS) -- \ -D clippy::fn_to_numeric_cast_any \ -D warnings \ - -A clippy::uninlined-format-args \ -A clippy::derive_partial_eq_without_eq \ -A clippy::from-over-into \ -A clippy::upper-case-acronyms \ From a6d5c7d7e0b5fc371445211f3ed3d911268a867a Mon Sep 17 00:00:00 2001 From: Jack McPherson Date: Thu, 6 Jul 2023 07:35:31 +0000 Subject: [PATCH 13/51] Correct checks for backfill completeness (#4465) ## Issue Addressed #4331 ## Proposed Changes - Use comparison rather than strict equality between the earliest epoch we know about and the backfill target (which will be the most recent WSP by default or genesis) - Add helper function `BackFillSync::would_complete` to achieve this in one location ## Additional Info - There's an ad hoc test for this in #4461 Co-authored-by: Age Manning --- .../network/src/sync/backfill_sync/mod.rs | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/beacon_node/network/src/sync/backfill_sync/mod.rs b/beacon_node/network/src/sync/backfill_sync/mod.rs index 460c8b1ee92..9f676ba017d 100644 --- a/beacon_node/network/src/sync/backfill_sync/mod.rs +++ b/beacon_node/network/src/sync/backfill_sync/mod.rs @@ -1098,12 +1098,7 @@ impl BackFillSync { match self.batches.entry(batch_id) { Entry::Occupied(_) => { // this batch doesn't need downloading, let this same function decide the next batch - if batch_id - == self - .beacon_chain - .genesis_backfill_slot - .epoch(T::EthSpec::slots_per_epoch()) - { + if self.would_complete(batch_id) { self.last_batch_downloaded = true; } @@ -1114,12 +1109,7 @@ impl BackFillSync { } Entry::Vacant(entry) => { entry.insert(BatchInfo::new(&batch_id, BACKFILL_EPOCHS_PER_BATCH)); - if batch_id - == self - .beacon_chain - .genesis_backfill_slot - .epoch(T::EthSpec::slots_per_epoch()) - { + if self.would_complete(batch_id) { self.last_batch_downloaded = true; } self.to_be_downloaded = self @@ -1151,14 +1141,8 @@ impl BackFillSync { /// Checks with the beacon chain if backfill sync has completed. fn check_completed(&mut self) -> bool { - if self.current_start - == self - .beacon_chain - .genesis_backfill_slot - .epoch(T::EthSpec::slots_per_epoch()) - { + if self.would_complete(self.current_start) { // Check that the beacon chain agrees - if let Some(anchor_info) = self.beacon_chain.store.get_anchor_info() { // Conditions that we have completed a backfill sync if anchor_info.block_backfill_complete(self.beacon_chain.genesis_backfill_slot) { @@ -1171,6 +1155,15 @@ impl BackFillSync { false } + /// Checks if backfill would complete by syncing to `start_epoch`. + fn would_complete(&self, start_epoch: Epoch) -> bool { + start_epoch + <= self + .beacon_chain + .genesis_backfill_slot + .epoch(T::EthSpec::slots_per_epoch()) + } + /// Updates the global network state indicating the current state of a backfill sync. fn set_state(&self, state: BackFillState) { *self.network_globals.backfill_state.write() = state; From 246d52d209ef1ddf5659450e37b4462d1c0555d7 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 6 Jul 2023 11:40:58 -0400 Subject: [PATCH 14/51] remove into gossip verified block --- .../beacon_chain/src/block_verification.rs | 34 ----------------- beacon_node/beacon_chain/src/lib.rs | 2 +- beacon_node/http_api/src/publish_blocks.rs | 37 +++++++++---------- 3 files changed, 19 insertions(+), 54 deletions(-) diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 492f492521e..206822de8cc 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -632,40 +632,6 @@ pub struct ExecutionPendingBlock { pub payload_verification_handle: PayloadVerificationHandle, } -pub trait IntoGossipVerifiedBlock: Sized { - fn into_gossip_verified_block( - self, - chain: &BeaconChain, - ) -> Result, BlockError>; - fn inner(&self) -> Arc>; -} - -impl IntoGossipVerifiedBlock for GossipVerifiedBlock { - fn into_gossip_verified_block( - self, - _chain: &BeaconChain, - ) -> Result, BlockError> { - Ok(self) - } - - fn inner(&self) -> Arc> { - self.block.clone() - } -} - -impl IntoGossipVerifiedBlock for Arc> { - fn into_gossip_verified_block( - self, - chain: &BeaconChain, - ) -> Result, BlockError> { - GossipVerifiedBlock::new(self, chain) - } - - fn inner(&self) -> Arc> { - self.clone() - } -} - /// Implemented on types that can be converted into a `ExecutionPendingBlock`. /// /// Used to allow functions to accept blocks at various stages of verification. diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index c5cf74e179c..d672c168288 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -64,7 +64,7 @@ pub use attestation_verification::Error as AttestationError; pub use beacon_fork_choice_store::{BeaconForkChoiceStore, Error as ForkChoiceStoreError}; pub use block_verification::{ get_block_root, BlockError, ExecutionPayloadError, GossipVerifiedBlock, - IntoExecutionPendingBlock, IntoGossipVerifiedBlock, + IntoExecutionPendingBlock, }; pub use canonical_head::{CachedHead, CanonicalHead, CanonicalHeadRwLock}; pub use eth1_chain::{Eth1Chain, Eth1ChainBackend}; diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 0f2f7b361c7..affca5f287f 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -1,7 +1,7 @@ use crate::metrics; use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now}; use beacon_chain::{ - BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, IntoGossipVerifiedBlock, + BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, GossipVerifiedBlock, NotifyExecutionLayer, }; use eth2::types::BroadcastValidation; @@ -10,7 +10,6 @@ use lighthouse_network::PubsubMessage; use network::NetworkMessage; use slog::{debug, error, info, warn, Logger}; use slot_clock::SlotClock; -use std::marker::PhantomData; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; @@ -21,28 +20,28 @@ use types::{ }; use warp::Rejection; -pub enum ProvenancedBlock> { +pub enum ProvenancedBlock { /// The payload was built using a local EE. - Local(B, PhantomData), + Local(Arc>), /// The payload was build using a remote builder (e.g., via a mev-boost /// compatible relay). - Builder(B, PhantomData), + Builder(Arc>), } -impl> ProvenancedBlock { - pub fn local(block: B) -> Self { - Self::Local(block, PhantomData) +impl ProvenancedBlock { + pub fn local(block: Arc>) -> Self { + Self::Local(block) } - pub fn builder(block: B) -> Self { - Self::Builder(block, PhantomData) + pub fn builder(block: Arc>) -> Self { + Self::Builder(block) } } /// Handles a request from the HTTP API for full blocks. -pub async fn publish_block>( +pub async fn publish_block( block_root: Option, - provenanced_block: ProvenancedBlock, + provenanced_block: ProvenancedBlock, chain: Arc>, network_tx: &UnboundedSender>, log: Logger, @@ -50,10 +49,10 @@ pub async fn publish_block>( ) -> Result<(), Rejection> { let seen_timestamp = timestamp_now(); let (block, is_locally_built_block) = match provenanced_block { - ProvenancedBlock::Local(block, _) => (block, true), - ProvenancedBlock::Builder(block, _) => (block, false), + ProvenancedBlock::Local(block) => (block, true), + ProvenancedBlock::Builder(block) => (block, false), }; - let beacon_block = block.inner(); + let beacon_block = block.clone(); let delay = get_block_delay_ms(seen_timestamp, beacon_block.message(), &chain.slot_clock); debug!(log, "Signed block received in HTTP API"; "slot" => beacon_block.slot()); @@ -75,7 +74,7 @@ pub async fn publish_block>( }; /* if we can form a `GossipVerifiedBlock`, we've passed our basic gossip checks */ - let gossip_verified_block = block.into_gossip_verified_block(&chain).map_err(|e| { + let gossip_verified_block = GossipVerifiedBlock::new(block, &chain).map_err(|e| { warn!(log, "Not publishing block, not gossip verified"; "slot" => beacon_block.slot(), "error" => ?e); warp_utils::reject::custom_bad_request(e.to_string()) })?; @@ -210,9 +209,9 @@ pub async fn publish_blinded_block( validation_level: BroadcastValidation, ) -> Result<(), Rejection> { let block_root = block.canonical_root(); - let full_block: ProvenancedBlock>> = + let full_block: ProvenancedBlock = reconstruct_block(chain.clone(), block_root, block, log.clone()).await?; - publish_block::( + publish_block::( Some(block_root), full_block, chain, @@ -231,7 +230,7 @@ pub async fn reconstruct_block( block_root: Hash256, block: SignedBeaconBlock>, log: Logger, -) -> Result>>, Rejection> { +) -> Result, Rejection> { let full_payload_opt = if let Ok(payload_header) = block.message().body().execution_payload() { let el = chain.execution_layer.as_ref().ok_or_else(|| { warp_utils::reject::custom_server_error("Missing execution layer".to_string()) From c4da1ba450257e3ac5c2f7a748b9ecd25115c3f9 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 7 Jul 2023 10:17:04 -0400 Subject: [PATCH 15/51] resolve merge issues --- beacon_node/beacon_chain/src/beacon_chain.rs | 4 +- beacon_node/beacon_chain/src/test_utils.rs | 8 +- beacon_node/beacon_chain/tests/tests.rs | 2 +- beacon_node/builder_client/src/lib.rs | 4 +- beacon_node/execution_layer/src/lib.rs | 9 +- beacon_node/http_api/src/lib.rs | 13 +- beacon_node/http_api/src/publish_blocks.rs | 182 ++++++++++-------- beacon_node/http_api/tests/tests.rs | 6 +- beacon_node/network/src/metrics.rs | 4 + consensus/fork_choice/src/fork_choice.rs | 3 +- .../state_processing/src/upgrade/deneb.rs | 1 + consensus/types/src/beacon_state.rs | 2 +- .../progressive_balances_cache.rs | 5 +- testing/ef_tests/src/cases/operations.rs | 7 +- 14 files changed, 142 insertions(+), 108 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index f08915eaf6a..13fd7e9108e 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2784,7 +2784,7 @@ impl BeaconChain { block_root: Hash256, unverified_block: B, notify_execution_layer: NotifyExecutionLayer, - publish_fn: impl FnOnce() -> Result<(), BlockError> + Send + 'static, + publish_fn: impl FnOnce() -> Result<(), BlockError> + Send + 'static, ) -> Result> { // Start the Prometheus timer. let _full_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_TIMES); @@ -2801,7 +2801,7 @@ impl BeaconChain { )?; //TODO(sean) error handling? - publish_fn()?; + publish_fn()?; let executed_block = self .clone() diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index c34cab1e7ab..125ff471d4b 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -812,9 +812,9 @@ where &self, state: BeaconState, slot: Slot, - ) -> (SignedBlindedBeaconBlock, BeaconState) { + ) -> (BlockContentsTuple>, BeaconState) { let (unblinded, new_state) = self.make_block(state, slot).await; - (unblinded.into(), new_state) + ((unblinded.0.into(), unblinded.1), new_state) } /// Returns a newly created block, signed by the proposer for the given slot. @@ -1839,7 +1839,9 @@ where self.set_current_slot(slot); let block_hash: SignedBeaconBlockHash = self .chain - .process_block(block_root, block.into(), NotifyExecutionLayer::Yes,|| Ok(()),) + .process_block(block_root, block.into(), NotifyExecutionLayer::Yes, || { + Ok(()) + }) .await? .try_into() .unwrap(); diff --git a/beacon_node/beacon_chain/tests/tests.rs b/beacon_node/beacon_chain/tests/tests.rs index 80e4992ead8..660bf41dc59 100644 --- a/beacon_node/beacon_chain/tests/tests.rs +++ b/beacon_node/beacon_chain/tests/tests.rs @@ -686,7 +686,7 @@ async fn run_skip_slot_test(skip_slots: u64) { harness_a.chain.head_snapshot().beacon_block_root, harness_a.get_head_block(), NotifyExecutionLayer::Yes, - || Ok(()) + || Ok(()), ) .await .unwrap(); diff --git a/beacon_node/builder_client/src/lib.rs b/beacon_node/builder_client/src/lib.rs index c78f686d02b..9a842340195 100644 --- a/beacon_node/builder_client/src/lib.rs +++ b/beacon_node/builder_client/src/lib.rs @@ -1,7 +1,7 @@ use eth2::types::builder_bid::SignedBuilderBid; use eth2::types::{ AbstractExecPayload, BlindedPayload, EthSpec, ExecutionBlockHash, ExecutionPayload, - ForkVersionedResponse, PublicKeyBytes, SignedBeaconBlock, SignedValidatorRegistrationData, + ForkVersionedResponse, PublicKeyBytes, SignedBlockContents, SignedValidatorRegistrationData, Slot, }; pub use eth2::Error; @@ -140,7 +140,7 @@ impl BuilderHttpClient { /// `POST /eth/v1/builder/blinded_blocks` pub async fn post_builder_blinded_blocks( &self, - blinded_block: &SignedBeaconBlock>, + blinded_block: &SignedBlockContents>, ) -> Result>, Error> { let mut path = self.server.full.clone(); diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 75c3f985dd5..7de53214122 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -13,6 +13,7 @@ pub use engine_api::*; pub use engine_api::{http, http::deposit_methods, http::HttpJsonRpc}; use engines::{Engine, EngineError}; pub use engines::{EngineState, ForkchoiceState}; +use eth2::types::SignedBlockContents; use eth2::types::{builder_bid::SignedBuilderBid, ForkVersionedResponse}; use ethers_core::abi::ethereum_types::FromStrRadixErr; use ethers_core::types::Transaction as EthersTransaction; @@ -47,10 +48,7 @@ use types::{ ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, ForkName, }; use types::{KzgProofs, Withdrawals}; -use types::{ - ProposerPreparationData, PublicKeyBytes, Signature, SignedBeaconBlock, Slot, Transaction, - Uint256, -}; +use types::{ProposerPreparationData, PublicKeyBytes, Signature, Slot, Transaction, Uint256}; mod block_hash; mod engine_api; @@ -1876,7 +1874,7 @@ impl ExecutionLayer { pub async fn propose_blinded_beacon_block( &self, block_root: Hash256, - block: &SignedBeaconBlock>, + block: &SignedBlockContents>, ) -> Result, Error> { debug!( self.log(), @@ -1924,6 +1922,7 @@ impl ExecutionLayer { "relay_response_ms" => duration.as_millis(), "block_root" => ?block_root, "parent_hash" => ?block + .signed_block() .message() .execution_payload() .map(|payload| format!("{}", payload.parent_hash())) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index c0d8a961535..6707719ae23 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -32,8 +32,8 @@ use beacon_chain::{ pub use block_id::BlockId; use directory::DEFAULT_ROOT_DIR; use eth2::types::{ - self as api_types, BroadcastValidation, EndpointVersion, ForkChoice, ForkChoiceNode, SignedBlockContents, - SkipRandaoVerification, ValidatorId, ValidatorStatus, + self as api_types, BroadcastValidation, EndpointVersion, ForkChoice, ForkChoiceNode, + SignedBlockContents, SkipRandaoVerification, ValidatorId, ValidatorStatus, }; use lighthouse_network::{types::SyncState, EnrExt, NetworkGlobals, PeerId, PubsubMessage}; use lighthouse_version::version_with_platform; @@ -63,9 +63,8 @@ use types::{ Attestation, AttestationData, AttestationShufflingId, AttesterSlashing, BeaconStateError, BlindedPayload, CommitteeCache, ConfigAndPreset, Epoch, EthSpec, ForkName, FullPayload, ProposerPreparationData, ProposerSlashing, RelativeEpoch, SignedAggregateAndProof, - SignedBeaconBlock, SignedBlsToExecutionChange, SignedContributionAndProof, - SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, SyncCommitteeMessage, - SyncContributionData, + SignedBlsToExecutionChange, SignedContributionAndProof, SignedValidatorRegistrationData, + SignedVoluntaryExit, Slot, SyncCommitteeMessage, SyncContributionData, }; use version::{ add_consensus_version_header, execution_optimistic_finalized_fork_versioned_response, @@ -1248,7 +1247,7 @@ pub fn serve( .and(log_filter.clone()) .then( |validation_level: api_types::BroadcastValidationQuery, - block_contents: SignedBlockContents, + block_contents: SignedBlockContents, chain: Arc>, network_tx: UnboundedSender>, log: Logger| async move { @@ -1289,7 +1288,7 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .and_then( - |block: SignedBeaconBlock>, + |block: SignedBlockContents>, chain: Arc>, network_tx: UnboundedSender>, log: Logger| async move { diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 3a4875113f4..0ddd72a7263 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -1,13 +1,13 @@ use crate::metrics; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; +use beacon_chain::blob_verification::BlockWrapper; use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now}; use beacon_chain::{ - BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, GossipVerifiedBlock, - NotifyExecutionLayer, AvailabilityProcessingStatus + AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, + GossipVerifiedBlock, NotifyExecutionLayer, }; -use eth2::types::SignedBlockContents; use eth2::types::BroadcastValidation; +use eth2::types::SignedBlockContents; use execution_layer::ProvenancedPayload; use lighthouse_network::PubsubMessage; use network::NetworkMessage; @@ -15,12 +15,11 @@ use slog::{debug, error, info, warn, Logger}; use slot_clock::SlotClock; use std::sync::Arc; use std::time::Duration; -use store::FixedVector; use tokio::sync::mpsc::UnboundedSender; use tree_hash::TreeHash; use types::{ AbstractExecPayload, BeaconBlockRef, BlindedPayload, EthSpec, ExecPayload, ExecutionBlockHash, - FullPayload, Hash256, SignedBeaconBlock, + FullPayload, Hash256, SignedBeaconBlock, SignedBlobSidecarList, VariableList, }; use warp::Rejection; @@ -33,11 +32,11 @@ pub enum ProvenancedBlock { } impl ProvenancedBlock { - pub fn local(block: Arc>) -> Self { + pub fn local(block: SignedBlockContents) -> Self { Self::Local(block) } - pub fn builder(block: Arc>) -> Self { + pub fn builder(block: SignedBlockContents) -> Self { Self::Builder(block) } } @@ -52,7 +51,8 @@ pub async fn publish_block( validation_level: BroadcastValidation, ) -> Result<(), Rejection> { let seen_timestamp = timestamp_now(); - let (block, maybe_blobs, is_locally_built_block) = match provenanced_block { + + let (block, blobs_opt, is_locally_built_block) = match provenanced_block { ProvenancedBlock::Local(block_contents) => { let (block, maybe_blobs) = block_contents.deconstruct(); (Arc::new(block), maybe_blobs, true) @@ -62,15 +62,12 @@ pub async fn publish_block( (Arc::new(block), maybe_blobs, false) } }; - let beacon_block = block.clone(); - let delay = get_block_delay_ms(seen_timestamp, beacon_block.message(), &chain.slot_clock); - //FIXME(sean) have to move this to prior to publishing because it's included in the blobs sidecar message. - //this may skew metrics - let block_root = block_root.unwrap_or_else(|| block.canonical_root()); - debug!(log, "Signed block received in HTTP API"; "slot" => beacon_block.slot()); + let delay = get_block_delay_ms(seen_timestamp, block.message(), &chain.slot_clock); + debug!(log, "Signed block received in HTTP API"; "slot" => block.slot()); /* actually publish a block */ let publish_block = move |block: Arc>, + blobs_opt: Option>, sender, log, seen_timestamp| { @@ -80,43 +77,59 @@ pub async fn publish_block( .unwrap_or_else(|| Duration::from_secs(0)); info!(log, "Signed block published to network via HTTP API"; "slot" => block.slot(), "publish_delay" => ?publish_delay); - // Send the block, regardless of whether or not it is valid. The API - // specification is very clear that this is the desired behaviour. - let wrapped_block: BlockWrapper = match block.as_ref() { - SignedBeaconBlock::Base(_) - | SignedBeaconBlock::Altair(_) - | SignedBeaconBlock::Merge(_) - | SignedBeaconBlock::Capella(_) => { - crate::publish_pubsub_message(network_tx, PubsubMessage::BeaconBlock(block.clone()))?; - block.into() - } - SignedBeaconBlock::Deneb(_) => { - crate::publish_pubsub_message(network_tx, PubsubMessage::BeaconBlock(block.clone()))?; - if let Some(signed_blobs) = maybe_blobs { - for (blob_index, blob) in signed_blobs.clone().into_iter().enumerate() { - crate::publish_pubsub_message( - network_tx, - PubsubMessage::BlobSidecar(Box::new((blob_index as u64, blob))), - )?; + // Send the block, regardless of whether or not it is valid. The API + // specification is very clear that this is the desired behaviour. + match block.as_ref() { + SignedBeaconBlock::Base(_) + | SignedBeaconBlock::Altair(_) + | SignedBeaconBlock::Merge(_) + | SignedBeaconBlock::Capella(_) => { + crate::publish_pubsub_message(&sender, PubsubMessage::BeaconBlock(block.clone())) + .map_err(|_| BlockError::BeaconChainError(BeaconChainError::UnableToPublish))?; + } + SignedBeaconBlock::Deneb(_) => { + crate::publish_pubsub_message(&sender, PubsubMessage::BeaconBlock(block.clone())) + .map_err(|_| BlockError::BeaconChainError(BeaconChainError::UnableToPublish))?; + if let Some(signed_blobs) = blobs_opt { + for (blob_index, blob) in signed_blobs.into_iter().enumerate() { + crate::publish_pubsub_message( + &sender, + PubsubMessage::BlobSidecar(Box::new((blob_index as u64, blob))), + ) + .map_err(|_| { + BlockError::BeaconChainError(BeaconChainError::UnableToPublish) + })?; + } } - let blobs = signed_blobs - .into_iter() - .map(|blob| Some(blob.message)) - .collect::>(); - BlockWrapper::BlockAndBlobs(block, FixedVector::from(blobs)) - } else { - block.into() } - } - }; - let message = PubsubMessage::BeaconBlock(block); - crate::publish_pubsub_message(&sender, message) - .map_err(|_| BeaconChainError::UnableToPublish.into()) + }; + Ok(()) }; + let mapped_blobs = blobs_opt.clone().map(|blobs| { + VariableList::from( + blobs + .into_iter() + .map(|blob| blob.message) + .collect::>(), + ) + }); + + /* only publish if gossip- and consensus-valid and equivocation-free */ + let chain_clone = chain.clone(); + let slot = block.message().slot(); + let block_clone = block.clone(); + let proposer_index = block.message().proposer_index(); + let sender_clone = network_tx.clone(); + let log_clone = log.clone(); + /* if we can form a `GossipVerifiedBlock`, we've passed our basic gossip checks */ - let gossip_verified_block = GossipVerifiedBlock::new(block, &chain).map_err(|e| { - warn!(log, "Not publishing block, not gossip verified"; "slot" => beacon_block.slot(), "error" => ?e); + let gossip_verified_block = GossipVerifiedBlock::new( + BlockWrapper::new(block.clone(), mapped_blobs), + &chain, + ) + .map_err(|e| { + warn!(log, "Not publishing block, not gossip verified"; "slot" => slot, "error" => ?e); warp_utils::reject::custom_bad_request(e.to_string()) })?; @@ -124,25 +137,24 @@ pub async fn publish_block( if let BroadcastValidation::Gossip = validation_level { publish_block( - beacon_block.clone(), - network_tx.clone(), + block.clone(), + blobs_opt.clone(), + sender_clone.clone(), log.clone(), seen_timestamp, ) .map_err(|_| warp_utils::reject::custom_server_error("unable to publish".into()))?; } - /* only publish if gossip- and consensus-valid and equivocation-free */ - let chain_clone = chain.clone(); - let block_clone = beacon_block.clone(); - let log_clone = log.clone(); - let sender_clone = network_tx.clone(); - let publish_fn = move || match validation_level { BroadcastValidation::Gossip => Ok(()), - BroadcastValidation::Consensus => { - publish_block(block_clone, sender_clone, log_clone, seen_timestamp) - } + BroadcastValidation::Consensus => publish_block( + block_clone, + blobs_opt, + sender_clone, + log_clone, + seen_timestamp, + ), BroadcastValidation::ConsensusAndEquivocation => { if chain_clone .observed_block_producers @@ -158,13 +170,16 @@ pub async fn publish_block( ); Err(BlockError::Slashable) } else { - publish_block(block_clone, sender_clone, log_clone, seen_timestamp) + publish_block( + block_clone, + blobs_opt, + sender_clone, + log_clone, + seen_timestamp, + ) } } }; - let block_clone = wrapped_block.block_cloned(); - let slot = block_clone.message().slot(); - let proposer_index = block_clone.message().proposer_index(); match chain .process_block( @@ -188,7 +203,7 @@ pub async fn publish_block( // Notify the validator monitor. chain.validator_monitor.read().register_api_block( seen_timestamp, - block_clone.message(), + block.message(), root, &chain.slot_clock, ); @@ -201,14 +216,7 @@ pub async fn publish_block( // blocks built with builders we consider the broadcast time to be // when the blinded block is published to the builder. if is_locally_built_block { - late_block_logging( - &chain, - seen_timestamp, - block_clone.message(), - root, - "local", - &log, - ) + late_block_logging(&chain, seen_timestamp, block.message(), root, "local", &log) } Ok(()) @@ -255,15 +263,14 @@ pub async fn publish_block( /// Handles a request from the HTTP API for blinded blocks. This converts blinded blocks into full /// blocks before publishing. pub async fn publish_blinded_block( - block: SignedBeaconBlock>, + block: SignedBlockContents>, chain: Arc>, network_tx: &UnboundedSender>, log: Logger, validation_level: BroadcastValidation, ) -> Result<(), Rejection> { - let block_root = block.canonical_root(); - let full_block: ProvenancedBlock = - reconstruct_block(chain.clone(), block_root, block, log.clone()).await?; + let block_root = block.signed_block().canonical_root(); + let full_block = reconstruct_block(chain.clone(), block_root, block, log.clone()).await?; publish_block::( Some(block_root), full_block, @@ -281,10 +288,12 @@ pub async fn publish_blinded_block( pub async fn reconstruct_block( chain: Arc>, block_root: Hash256, - block: SignedBeaconBlock>, + block: SignedBlockContents>, log: Logger, -) -> Result, Rejection> { - let full_payload_opt = if let Ok(payload_header) = block.message().body().execution_payload() { +) -> Result, Rejection> { + let full_payload_opt = if let Ok(payload_header) = + block.signed_block().message().body().execution_payload() + { let el = chain.execution_layer.as_ref().ok_or_else(|| { warp_utils::reject::custom_server_error("Missing execution layer".to_string()) })?; @@ -292,9 +301,12 @@ pub async fn reconstruct_block( // If the execution block hash is zero, use an empty payload. let full_payload = if payload_header.block_hash() == ExecutionBlockHash::zero() { let payload = FullPayload::default_at_fork( - chain - .spec - .fork_name_at_epoch(block.slot().epoch(T::EthSpec::slots_per_epoch())), + chain.spec.fork_name_at_epoch( + block + .signed_block() + .slot() + .epoch(T::EthSpec::slots_per_epoch()), + ), ) .map_err(|e| { warp_utils::reject::custom_server_error(format!( @@ -319,7 +331,7 @@ pub async fn reconstruct_block( late_block_logging( &chain, timestamp_now(), - block.message(), + block.signed_block().message(), block_root, "builder", &log, @@ -347,14 +359,20 @@ pub async fn reconstruct_block( // A block without a payload is pre-merge and we consider it locally // built. None => block + .deconstruct() + .0 .try_into_full_block(None) .map(SignedBlockContents::Block) .map(ProvenancedBlock::local), Some(ProvenancedPayload::Local(full_payload)) => block + .deconstruct() + .0 .try_into_full_block(Some(full_payload)) .map(SignedBlockContents::Block) .map(ProvenancedBlock::local), Some(ProvenancedPayload::Builder(full_payload)) => block + .deconstruct() + .0 .try_into_full_block(Some(full_payload)) .map(SignedBlockContents::Block) .map(ProvenancedBlock::builder), diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 9d35c710adf..2bcb1313395 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -1264,7 +1264,11 @@ impl ApiTester { .await .0; - assert!(self.client.post_beacon_blocks(&SignedBlockContents::from(block)).await.is_err()); + assert!(self + .client + .post_beacon_blocks(&SignedBlockContents::from(block)) + .await + .is_err()); assert!( self.network_rx.network_recv.recv().await.is_some(), diff --git a/beacon_node/network/src/metrics.rs b/beacon_node/network/src/metrics.rs index 62e59d14e79..28f80a1dcaf 100644 --- a/beacon_node/network/src/metrics.rs +++ b/beacon_node/network/src/metrics.rs @@ -171,6 +171,10 @@ lazy_static! { "beacon_processor_bls_to_execution_change_imported_total", "Total number of address changes imported to the op pool." ); +} + +// Need to split up this `lazy_static!` due to recursion limits. +lazy_static! { // Rpc blocks. pub static ref BEACON_PROCESSOR_RPC_BLOCK_QUEUE_TOTAL: Result = try_create_int_gauge( "beacon_processor_rpc_block_queue_total", diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index 8ed312923d2..536100a02ea 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -770,7 +770,8 @@ where (parent_justified, parent_finalized) } else { let justification_and_finalization_state = match block { - BeaconBlockRef::Deneb(_)| BeaconBlockRef::Capella(_) + BeaconBlockRef::Deneb(_) + | BeaconBlockRef::Capella(_) | BeaconBlockRef::Merge(_) | BeaconBlockRef::Altair(_) => match progressive_balances_mode { ProgressiveBalancesMode::Disabled => { diff --git a/consensus/state_processing/src/upgrade/deneb.rs b/consensus/state_processing/src/upgrade/deneb.rs index 76dbcd068a1..c253a8c1627 100644 --- a/consensus/state_processing/src/upgrade/deneb.rs +++ b/consensus/state_processing/src/upgrade/deneb.rs @@ -63,6 +63,7 @@ pub fn upgrade_to_deneb( historical_summaries: pre.historical_summaries.clone(), // Caches total_active_balance: pre.total_active_balance, + progressive_balances_cache: mem::take(&mut pre.progressive_balances_cache), committee_caches: mem::take(&mut pre.committee_caches), pubkey_cache: mem::take(&mut pre.pubkey_cache), exit_cache: mem::take(&mut pre.exit_cache), diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index b09f2aa500a..01d2f40e69f 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -1207,7 +1207,7 @@ impl BeaconState { &mut state.balances, &mut state.progressive_balances_cache, ), - BeaconState::Deneb(state) => ( + BeaconState::Deneb(state) => ( &mut state.validators, &mut state.balances, &mut state.progressive_balances_cache, diff --git a/consensus/types/src/beacon_state/progressive_balances_cache.rs b/consensus/types/src/beacon_state/progressive_balances_cache.rs index 9f5c223d573..3e776965cb8 100644 --- a/consensus/types/src/beacon_state/progressive_balances_cache.rs +++ b/consensus/types/src/beacon_state/progressive_balances_cache.rs @@ -179,6 +179,9 @@ impl ProgressiveBalancesMode { pub fn is_progressive_balances_enabled(state: &BeaconState) -> bool { match state { BeaconState::Base(_) => false, - BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) => true, + BeaconState::Altair(_) + | BeaconState::Merge(_) + | BeaconState::Capella(_) + | BeaconState::Deneb(_) => true, } } diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index 59aa51228f0..e823f6273ca 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -4,8 +4,8 @@ use crate::case_result::compare_beacon_state_results_without_caches; use crate::decode::{ssz_decode_file, ssz_decode_file_with, ssz_decode_state, yaml_decode_file}; use crate::testing_spec; use serde_derive::Deserialize; -use state_processing::common::update_progressive_balances_cache::initialize_progressive_balances_cache; use ssz::Decode; +use state_processing::common::update_progressive_balances_cache::initialize_progressive_balances_cache; use state_processing::{ per_block_processing::{ errors::BlockProcessingError, @@ -98,7 +98,10 @@ impl Operation for Attestation { &mut ctxt, spec, ), - BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_)| BeaconState::Deneb(_) => { + BeaconState::Altair(_) + | BeaconState::Merge(_) + | BeaconState::Capella(_) + | BeaconState::Deneb(_) => { initialize_progressive_balances_cache(state, None, spec)?; altair::process_attestation(state, self, 0, &mut ctxt, VerifySignatures::True, spec) } From ea2420d193ee758ce372a430b857e4b2724cd7e4 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Sat, 8 Jul 2023 13:16:06 +0000 Subject: [PATCH 16/51] Bump default checkpoint sync timeout to 3 minutes (#4466) ## Issue Addressed [Users on Twitter](https://twitter.com/ashekhirin/status/1676334843192397824) are getting checkpoint sync URL timeouts with the default of 60s, so this PR increases the default timeout to 3 minutes. I've also added a short section to the book about adjusting the timeout with `--checkpoint-sync-url-timeout`. --- beacon_node/src/cli.rs | 2 +- book/src/checkpoint-sync.md | 11 +++++++++++ lighthouse/tests/beacon_node.rs | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 646356b6cb4..fc53f578880 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -848,7 +848,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .help("Set the timeout for checkpoint sync calls to remote beacon node HTTP endpoint.") .value_name("SECONDS") .takes_value(true) - .default_value("60") + .default_value("180") ) .arg( Arg::with_name("reconstruct-historic-states") diff --git a/book/src/checkpoint-sync.md b/book/src/checkpoint-sync.md index d5c8b18e577..57883828947 100644 --- a/book/src/checkpoint-sync.md +++ b/book/src/checkpoint-sync.md @@ -48,6 +48,17 @@ The Ethereum community provides various [public endpoints](https://eth-clients.g lighthouse bn --checkpoint-sync-url https://example.com/ ... ``` +### Adjusting the timeout + +If the beacon node fails to start due to a timeout from the checkpoint sync server, you can try +running it again with a longer timeout by adding the flag `--checkpoint-sync-url-timeout`. + +``` +lighthouse bn --checkpoint-sync-url-timeout 300 --checkpoint-sync-url https://example.com/ ... +``` + +The flag takes a value in seconds. For more information see `lighthouse bn --help`. + ## Backfilling Blocks Once forwards sync completes, Lighthouse will commence a "backfill sync" to download the blocks diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 9b6d23ddcf5..4badcef3cf2 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -177,7 +177,7 @@ fn checkpoint_sync_url_timeout_default() { CommandLineTest::new() .run_with_zero_port() .with_config(|config| { - assert_eq!(config.chain.checkpoint_sync_url_timeout, 60); + assert_eq!(config.chain.checkpoint_sync_url_timeout, 180); }); } From c25825a5393aca2cde6e33dd673f8c074c2b543b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 10 Jul 2023 07:45:54 +0000 Subject: [PATCH 17/51] Move the `BeaconProcessor` into a new crate (#4435) *Replaces #4434. It is identical, but this PR has a smaller diff due to a curated commit history.* ## Issue Addressed NA ## Proposed Changes This PR moves the scheduling logic for the `BeaconProcessor` into a new crate in `beacon_node/beacon_processor`. Previously it existed in the `beacon_node/network` crate. This addresses a circular-dependency problem where it's not possible to use the `BeaconProcessor` from the `beacon_chain` crate. The `network` crate depends on the `beacon_chain` crate (`network -> beacon_chain`), but importing the `BeaconProcessor` into the `beacon_chain` crate would create a circular dependancy of `beacon_chain -> network`. The `BeaconProcessor` was designed to provide queuing and prioritized scheduling for messages from the network. It has proven to be quite valuable and I believe we'd make Lighthouse more stable and effective by using it elsewhere. In particular, I think we should use the `BeaconProcessor` for: 1. HTTP API requests. 1. Scheduled tasks in the `BeaconChain` (e.g., state advance). Using the `BeaconProcessor` for these tasks would help prevent the BN from becoming overwhelmed and would also help it to prioritize operations (e.g., choosing to process blocks from gossip before responding to low-priority HTTP API requests). ## Additional Info This PR is intended to have zero impact on runtime behaviour. It aims to simply separate the *scheduling* code (i.e., the `BeaconProcessor`) from the *business logic* in the `network` crate (i.e., the `Worker` impls). Future PRs (see #4462) can build upon these works to actually use the `BeaconProcessor` for more operations. I've gone to some effort to use `git mv` to make the diff look more like "file was moved and modified" rather than "file was deleted and a new one added". This should reduce review burden and help maintain commit attribution. --- Cargo.lock | 28 + Cargo.toml | 1 + beacon_node/Cargo.toml | 2 +- beacon_node/beacon_processor/Cargo.toml | 24 + .../mod.rs => beacon_processor/src/lib.rs} | 1264 ++++------------- beacon_node/beacon_processor/src/metrics.rs | 141 ++ .../src}/work_reprocessing_queue.rs | 243 ++-- beacon_node/client/Cargo.toml | 2 + beacon_node/client/src/builder.rs | 42 +- beacon_node/network/Cargo.toml | 6 +- .../src/beacon_processor/worker/mod.rs | 51 - beacon_node/network/src/lib.rs | 2 +- beacon_node/network/src/metrics.rs | 117 +- .../gossip_methods.rs | 202 ++- .../src/network_beacon_processor/mod.rs | 590 ++++++++ .../rpc_methods.rs | 14 +- .../sync_methods.rs | 109 +- .../tests.rs | 139 +- beacon_node/network/src/router.rs | 258 ++-- beacon_node/network/src/service.rs | 7 +- beacon_node/network/src/service/tests.rs | 23 +- .../network/src/sync/backfill_sync/mod.rs | 6 +- .../network/src/sync/block_lookups/mod.rs | 20 +- .../network/src/sync/block_lookups/tests.rs | 21 +- beacon_node/network/src/sync/manager.rs | 45 +- .../network/src/sync/network_context.rs | 29 +- .../network/src/sync/range_sync/chain.rs | 10 +- .../network/src/sync/range_sync/range.rs | 17 +- 28 files changed, 1741 insertions(+), 1672 deletions(-) create mode 100644 beacon_node/beacon_processor/Cargo.toml rename beacon_node/{network/src/beacon_processor/mod.rs => beacon_processor/src/lib.rs} (58%) create mode 100644 beacon_node/beacon_processor/src/metrics.rs rename beacon_node/{network/src/beacon_processor => beacon_processor/src}/work_reprocessing_queue.rs (85%) delete mode 100644 beacon_node/network/src/beacon_processor/worker/mod.rs rename beacon_node/network/src/{beacon_processor/worker => network_beacon_processor}/gossip_methods.rs (95%) create mode 100644 beacon_node/network/src/network_beacon_processor/mod.rs rename beacon_node/network/src/{beacon_processor/worker => network_beacon_processor}/rpc_methods.rs (98%) rename beacon_node/network/src/{beacon_processor/worker => network_beacon_processor}/sync_methods.rs (90%) rename beacon_node/network/src/{beacon_processor => network_beacon_processor}/tests.rs (89%) diff --git a/Cargo.lock b/Cargo.lock index e360bdd62e2..532f7ff2042 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -700,6 +700,30 @@ dependencies = [ "unused_port", ] +[[package]] +name = "beacon_processor" +version = "0.1.0" +dependencies = [ + "derivative", + "ethereum_ssz", + "fnv", + "futures", + "hex", + "itertools", + "lazy_static", + "lighthouse_metrics", + "lighthouse_network", + "logging", + "parking_lot 0.12.1", + "slog", + "slot_clock", + "strum", + "task_executor", + "tokio", + "tokio-util 0.6.10", + "types", +] + [[package]] name = "bincode" version = "1.3.3" @@ -1148,6 +1172,7 @@ name = "client" version = "0.2.0" dependencies = [ "beacon_chain", + "beacon_processor", "directory", "dirs", "environment", @@ -1165,6 +1190,7 @@ dependencies = [ "logging", "monitoring_api", "network", + "num_cpus", "operation_pool", "parking_lot 0.12.1", "sensitive_url", @@ -5405,6 +5431,7 @@ name = "network" version = "0.2.0" dependencies = [ "beacon_chain", + "beacon_processor", "delay_map", "derivative", "environment", @@ -5428,6 +5455,7 @@ dependencies = [ "matches", "num_cpus", "operation_pool", + "parking_lot 0.12.1", "rand 0.8.5", "rlp", "slog", diff --git a/Cargo.toml b/Cargo.toml index 5c39e01ed13..775842a8c7a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "beacon_node", "beacon_node/beacon_chain", + "beacon_node/beacon_processor", "beacon_node/builder_client", "beacon_node/client", "beacon_node/eth1", diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 7c74365418b..f55c724dc39 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -41,4 +41,4 @@ monitoring_api = { path = "../common/monitoring_api" } sensitive_url = { path = "../common/sensitive_url" } http_api = { path = "http_api" } unused_port = { path = "../common/unused_port" } -strum = "0.24.1" +strum = "0.24.1" \ No newline at end of file diff --git a/beacon_node/beacon_processor/Cargo.toml b/beacon_node/beacon_processor/Cargo.toml new file mode 100644 index 00000000000..5c5200e101e --- /dev/null +++ b/beacon_node/beacon_processor/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "beacon_processor" +version = "0.1.0" +edition = "2021" + +[dependencies] +slog = { version = "2.5.2", features = ["max_level_trace"] } +itertools = "0.10.0" +logging = { path = "../../common/logging" } +tokio = { version = "1.14.0", features = ["full"] } +tokio-util = { version = "0.6.3", features = ["time"] } +futures = "0.3.7" +fnv = "1.0.7" +strum = "0.24.0" +task_executor = { path = "../../common/task_executor" } +slot_clock = { path = "../../common/slot_clock" } +lighthouse_network = { path = "../lighthouse_network" } +hex = "0.4.2" +derivative = "2.2.0" +types = { path = "../../consensus/types" } +ethereum_ssz = "0.5.0" +lazy_static = "1.4.0" +lighthouse_metrics = { path = "../../common/lighthouse_metrics" } +parking_lot = "0.12.0" \ No newline at end of file diff --git a/beacon_node/network/src/beacon_processor/mod.rs b/beacon_node/beacon_processor/src/lib.rs similarity index 58% rename from beacon_node/network/src/beacon_processor/mod.rs rename to beacon_node/beacon_processor/src/lib.rs index 84d8e1b07a8..88066f2a305 100644 --- a/beacon_node/network/src/beacon_processor/mod.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -38,51 +38,34 @@ //! checks the queues to see if there are more parcels of work that can be spawned in a new worker //! task. -use crate::sync::manager::BlockProcessType; -use crate::{metrics, service::NetworkMessage, sync::SyncMessage}; -use beacon_chain::parking_lot::Mutex; -use beacon_chain::{BeaconChain, BeaconChainTypes, GossipVerifiedBlock, NotifyExecutionLayer}; -use derivative::Derivative; +use crate::work_reprocessing_queue::{ + spawn_reprocess_scheduler, QueuedAggregate, QueuedBackfillBatch, QueuedGossipBlock, + QueuedLightClientUpdate, QueuedRpcBlock, QueuedUnaggregate, ReadyWork, ReprocessQueueMessage, +}; use futures::stream::{Stream, StreamExt}; use futures::task::Poll; -use lighthouse_network::rpc::LightClientBootstrapRequest; -use lighthouse_network::{ - rpc::{BlocksByRangeRequest, BlocksByRootRequest, StatusMessage}, - Client, MessageId, NetworkGlobals, PeerId, PeerRequestId, -}; +use lighthouse_network::NetworkGlobals; +use lighthouse_network::{MessageId, PeerId}; use logging::TimeLatch; +use parking_lot::Mutex; use slog::{crit, debug, error, trace, warn, Logger}; -use std::collections::VecDeque; +use slot_clock::SlotClock; +use std::cmp; +use std::collections::{HashSet, VecDeque}; +use std::fmt; use std::future::Future; -use std::path::PathBuf; use std::pin::Pin; -use std::sync::{Arc, Weak}; +use std::sync::Arc; use std::task::Context; use std::time::Duration; -use std::{cmp, collections::HashSet}; use task_executor::TaskExecutor; use tokio::sync::mpsc; use tokio::sync::mpsc::error::TrySendError; -use types::{ - Attestation, AttesterSlashing, Hash256, LightClientFinalityUpdate, LightClientOptimisticUpdate, - ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBlsToExecutionChange, - SignedContributionAndProof, SignedVoluntaryExit, SubnetId, SyncCommitteeMessage, SyncSubnetId, -}; -use work_reprocessing_queue::{ - spawn_reprocess_scheduler, QueuedAggregate, QueuedLightClientUpdate, QueuedRpcBlock, - QueuedUnaggregate, ReadyWork, -}; - -use worker::{Toolbox, Worker}; - -mod tests; -mod work_reprocessing_queue; -mod worker; +use types::{Attestation, EthSpec, Hash256, SignedAggregateAndProof, Slot, SubnetId}; +use work_reprocessing_queue::IgnoredRpcBlock; -use crate::beacon_processor::work_reprocessing_queue::{ - QueuedBackfillBatch, QueuedGossipBlock, ReprocessQueueMessage, -}; -pub use worker::{ChainSegmentProcessId, GossipAggregatePackage, GossipAttestationPackage}; +mod metrics; +pub mod work_reprocessing_queue; /// The maximum size of the channel for work events to the `BeaconProcessor`. /// @@ -96,7 +79,7 @@ pub const MAX_WORK_EVENT_QUEUE_LEN: usize = 16_384; const MAX_IDLE_QUEUE_LEN: usize = 16_384; /// The maximum size of the channel for re-processing work events. -const MAX_SCHEDULED_WORK_QUEUE_LEN: usize = 3 * MAX_WORK_EVENT_QUEUE_LEN / 4; +pub const MAX_SCHEDULED_WORK_QUEUE_LEN: usize = 3 * MAX_WORK_EVENT_QUEUE_LEN / 4; /// The maximum number of queued `Attestation` objects that will be stored before we start dropping /// them. @@ -221,6 +204,7 @@ pub const GOSSIP_SYNC_CONTRIBUTION: &str = "gossip_sync_contribution"; pub const GOSSIP_LIGHT_CLIENT_FINALITY_UPDATE: &str = "light_client_finality_update"; pub const GOSSIP_LIGHT_CLIENT_OPTIMISTIC_UPDATE: &str = "light_client_optimistic_update"; pub const RPC_BLOCK: &str = "rpc_block"; +pub const IGNORED_RPC_BLOCK: &str = "ignored_rpc_block"; pub const CHAIN_SEGMENT: &str = "chain_segment"; pub const CHAIN_SEGMENT_BACKFILL: &str = "chain_segment_backfill"; pub const STATUS_PROCESSING: &str = "status_processing"; @@ -366,394 +350,104 @@ impl DuplicateCache { } /// An event to be processed by the manager task. -#[derive(Derivative)] -#[derivative(Debug(bound = "T: BeaconChainTypes"))] -pub struct WorkEvent { - drop_during_sync: bool, - work: Work, +#[derive(Debug)] +pub struct WorkEvent { + pub drop_during_sync: bool, + pub work: Work, } -impl WorkEvent { - /// Create a new `Work` event for some unaggregated attestation. - pub fn unaggregated_attestation( - message_id: MessageId, - peer_id: PeerId, - attestation: Attestation, - subnet_id: SubnetId, - should_import: bool, - seen_timestamp: Duration, - ) -> Self { - Self { - drop_during_sync: true, - work: Work::GossipAttestation { - message_id, - peer_id, - attestation: Box::new(attestation), - subnet_id, - should_import, - seen_timestamp, - }, - } - } - - /// Create a new `Work` event for some aggregated attestation. - pub fn aggregated_attestation( - message_id: MessageId, - peer_id: PeerId, - aggregate: SignedAggregateAndProof, - seen_timestamp: Duration, - ) -> Self { - Self { - drop_during_sync: true, - work: Work::GossipAggregate { - message_id, - peer_id, - aggregate: Box::new(aggregate), - seen_timestamp, - }, - } - } - - /// Create a new `Work` event for some block. - pub fn gossip_beacon_block( - message_id: MessageId, - peer_id: PeerId, - peer_client: Client, - block: Arc>, - seen_timestamp: Duration, - ) -> Self { - Self { - drop_during_sync: false, - work: Work::GossipBlock { - message_id, - peer_id, - peer_client, - block, - seen_timestamp, - }, - } - } - - /// Create a new `Work` event for some sync committee signature. - pub fn gossip_sync_signature( - message_id: MessageId, - peer_id: PeerId, - sync_signature: SyncCommitteeMessage, - subnet_id: SyncSubnetId, - seen_timestamp: Duration, - ) -> Self { - Self { - drop_during_sync: true, - work: Work::GossipSyncSignature { - message_id, - peer_id, - sync_signature: Box::new(sync_signature), - subnet_id, - seen_timestamp, - }, - } - } - - /// Create a new `Work` event for some sync committee contribution. - pub fn gossip_sync_contribution( - message_id: MessageId, - peer_id: PeerId, - sync_contribution: SignedContributionAndProof, - seen_timestamp: Duration, - ) -> Self { - Self { - drop_during_sync: true, - work: Work::GossipSyncContribution { - message_id, - peer_id, - sync_contribution: Box::new(sync_contribution), - seen_timestamp, - }, - } - } - - /// Create a new `Work` event for some exit. - pub fn gossip_voluntary_exit( - message_id: MessageId, - peer_id: PeerId, - voluntary_exit: Box, - ) -> Self { - Self { - drop_during_sync: false, - work: Work::GossipVoluntaryExit { - message_id, - peer_id, - voluntary_exit, - }, - } - } - - /// Create a new `Work` event for some proposer slashing. - pub fn gossip_proposer_slashing( - message_id: MessageId, - peer_id: PeerId, - proposer_slashing: Box, - ) -> Self { - Self { - drop_during_sync: false, - work: Work::GossipProposerSlashing { - message_id, - peer_id, - proposer_slashing, - }, - } - } - - /// Create a new `Work` event for some light client finality update. - pub fn gossip_light_client_finality_update( - message_id: MessageId, - peer_id: PeerId, - light_client_finality_update: Box>, - seen_timestamp: Duration, - ) -> Self { - Self { - drop_during_sync: true, - work: Work::GossipLightClientFinalityUpdate { - message_id, - peer_id, - light_client_finality_update, - seen_timestamp, - }, - } - } - - /// Create a new `Work` event for some light client optimistic update. - pub fn gossip_light_client_optimistic_update( - message_id: MessageId, - peer_id: PeerId, - light_client_optimistic_update: Box>, - seen_timestamp: Duration, - ) -> Self { - Self { - drop_during_sync: true, - work: Work::GossipLightClientOptimisticUpdate { - message_id, - peer_id, - light_client_optimistic_update, - seen_timestamp, - }, - } - } - - /// Create a new `Work` event for some attester slashing. - pub fn gossip_attester_slashing( - message_id: MessageId, - peer_id: PeerId, - attester_slashing: Box>, - ) -> Self { - Self { - drop_during_sync: false, - work: Work::GossipAttesterSlashing { - message_id, - peer_id, - attester_slashing, - }, - } - } - - /// Create a new `Work` event for some BLS to execution change. - pub fn gossip_bls_to_execution_change( - message_id: MessageId, - peer_id: PeerId, - bls_to_execution_change: Box, - ) -> Self { - Self { - drop_during_sync: false, - work: Work::GossipBlsToExecutionChange { - message_id, - peer_id, - bls_to_execution_change, - }, - } - } - - /// Create a new `Work` event for some block, where the result from computation (if any) is - /// sent to the other side of `result_tx`. - pub fn rpc_beacon_block( - block_root: Hash256, - block: Arc>, - seen_timestamp: Duration, - process_type: BlockProcessType, - ) -> Self { - Self { - drop_during_sync: false, - work: Work::RpcBlock { - block_root, - block, - seen_timestamp, - process_type, - should_process: true, - }, - } - } - - /// Create a new work event to import `blocks` as a beacon chain segment. - pub fn chain_segment( - process_id: ChainSegmentProcessId, - blocks: Vec>>, - ) -> Self { - Self { - drop_during_sync: false, - work: Work::ChainSegment { process_id, blocks }, - } - } - - /// Create a new work event to process `StatusMessage`s from the RPC network. - pub fn status_message(peer_id: PeerId, message: StatusMessage) -> Self { - Self { - drop_during_sync: false, - work: Work::Status { peer_id, message }, - } - } - - /// Create a new work event to process `BlocksByRangeRequest`s from the RPC network. - pub fn blocks_by_range_request( - peer_id: PeerId, - request_id: PeerRequestId, - request: BlocksByRangeRequest, - ) -> Self { - Self { - drop_during_sync: false, - work: Work::BlocksByRangeRequest { - peer_id, - request_id, - request, - }, - } - } - - /// Create a new work event to process `BlocksByRootRequest`s from the RPC network. - pub fn blocks_by_roots_request( - peer_id: PeerId, - request_id: PeerRequestId, - request: BlocksByRootRequest, - ) -> Self { - Self { - drop_during_sync: false, - work: Work::BlocksByRootsRequest { - peer_id, - request_id, - request, - }, - } - } - - /// Create a new work event to process `LightClientBootstrap`s from the RPC network. - pub fn lightclient_bootstrap_request( - peer_id: PeerId, - request_id: PeerRequestId, - request: LightClientBootstrapRequest, - ) -> Self { - Self { - drop_during_sync: true, - work: Work::LightClientBootstrapRequest { - peer_id, - request_id, - request, - }, - } - } - +impl WorkEvent { /// Get a `str` representation of the type of work this `WorkEvent` contains. pub fn work_type(&self) -> &'static str { self.work.str_id() } } -impl std::convert::From> for WorkEvent { - fn from(ready_work: ReadyWork) -> Self { +impl std::convert::From for WorkEvent { + fn from(ready_work: ReadyWork) -> Self { match ready_work { ReadyWork::Block(QueuedGossipBlock { - peer_id, - block, - seen_timestamp, + beacon_block_slot, + beacon_block_root, + process_fn, }) => Self { drop_during_sync: false, work: Work::DelayedImportBlock { - peer_id, - block, - seen_timestamp, + beacon_block_slot, + beacon_block_root, + process_fn, }, }, ReadyWork::RpcBlock(QueuedRpcBlock { - block_root, - block, - seen_timestamp, - process_type, - should_process, + beacon_block_root: _, + process_fn, + ignore_fn: _, }) => Self { drop_during_sync: false, - work: Work::RpcBlock { - block_root, - block, - seen_timestamp, - process_type, - should_process, - }, + work: Work::RpcBlock { process_fn }, + }, + ReadyWork::IgnoredRpcBlock(IgnoredRpcBlock { process_fn }) => Self { + drop_during_sync: false, + work: Work::IgnoredRpcBlock { process_fn }, }, ReadyWork::Unaggregate(QueuedUnaggregate { - peer_id, - message_id, - attestation, - subnet_id, - should_import, - seen_timestamp, + beacon_block_root: _, + process_fn, }) => Self { drop_during_sync: true, - work: Work::UnknownBlockAttestation { - message_id, - peer_id, - attestation, - subnet_id, - should_import, - seen_timestamp, - }, + work: Work::UnknownBlockAttestation { process_fn }, }, ReadyWork::Aggregate(QueuedAggregate { - peer_id, - message_id, - attestation, - seen_timestamp, + process_fn, + beacon_block_root: _, }) => Self { drop_during_sync: true, - work: Work::UnknownBlockAggregate { - message_id, - peer_id, - aggregate: attestation, - seen_timestamp, - }, + work: Work::UnknownBlockAggregate { process_fn }, }, ReadyWork::LightClientUpdate(QueuedLightClientUpdate { - peer_id, - message_id, - light_client_optimistic_update, - seen_timestamp, - .. + parent_root, + process_fn, }) => Self { drop_during_sync: true, work: Work::UnknownLightClientOptimisticUpdate { - message_id, - peer_id, - light_client_optimistic_update, - seen_timestamp, + parent_root, + process_fn, }, }, - ReadyWork::BackfillSync(QueuedBackfillBatch { process_id, blocks }) => { - WorkEvent::chain_segment(process_id, blocks) - } + ReadyWork::BackfillSync(QueuedBackfillBatch(process_fn)) => Self { + drop_during_sync: false, + work: Work::ChainSegmentBackfill(process_fn), + }, } } } -pub struct BeaconProcessorSend(pub mpsc::Sender>); +/// Items required to verify a batch of unaggregated gossip attestations. +#[derive(Debug)] +pub struct GossipAttestationPackage { + pub message_id: MessageId, + pub peer_id: PeerId, + pub attestation: Box>, + pub subnet_id: SubnetId, + pub should_import: bool, + pub seen_timestamp: Duration, +} + +/// Items required to verify a batch of aggregated gossip attestations. +#[derive(Debug)] +pub struct GossipAggregatePackage { + pub message_id: MessageId, + pub peer_id: PeerId, + pub aggregate: Box>, + pub beacon_block_root: Hash256, + pub seen_timestamp: Duration, +} -impl BeaconProcessorSend { - pub fn try_send(&self, message: WorkEvent) -> Result<(), Box>>> { +#[derive(Clone)] +pub struct BeaconProcessorSend(pub mpsc::Sender>); + +impl BeaconProcessorSend { + pub fn try_send(&self, message: WorkEvent) -> Result<(), TrySendError>> { let work_type = message.work_type(); match self.0.try_send(message) { Ok(res) => Ok(res), @@ -762,146 +456,82 @@ impl BeaconProcessorSend { &metrics::BEACON_PROCESSOR_SEND_ERROR_PER_WORK_TYPE, &[work_type], ); - Err(Box::new(e)) + Err(e) } } } } -/// A consensus message (or multiple) from the network that requires processing. -#[derive(Derivative)] -#[derivative(Debug(bound = "T: BeaconChainTypes"))] -pub enum Work { +pub type AsyncFn = Pin + Send + Sync>>; +pub type BlockingFn = Box; +pub type BlockingFnWithManualSendOnIdle = Box; + +/// Indicates the type of work to be performed and therefore its priority and +/// queuing specifics. +pub enum Work { GossipAttestation { - message_id: MessageId, - peer_id: PeerId, - attestation: Box>, - subnet_id: SubnetId, - should_import: bool, - seen_timestamp: Duration, + attestation: GossipAttestationPackage, + process_individual: Box) + Send + Sync>, + process_batch: Box>) + Send + Sync>, }, UnknownBlockAttestation { - message_id: MessageId, - peer_id: PeerId, - attestation: Box>, - subnet_id: SubnetId, - should_import: bool, - seen_timestamp: Duration, + process_fn: BlockingFn, }, GossipAttestationBatch { - packages: Vec>, + attestations: Vec>, + process_batch: Box>) + Send + Sync>, }, GossipAggregate { - message_id: MessageId, - peer_id: PeerId, - aggregate: Box>, - seen_timestamp: Duration, + aggregate: GossipAggregatePackage, + process_individual: Box) + Send + Sync>, + process_batch: Box>) + Send + Sync>, }, UnknownBlockAggregate { - message_id: MessageId, - peer_id: PeerId, - aggregate: Box>, - seen_timestamp: Duration, + process_fn: BlockingFn, }, UnknownLightClientOptimisticUpdate { - message_id: MessageId, - peer_id: PeerId, - light_client_optimistic_update: Box>, - seen_timestamp: Duration, + parent_root: Hash256, + process_fn: BlockingFn, }, GossipAggregateBatch { - packages: Vec>, - }, - GossipBlock { - message_id: MessageId, - peer_id: PeerId, - peer_client: Client, - block: Arc>, - seen_timestamp: Duration, + aggregates: Vec>, + process_batch: Box>) + Send + Sync>, }, + GossipBlock(AsyncFn), DelayedImportBlock { - peer_id: PeerId, - block: Box>, - seen_timestamp: Duration, - }, - GossipVoluntaryExit { - message_id: MessageId, - peer_id: PeerId, - voluntary_exit: Box, - }, - GossipProposerSlashing { - message_id: MessageId, - peer_id: PeerId, - proposer_slashing: Box, - }, - GossipAttesterSlashing { - message_id: MessageId, - peer_id: PeerId, - attester_slashing: Box>, - }, - GossipSyncSignature { - message_id: MessageId, - peer_id: PeerId, - sync_signature: Box, - subnet_id: SyncSubnetId, - seen_timestamp: Duration, - }, - GossipSyncContribution { - message_id: MessageId, - peer_id: PeerId, - sync_contribution: Box>, - seen_timestamp: Duration, - }, - GossipLightClientFinalityUpdate { - message_id: MessageId, - peer_id: PeerId, - light_client_finality_update: Box>, - seen_timestamp: Duration, - }, - GossipLightClientOptimisticUpdate { - message_id: MessageId, - peer_id: PeerId, - light_client_optimistic_update: Box>, - seen_timestamp: Duration, + beacon_block_slot: Slot, + beacon_block_root: Hash256, + process_fn: AsyncFn, }, + GossipVoluntaryExit(BlockingFn), + GossipProposerSlashing(BlockingFn), + GossipAttesterSlashing(BlockingFn), + GossipSyncSignature(BlockingFn), + GossipSyncContribution(BlockingFn), + GossipLightClientFinalityUpdate(BlockingFn), + GossipLightClientOptimisticUpdate(BlockingFn), RpcBlock { - block_root: Hash256, - block: Arc>, - seen_timestamp: Duration, - process_type: BlockProcessType, - should_process: bool, + process_fn: AsyncFn, }, - ChainSegment { - process_id: ChainSegmentProcessId, - blocks: Vec>>, - }, - Status { - peer_id: PeerId, - message: StatusMessage, - }, - BlocksByRangeRequest { - peer_id: PeerId, - request_id: PeerRequestId, - request: BlocksByRangeRequest, - }, - BlocksByRootsRequest { - peer_id: PeerId, - request_id: PeerRequestId, - request: BlocksByRootRequest, - }, - GossipBlsToExecutionChange { - message_id: MessageId, - peer_id: PeerId, - bls_to_execution_change: Box, - }, - LightClientBootstrapRequest { - peer_id: PeerId, - request_id: PeerRequestId, - request: LightClientBootstrapRequest, + IgnoredRpcBlock { + process_fn: BlockingFn, }, + ChainSegment(AsyncFn), + ChainSegmentBackfill(AsyncFn), + Status(BlockingFn), + BlocksByRangeRequest(BlockingFnWithManualSendOnIdle), + BlocksByRootsRequest(BlockingFnWithManualSendOnIdle), + GossipBlsToExecutionChange(BlockingFn), + LightClientBootstrapRequest(BlockingFn), } -impl Work { +impl fmt::Debug for Work { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.str_id()) + } +} + +impl Work { /// Provides a `&str` that uniquely identifies each enum variant. fn str_id(&self) -> &'static str { match self { @@ -909,58 +539,56 @@ impl Work { Work::GossipAttestationBatch { .. } => GOSSIP_ATTESTATION_BATCH, Work::GossipAggregate { .. } => GOSSIP_AGGREGATE, Work::GossipAggregateBatch { .. } => GOSSIP_AGGREGATE_BATCH, - Work::GossipBlock { .. } => GOSSIP_BLOCK, + Work::GossipBlock(_) => GOSSIP_BLOCK, Work::DelayedImportBlock { .. } => DELAYED_IMPORT_BLOCK, - Work::GossipVoluntaryExit { .. } => GOSSIP_VOLUNTARY_EXIT, - Work::GossipProposerSlashing { .. } => GOSSIP_PROPOSER_SLASHING, - Work::GossipAttesterSlashing { .. } => GOSSIP_ATTESTER_SLASHING, - Work::GossipSyncSignature { .. } => GOSSIP_SYNC_SIGNATURE, - Work::GossipSyncContribution { .. } => GOSSIP_SYNC_CONTRIBUTION, - Work::GossipLightClientFinalityUpdate { .. } => GOSSIP_LIGHT_CLIENT_FINALITY_UPDATE, - Work::GossipLightClientOptimisticUpdate { .. } => GOSSIP_LIGHT_CLIENT_OPTIMISTIC_UPDATE, + Work::GossipVoluntaryExit(_) => GOSSIP_VOLUNTARY_EXIT, + Work::GossipProposerSlashing(_) => GOSSIP_PROPOSER_SLASHING, + Work::GossipAttesterSlashing(_) => GOSSIP_ATTESTER_SLASHING, + Work::GossipSyncSignature(_) => GOSSIP_SYNC_SIGNATURE, + Work::GossipSyncContribution(_) => GOSSIP_SYNC_CONTRIBUTION, + Work::GossipLightClientFinalityUpdate(_) => GOSSIP_LIGHT_CLIENT_FINALITY_UPDATE, + Work::GossipLightClientOptimisticUpdate(_) => GOSSIP_LIGHT_CLIENT_OPTIMISTIC_UPDATE, Work::RpcBlock { .. } => RPC_BLOCK, - Work::ChainSegment { - process_id: ChainSegmentProcessId::BackSyncBatchId { .. }, - .. - } => CHAIN_SEGMENT_BACKFILL, + Work::IgnoredRpcBlock { .. } => IGNORED_RPC_BLOCK, Work::ChainSegment { .. } => CHAIN_SEGMENT, - Work::Status { .. } => STATUS_PROCESSING, - Work::BlocksByRangeRequest { .. } => BLOCKS_BY_RANGE_REQUEST, - Work::BlocksByRootsRequest { .. } => BLOCKS_BY_ROOTS_REQUEST, - Work::LightClientBootstrapRequest { .. } => LIGHT_CLIENT_BOOTSTRAP_REQUEST, + Work::ChainSegmentBackfill(_) => CHAIN_SEGMENT_BACKFILL, + Work::Status(_) => STATUS_PROCESSING, + Work::BlocksByRangeRequest(_) => BLOCKS_BY_RANGE_REQUEST, + Work::BlocksByRootsRequest(_) => BLOCKS_BY_ROOTS_REQUEST, + Work::LightClientBootstrapRequest(_) => LIGHT_CLIENT_BOOTSTRAP_REQUEST, Work::UnknownBlockAttestation { .. } => UNKNOWN_BLOCK_ATTESTATION, Work::UnknownBlockAggregate { .. } => UNKNOWN_BLOCK_AGGREGATE, - Work::GossipBlsToExecutionChange { .. } => GOSSIP_BLS_TO_EXECUTION_CHANGE, + Work::GossipBlsToExecutionChange(_) => GOSSIP_BLS_TO_EXECUTION_CHANGE, Work::UnknownLightClientOptimisticUpdate { .. } => UNKNOWN_LIGHT_CLIENT_UPDATE, } } } /// Unifies all the messages processed by the `BeaconProcessor`. -enum InboundEvent { +enum InboundEvent { /// A worker has completed a task and is free. WorkerIdle, /// There is new work to be done. - WorkEvent(WorkEvent), + WorkEvent(WorkEvent), /// A work event that was queued for re-processing has become ready. - ReprocessingWork(WorkEvent), + ReprocessingWork(WorkEvent), } /// Combines the various incoming event streams for the `BeaconProcessor` into a single stream. /// /// This struct has a similar purpose to `tokio::select!`, however it allows for more fine-grained /// control (specifically in the ordering of event processing). -struct InboundEvents { +struct InboundEvents { /// Used by workers when they finish a task. idle_rx: mpsc::Receiver<()>, /// Used by upstream processes to send new work to the `BeaconProcessor`. - event_rx: mpsc::Receiver>, + event_rx: mpsc::Receiver>, /// Used internally for queuing work ready to be re-processed. - reprocess_work_rx: mpsc::Receiver>, + reprocess_work_rx: mpsc::Receiver, } -impl Stream for InboundEvents { - type Item = InboundEvent; +impl Stream for InboundEvents { + type Item = InboundEvent; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { // Always check for idle workers before anything else. This allows us to ensure that a big @@ -1001,31 +629,20 @@ impl Stream for InboundEvents { } } -/// Defines if and where we will store the SSZ files of invalid blocks. -#[derive(Clone)] -pub enum InvalidBlockStorage { - Enabled(PathBuf), - Disabled, -} - /// A mutli-threaded processor for messages received on the network /// that need to be processed by the `BeaconChain` /// /// See module level documentation for more information. -pub struct BeaconProcessor { - pub beacon_chain: Weak>, - pub network_tx: mpsc::UnboundedSender>, - pub sync_tx: mpsc::UnboundedSender>, - pub network_globals: Arc>, +pub struct BeaconProcessor { + pub network_globals: Arc>, pub executor: TaskExecutor, pub max_workers: usize, pub current_workers: usize, - pub importing_blocks: DuplicateCache, - pub invalid_block_storage: InvalidBlockStorage, + pub enable_backfill_rate_limiting: bool, pub log: Logger, } -impl BeaconProcessor { +impl BeaconProcessor { /// Spawns the "manager" task which checks the receiver end of the returned `Sender` for /// messages which contain some new work which will be: /// @@ -1037,10 +654,13 @@ impl BeaconProcessor { /// /// The optional `work_journal_tx` allows for an outside process to receive a log of all work /// events processed by `self`. This should only be used during testing. - pub fn spawn_manager( + pub fn spawn_manager( mut self, - event_rx: mpsc::Receiver>, + event_rx: mpsc::Receiver>, + work_reprocessing_tx: mpsc::Sender, + work_reprocessing_rx: mpsc::Receiver, work_journal_tx: Option>, + slot_clock: S, ) { // Used by workers to communicate that they are finished a task. let (idle_tx, idle_rx) = mpsc::channel::<()>(MAX_IDLE_QUEUE_LEN); @@ -1093,20 +713,15 @@ impl BeaconProcessor { let mut lcbootstrap_queue = FifoQueue::new(MAX_LIGHT_CLIENT_BOOTSTRAP_QUEUE_LEN); - let chain = match self.beacon_chain.upgrade() { - Some(chain) => chain, - // No need to proceed any further if the beacon chain has been dropped, the client - // is shutting down. - None => return, - }; - // Channels for sending work to the re-process scheduler (`work_reprocessing_tx`) and to // receive them back once they are ready (`ready_work_rx`). - let (ready_work_tx, ready_work_rx) = mpsc::channel(MAX_SCHEDULED_WORK_QUEUE_LEN); - let work_reprocessing_tx = spawn_reprocess_scheduler( + let (ready_work_tx, ready_work_rx) = + mpsc::channel::(MAX_SCHEDULED_WORK_QUEUE_LEN); + spawn_reprocess_scheduler( ready_work_tx, + work_reprocessing_rx, &self.executor, - chain.slot_clock.clone(), + slot_clock, self.log.clone(), ); @@ -1121,7 +736,7 @@ impl BeaconProcessor { reprocess_work_rx: ready_work_rx, }; - let enable_backfill_rate_limiting = chain.config.enable_backfill_rate_limiting; + let enable_backfill_rate_limiting = self.enable_backfill_rate_limiting; loop { let work_event = match inbound_events.next().await { @@ -1209,33 +824,29 @@ impl BeaconProcessor { .as_ref() .map_or(false, |event| event.drop_during_sync); + let idle_tx = idle_tx.clone(); match work_event { // There is no new work event, but we are able to spawn a new worker. // // We don't check the `work.drop_during_sync` here. We assume that if it made // it into the queue at any point then we should process it. None if can_spawn => { - let toolbox = Toolbox { - idle_tx: idle_tx.clone(), - work_reprocessing_tx: work_reprocessing_tx.clone(), - }; - // Check for chain segments first, they're the most efficient way to get // blocks into the system. if let Some(item) = chain_segment_queue.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); // Check sync blocks before gossip blocks, since we've already explicitly // requested these blocks. } else if let Some(item) = rpc_block_queue.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); // Check delayed blocks before gossip blocks, the gossip blocks might rely // on the delayed ones. } else if let Some(item) = delayed_block_queue.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); // Check gossip blocks before gossip attestations, since a block might be // required to verify some attestations. } else if let Some(item) = gossip_block_queue.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); // Check the aggregates, *then* the unaggregates since we assume that // aggregates are more valuable to local validators and effectively give us // more information with less signature verification time. @@ -1246,7 +857,7 @@ impl BeaconProcessor { if batch_size < 2 { // One single aggregate is in the queue, process it individually. if let Some(item) = aggregate_queue.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); } } else { // Collect two or more aggregates into a batch, so they can take @@ -1254,32 +865,45 @@ impl BeaconProcessor { // // Note: this will convert the `Work::GossipAggregate` item into a // `Work::GossipAggregateBatch` item. - let mut packages = Vec::with_capacity(batch_size); + let mut aggregates = Vec::with_capacity(batch_size); + let mut process_batch_opt = None; for _ in 0..batch_size { if let Some(item) = aggregate_queue.pop() { match item { Work::GossipAggregate { - message_id, - peer_id, aggregate, - seen_timestamp, + process_individual: _, + process_batch, } => { - packages.push(GossipAggregatePackage::new( - message_id, - peer_id, - aggregate, - seen_timestamp, - )); + aggregates.push(aggregate); + if process_batch_opt.is_none() { + process_batch_opt = Some(process_batch); + } } _ => { - error!(self.log, "Invalid item in aggregate queue") + error!(self.log, "Invalid item in aggregate queue"); } } } } - // Process all aggregates with a single worker. - self.spawn_worker(Work::GossipAggregateBatch { packages }, toolbox) + if let Some(process_batch) = process_batch_opt { + // Process all aggregates with a single worker. + self.spawn_worker( + Work::GossipAggregateBatch { + aggregates, + process_batch, + }, + idle_tx, + ) + } else { + // There is no good reason for this to + // happen, it is a serious logic error. + // Since we only form batches when multiple + // work items exist, we should always have a + // work closure at this point. + crit!(self.log, "Missing aggregate work"); + } } // Check the unaggregated attestation queue. // @@ -1293,7 +917,7 @@ impl BeaconProcessor { if batch_size < 2 { // One single attestation is in the queue, process it individually. if let Some(item) = attestation_queue.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); } } else { // Collect two or more attestations into a batch, so they can take @@ -1301,26 +925,20 @@ impl BeaconProcessor { // // Note: this will convert the `Work::GossipAttestation` item into a // `Work::GossipAttestationBatch` item. - let mut packages = Vec::with_capacity(batch_size); + let mut attestations = Vec::with_capacity(batch_size); + let mut process_batch_opt = None; for _ in 0..batch_size { if let Some(item) = attestation_queue.pop() { match item { Work::GossipAttestation { - message_id, - peer_id, attestation, - subnet_id, - should_import, - seen_timestamp, + process_individual: _, + process_batch, } => { - packages.push(GossipAttestationPackage::new( - message_id, - peer_id, - attestation, - subnet_id, - should_import, - seen_timestamp, - )); + attestations.push(attestation); + if process_batch_opt.is_none() { + process_batch_opt = Some(process_batch); + } } _ => error!( self.log, @@ -1330,54 +948,66 @@ impl BeaconProcessor { } } - // Process all attestations with a single worker. - self.spawn_worker( - Work::GossipAttestationBatch { packages }, - toolbox, - ) + if let Some(process_batch) = process_batch_opt { + // Process all attestations with a single worker. + self.spawn_worker( + Work::GossipAttestationBatch { + attestations, + process_batch, + }, + idle_tx, + ) + } else { + // There is no good reason for this to + // happen, it is a serious logic error. + // Since we only form batches when multiple + // work items exist, we should always have a + // work closure at this point. + crit!(self.log, "Missing attestations work"); + } } // Check sync committee messages after attestations as their rewards are lesser // and they don't influence fork choice. } else if let Some(item) = sync_contribution_queue.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); } else if let Some(item) = sync_message_queue.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); // Aggregates and unaggregates queued for re-processing are older and we // care about fresher ones, so check those first. } else if let Some(item) = unknown_block_aggregate_queue.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); } else if let Some(item) = unknown_block_attestation_queue.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); // Check RPC methods next. Status messages are needed for sync so // prioritize them over syncing requests from other peers (BlocksByRange // and BlocksByRoot) } else if let Some(item) = status_queue.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); } else if let Some(item) = bbrange_queue.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); } else if let Some(item) = bbroots_queue.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); // Check slashings after all other consensus messages so we prioritize // following head. // // Check attester slashings before proposer slashings since they have the // potential to slash multiple validators at once. } else if let Some(item) = gossip_attester_slashing_queue.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); } else if let Some(item) = gossip_proposer_slashing_queue.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); // Check exits and address changes late since our validators don't get // rewards from them. } else if let Some(item) = gossip_voluntary_exit_queue.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); } else if let Some(item) = gossip_bls_to_execution_change_queue.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); // Handle backfill sync chain segments. } else if let Some(item) = backfill_chain_segment.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); // This statement should always be the final else statement. } else if let Some(item) = lcbootstrap_queue.pop() { - self.spawn_worker(item, toolbox); + self.spawn_worker(item, idle_tx); } else { // Let the journal know that a worker is freed and there's nothing else // for it to do. @@ -1419,13 +1049,9 @@ impl BeaconProcessor { // it. Some(WorkEvent { work, .. }) => { let work_id = work.str_id(); - let toolbox = Toolbox { - idle_tx: idle_tx.clone(), - work_reprocessing_tx: work_reprocessing_tx.clone(), - }; match work { - _ if can_spawn => self.spawn_worker(work, toolbox), + _ if can_spawn => self.spawn_worker(work, idle_tx), Work::GossipAttestation { .. } => attestation_queue.push(work), // Attestation batches are formed internally within the // `BeaconProcessor`, they are not sent from external services. @@ -1467,16 +1093,15 @@ impl BeaconProcessor { Work::GossipLightClientOptimisticUpdate { .. } => { optimistic_update_queue.push(work, work_id, &self.log) } - Work::RpcBlock { .. } => rpc_block_queue.push(work, work_id, &self.log), - Work::ChainSegment { ref process_id, .. } => match process_id { - ChainSegmentProcessId::RangeBatchId { .. } - | ChainSegmentProcessId::ParentLookup { .. } => { - chain_segment_queue.push(work, work_id, &self.log) - } - ChainSegmentProcessId::BackSyncBatchId { .. } => { - backfill_chain_segment.push(work, work_id, &self.log) - } - }, + Work::RpcBlock { .. } | Work::IgnoredRpcBlock { .. } => { + rpc_block_queue.push(work, work_id, &self.log) + } + Work::ChainSegment { .. } => { + chain_segment_queue.push(work, work_id, &self.log) + } + Work::ChainSegmentBackfill { .. } => { + backfill_chain_segment.push(work, work_id, &self.log) + } Work::Status { .. } => status_queue.push(work, work_id, &self.log), Work::BlocksByRangeRequest { .. } => { bbrange_queue.push(work, work_id, &self.log) @@ -1583,10 +1208,7 @@ impl BeaconProcessor { /// Spawns a blocking worker thread to process some `Work`. /// /// Sends an message on `idle_tx` when the work is complete and the task is stopping. - fn spawn_worker(&mut self, work: Work, toolbox: Toolbox) { - let idle_tx = toolbox.idle_tx; - let work_reprocessing_tx = toolbox.work_reprocessing_tx; - + fn spawn_worker(&mut self, work: Work, idle_tx: mpsc::Sender<()>) { let work_id = work.str_id(); let worker_timer = metrics::start_timer_vec(&metrics::BEACON_PROCESSOR_WORKER_TIME, &[work_id]); @@ -1609,27 +1231,8 @@ impl BeaconProcessor { let worker_id = self.current_workers; self.current_workers = self.current_workers.saturating_add(1); - let chain = if let Some(chain) = self.beacon_chain.upgrade() { - chain - } else { - debug!( - self.log, - "Beacon chain dropped, shutting down"; - ); - return; - }; - let executor = self.executor.clone(); - let worker = Worker { - chain, - network_tx: self.network_tx.clone(), - sync_tx: self.sync_tx.clone(), - log: self.log.clone(), - }; - - let duplicate_cache = self.importing_blocks.clone(); - trace!( self.log, "Spawning beacon processor worker"; @@ -1638,349 +1241,72 @@ impl BeaconProcessor { ); let task_spawner = TaskSpawner { - executor: executor.clone(), + executor, send_idle_on_drop, }; - let sub_executor = executor; match work { - /* - * Individual unaggregated attestation verification. - */ Work::GossipAttestation { - message_id, - peer_id, attestation, - subnet_id, - should_import, - seen_timestamp, + process_individual, + process_batch: _, } => task_spawner.spawn_blocking(move || { - worker.process_gossip_attestation( - message_id, - peer_id, - attestation, - subnet_id, - should_import, - Some(work_reprocessing_tx), - seen_timestamp, - ) + process_individual(attestation); }), - /* - * Batched unaggregated attestation verification. - */ - Work::GossipAttestationBatch { packages } => task_spawner.spawn_blocking(|| { - worker.process_gossip_attestation_batch(packages, Some(work_reprocessing_tx)) + Work::GossipAttestationBatch { + attestations, + process_batch, + } => task_spawner.spawn_blocking(move || { + process_batch(attestations); }), - /* - * Individual aggregated attestation verification. - */ Work::GossipAggregate { - message_id, - peer_id, aggregate, - seen_timestamp, + process_individual, + process_batch: _, } => task_spawner.spawn_blocking(move || { - worker.process_gossip_aggregate( - message_id, - peer_id, - aggregate, - Some(work_reprocessing_tx), - seen_timestamp, - ) - }), - /* - * Batched aggregated attestation verification. - */ - Work::GossipAggregateBatch { packages } => task_spawner.spawn_blocking(|| { - worker.process_gossip_aggregate_batch(packages, Some(work_reprocessing_tx)) + process_individual(aggregate); }), - /* - * Verification for beacon blocks received on gossip. - */ - Work::GossipBlock { - message_id, - peer_id, - peer_client, - block, - seen_timestamp, - } => { - let invalid_block_storage = self.invalid_block_storage.clone(); - task_spawner.spawn_async(async move { - worker - .process_gossip_block( - message_id, - peer_id, - peer_client, - block, - work_reprocessing_tx, - duplicate_cache, - invalid_block_storage, - seen_timestamp, - ) - .await - }) - } - /* - * Import for blocks that we received earlier than their intended slot. - */ - Work::DelayedImportBlock { - peer_id, - block, - seen_timestamp, - } => { - let invalid_block_storage = self.invalid_block_storage.clone(); - task_spawner.spawn_async(worker.process_gossip_verified_block( - peer_id, - *block, - work_reprocessing_tx, - invalid_block_storage, - seen_timestamp, - )) - } - /* - * Voluntary exits received on gossip. - */ - Work::GossipVoluntaryExit { - message_id, - peer_id, - voluntary_exit, + Work::GossipAggregateBatch { + aggregates, + process_batch, } => task_spawner.spawn_blocking(move || { - worker.process_gossip_voluntary_exit(message_id, peer_id, *voluntary_exit) + process_batch(aggregates); }), - /* - * Proposer slashings received on gossip. - */ - Work::GossipProposerSlashing { - message_id, - peer_id, - proposer_slashing, - } => task_spawner.spawn_blocking(move || { - worker.process_gossip_proposer_slashing(message_id, peer_id, *proposer_slashing) - }), - /* - * Attester slashings received on gossip. - */ - Work::GossipAttesterSlashing { - message_id, - peer_id, - attester_slashing, - } => task_spawner.spawn_blocking(move || { - worker.process_gossip_attester_slashing(message_id, peer_id, *attester_slashing) + Work::ChainSegment(process_fn) => task_spawner.spawn_async(async move { + process_fn.await; }), - /* - * Sync committee message verification. - */ - Work::GossipSyncSignature { - message_id, - peer_id, - sync_signature, - subnet_id, - seen_timestamp, - } => task_spawner.spawn_blocking(move || { - worker.process_gossip_sync_committee_signature( - message_id, - peer_id, - *sync_signature, - subnet_id, - seen_timestamp, - ) - }), - /* - * Sync contribution verification. - */ - Work::GossipSyncContribution { - message_id, - peer_id, - sync_contribution, - seen_timestamp, - } => task_spawner.spawn_blocking(move || { - worker.process_sync_committee_contribution( - message_id, - peer_id, - *sync_contribution, - seen_timestamp, - ) - }), - /* - * BLS to execution change verification. - */ - Work::GossipBlsToExecutionChange { - message_id, - peer_id, - bls_to_execution_change, - } => task_spawner.spawn_blocking(move || { - worker.process_gossip_bls_to_execution_change( - message_id, - peer_id, - *bls_to_execution_change, - ) - }), - /* - * Light client finality update verification. - */ - Work::GossipLightClientFinalityUpdate { - message_id, - peer_id, - light_client_finality_update, - seen_timestamp, - } => task_spawner.spawn_blocking(move || { - worker.process_gossip_finality_update( - message_id, - peer_id, - *light_client_finality_update, - seen_timestamp, - ) - }), - /* - * Light client optimistic update verification. - */ - Work::GossipLightClientOptimisticUpdate { - message_id, - peer_id, - light_client_optimistic_update, - seen_timestamp, - } => task_spawner.spawn_blocking(move || { - worker.process_gossip_optimistic_update( - message_id, - peer_id, - *light_client_optimistic_update, - Some(work_reprocessing_tx), - seen_timestamp, - ) + Work::UnknownBlockAttestation { process_fn } => task_spawner.spawn_blocking(process_fn), + Work::UnknownBlockAggregate { process_fn } => task_spawner.spawn_blocking(process_fn), + Work::UnknownLightClientOptimisticUpdate { + parent_root: _, + process_fn, + } => task_spawner.spawn_blocking(process_fn), + Work::DelayedImportBlock { + beacon_block_slot: _, + beacon_block_root: _, + process_fn, + } => task_spawner.spawn_async(process_fn), + Work::RpcBlock { process_fn } => task_spawner.spawn_async(process_fn), + Work::IgnoredRpcBlock { process_fn } => task_spawner.spawn_blocking(process_fn), + Work::GossipBlock(work) => task_spawner.spawn_async(async move { + work.await; }), - /* - * Verification for beacon blocks received during syncing via RPC. - */ - Work::RpcBlock { - block_root, - block, - seen_timestamp, - process_type, - should_process, - } => task_spawner.spawn_async(worker.process_rpc_block( - block_root, - block, - seen_timestamp, - process_type, - work_reprocessing_tx, - duplicate_cache, - should_process, - )), - /* - * Verification for a chain segment (multiple blocks). - */ - Work::ChainSegment { process_id, blocks } => { - let notify_execution_layer = if self - .network_globals - .sync_state - .read() - .is_syncing_finalized() - { - NotifyExecutionLayer::No - } else { - NotifyExecutionLayer::Yes - }; - - task_spawner.spawn_async(async move { - worker - .process_chain_segment(process_id, blocks, notify_execution_layer) - .await - }) + Work::BlocksByRangeRequest(work) | Work::BlocksByRootsRequest(work) => { + task_spawner.spawn_blocking_with_manual_send_idle(work) } - /* - * Processing of Status Messages. - */ - Work::Status { peer_id, message } => { - task_spawner.spawn_blocking(move || worker.process_status(peer_id, message)) + Work::ChainSegmentBackfill(process_fn) => task_spawner.spawn_async(process_fn), + Work::GossipVoluntaryExit(process_fn) + | Work::GossipProposerSlashing(process_fn) + | Work::GossipAttesterSlashing(process_fn) + | Work::GossipSyncSignature(process_fn) + | Work::GossipSyncContribution(process_fn) + | Work::GossipLightClientFinalityUpdate(process_fn) + | Work::GossipLightClientOptimisticUpdate(process_fn) + | Work::Status(process_fn) + | Work::GossipBlsToExecutionChange(process_fn) + | Work::LightClientBootstrapRequest(process_fn) => { + task_spawner.spawn_blocking(process_fn) } - /* - * Processing of range syncing requests from other peers. - */ - Work::BlocksByRangeRequest { - peer_id, - request_id, - request, - } => task_spawner.spawn_blocking_with_manual_send_idle(move |send_idle_on_drop| { - worker.handle_blocks_by_range_request( - sub_executor, - send_idle_on_drop, - peer_id, - request_id, - request, - ) - }), - /* - * Processing of blocks by roots requests from other peers. - */ - Work::BlocksByRootsRequest { - peer_id, - request_id, - request, - } => task_spawner.spawn_blocking_with_manual_send_idle(move |send_idle_on_drop| { - worker.handle_blocks_by_root_request( - sub_executor, - send_idle_on_drop, - peer_id, - request_id, - request, - ) - }), - /* - * Processing of lightclient bootstrap requests from other peers. - */ - Work::LightClientBootstrapRequest { - peer_id, - request_id, - request, - } => task_spawner.spawn_blocking(move || { - worker.handle_light_client_bootstrap(peer_id, request_id, request) - }), - Work::UnknownBlockAttestation { - message_id, - peer_id, - attestation, - subnet_id, - should_import, - seen_timestamp, - } => task_spawner.spawn_blocking(move || { - worker.process_gossip_attestation( - message_id, - peer_id, - attestation, - subnet_id, - should_import, - None, // Do not allow this attestation to be re-processed beyond this point. - seen_timestamp, - ) - }), - Work::UnknownBlockAggregate { - message_id, - peer_id, - aggregate, - seen_timestamp, - } => task_spawner.spawn_blocking(move || { - worker.process_gossip_aggregate( - message_id, - peer_id, - aggregate, - None, - seen_timestamp, - ) - }), - Work::UnknownLightClientOptimisticUpdate { - message_id, - peer_id, - light_client_optimistic_update, - seen_timestamp, - } => task_spawner.spawn_blocking(move || { - worker.process_gossip_optimistic_update( - message_id, - peer_id, - *light_client_optimistic_update, - None, - seen_timestamp, - ) - }), }; } } diff --git a/beacon_node/beacon_processor/src/metrics.rs b/beacon_node/beacon_processor/src/metrics.rs new file mode 100644 index 00000000000..65ab0bd8fc5 --- /dev/null +++ b/beacon_node/beacon_processor/src/metrics.rs @@ -0,0 +1,141 @@ +pub use lighthouse_metrics::*; + +lazy_static::lazy_static! { + + /* + * Gossip processor + */ + pub static ref BEACON_PROCESSOR_WORK_EVENTS_RX_COUNT: Result = try_create_int_counter_vec( + "beacon_processor_work_events_rx_count", + "Count of work events received (but not necessarily processed)", + &["type"] + ); + pub static ref BEACON_PROCESSOR_WORK_EVENTS_IGNORED_COUNT: Result = try_create_int_counter_vec( + "beacon_processor_work_events_ignored_count", + "Count of work events purposefully ignored", + &["type"] + ); + pub static ref BEACON_PROCESSOR_WORK_EVENTS_STARTED_COUNT: Result = try_create_int_counter_vec( + "beacon_processor_work_events_started_count", + "Count of work events which have been started by a worker", + &["type"] + ); + pub static ref BEACON_PROCESSOR_WORKER_TIME: Result = try_create_histogram_vec( + "beacon_processor_worker_time", + "Time taken for a worker to fully process some parcel of work.", + &["type"] + ); + pub static ref BEACON_PROCESSOR_WORKERS_SPAWNED_TOTAL: Result = try_create_int_counter( + "beacon_processor_workers_spawned_total", + "The number of workers ever spawned by the gossip processing pool." + ); + pub static ref BEACON_PROCESSOR_WORKERS_ACTIVE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_workers_active_total", + "Count of active workers in the gossip processing pool." + ); + pub static ref BEACON_PROCESSOR_IDLE_EVENTS_TOTAL: Result = try_create_int_counter( + "beacon_processor_idle_events_total", + "Count of idle events processed by the gossip processor manager." + ); + pub static ref BEACON_PROCESSOR_EVENT_HANDLING_SECONDS: Result = try_create_histogram( + "beacon_processor_event_handling_seconds", + "Time spent handling a new message and allocating it to a queue or worker." + ); + // Gossip blocks. + pub static ref BEACON_PROCESSOR_GOSSIP_BLOCK_QUEUE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_gossip_block_queue_total", + "Count of blocks from gossip waiting to be verified." + ); + // Gossip Exits. + pub static ref BEACON_PROCESSOR_EXIT_QUEUE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_exit_queue_total", + "Count of exits from gossip waiting to be verified." + ); + // Gossip proposer slashings. + pub static ref BEACON_PROCESSOR_PROPOSER_SLASHING_QUEUE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_proposer_slashing_queue_total", + "Count of proposer slashings from gossip waiting to be verified." + ); + // Gossip attester slashings. + pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_QUEUE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_attester_slashing_queue_total", + "Count of attester slashings from gossip waiting to be verified." + ); + // Gossip BLS to execution changes. + pub static ref BEACON_PROCESSOR_BLS_TO_EXECUTION_CHANGE_QUEUE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_bls_to_execution_change_queue_total", + "Count of address changes from gossip waiting to be verified." + ); + // Rpc blocks. + pub static ref BEACON_PROCESSOR_RPC_BLOCK_QUEUE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_rpc_block_queue_total", + "Count of blocks from the rpc waiting to be verified." + ); + // Chain segments. + pub static ref BEACON_PROCESSOR_CHAIN_SEGMENT_QUEUE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_chain_segment_queue_total", + "Count of chain segments from the rpc waiting to be verified." + ); + pub static ref BEACON_PROCESSOR_BACKFILL_CHAIN_SEGMENT_QUEUE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_backfill_chain_segment_queue_total", + "Count of backfill chain segments from the rpc waiting to be verified." + ); + // Unaggregated attestations. + pub static ref BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_QUEUE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_unaggregated_attestation_queue_total", + "Count of unagg. attestations waiting to be processed." + ); + // Aggregated attestations. + pub static ref BEACON_PROCESSOR_AGGREGATED_ATTESTATION_QUEUE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_aggregated_attestation_queue_total", + "Count of agg. attestations waiting to be processed." + ); + // Sync committee messages. + pub static ref BEACON_PROCESSOR_SYNC_MESSAGE_QUEUE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_sync_message_queue_total", + "Count of sync committee messages waiting to be processed." + ); + // Sync contribution. + pub static ref BEACON_PROCESSOR_SYNC_CONTRIBUTION_QUEUE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_sync_contribution_queue_total", + "Count of sync committee contributions waiting to be processed." + ); + + /* + * Attestation reprocessing queue metrics. + */ + pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_TOTAL: Result = + try_create_int_gauge_vec( + "beacon_processor_reprocessing_queue_total", + "Count of items in a reprocessing queue.", + &["type"] + ); + pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_EXPIRED_ATTESTATIONS: Result = try_create_int_counter( + "beacon_processor_reprocessing_queue_expired_attestations", + "Number of queued attestations which have expired before a matching block has been found." + ); + pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_MATCHED_ATTESTATIONS: Result = try_create_int_counter( + "beacon_processor_reprocessing_queue_matched_attestations", + "Number of queued attestations where as matching block has been imported." + ); + + /* + * Light client update reprocessing queue metrics. + */ + pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_EXPIRED_OPTIMISTIC_UPDATES: Result = try_create_int_counter( + "beacon_processor_reprocessing_queue_expired_optimistic_updates", + "Number of queued light client optimistic updates which have expired before a matching block has been found." + ); + pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_MATCHED_OPTIMISTIC_UPDATES: Result = try_create_int_counter( + "beacon_processor_reprocessing_queue_matched_optimistic_updates", + "Number of queued light client optimistic updates where as matching block has been imported." + ); + + /// Errors and Debugging Stats + pub static ref BEACON_PROCESSOR_SEND_ERROR_PER_WORK_TYPE: Result = + try_create_int_counter_vec( + "beacon_processor_send_error_per_work_type", + "Total number of beacon processor send error per work type", + &["type"] + ); +} diff --git a/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs b/beacon_node/beacon_processor/src/work_reprocessing_queue.rs similarity index 85% rename from beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs rename to beacon_node/beacon_processor/src/work_reprocessing_queue.rs index 427be6d5138..608f634d537 100644 --- a/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs +++ b/beacon_node/beacon_processor/src/work_reprocessing_queue.rs @@ -10,23 +10,18 @@ //! //! Aggregated and unaggregated attestations that failed verification due to referencing an unknown //! block will be re-queued until their block is imported, or until they expire. -use super::MAX_SCHEDULED_WORK_QUEUE_LEN; -use crate::beacon_processor::{ChainSegmentProcessId, Work, WorkEvent}; use crate::metrics; -use crate::sync::manager::BlockProcessType; -use beacon_chain::{BeaconChainTypes, GossipVerifiedBlock, MAXIMUM_GOSSIP_CLOCK_DISPARITY}; +use crate::{AsyncFn, BlockingFn, Work, WorkEvent}; use fnv::FnvHashMap; use futures::task::Poll; use futures::{Stream, StreamExt}; use itertools::Itertools; -use lighthouse_network::{MessageId, PeerId}; use logging::TimeLatch; use slog::{crit, debug, error, trace, warn, Logger}; use slot_clock::SlotClock; use std::collections::{HashMap, HashSet}; use std::future::Future; use std::pin::Pin; -use std::sync::Arc; use std::task::Context; use std::time::Duration; use strum::AsRefStr; @@ -34,10 +29,7 @@ use task_executor::TaskExecutor; use tokio::sync::mpsc::{self, Receiver, Sender}; use tokio::time::error::Error as TimeError; use tokio_util::time::delay_queue::{DelayQueue, Key as DelayKey}; -use types::{ - Attestation, EthSpec, Hash256, LightClientOptimisticUpdate, SignedAggregateAndProof, - SignedBeaconBlock, SubnetId, -}; +use types::{EthSpec, Hash256, Slot}; const TASK_NAME: &str = "beacon_processor_reprocess_queue"; const GOSSIP_BLOCKS: &str = "gossip_blocks"; @@ -47,7 +39,7 @@ const LIGHT_CLIENT_UPDATES: &str = "lc_updates"; /// Queue blocks for re-processing with an `ADDITIONAL_QUEUED_BLOCK_DELAY` after the slot starts. /// This is to account for any slight drift in the system clock. -const ADDITIONAL_QUEUED_BLOCK_DELAY: Duration = Duration::from_millis(5); +pub const ADDITIONAL_QUEUED_BLOCK_DELAY: Duration = Duration::from_millis(5); /// For how long to queue aggregated and unaggregated attestations for re-processing. pub const QUEUED_ATTESTATION_DELAY: Duration = Duration::from_secs(12); @@ -84,12 +76,12 @@ pub const BACKFILL_SCHEDULE_IN_SLOT: [(u32, u32); 3] = [ /// Messages that the scheduler can receive. #[derive(AsRefStr)] -pub enum ReprocessQueueMessage { +pub enum ReprocessQueueMessage { /// A block that has been received early and we should queue for later processing. - EarlyBlock(QueuedGossipBlock), + EarlyBlock(QueuedGossipBlock), /// A gossip block for hash `X` is being imported, we should queue the rpc block for the same /// hash until the gossip block is imported. - RpcBlock(QueuedRpcBlock), + RpcBlock(QueuedRpcBlock), /// A block that was successfully processed. We use this to handle attestations and light client updates /// for unknown blocks. BlockImported { @@ -97,139 +89,127 @@ pub enum ReprocessQueueMessage { parent_root: Hash256, }, /// An unaggregated attestation that references an unknown block. - UnknownBlockUnaggregate(QueuedUnaggregate), + UnknownBlockUnaggregate(QueuedUnaggregate), /// An aggregated attestation that references an unknown block. - UnknownBlockAggregate(QueuedAggregate), + UnknownBlockAggregate(QueuedAggregate), /// A light client optimistic update that references a parent root that has not been seen as a parent. - UnknownLightClientOptimisticUpdate(QueuedLightClientUpdate), + UnknownLightClientOptimisticUpdate(QueuedLightClientUpdate), /// A new backfill batch that needs to be scheduled for processing. - BackfillSync(QueuedBackfillBatch), + BackfillSync(QueuedBackfillBatch), } /// Events sent by the scheduler once they are ready for re-processing. -pub enum ReadyWork { - Block(QueuedGossipBlock), - RpcBlock(QueuedRpcBlock), - Unaggregate(QueuedUnaggregate), - Aggregate(QueuedAggregate), - LightClientUpdate(QueuedLightClientUpdate), - BackfillSync(QueuedBackfillBatch), +pub enum ReadyWork { + Block(QueuedGossipBlock), + RpcBlock(QueuedRpcBlock), + IgnoredRpcBlock(IgnoredRpcBlock), + Unaggregate(QueuedUnaggregate), + Aggregate(QueuedAggregate), + LightClientUpdate(QueuedLightClientUpdate), + BackfillSync(QueuedBackfillBatch), } /// An Attestation for which the corresponding block was not seen while processing, queued for /// later. -pub struct QueuedUnaggregate { - pub peer_id: PeerId, - pub message_id: MessageId, - pub attestation: Box>, - pub subnet_id: SubnetId, - pub should_import: bool, - pub seen_timestamp: Duration, +pub struct QueuedUnaggregate { + pub beacon_block_root: Hash256, + pub process_fn: BlockingFn, } /// An aggregated attestation for which the corresponding block was not seen while processing, queued for /// later. -pub struct QueuedAggregate { - pub peer_id: PeerId, - pub message_id: MessageId, - pub attestation: Box>, - pub seen_timestamp: Duration, +pub struct QueuedAggregate { + pub beacon_block_root: Hash256, + pub process_fn: BlockingFn, } /// A light client update for which the corresponding parent block was not seen while processing, /// queued for later. -pub struct QueuedLightClientUpdate { - pub peer_id: PeerId, - pub message_id: MessageId, - pub light_client_optimistic_update: Box>, +pub struct QueuedLightClientUpdate { pub parent_root: Hash256, - pub seen_timestamp: Duration, + pub process_fn: BlockingFn, } /// A block that arrived early and has been queued for later import. -pub struct QueuedGossipBlock { - pub peer_id: PeerId, - pub block: Box>, - pub seen_timestamp: Duration, +pub struct QueuedGossipBlock { + pub beacon_block_slot: Slot, + pub beacon_block_root: Hash256, + pub process_fn: AsyncFn, } /// A block that arrived for processing when the same block was being imported over gossip. /// It is queued for later import. -pub struct QueuedRpcBlock { - pub block_root: Hash256, - pub block: Arc>, - pub process_type: BlockProcessType, - pub seen_timestamp: Duration, - /// Indicates if the beacon chain should process this block or not. - /// We use this to ignore block processing when rpc block queues are full. - pub should_process: bool, +pub struct QueuedRpcBlock { + pub beacon_block_root: Hash256, + /// Processes/imports the block. + pub process_fn: AsyncFn, + /// Ignores the block. + pub ignore_fn: BlockingFn, } -/// A backfill batch work that has been queued for processing later. -#[derive(Clone)] -pub struct QueuedBackfillBatch { - pub process_id: ChainSegmentProcessId, - pub blocks: Vec>>, +/// A block that arrived for processing when the same block was being imported over gossip. +/// It is queued for later import. +pub struct IgnoredRpcBlock { + pub process_fn: BlockingFn, } -impl TryFrom> for QueuedBackfillBatch { +/// A backfill batch work that has been queued for processing later. +pub struct QueuedBackfillBatch(pub AsyncFn); + +impl TryFrom> for QueuedBackfillBatch { type Error = WorkEvent; fn try_from(event: WorkEvent) -> Result> { match event { WorkEvent { - work: - Work::ChainSegment { - process_id: process_id @ ChainSegmentProcessId::BackSyncBatchId(_), - blocks, - }, + work: Work::ChainSegmentBackfill(process_fn), .. - } => Ok(QueuedBackfillBatch { process_id, blocks }), + } => Ok(QueuedBackfillBatch(process_fn)), _ => Err(event), } } } -impl From> for WorkEvent { - fn from(queued_backfill_batch: QueuedBackfillBatch) -> WorkEvent { - WorkEvent::chain_segment( - queued_backfill_batch.process_id, - queued_backfill_batch.blocks, - ) +impl From for WorkEvent { + fn from(queued_backfill_batch: QueuedBackfillBatch) -> WorkEvent { + WorkEvent { + drop_during_sync: false, + work: Work::ChainSegmentBackfill(queued_backfill_batch.0), + } } } /// Unifies the different messages processed by the block delay queue. -enum InboundEvent { +enum InboundEvent { /// A gossip block that was queued for later processing and is ready for import. - ReadyGossipBlock(QueuedGossipBlock), + ReadyGossipBlock(QueuedGossipBlock), /// A rpc block that was queued because the same gossip block was being imported /// will now be retried for import. - ReadyRpcBlock(QueuedRpcBlock), + ReadyRpcBlock(QueuedRpcBlock), /// An aggregated or unaggregated attestation is ready for re-processing. ReadyAttestation(QueuedAttestationId), /// A light client update that is ready for re-processing. ReadyLightClientUpdate(QueuedLightClientUpdateId), /// A backfill batch that was queued is ready for processing. - ReadyBackfillSync(QueuedBackfillBatch), + ReadyBackfillSync(QueuedBackfillBatch), /// A `DelayQueue` returned an error. DelayQueueError(TimeError, &'static str), /// A message sent to the `ReprocessQueue` - Msg(ReprocessQueueMessage), + Msg(ReprocessQueueMessage), } /// Manages scheduling works that need to be later re-processed. -struct ReprocessQueue { +struct ReprocessQueue { /// Receiver of messages relevant to schedule works for reprocessing. - work_reprocessing_rx: Receiver>, + work_reprocessing_rx: Receiver, /// Sender of works once they become ready - ready_work_tx: Sender>, + ready_work_tx: Sender, /* Queues */ /// Queue to manage scheduled early blocks. - gossip_block_delay_queue: DelayQueue>, + gossip_block_delay_queue: DelayQueue, /// Queue to manage scheduled early blocks. - rpc_block_delay_queue: DelayQueue>, + rpc_block_delay_queue: DelayQueue, /// Queue to manage scheduled attestations. attestations_delay_queue: DelayQueue, /// Queue to manage scheduled light client updates. @@ -239,17 +219,17 @@ struct ReprocessQueue { /// Queued blocks. queued_gossip_block_roots: HashSet, /// Queued aggregated attestations. - queued_aggregates: FnvHashMap, DelayKey)>, + queued_aggregates: FnvHashMap, /// Queued attestations. - queued_unaggregates: FnvHashMap, DelayKey)>, + queued_unaggregates: FnvHashMap, /// Attestations (aggregated and unaggregated) per root. awaiting_attestations_per_root: HashMap>, /// Queued Light Client Updates. - queued_lc_updates: FnvHashMap, DelayKey)>, + queued_lc_updates: FnvHashMap, /// Light Client Updates per parent_root. awaiting_lc_updates_per_parent_root: HashMap>, /// Queued backfill batches - queued_backfill_batches: Vec>, + queued_backfill_batches: Vec, /* Aux */ /// Next attestation id, used for both aggregated and unaggregated attestations @@ -260,7 +240,7 @@ struct ReprocessQueue { attestation_delay_debounce: TimeLatch, lc_update_delay_debounce: TimeLatch, next_backfill_batch_event: Option>>, - slot_clock: Pin>, + slot_clock: Pin>, } pub type QueuedLightClientUpdateId = usize; @@ -271,20 +251,20 @@ enum QueuedAttestationId { Unaggregate(usize), } -impl QueuedAggregate { +impl QueuedAggregate { pub fn beacon_block_root(&self) -> &Hash256 { - &self.attestation.message.aggregate.data.beacon_block_root + &self.beacon_block_root } } -impl QueuedUnaggregate { +impl QueuedUnaggregate { pub fn beacon_block_root(&self) -> &Hash256 { - &self.attestation.data.beacon_block_root + &self.beacon_block_root } } -impl Stream for ReprocessQueue { - type Item = InboundEvent; +impl Stream for ReprocessQueue { + type Item = InboundEvent; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { // NOTE: implementing `Stream` is not necessary but allows to maintain the future selection @@ -375,16 +355,13 @@ impl Stream for ReprocessQueue { /// Starts the job that manages scheduling works that need re-processing. The returned `Sender` /// gives the communicating channel to receive those works. Once a work is ready, it is sent back /// via `ready_work_tx`. -pub fn spawn_reprocess_scheduler( - ready_work_tx: Sender>, +pub fn spawn_reprocess_scheduler( + ready_work_tx: Sender, + work_reprocessing_rx: Receiver, executor: &TaskExecutor, - slot_clock: T::SlotClock, + slot_clock: S, log: Logger, -) -> Sender> { - let (work_reprocessing_tx, work_reprocessing_rx) = mpsc::channel(MAX_SCHEDULED_WORK_QUEUE_LEN); - // Basic sanity check. - assert!(ADDITIONAL_QUEUED_BLOCK_DELAY < MAXIMUM_GOSSIP_CLOCK_DISPARITY); - +) { let mut queue = ReprocessQueue { work_reprocessing_rx, ready_work_tx, @@ -423,19 +400,17 @@ pub fn spawn_reprocess_scheduler( }, TASK_NAME, ); - - work_reprocessing_tx } -impl ReprocessQueue { - fn handle_message(&mut self, msg: InboundEvent, slot_clock: &T::SlotClock, log: &Logger) { +impl ReprocessQueue { + fn handle_message(&mut self, msg: InboundEvent, slot_clock: &S, log: &Logger) { use ReprocessQueueMessage::*; match msg { // Some block has been indicated as "early" and should be processed when the // appropriate slot arrives. InboundEvent::Msg(EarlyBlock(early_block)) => { - let block_slot = early_block.block.block.slot(); - let block_root = early_block.block.block_root; + let block_slot = early_block.beacon_block_slot; + let block_root = early_block.beacon_block_root; // Don't add the same block to the queue twice. This prevents DoS attacks. if self.queued_gossip_block_roots.contains(&block_root) { @@ -494,7 +469,7 @@ impl ReprocessQueue { // for the same block hash is being imported. We wait for `QUEUED_RPC_BLOCK_DELAY` // and then send the rpc block back for processing assuming the gossip import // has completed by then. - InboundEvent::Msg(RpcBlock(mut rpc_block)) => { + InboundEvent::Msg(RpcBlock(rpc_block)) => { // Check to ensure this won't over-fill the queue. if self.rpc_block_delay_queue.len() >= MAXIMUM_QUEUED_BLOCKS { if self.rpc_block_debounce.elapsed() { @@ -507,10 +482,11 @@ impl ReprocessQueue { } // Return the block to the beacon processor signalling to // ignore processing for this block - rpc_block.should_process = false; if self .ready_work_tx - .try_send(ReadyWork::RpcBlock(rpc_block)) + .try_send(ReadyWork::IgnoredRpcBlock(IgnoredRpcBlock { + process_fn: rpc_block.ignore_fn, + })) .is_err() { error!( @@ -529,7 +505,7 @@ impl ReprocessQueue { debug!( log, "Sending rpc block for reprocessing"; - "block_root" => %queued_rpc_block.block.canonical_root() + "block_root" => %queued_rpc_block.beacon_block_root ); if self .ready_work_tx @@ -767,7 +743,7 @@ impl ReprocessQueue { } // A block that was queued for later processing is now ready to be processed. InboundEvent::ReadyGossipBlock(ready_block) => { - let block_root = ready_block.block.block_root; + let block_root = ready_block.beacon_block_root; if !self.queued_gossip_block_roots.remove(&block_root) { // Log an error to alert that we've made a bad assumption about how this @@ -885,18 +861,28 @@ impl ReprocessQueue { "millis_from_slot_start" => millis_from_slot_start ); - if self + match self .ready_work_tx - .try_send(ReadyWork::BackfillSync(queued_backfill_batch.clone())) - .is_err() + .try_send(ReadyWork::BackfillSync(queued_backfill_batch)) { - error!( + // The message was sent successfully. + Ok(()) => (), + // The message was not sent, recover it from the returned `Err`. + Err(mpsc::error::TrySendError::Full(ReadyWork::BackfillSync(batch))) + | Err(mpsc::error::TrySendError::Closed(ReadyWork::BackfillSync(batch))) => { + error!( + log, + "Failed to send scheduled backfill work"; + "info" => "sending work back to queue" + ); + self.queued_backfill_batches.insert(0, batch) + } + // The message was not sent and we didn't get the correct + // return result. This is a logic error. + _ => crit!( log, - "Failed to send scheduled backfill work"; - "info" => "sending work back to queue" - ); - self.queued_backfill_batches - .insert(0, queued_backfill_batch); + "Unexpected return from try_send error"; + ), } } } @@ -927,7 +913,7 @@ impl ReprocessQueue { // only recompute the `next_backfill_batch_event` if there are backfill batches in the queue if !self.queued_backfill_batches.is_empty() { self.next_backfill_batch_event = Some(Box::pin(tokio::time::sleep( - ReprocessQueue::::duration_until_next_backfill_batch_event(&self.slot_clock), + ReprocessQueue::::duration_until_next_backfill_batch_event(&self.slot_clock), ))); } else { self.next_backfill_batch_event = None @@ -936,7 +922,7 @@ impl ReprocessQueue { /// Returns duration until the next scheduled processing time. The schedule ensure that backfill /// processing is done in windows of time that aren't critical - fn duration_until_next_backfill_batch_event(slot_clock: &T::SlotClock) -> Duration { + fn duration_until_next_backfill_batch_event(slot_clock: &S) -> Duration { let slot_duration = slot_clock.slot_duration(); slot_clock .millis_from_current_slot_start() @@ -966,16 +952,9 @@ impl ReprocessQueue { #[cfg(test)] mod tests { use super::*; - use beacon_chain::builder::Witness; - use beacon_chain::eth1_chain::CachingEth1Backend; use slot_clock::TestingSlotClock; - use store::MemoryStore; - use types::MainnetEthSpec as E; use types::Slot; - type TestBeaconChainType = - Witness, E, MemoryStore, MemoryStore>; - #[test] fn backfill_processing_schedule_calculation() { let slot_duration = Duration::from_secs(12); @@ -988,7 +967,7 @@ mod tests { for &event_duration_from_slot_start in event_times.iter() { let duration_to_next_event = - ReprocessQueue::::duration_until_next_backfill_batch_event( + ReprocessQueue::::duration_until_next_backfill_batch_event( &slot_clock, ); @@ -1005,7 +984,7 @@ mod tests { // check for next event beyond the current slot let duration_to_next_slot = slot_clock.duration_to_next_slot().unwrap(); let duration_to_next_event = - ReprocessQueue::::duration_until_next_backfill_batch_event( + ReprocessQueue::::duration_until_next_backfill_batch_event( &slot_clock, ); assert_eq!( diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 64c79ea668b..72b6a6c7d47 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -43,3 +43,5 @@ slasher = { path = "../../slasher" } slasher_service = { path = "../../slasher/service" } monitoring_api = {path = "../../common/monitoring_api"} execution_layer = { path = "../execution_layer" } +beacon_processor = { path = "../beacon_processor" } +num_cpus = "1.13.0" diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index e05b92a2779..f3ed7f65a90 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -13,6 +13,10 @@ use beacon_chain::{ store::{HotColdDB, ItemStore, LevelDB, StoreConfig}, BeaconChain, BeaconChainTypes, Eth1ChainBackend, ServerSentEventHandler, }; +use beacon_processor::{ + work_reprocessing_queue::ReprocessQueueMessage, BeaconProcessor, BeaconProcessorSend, + WorkEvent, MAX_SCHEDULED_WORK_QUEUE_LEN, MAX_WORK_EVENT_QUEUE_LEN, +}; use environment::RuntimeContext; use eth1::{Config as Eth1Config, Service as Eth1Service}; use eth2::{ @@ -27,12 +31,13 @@ use network::{NetworkConfig, NetworkSenders, NetworkService}; use slasher::Slasher; use slasher_service::SlasherService; use slog::{debug, info, warn, Logger}; +use std::cmp; use std::net::TcpListener; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; use timer::spawn_timer; -use tokio::sync::oneshot; +use tokio::sync::{mpsc, oneshot}; use types::{ test_utils::generate_deterministic_keypairs, BeaconState, ChainSpec, EthSpec, ExecutionBlockHash, Hash256, SignedBeaconBlock, @@ -72,6 +77,10 @@ pub struct ClientBuilder { http_metrics_config: http_metrics::Config, slasher: Option>>, eth_spec_instance: T::EthSpec, + beacon_processor_send: BeaconProcessorSend, + beacon_processor_receive: mpsc::Receiver>, + work_reprocessing_tx: mpsc::Sender, + work_reprocessing_rx: mpsc::Receiver, } impl @@ -87,6 +96,10 @@ where /// /// The `eth_spec_instance` parameter is used to concretize `TEthSpec`. pub fn new(eth_spec_instance: TEthSpec) -> Self { + let (beacon_processor_send, beacon_processor_receive) = + mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN); + let (work_reprocessing_tx, work_reprocessing_rx) = + mpsc::channel(MAX_SCHEDULED_WORK_QUEUE_LEN); Self { slot_clock: None, store: None, @@ -104,6 +117,10 @@ where http_metrics_config: <_>::default(), slasher: None, eth_spec_instance, + beacon_processor_send: BeaconProcessorSend(beacon_processor_send), + beacon_processor_receive, + work_reprocessing_tx, + work_reprocessing_rx, } } @@ -568,6 +585,8 @@ where gossipsub_registry .as_mut() .map(|registry| registry.sub_registry_with_prefix("gossipsub")), + self.beacon_processor_send.clone(), + self.work_reprocessing_tx.clone(), ) .await .map_err(|e| format!("Failed to start network: {:?}", e))?; @@ -755,6 +774,27 @@ where } if let Some(beacon_chain) = self.beacon_chain.as_ref() { + if let Some(network_globals) = &self.network_globals { + let beacon_processor_context = runtime_context.service_context("bproc".into()); + BeaconProcessor { + network_globals: network_globals.clone(), + executor: beacon_processor_context.executor.clone(), + max_workers: cmp::max(1, num_cpus::get()), + current_workers: 0, + enable_backfill_rate_limiting: beacon_chain + .config + .enable_backfill_rate_limiting, + log: beacon_processor_context.log().clone(), + } + .spawn_manager( + self.beacon_processor_receive, + self.work_reprocessing_tx, + self.work_reprocessing_rx, + None, + beacon_chain.slot_clock.clone(), + ); + } + let state_advance_context = runtime_context.service_context("state_advance".into()); let state_advance_log = state_advance_context.log().clone(); spawn_state_advance_timer( diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index aa1827787c6..a5cc12bbc55 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -11,7 +11,6 @@ matches = "0.1.8" exit-future = "0.2.0" slog-term = "2.6.0" slog-async = "2.5.0" -environment = { path = "../../lighthouse/environment" } [dependencies] beacon_chain = { path = "../beacon_chain" } @@ -46,4 +45,7 @@ derivative = "2.2.0" delay_map = "0.3.0" ethereum-types = { version = "0.14.1", optional = true } operation_pool = { path = "../operation_pool" } -execution_layer = { path = "../execution_layer" } \ No newline at end of file +execution_layer = { path = "../execution_layer" } +beacon_processor = { path = "../beacon_processor" } +parking_lot = "0.12.0" +environment = { path = "../../lighthouse/environment" } \ No newline at end of file diff --git a/beacon_node/network/src/beacon_processor/worker/mod.rs b/beacon_node/network/src/beacon_processor/worker/mod.rs deleted file mode 100644 index 1cbc64b6329..00000000000 --- a/beacon_node/network/src/beacon_processor/worker/mod.rs +++ /dev/null @@ -1,51 +0,0 @@ -use super::work_reprocessing_queue::ReprocessQueueMessage; -use crate::{service::NetworkMessage, sync::SyncMessage}; -use beacon_chain::{BeaconChain, BeaconChainTypes}; -use slog::{debug, Logger}; -use std::sync::Arc; -use tokio::sync::mpsc; - -mod gossip_methods; -mod rpc_methods; -mod sync_methods; - -pub use gossip_methods::{GossipAggregatePackage, GossipAttestationPackage}; -pub use sync_methods::ChainSegmentProcessId; - -pub(crate) const FUTURE_SLOT_TOLERANCE: u64 = 1; - -/// Contains the context necessary to import blocks, attestations, etc to the beacon chain. -pub struct Worker { - pub chain: Arc>, - pub network_tx: mpsc::UnboundedSender>, - pub sync_tx: mpsc::UnboundedSender>, - pub log: Logger, -} - -impl Worker { - /// Send a message to `sync_tx`. - /// - /// Creates a log if there is an internal error. - fn send_sync_message(&self, message: SyncMessage) { - self.sync_tx.send(message).unwrap_or_else(|e| { - debug!(self.log, "Could not send message to the sync service"; - "error" => %e) - }); - } - - /// Send a message to `network_tx`. - /// - /// Creates a log if there is an internal error. - fn send_network_message(&self, message: NetworkMessage) { - self.network_tx.send(message).unwrap_or_else(|e| { - debug!(self.log, "Could not send message to the network service. Likely shutdown"; - "error" => %e) - }); - } -} - -/// Contains the necessary items for a worker to do their job. -pub struct Toolbox { - pub idle_tx: mpsc::Sender<()>, - pub work_reprocessing_tx: mpsc::Sender>, -} diff --git a/beacon_node/network/src/lib.rs b/beacon_node/network/src/lib.rs index 648c636acca..da64368b16d 100644 --- a/beacon_node/network/src/lib.rs +++ b/beacon_node/network/src/lib.rs @@ -6,10 +6,10 @@ pub mod error; #[allow(clippy::mutable_key_type)] // PeerId in hashmaps are no longer permitted by clippy pub mod service; -mod beacon_processor; #[allow(clippy::mutable_key_type)] // PeerId in hashmaps are no longer permitted by clippy mod metrics; mod nat; +mod network_beacon_processor; mod persisted_dht; mod router; mod status; diff --git a/beacon_node/network/src/metrics.rs b/beacon_node/network/src/metrics.rs index 27d7dc9625d..0144824861d 100644 --- a/beacon_node/network/src/metrics.rs +++ b/beacon_node/network/src/metrics.rs @@ -49,47 +49,8 @@ lazy_static! { /* * Gossip processor */ - pub static ref BEACON_PROCESSOR_WORK_EVENTS_RX_COUNT: Result = try_create_int_counter_vec( - "beacon_processor_work_events_rx_count", - "Count of work events received (but not necessarily processed)", - &["type"] - ); - pub static ref BEACON_PROCESSOR_WORK_EVENTS_IGNORED_COUNT: Result = try_create_int_counter_vec( - "beacon_processor_work_events_ignored_count", - "Count of work events purposefully ignored", - &["type"] - ); - pub static ref BEACON_PROCESSOR_WORK_EVENTS_STARTED_COUNT: Result = try_create_int_counter_vec( - "beacon_processor_work_events_started_count", - "Count of work events which have been started by a worker", - &["type"] - ); - pub static ref BEACON_PROCESSOR_WORKER_TIME: Result = try_create_histogram_vec( - "beacon_processor_worker_time", - "Time taken for a worker to fully process some parcel of work.", - &["type"] - ); - pub static ref BEACON_PROCESSOR_WORKERS_SPAWNED_TOTAL: Result = try_create_int_counter( - "beacon_processor_workers_spawned_total", - "The number of workers ever spawned by the gossip processing pool." - ); - pub static ref BEACON_PROCESSOR_WORKERS_ACTIVE_TOTAL: Result = try_create_int_gauge( - "beacon_processor_workers_active_total", - "Count of active workers in the gossip processing pool." - ); - pub static ref BEACON_PROCESSOR_IDLE_EVENTS_TOTAL: Result = try_create_int_counter( - "beacon_processor_idle_events_total", - "Count of idle events processed by the gossip processor manager." - ); - pub static ref BEACON_PROCESSOR_EVENT_HANDLING_SECONDS: Result = try_create_histogram( - "beacon_processor_event_handling_seconds", - "Time spent handling a new message and allocating it to a queue or worker." - ); + // Gossip blocks. - pub static ref BEACON_PROCESSOR_GOSSIP_BLOCK_QUEUE_TOTAL: Result = try_create_int_gauge( - "beacon_processor_gossip_block_queue_total", - "Count of blocks from gossip waiting to be verified." - ); pub static ref BEACON_PROCESSOR_GOSSIP_BLOCK_VERIFIED_TOTAL: Result = try_create_int_counter( "beacon_processor_gossip_block_verified_total", "Total number of gossip blocks verified for propagation." @@ -107,10 +68,6 @@ lazy_static! { "Whenever a gossip block is received early this metrics is set to how early that block was." ); // Gossip Exits. - pub static ref BEACON_PROCESSOR_EXIT_QUEUE_TOTAL: Result = try_create_int_gauge( - "beacon_processor_exit_queue_total", - "Count of exits from gossip waiting to be verified." - ); pub static ref BEACON_PROCESSOR_EXIT_VERIFIED_TOTAL: Result = try_create_int_counter( "beacon_processor_exit_verified_total", "Total number of voluntary exits verified for propagation." @@ -120,10 +77,6 @@ lazy_static! { "Total number of voluntary exits imported to the op pool." ); // Gossip proposer slashings. - pub static ref BEACON_PROCESSOR_PROPOSER_SLASHING_QUEUE_TOTAL: Result = try_create_int_gauge( - "beacon_processor_proposer_slashing_queue_total", - "Count of proposer slashings from gossip waiting to be verified." - ); pub static ref BEACON_PROCESSOR_PROPOSER_SLASHING_VERIFIED_TOTAL: Result = try_create_int_counter( "beacon_processor_proposer_slashing_verified_total", "Total number of proposer slashings verified for propagation." @@ -133,10 +86,6 @@ lazy_static! { "Total number of proposer slashings imported to the op pool." ); // Gossip attester slashings. - pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_QUEUE_TOTAL: Result = try_create_int_gauge( - "beacon_processor_attester_slashing_queue_total", - "Count of attester slashings from gossip waiting to be verified." - ); pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_VERIFIED_TOTAL: Result = try_create_int_counter( "beacon_processor_attester_slashing_verified_total", "Total number of attester slashings verified for propagation." @@ -146,10 +95,6 @@ lazy_static! { "Total number of attester slashings imported to the op pool." ); // Gossip BLS to execution changes. - pub static ref BEACON_PROCESSOR_BLS_TO_EXECUTION_CHANGE_QUEUE_TOTAL: Result = try_create_int_gauge( - "beacon_processor_bls_to_execution_change_queue_total", - "Count of address changes from gossip waiting to be verified." - ); pub static ref BEACON_PROCESSOR_BLS_TO_EXECUTION_CHANGE_VERIFIED_TOTAL: Result = try_create_int_counter( "beacon_processor_bls_to_execution_change_verified_total", "Total number of address changes verified for propagation." @@ -159,23 +104,11 @@ lazy_static! { "Total number of address changes imported to the op pool." ); // Rpc blocks. - pub static ref BEACON_PROCESSOR_RPC_BLOCK_QUEUE_TOTAL: Result = try_create_int_gauge( - "beacon_processor_rpc_block_queue_total", - "Count of blocks from the rpc waiting to be verified." - ); pub static ref BEACON_PROCESSOR_RPC_BLOCK_IMPORTED_TOTAL: Result = try_create_int_counter( "beacon_processor_rpc_block_imported_total", "Total number of gossip blocks imported to fork choice, etc." ); // Chain segments. - pub static ref BEACON_PROCESSOR_CHAIN_SEGMENT_QUEUE_TOTAL: Result = try_create_int_gauge( - "beacon_processor_chain_segment_queue_total", - "Count of chain segments from the rpc waiting to be verified." - ); - pub static ref BEACON_PROCESSOR_BACKFILL_CHAIN_SEGMENT_QUEUE_TOTAL: Result = try_create_int_gauge( - "beacon_processor_backfill_chain_segment_queue_total", - "Count of backfill chain segments from the rpc waiting to be verified." - ); pub static ref BEACON_PROCESSOR_CHAIN_SEGMENT_SUCCESS_TOTAL: Result = try_create_int_counter( "beacon_processor_chain_segment_success_total", "Total number of chain segments successfully processed." @@ -193,10 +126,6 @@ lazy_static! { "Total number of backfill chain segments that failed processing." ); // Unaggregated attestations. - pub static ref BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_QUEUE_TOTAL: Result = try_create_int_gauge( - "beacon_processor_unaggregated_attestation_queue_total", - "Count of unagg. attestations waiting to be processed." - ); pub static ref BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_VERIFIED_TOTAL: Result = try_create_int_counter( "beacon_processor_unaggregated_attestation_verified_total", "Total number of unaggregated attestations verified for gossip." @@ -210,10 +139,6 @@ lazy_static! { "Total number of unaggregated attestations that referenced an unknown block and were re-queued." ); // Aggregated attestations. - pub static ref BEACON_PROCESSOR_AGGREGATED_ATTESTATION_QUEUE_TOTAL: Result = try_create_int_gauge( - "beacon_processor_aggregated_attestation_queue_total", - "Count of agg. attestations waiting to be processed." - ); pub static ref BEACON_PROCESSOR_AGGREGATED_ATTESTATION_VERIFIED_TOTAL: Result = try_create_int_counter( "beacon_processor_aggregated_attestation_verified_total", "Total number of aggregated attestations verified for gossip." @@ -227,10 +152,6 @@ lazy_static! { "Total number of aggregated attestations that referenced an unknown block and were re-queued." ); // Sync committee messages. - pub static ref BEACON_PROCESSOR_SYNC_MESSAGE_QUEUE_TOTAL: Result = try_create_int_gauge( - "beacon_processor_sync_message_queue_total", - "Count of sync committee messages waiting to be processed." - ); pub static ref BEACON_PROCESSOR_SYNC_MESSAGE_VERIFIED_TOTAL: Result = try_create_int_counter( "beacon_processor_sync_message_verified_total", "Total number of sync committee messages verified for gossip." @@ -240,10 +161,6 @@ lazy_static! { "Total number of sync committee messages imported to fork choice, etc." ); // Sync contribution. - pub static ref BEACON_PROCESSOR_SYNC_CONTRIBUTION_QUEUE_TOTAL: Result = try_create_int_gauge( - "beacon_processor_sync_contribution_queue_total", - "Count of sync committee contributions waiting to be processed." - ); pub static ref BEACON_PROCESSOR_SYNC_CONTRIBUTION_VERIFIED_TOTAL: Result = try_create_int_counter( "beacon_processor_sync_contribution_verified_total", "Total number of sync committee contributions verified for gossip." @@ -279,12 +196,6 @@ lazy_static! { "Gossipsub light_client_optimistic_update errors per error type", &["type"] ); - pub static ref BEACON_PROCESSOR_SEND_ERROR_PER_WORK_TYPE: Result = - try_create_int_counter_vec( - "beacon_processor_send_error_per_work_type", - "Total number of beacon processor send error per work type", - &["type"] - ); /* @@ -371,35 +282,9 @@ lazy_static! { "Count of times when a gossip block arrived from the network later than the attestation deadline.", ); - /* - * Attestation reprocessing queue metrics. - */ - pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_TOTAL: Result = - try_create_int_gauge_vec( - "beacon_processor_reprocessing_queue_total", - "Count of items in a reprocessing queue.", - &["type"] - ); - pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_EXPIRED_ATTESTATIONS: Result = try_create_int_counter( - "beacon_processor_reprocessing_queue_expired_attestations", - "Number of queued attestations which have expired before a matching block has been found." - ); - pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_MATCHED_ATTESTATIONS: Result = try_create_int_counter( - "beacon_processor_reprocessing_queue_matched_attestations", - "Number of queued attestations where as matching block has been imported." - ); - /* * Light client update reprocessing queue metrics. */ - pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_EXPIRED_OPTIMISTIC_UPDATES: Result = try_create_int_counter( - "beacon_processor_reprocessing_queue_expired_optimistic_updates", - "Number of queued light client optimistic updates which have expired before a matching block has been found." - ); - pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_MATCHED_OPTIMISTIC_UPDATES: Result = try_create_int_counter( - "beacon_processor_reprocessing_queue_matched_optimistic_updates", - "Number of queued light client optimistic updates where as matching block has been imported." - ); pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_SENT_OPTIMISTIC_UPDATES: Result = try_create_int_counter( "beacon_processor_reprocessing_queue_sent_optimistic_updates", "Number of queued light client optimistic updates where as matching block has been imported." diff --git a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs similarity index 95% rename from beacon_node/network/src/beacon_processor/worker/gossip_methods.rs rename to beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 91ec81b18d3..cde4da9ffcc 100644 --- a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1,4 +1,9 @@ -use crate::{metrics, service::NetworkMessage, sync::SyncMessage}; +use crate::{ + metrics, + network_beacon_processor::{InvalidBlockStorage, NetworkBeaconProcessor}, + service::NetworkMessage, + sync::SyncMessage, +}; use beacon_chain::store::Error; use beacon_chain::{ @@ -30,14 +35,13 @@ use types::{ SyncCommitteeMessage, SyncSubnetId, }; -use super::{ - super::work_reprocessing_queue::{ +use beacon_processor::{ + work_reprocessing_queue::{ QueuedAggregate, QueuedGossipBlock, QueuedLightClientUpdate, QueuedUnaggregate, ReprocessQueueMessage, }, - Worker, + DuplicateCache, GossipAggregatePackage, GossipAttestationPackage, }; -use crate::beacon_processor::{DuplicateCache, InvalidBlockStorage}; /// Set to `true` to introduce stricter penalties for peers who send some types of late consensus /// messages. @@ -144,65 +148,7 @@ impl FailedAtt { } } -/// Items required to verify a batch of unaggregated gossip attestations. -#[derive(Debug)] -pub struct GossipAttestationPackage { - message_id: MessageId, - peer_id: PeerId, - attestation: Box>, - subnet_id: SubnetId, - should_import: bool, - seen_timestamp: Duration, -} - -impl GossipAttestationPackage { - pub fn new( - message_id: MessageId, - peer_id: PeerId, - attestation: Box>, - subnet_id: SubnetId, - should_import: bool, - seen_timestamp: Duration, - ) -> Self { - Self { - message_id, - peer_id, - attestation, - subnet_id, - should_import, - seen_timestamp, - } - } -} - -/// Items required to verify a batch of aggregated gossip attestations. -#[derive(Debug)] -pub struct GossipAggregatePackage { - message_id: MessageId, - peer_id: PeerId, - aggregate: Box>, - beacon_block_root: Hash256, - seen_timestamp: Duration, -} - -impl GossipAggregatePackage { - pub fn new( - message_id: MessageId, - peer_id: PeerId, - aggregate: Box>, - seen_timestamp: Duration, - ) -> Self { - Self { - message_id, - peer_id, - beacon_block_root: aggregate.message.aggregate.data.beacon_block_root, - aggregate, - seen_timestamp, - } - } -} - -impl Worker { +impl NetworkBeaconProcessor { /* Auxiliary functions */ /// Penalizes a peer for misbehaviour. @@ -245,13 +191,13 @@ impl Worker { /// Raises a log if there are errors. #[allow(clippy::too_many_arguments)] pub fn process_gossip_attestation( - self, + self: Arc, message_id: MessageId, peer_id: PeerId, attestation: Box>, subnet_id: SubnetId, should_import: bool, - reprocess_tx: Option>>, + reprocess_tx: Option>, seen_timestamp: Duration, ) { let result = match self @@ -277,9 +223,9 @@ impl Worker { } pub fn process_gossip_attestation_batch( - self, + self: Arc, packages: Vec>, - reprocess_tx: Option>>, + reprocess_tx: Option>, ) { let attestations_and_subnets = packages .iter() @@ -348,12 +294,12 @@ impl Worker { // cant' be mixed-up) and creating a struct would result in more complexity. #[allow(clippy::too_many_arguments)] fn process_gossip_attestation_result( - &self, + self: &Arc, result: Result, RejectedUnaggregate>, message_id: MessageId, peer_id: PeerId, subnet_id: SubnetId, - reprocess_tx: Option>>, + reprocess_tx: Option>, should_import: bool, seen_timestamp: Duration, ) { @@ -456,11 +402,11 @@ impl Worker { /// /// Raises a log if there are errors. pub fn process_gossip_aggregate( - self, + self: Arc, message_id: MessageId, peer_id: PeerId, aggregate: Box>, - reprocess_tx: Option>>, + reprocess_tx: Option>, seen_timestamp: Duration, ) { let beacon_block_root = aggregate.message.aggregate.data.beacon_block_root; @@ -490,9 +436,9 @@ impl Worker { } pub fn process_gossip_aggregate_batch( - self, + self: Arc, packages: Vec>, - reprocess_tx: Option>>, + reprocess_tx: Option>, ) { let aggregates = packages.iter().map(|package| package.aggregate.as_ref()); @@ -555,12 +501,12 @@ impl Worker { } fn process_gossip_aggregate_result( - &self, + self: &Arc, result: Result, RejectedAggregate>, beacon_block_root: Hash256, message_id: MessageId, peer_id: PeerId, - reprocess_tx: Option>>, + reprocess_tx: Option>, seen_timestamp: Duration, ) { match result { @@ -659,12 +605,12 @@ impl Worker { /// Raises a log if there are errors. #[allow(clippy::too_many_arguments)] pub async fn process_gossip_block( - self, + self: Arc, message_id: MessageId, peer_id: PeerId, peer_client: Client, block: Arc>, - reprocess_tx: mpsc::Sender>, + reprocess_tx: mpsc::Sender, duplicate_cache: DuplicateCache, invalid_block_storage: InvalidBlockStorage, seen_duration: Duration, @@ -708,12 +654,12 @@ impl Worker { /// /// Returns the `GossipVerifiedBlock` if verification passes and raises a log if there are errors. pub async fn process_gossip_unverified_block( - &self, + self: &Arc, message_id: MessageId, peer_id: PeerId, peer_client: Client, block: Arc>, - reprocess_tx: mpsc::Sender>, + reprocess_tx: mpsc::Sender, seen_duration: Duration, ) -> Option> { let block_delay = @@ -911,11 +857,25 @@ impl Worker { metrics::inc_counter(&metrics::BEACON_PROCESSOR_GOSSIP_BLOCK_REQUEUED_TOTAL); + let inner_self = self.clone(); + let process_fn = Box::pin(async move { + let reprocess_tx = inner_self.reprocess_tx.clone(); + let invalid_block_storage = inner_self.invalid_block_storage.clone(); + inner_self + .process_gossip_verified_block( + peer_id, + verified_block, + reprocess_tx, + invalid_block_storage, + seen_duration, + ) + .await; + }); if reprocess_tx .try_send(ReprocessQueueMessage::EarlyBlock(QueuedGossipBlock { - peer_id, - block: Box::new(verified_block), - seen_timestamp: seen_duration, + beacon_block_slot: block_slot, + beacon_block_root: block_root, + process_fn, })) .is_err() { @@ -948,10 +908,10 @@ impl Worker { /// /// Raises a log if there are errors. pub async fn process_gossip_verified_block( - self, + self: Arc, peer_id: PeerId, verified_block: GossipVerifiedBlock, - reprocess_tx: mpsc::Sender>, + reprocess_tx: mpsc::Sender, invalid_block_storage: InvalidBlockStorage, // This value is not used presently, but it might come in handy for debugging. _seen_duration: Duration, @@ -1051,7 +1011,7 @@ impl Worker { } pub fn process_gossip_voluntary_exit( - self, + self: &Arc, message_id: MessageId, peer_id: PeerId, voluntary_exit: SignedVoluntaryExit, @@ -1109,7 +1069,7 @@ impl Worker { } pub fn process_gossip_proposer_slashing( - self, + self: &Arc, message_id: MessageId, peer_id: PeerId, proposer_slashing: ProposerSlashing, @@ -1171,7 +1131,7 @@ impl Worker { } pub fn process_gossip_attester_slashing( - self, + self: &Arc, message_id: MessageId, peer_id: PeerId, attester_slashing: AttesterSlashing, @@ -1225,7 +1185,7 @@ impl Worker { } pub fn process_gossip_bls_to_execution_change( - self, + self: &Arc, message_id: MessageId, peer_id: PeerId, bls_to_execution_change: SignedBlsToExecutionChange, @@ -1308,7 +1268,7 @@ impl Worker { /// /// Raises a log if there are errors. pub fn process_gossip_sync_committee_signature( - self, + self: &Arc, message_id: MessageId, peer_id: PeerId, sync_signature: SyncCommitteeMessage, @@ -1371,7 +1331,7 @@ impl Worker { /// /// Raises a log if there are errors. pub fn process_sync_committee_contribution( - self, + self: &Arc, message_id: MessageId, peer_id: PeerId, sync_contribution: SignedContributionAndProof, @@ -1426,7 +1386,7 @@ impl Worker { } pub fn process_gossip_finality_update( - self, + self: &Arc, message_id: MessageId, peer_id: PeerId, light_client_finality_update: LightClientFinalityUpdate, @@ -1492,11 +1452,11 @@ impl Worker { } pub fn process_gossip_optimistic_update( - self, + self: &Arc, message_id: MessageId, peer_id: PeerId, light_client_optimistic_update: LightClientOptimisticUpdate, - reprocess_tx: Option>>, + reprocess_tx: Option>, seen_timestamp: Duration, ) { match self.chain.verify_optimistic_update_for_gossip( @@ -1527,15 +1487,19 @@ impl Worker { ); if let Some(sender) = reprocess_tx { + let processor = self.clone(); let msg = ReprocessQueueMessage::UnknownLightClientOptimisticUpdate( QueuedLightClientUpdate { - peer_id, - message_id, - light_client_optimistic_update: Box::new( - light_client_optimistic_update, - ), parent_root, - seen_timestamp, + process_fn: Box::new(move || { + processor.process_gossip_optimistic_update( + message_id, + peer_id, + light_client_optimistic_update, + None, // Do not reprocess this message again. + seen_timestamp, + ) + }), }, ); @@ -1624,11 +1588,11 @@ impl Worker { /// Handle an error whilst verifying an `Attestation` or `SignedAggregateAndProof` from the /// network. fn handle_attestation_verification_failure( - &self, + self: &Arc, peer_id: PeerId, message_id: MessageId, failed_att: FailedAtt, - reprocess_tx: Option>>, + reprocess_tx: Option>, error: AttnError, seen_timestamp: Duration, ) { @@ -1860,11 +1824,18 @@ impl Worker { metrics::inc_counter( &metrics::BEACON_PROCESSOR_AGGREGATED_ATTESTATION_REQUEUED_TOTAL, ); + let processor = self.clone(); ReprocessQueueMessage::UnknownBlockAggregate(QueuedAggregate { - peer_id, - message_id, - attestation, - seen_timestamp, + beacon_block_root: *beacon_block_root, + process_fn: Box::new(move || { + processor.process_gossip_aggregate( + message_id, + peer_id, + attestation, + None, // Do not allow this attestation to be re-processed beyond this point. + seen_timestamp, + ) + }), }) } FailedAtt::Unaggregate { @@ -1876,13 +1847,20 @@ impl Worker { metrics::inc_counter( &metrics::BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_REQUEUED_TOTAL, ); + let processor = self.clone(); ReprocessQueueMessage::UnknownBlockUnaggregate(QueuedUnaggregate { - peer_id, - message_id, - attestation, - subnet_id, - should_import, - seen_timestamp, + beacon_block_root: *beacon_block_root, + process_fn: Box::new(move || { + processor.process_gossip_attestation( + message_id, + peer_id, + attestation, + subnet_id, + should_import, + None, // Do not allow this attestation to be re-processed beyond this point. + seen_timestamp, + ) + }), }) } }; diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs new file mode 100644 index 00000000000..7f0ef1fb817 --- /dev/null +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -0,0 +1,590 @@ +use crate::{ + service::NetworkMessage, + sync::{manager::BlockProcessType, SyncMessage}, +}; +use beacon_chain::{ + builder::Witness, eth1_chain::CachingEth1Backend, test_utils::BeaconChainHarness, BeaconChain, +}; +use beacon_chain::{BeaconChainTypes, NotifyExecutionLayer}; +use beacon_processor::{ + work_reprocessing_queue::ReprocessQueueMessage, BeaconProcessorSend, DuplicateCache, + GossipAggregatePackage, GossipAttestationPackage, Work, WorkEvent as BeaconWorkEvent, + MAX_SCHEDULED_WORK_QUEUE_LEN, MAX_WORK_EVENT_QUEUE_LEN, +}; +use environment::null_logger; +use lighthouse_network::{ + rpc::{BlocksByRangeRequest, BlocksByRootRequest, LightClientBootstrapRequest, StatusMessage}, + Client, MessageId, NetworkGlobals, PeerId, PeerRequestId, +}; +use slog::{debug, Logger}; +use slot_clock::ManualSlotClock; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; +use store::MemoryStore; +use task_executor::test_utils::TestRuntime; +use task_executor::TaskExecutor; +use tokio::sync::mpsc::{self, error::TrySendError}; +use types::*; + +pub use sync_methods::ChainSegmentProcessId; + +pub type Error = TrySendError>; + +mod gossip_methods; +mod rpc_methods; +mod sync_methods; +mod tests; + +pub(crate) const FUTURE_SLOT_TOLERANCE: u64 = 1; + +/// Defines if and where we will store the SSZ files of invalid blocks. +#[derive(Clone)] +pub enum InvalidBlockStorage { + Enabled(PathBuf), + Disabled, +} + +/// Provides an interface to a `BeaconProcessor` running in some other thread. +/// The wider `networking` crate should use this struct to interface with the +/// beacon processor. +pub struct NetworkBeaconProcessor { + pub beacon_processor_send: BeaconProcessorSend, + pub duplicate_cache: DuplicateCache, + pub chain: Arc>, + pub network_tx: mpsc::UnboundedSender>, + pub sync_tx: mpsc::UnboundedSender>, + pub reprocess_tx: mpsc::Sender, + pub network_globals: Arc>, + pub invalid_block_storage: InvalidBlockStorage, + pub executor: TaskExecutor, + pub log: Logger, +} + +impl NetworkBeaconProcessor { + fn try_send(&self, event: BeaconWorkEvent) -> Result<(), Error> { + self.beacon_processor_send + .try_send(event) + .map_err(Into::into) + } + + /// Create a new `Work` event for some unaggregated attestation. + pub fn send_unaggregated_attestation( + self: &Arc, + message_id: MessageId, + peer_id: PeerId, + attestation: Attestation, + subnet_id: SubnetId, + should_import: bool, + seen_timestamp: Duration, + ) -> Result<(), Error> { + // Define a closure for processing individual attestations. + let processor = self.clone(); + let process_individual = move |package: GossipAttestationPackage| { + let reprocess_tx = processor.reprocess_tx.clone(); + processor.process_gossip_attestation( + package.message_id, + package.peer_id, + package.attestation, + package.subnet_id, + package.should_import, + Some(reprocess_tx), + package.seen_timestamp, + ) + }; + + // Define a closure for processing batches of attestations. + let processor = self.clone(); + let process_batch = move |attestations| { + let reprocess_tx = processor.reprocess_tx.clone(); + processor.process_gossip_attestation_batch(attestations, Some(reprocess_tx)) + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: true, + work: Work::GossipAttestation { + attestation: GossipAttestationPackage { + message_id, + peer_id, + attestation: Box::new(attestation), + subnet_id, + should_import, + seen_timestamp, + }, + process_individual: Box::new(process_individual), + process_batch: Box::new(process_batch), + }, + }) + } + + /// Create a new `Work` event for some aggregated attestation. + pub fn send_aggregated_attestation( + self: &Arc, + message_id: MessageId, + peer_id: PeerId, + aggregate: SignedAggregateAndProof, + seen_timestamp: Duration, + ) -> Result<(), Error> { + // Define a closure for processing individual attestations. + let processor = self.clone(); + let process_individual = move |package: GossipAggregatePackage| { + let reprocess_tx = processor.reprocess_tx.clone(); + processor.process_gossip_aggregate( + package.message_id, + package.peer_id, + package.aggregate, + Some(reprocess_tx), + package.seen_timestamp, + ) + }; + + // Define a closure for processing batches of attestations. + let processor = self.clone(); + let process_batch = move |aggregates| { + let reprocess_tx = processor.reprocess_tx.clone(); + processor.process_gossip_aggregate_batch(aggregates, Some(reprocess_tx)) + }; + + let beacon_block_root = aggregate.message.aggregate.data.beacon_block_root; + self.try_send(BeaconWorkEvent { + drop_during_sync: true, + work: Work::GossipAggregate { + aggregate: GossipAggregatePackage { + message_id, + peer_id, + aggregate: Box::new(aggregate), + beacon_block_root, + seen_timestamp, + }, + process_individual: Box::new(process_individual), + process_batch: Box::new(process_batch), + }, + }) + } + + /// Create a new `Work` event for some block. + pub fn send_gossip_beacon_block( + self: &Arc, + message_id: MessageId, + peer_id: PeerId, + peer_client: Client, + block: Arc>, + seen_timestamp: Duration, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = async move { + let reprocess_tx = processor.reprocess_tx.clone(); + let invalid_block_storage = processor.invalid_block_storage.clone(); + let duplicate_cache = processor.duplicate_cache.clone(); + processor + .process_gossip_block( + message_id, + peer_id, + peer_client, + block, + reprocess_tx, + duplicate_cache, + invalid_block_storage, + seen_timestamp, + ) + .await + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::GossipBlock(Box::pin(process_fn)), + }) + } + + /// Create a new `Work` event for some sync committee signature. + pub fn send_gossip_sync_signature( + self: &Arc, + message_id: MessageId, + peer_id: PeerId, + sync_signature: SyncCommitteeMessage, + subnet_id: SyncSubnetId, + seen_timestamp: Duration, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = move || { + processor.process_gossip_sync_committee_signature( + message_id, + peer_id, + sync_signature, + subnet_id, + seen_timestamp, + ) + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: true, + work: Work::GossipSyncSignature(Box::new(process_fn)), + }) + } + + /// Create a new `Work` event for some sync committee contribution. + pub fn send_gossip_sync_contribution( + self: &Arc, + message_id: MessageId, + peer_id: PeerId, + sync_contribution: SignedContributionAndProof, + seen_timestamp: Duration, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = move || { + processor.process_sync_committee_contribution( + message_id, + peer_id, + sync_contribution, + seen_timestamp, + ) + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: true, + work: Work::GossipSyncContribution(Box::new(process_fn)), + }) + } + + /// Create a new `Work` event for some exit. + pub fn send_gossip_voluntary_exit( + self: &Arc, + message_id: MessageId, + peer_id: PeerId, + voluntary_exit: Box, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = + move || processor.process_gossip_voluntary_exit(message_id, peer_id, *voluntary_exit); + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::GossipVoluntaryExit(Box::new(process_fn)), + }) + } + + /// Create a new `Work` event for some proposer slashing. + pub fn send_gossip_proposer_slashing( + self: &Arc, + message_id: MessageId, + peer_id: PeerId, + proposer_slashing: Box, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = move || { + processor.process_gossip_proposer_slashing(message_id, peer_id, *proposer_slashing) + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::GossipProposerSlashing(Box::new(process_fn)), + }) + } + + /// Create a new `Work` event for some light client finality update. + pub fn send_gossip_light_client_finality_update( + self: &Arc, + message_id: MessageId, + peer_id: PeerId, + light_client_finality_update: LightClientFinalityUpdate, + seen_timestamp: Duration, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = move || { + processor.process_gossip_finality_update( + message_id, + peer_id, + light_client_finality_update, + seen_timestamp, + ) + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: true, + work: Work::GossipLightClientFinalityUpdate(Box::new(process_fn)), + }) + } + + /// Create a new `Work` event for some light client optimistic update. + pub fn send_gossip_light_client_optimistic_update( + self: &Arc, + message_id: MessageId, + peer_id: PeerId, + light_client_optimistic_update: LightClientOptimisticUpdate, + seen_timestamp: Duration, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = move || { + let reprocess_tx = processor.reprocess_tx.clone(); + processor.process_gossip_optimistic_update( + message_id, + peer_id, + light_client_optimistic_update, + Some(reprocess_tx), + seen_timestamp, + ) + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: true, + work: Work::GossipLightClientOptimisticUpdate(Box::new(process_fn)), + }) + } + + /// Create a new `Work` event for some attester slashing. + pub fn send_gossip_attester_slashing( + self: &Arc, + message_id: MessageId, + peer_id: PeerId, + attester_slashing: Box>, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = move || { + processor.process_gossip_attester_slashing(message_id, peer_id, *attester_slashing) + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::GossipAttesterSlashing(Box::new(process_fn)), + }) + } + + /// Create a new `Work` event for some BLS to execution change. + pub fn send_gossip_bls_to_execution_change( + self: &Arc, + message_id: MessageId, + peer_id: PeerId, + bls_to_execution_change: Box, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = move || { + processor.process_gossip_bls_to_execution_change( + message_id, + peer_id, + *bls_to_execution_change, + ) + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::GossipBlsToExecutionChange(Box::new(process_fn)), + }) + } + + /// Create a new `Work` event for some block, where the result from computation (if any) is + /// sent to the other side of `result_tx`. + pub fn send_rpc_beacon_block( + self: &Arc, + block_root: Hash256, + block: Arc>, + seen_timestamp: Duration, + process_type: BlockProcessType, + ) -> Result<(), Error> { + let process_fn = self.clone().generate_rpc_beacon_block_process_fn( + block_root, + block, + seen_timestamp, + process_type, + ); + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::RpcBlock { process_fn }, + }) + } + + /// Create a new work event to import `blocks` as a beacon chain segment. + pub fn send_chain_segment( + self: &Arc, + process_id: ChainSegmentProcessId, + blocks: Vec>>, + ) -> Result<(), Error> { + let is_backfill = matches!(&process_id, ChainSegmentProcessId::BackSyncBatchId { .. }); + let processor = self.clone(); + let process_fn = async move { + let notify_execution_layer = if processor + .network_globals + .sync_state + .read() + .is_syncing_finalized() + { + NotifyExecutionLayer::No + } else { + NotifyExecutionLayer::Yes + }; + processor + .process_chain_segment(process_id, blocks, notify_execution_layer) + .await; + }; + let process_fn = Box::pin(process_fn); + + // Back-sync batches are dispatched with a different `Work` variant so + // they can be rate-limited. + let work = if is_backfill { + Work::ChainSegmentBackfill(process_fn) + } else { + Work::ChainSegment(process_fn) + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work, + }) + } + + /// Create a new work event to process `StatusMessage`s from the RPC network. + pub fn send_status_message( + self: &Arc, + peer_id: PeerId, + message: StatusMessage, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = move || processor.process_status(peer_id, message); + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::Status(Box::new(process_fn)), + }) + } + + /// Create a new work event to process `BlocksByRangeRequest`s from the RPC network. + pub fn send_blocks_by_range_request( + self: &Arc, + peer_id: PeerId, + request_id: PeerRequestId, + request: BlocksByRangeRequest, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = move |send_idle_on_drop| { + let executor = processor.executor.clone(); + processor.handle_blocks_by_range_request( + executor, + send_idle_on_drop, + peer_id, + request_id, + request, + ) + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::BlocksByRangeRequest(Box::new(process_fn)), + }) + } + + /// Create a new work event to process `BlocksByRootRequest`s from the RPC network. + pub fn send_blocks_by_roots_request( + self: &Arc, + peer_id: PeerId, + request_id: PeerRequestId, + request: BlocksByRootRequest, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = move |send_idle_on_drop| { + let executor = processor.executor.clone(); + processor.handle_blocks_by_root_request( + executor, + send_idle_on_drop, + peer_id, + request_id, + request, + ) + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::BlocksByRootsRequest(Box::new(process_fn)), + }) + } + + /// Create a new work event to process `LightClientBootstrap`s from the RPC network. + pub fn send_lightclient_bootstrap_request( + self: &Arc, + peer_id: PeerId, + request_id: PeerRequestId, + request: LightClientBootstrapRequest, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = + move || processor.handle_light_client_bootstrap(peer_id, request_id, request); + + self.try_send(BeaconWorkEvent { + drop_during_sync: true, + work: Work::LightClientBootstrapRequest(Box::new(process_fn)), + }) + } + + /// Send a message to `sync_tx`. + /// + /// Creates a log if there is an internal error. + fn send_sync_message(&self, message: SyncMessage) { + self.sync_tx.send(message).unwrap_or_else(|e| { + debug!(self.log, "Could not send message to the sync service"; + "error" => %e) + }); + } + + /// Send a message to `network_tx`. + /// + /// Creates a log if there is an internal error. + fn send_network_message(&self, message: NetworkMessage) { + self.network_tx.send(message).unwrap_or_else(|e| { + debug!(self.log, "Could not send message to the network service. Likely shutdown"; + "error" => %e) + }); + } +} + +type TestBeaconChainType = + Witness, E, MemoryStore, MemoryStore>; + +impl NetworkBeaconProcessor> { + // Instantiates a mostly non-functional version of `Self` and returns the + // event receiver that would normally go to the beacon processor. This is + // useful for testing that messages are actually being sent to the beacon + // processor (but not much else). + pub fn null_for_testing( + network_globals: Arc>, + ) -> (Self, mpsc::Receiver>) { + let (beacon_processor_send, beacon_processor_receive) = + mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN); + let (network_tx, _network_rx) = mpsc::unbounded_channel(); + let (sync_tx, _sync_rx) = mpsc::unbounded_channel(); + let (reprocess_tx, _reprocess_rx) = mpsc::channel(MAX_SCHEDULED_WORK_QUEUE_LEN); + let log = null_logger().unwrap(); + let harness: BeaconChainHarness> = + BeaconChainHarness::builder(E::default()) + .spec(E::default_spec()) + .deterministic_keypairs(8) + .logger(log.clone()) + .fresh_ephemeral_store() + .mock_execution_layer() + .build(); + let runtime = TestRuntime::default(); + + let network_beacon_processor = Self { + beacon_processor_send: BeaconProcessorSend(beacon_processor_send), + duplicate_cache: DuplicateCache::default(), + chain: harness.chain, + network_tx, + sync_tx, + reprocess_tx, + network_globals, + invalid_block_storage: InvalidBlockStorage::Disabled, + executor: runtime.task_executor.clone(), + log, + }; + + (network_beacon_processor, beacon_processor_receive) + } +} + +#[cfg(test)] +mod test { + #[test] + fn queued_block_delay_is_sane() { + assert!( + beacon_processor::work_reprocessing_queue::ADDITIONAL_QUEUED_BLOCK_DELAY + < beacon_chain::MAXIMUM_GOSSIP_CLOCK_DISPARITY + ); + } +} diff --git a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs similarity index 98% rename from beacon_node/network/src/beacon_processor/worker/rpc_methods.rs rename to beacon_node/network/src/network_beacon_processor/rpc_methods.rs index 83baa0417bc..19b0a60a43e 100644 --- a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs @@ -1,21 +1,21 @@ -use crate::beacon_processor::{worker::FUTURE_SLOT_TOLERANCE, SendOnDrop}; +use crate::network_beacon_processor::{NetworkBeaconProcessor, FUTURE_SLOT_TOLERANCE}; use crate::service::NetworkMessage; use crate::status::ToStatusMessage; use crate::sync::SyncMessage; use beacon_chain::{BeaconChainError, BeaconChainTypes, HistoricalBlockError, WhenSlotSkipped}; +use beacon_processor::SendOnDrop; use itertools::process_results; use lighthouse_network::rpc::StatusMessage; use lighthouse_network::rpc::*; use lighthouse_network::{PeerId, PeerRequestId, ReportSource, Response, SyncInfo}; use slog::{debug, error, warn}; use slot_clock::SlotClock; +use std::sync::Arc; use task_executor::TaskExecutor; use tokio_stream::StreamExt; use types::{light_client_bootstrap::LightClientBootstrap, Epoch, EthSpec, Hash256, Slot}; -use super::Worker; - -impl Worker { +impl NetworkBeaconProcessor { /* Auxiliary functions */ /// Disconnects and ban's a peer, sending a Goodbye request with the associated reason. @@ -124,7 +124,7 @@ impl Worker { /// Handle a `BlocksByRoot` request from the peer. pub fn handle_blocks_by_root_request( - self, + self: Arc, executor: TaskExecutor, send_on_drop: SendOnDrop, peer_id: PeerId, @@ -210,7 +210,7 @@ impl Worker { /// Handle a `BlocksByRoot` request from the peer. pub fn handle_light_client_bootstrap( - self, + self: &Arc, peer_id: PeerId, request_id: PeerRequestId, request: LightClientBootstrapRequest, @@ -283,7 +283,7 @@ impl Worker { /// Handle a `BlocksByRange` request from the peer. pub fn handle_blocks_by_range_request( - self, + self: Arc, executor: TaskExecutor, send_on_drop: SendOnDrop, peer_id: PeerId, diff --git a/beacon_node/network/src/beacon_processor/worker/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs similarity index 90% rename from beacon_node/network/src/beacon_processor/worker/sync_methods.rs rename to beacon_node/network/src/network_beacon_processor/sync_methods.rs index ac59b1daa93..c33e2acf542 100644 --- a/beacon_node/network/src/beacon_processor/worker/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -1,17 +1,21 @@ use std::time::Duration; -use super::{super::work_reprocessing_queue::ReprocessQueueMessage, Worker}; -use crate::beacon_processor::work_reprocessing_queue::QueuedRpcBlock; -use crate::beacon_processor::worker::FUTURE_SLOT_TOLERANCE; -use crate::beacon_processor::DuplicateCache; use crate::metrics; -use crate::sync::manager::{BlockProcessType, SyncMessage}; -use crate::sync::{BatchProcessResult, ChainId}; +use crate::network_beacon_processor::{NetworkBeaconProcessor, FUTURE_SLOT_TOLERANCE}; +use crate::sync::BatchProcessResult; +use crate::sync::{ + manager::{BlockProcessType, SyncMessage}, + ChainId, +}; use beacon_chain::{ observed_block_producers::Error as ObserveError, validator_monitor::get_block_delay_ms, BeaconChainError, BeaconChainTypes, BlockError, ChainSegmentResult, HistoricalBlockError, NotifyExecutionLayer, }; +use beacon_processor::{ + work_reprocessing_queue::{QueuedRpcBlock, ReprocessQueueMessage}, + AsyncFn, BlockingFn, DuplicateCache, +}; use lighthouse_network::PeerAction; use slog::{debug, error, info, warn}; use slot_clock::SlotClock; @@ -39,27 +43,71 @@ struct ChainSegmentFailed { peer_action: Option, } -impl Worker { - /// Attempt to process a block received from a direct RPC request. - #[allow(clippy::too_many_arguments)] - pub async fn process_rpc_block( - self, +impl NetworkBeaconProcessor { + /// Returns an async closure which processes a beacon block recieved via RPC. + /// + /// This separate function was required to prevent a cycle during compiler + /// type checking. + pub fn generate_rpc_beacon_block_process_fn( + self: Arc, block_root: Hash256, block: Arc>, seen_timestamp: Duration, process_type: BlockProcessType, - reprocess_tx: mpsc::Sender>, - duplicate_cache: DuplicateCache, - should_process: bool, - ) { - if !should_process { + ) -> AsyncFn { + let process_fn = async move { + let reprocess_tx = self.reprocess_tx.clone(); + let duplicate_cache = self.duplicate_cache.clone(); + self.process_rpc_block( + block_root, + block, + seen_timestamp, + process_type, + reprocess_tx, + duplicate_cache, + ) + .await; + }; + Box::pin(process_fn) + } + + /// Returns the `process_fn` and `ignore_fn` required when requeuing an RPC block. + pub fn generate_rpc_beacon_block_fns( + self: Arc, + block_root: Hash256, + block: Arc>, + seen_timestamp: Duration, + process_type: BlockProcessType, + ) -> (AsyncFn, BlockingFn) { + // An async closure which will import the block. + let process_fn = self.clone().generate_rpc_beacon_block_process_fn( + block_root, + block, + seen_timestamp, + process_type.clone(), + ); + // A closure which will ignore the block. + let ignore_fn = move || { // Sync handles these results self.send_sync_message(SyncMessage::BlockProcessed { process_type, result: crate::sync::manager::BlockProcessResult::Ignored, }); - return; - } + }; + (process_fn, Box::new(ignore_fn)) + } + + /// Attempt to process a block received from a direct RPC request. + #[allow(clippy::too_many_arguments)] + pub async fn process_rpc_block( + self: Arc>, + block_root: Hash256, + block: Arc>, + seen_timestamp: Duration, + process_type: BlockProcessType, + reprocess_tx: mpsc::Sender, + duplicate_cache: DuplicateCache, + ) { // Check if the block is already being imported through another source let handle = match duplicate_cache.check_and_insert(block_root) { Some(handle) => handle, @@ -70,13 +118,18 @@ impl Worker { "action" => "sending rpc block to reprocessing queue", "block_root" => %block_root, ); + // Send message to work reprocess queue to retry the block - let reprocess_msg = ReprocessQueueMessage::RpcBlock(QueuedRpcBlock { + let (process_fn, ignore_fn) = self.clone().generate_rpc_beacon_block_fns( block_root, - block: block.clone(), - process_type, + block, seen_timestamp, - should_process: true, + process_type, + ); + let reprocess_msg = ReprocessQueueMessage::RpcBlock(QueuedRpcBlock { + beacon_block_root: block_root, + process_fn, + ignore_fn, }); if reprocess_tx.try_send(reprocess_msg).is_err() { @@ -130,12 +183,16 @@ impl Worker { ); // Send message to work reprocess queue to retry the block - let reprocess_msg = ReprocessQueueMessage::RpcBlock(QueuedRpcBlock { + let (process_fn, ignore_fn) = self.clone().generate_rpc_beacon_block_fns( block_root, - block: block.clone(), - process_type, + block, seen_timestamp, - should_process: true, + process_type, + ); + let reprocess_msg = ReprocessQueueMessage::RpcBlock(QueuedRpcBlock { + beacon_block_root: block_root, + process_fn, + ignore_fn, }); if reprocess_tx.try_send(reprocess_msg).is_err() { diff --git a/beacon_node/network/src/beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs similarity index 89% rename from beacon_node/network/src/beacon_processor/tests.rs rename to beacon_node/network/src/network_beacon_processor/tests.rs index b93e83ad785..b8d5db568ec 100644 --- a/beacon_node/network/src/beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -1,20 +1,23 @@ #![cfg(not(debug_assertions))] // Tests are too slow in debug. #![cfg(test)] -use crate::beacon_processor::work_reprocessing_queue::{ - QUEUED_ATTESTATION_DELAY, QUEUED_RPC_BLOCK_DELAY, +use crate::{ + network_beacon_processor::{ + ChainSegmentProcessId, DuplicateCache, InvalidBlockStorage, NetworkBeaconProcessor, + }, + service::NetworkMessage, + sync::{manager::BlockProcessType, SyncMessage}, }; -use crate::beacon_processor::*; -use crate::{service::NetworkMessage, sync::SyncMessage}; use beacon_chain::test_utils::{ AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType, }; use beacon_chain::{BeaconChain, ChainConfig, MAXIMUM_GOSSIP_CLOCK_DISPARITY}; +use beacon_processor::{work_reprocessing_queue::*, *}; use lighthouse_network::{ discv5::enr::{CombinedKey, EnrBuilder}, rpc::methods::{MetaData, MetaDataV2}, types::{EnrAttestationBitfield, EnrSyncCommitteeBitfield}, - MessageId, NetworkGlobals, PeerId, + Client, MessageId, NetworkGlobals, PeerId, }; use slot_clock::SlotClock; use std::cmp; @@ -23,8 +26,8 @@ use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; use types::{ - Attestation, AttesterSlashing, Epoch, EthSpec, MainnetEthSpec, ProposerSlashing, - SignedBeaconBlock, SignedVoluntaryExit, SubnetId, + Attestation, AttesterSlashing, Epoch, EthSpec, Hash256, MainnetEthSpec, ProposerSlashing, + SignedAggregateAndProof, SignedBeaconBlock, SignedVoluntaryExit, SubnetId, }; type E = MainnetEthSpec; @@ -51,11 +54,12 @@ struct TestRig { attester_slashing: AttesterSlashing, proposer_slashing: ProposerSlashing, voluntary_exit: SignedVoluntaryExit, - beacon_processor_tx: mpsc::Sender>, + beacon_processor_tx: BeaconProcessorSend, work_journal_rx: mpsc::Receiver<&'static str>, _network_rx: mpsc::UnboundedReceiver>, _sync_rx: mpsc::UnboundedReceiver>, duplicate_cache: DuplicateCache, + network_beacon_processor: Arc>, _harness: BeaconChainHarness, } @@ -64,7 +68,7 @@ struct TestRig { impl Drop for TestRig { fn drop(&mut self) { // Causes the beacon processor to shutdown. - self.beacon_processor_tx = mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN).0; + self.beacon_processor_tx = BeaconProcessorSend(mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN).0); } } @@ -169,6 +173,7 @@ impl TestRig { let log = harness.logger().clone(); let (beacon_processor_tx, beacon_processor_rx) = mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN); + let beacon_processor_tx = BeaconProcessorSend(beacon_processor_tx); let (sync_tx, _sync_rx) = mpsc::unbounded_channel(); // Default metadata @@ -191,22 +196,40 @@ impl TestRig { let executor = harness.runtime.task_executor.clone(); + let (work_reprocessing_tx, work_reprocessing_rx) = + mpsc::channel(MAX_SCHEDULED_WORK_QUEUE_LEN); let (work_journal_tx, work_journal_rx) = mpsc::channel(16_364); let duplicate_cache = DuplicateCache::default(); - BeaconProcessor { - beacon_chain: Arc::downgrade(&chain), + let network_beacon_processor = NetworkBeaconProcessor { + beacon_processor_send: beacon_processor_tx.clone(), + duplicate_cache: duplicate_cache.clone(), + chain: harness.chain.clone(), network_tx, sync_tx, + reprocess_tx: work_reprocessing_tx.clone(), + network_globals: network_globals.clone(), + invalid_block_storage: InvalidBlockStorage::Disabled, + executor: executor.clone(), + log: log.clone(), + }; + let network_beacon_processor = Arc::new(network_beacon_processor); + + BeaconProcessor { network_globals, executor, max_workers: cmp::max(1, num_cpus::get()), current_workers: 0, - importing_blocks: duplicate_cache.clone(), - invalid_block_storage: InvalidBlockStorage::Disabled, + enable_backfill_rate_limiting: harness.chain.config.enable_backfill_rate_limiting, log: log.clone(), } - .spawn_manager(beacon_processor_rx, Some(work_journal_tx)); + .spawn_manager( + beacon_processor_rx, + work_reprocessing_tx, + work_reprocessing_rx, + Some(work_journal_tx), + harness.chain.slot_clock.clone(), + ); Self { chain, @@ -222,6 +245,7 @@ impl TestRig { _network_rx, _sync_rx, duplicate_cache, + network_beacon_processor, _harness: harness, } } @@ -235,102 +259,105 @@ impl TestRig { } pub fn enqueue_gossip_block(&self) { - self.beacon_processor_tx - .try_send(WorkEvent::gossip_beacon_block( + self.network_beacon_processor + .send_gossip_beacon_block( junk_message_id(), junk_peer_id(), Client::default(), self.next_block.clone(), Duration::from_secs(0), - )) + ) .unwrap(); } pub fn enqueue_rpc_block(&self) { - let event = WorkEvent::rpc_beacon_block( - self.next_block.canonical_root(), - self.next_block.clone(), - std::time::Duration::default(), - BlockProcessType::ParentLookup { - chain_hash: Hash256::random(), - }, - ); - self.beacon_processor_tx.try_send(event).unwrap(); + self.network_beacon_processor + .send_rpc_beacon_block( + self.next_block.canonical_root(), + self.next_block.clone(), + std::time::Duration::default(), + BlockProcessType::ParentLookup { + chain_hash: Hash256::random(), + }, + ) + .unwrap(); } pub fn enqueue_single_lookup_rpc_block(&self) { - let event = WorkEvent::rpc_beacon_block( - self.next_block.canonical_root(), - self.next_block.clone(), - std::time::Duration::default(), - BlockProcessType::SingleBlock { id: 1 }, - ); - self.beacon_processor_tx.try_send(event).unwrap(); + self.network_beacon_processor + .send_rpc_beacon_block( + self.next_block.canonical_root(), + self.next_block.clone(), + std::time::Duration::default(), + BlockProcessType::SingleBlock { id: 1 }, + ) + .unwrap(); } pub fn enqueue_backfill_batch(&self) { - let event = WorkEvent::chain_segment( - ChainSegmentProcessId::BackSyncBatchId(Epoch::default()), - Vec::default(), - ); - self.beacon_processor_tx.try_send(event).unwrap(); + self.network_beacon_processor + .send_chain_segment( + ChainSegmentProcessId::BackSyncBatchId(Epoch::default()), + Vec::default(), + ) + .unwrap(); } pub fn enqueue_unaggregated_attestation(&self) { let (attestation, subnet_id) = self.attestations.first().unwrap().clone(); - self.beacon_processor_tx - .try_send(WorkEvent::unaggregated_attestation( + self.network_beacon_processor + .send_unaggregated_attestation( junk_message_id(), junk_peer_id(), attestation, subnet_id, true, Duration::from_secs(0), - )) + ) .unwrap(); } pub fn enqueue_gossip_attester_slashing(&self) { - self.beacon_processor_tx - .try_send(WorkEvent::gossip_attester_slashing( + self.network_beacon_processor + .send_gossip_attester_slashing( junk_message_id(), junk_peer_id(), Box::new(self.attester_slashing.clone()), - )) + ) .unwrap(); } pub fn enqueue_gossip_proposer_slashing(&self) { - self.beacon_processor_tx - .try_send(WorkEvent::gossip_proposer_slashing( + self.network_beacon_processor + .send_gossip_proposer_slashing( junk_message_id(), junk_peer_id(), Box::new(self.proposer_slashing.clone()), - )) + ) .unwrap(); } pub fn enqueue_gossip_voluntary_exit(&self) { - self.beacon_processor_tx - .try_send(WorkEvent::gossip_voluntary_exit( + self.network_beacon_processor + .send_gossip_voluntary_exit( junk_message_id(), junk_peer_id(), Box::new(self.voluntary_exit.clone()), - )) + ) .unwrap(); } pub fn enqueue_next_block_unaggregated_attestation(&self) { let (attestation, subnet_id) = self.next_block_attestations.first().unwrap().clone(); - self.beacon_processor_tx - .try_send(WorkEvent::unaggregated_attestation( + self.network_beacon_processor + .send_unaggregated_attestation( junk_message_id(), junk_peer_id(), attestation, subnet_id, true, Duration::from_secs(0), - )) + ) .unwrap(); } @@ -340,13 +367,13 @@ impl TestRig { .first() .unwrap() .clone(); - self.beacon_processor_tx - .try_send(WorkEvent::aggregated_attestation( + self.network_beacon_processor + .send_aggregated_attestation( junk_message_id(), junk_peer_id(), aggregate, Duration::from_secs(0), - )) + ) .unwrap(); } diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 7a91f2d0b1a..c8332705cfa 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -5,16 +5,16 @@ //! syncing-related responses to the Sync manager. #![allow(clippy::unit_arg)] -use crate::beacon_processor::{ - BeaconProcessor, BeaconProcessorSend, InvalidBlockStorage, WorkEvent as BeaconWorkEvent, - MAX_WORK_EVENT_QUEUE_LEN, -}; use crate::error; +use crate::network_beacon_processor::{InvalidBlockStorage, NetworkBeaconProcessor}; use crate::service::{NetworkMessage, RequestId}; use crate::status::status_message; use crate::sync::manager::RequestId as SyncId; use crate::sync::SyncMessage; use beacon_chain::{BeaconChain, BeaconChainTypes}; +use beacon_processor::{ + work_reprocessing_queue::ReprocessQueueMessage, BeaconProcessorSend, DuplicateCache, +}; use futures::prelude::*; use lighthouse_network::rpc::*; use lighthouse_network::{ @@ -23,7 +23,6 @@ use lighthouse_network::{ use logging::TimeLatch; use slog::{debug, o, trace}; use slog::{error, warn}; -use std::cmp; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::sync::mpsc; @@ -41,7 +40,7 @@ pub struct Router { /// A network context to return and handle RPC requests. network: HandlerNetworkContext, /// A multi-threaded, non-blocking processor for applying messages to the beacon chain. - beacon_processor_send: BeaconProcessorSend, + network_beacon_processor: Arc>, /// The `Router` logger. log: slog::Logger, /// Provides de-bounce functionality for logging. @@ -80,12 +79,15 @@ pub enum RouterMessage { impl Router { /// Initializes and runs the Router. + #[allow(clippy::too_many_arguments)] pub fn spawn( beacon_chain: Arc>, network_globals: Arc>, network_send: mpsc::UnboundedSender>, executor: task_executor::TaskExecutor, invalid_block_storage: InvalidBlockStorage, + beacon_processor_send: BeaconProcessorSend, + beacon_processor_reprocess_tx: mpsc::Sender, log: slog::Logger, ) -> error::Result>> { let message_handler_log = log.new(o!("service"=> "router")); @@ -93,42 +95,41 @@ impl Router { let (handler_send, handler_recv) = mpsc::unbounded_channel(); - let (beacon_processor_send, beacon_processor_receive) = - mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN); - let sync_logger = log.new(o!("service"=> "sync")); + // generate the message channel + let (sync_send, sync_recv) = mpsc::unbounded_channel::>(); + + let network_beacon_processor = NetworkBeaconProcessor { + beacon_processor_send, + duplicate_cache: DuplicateCache::default(), + chain: beacon_chain.clone(), + network_tx: network_send.clone(), + sync_tx: sync_send.clone(), + reprocess_tx: beacon_processor_reprocess_tx, + network_globals: network_globals.clone(), + invalid_block_storage, + executor: executor.clone(), + log: log.clone(), + }; + let network_beacon_processor = Arc::new(network_beacon_processor); // spawn the sync thread - let sync_send = crate::sync::manager::spawn( + crate::sync::manager::spawn( executor.clone(), beacon_chain.clone(), - network_globals.clone(), network_send.clone(), - BeaconProcessorSend(beacon_processor_send.clone()), + network_beacon_processor.clone(), + sync_recv, sync_logger, ); - BeaconProcessor { - beacon_chain: Arc::downgrade(&beacon_chain), - network_tx: network_send.clone(), - sync_tx: sync_send.clone(), - network_globals: network_globals.clone(), - executor: executor.clone(), - max_workers: cmp::max(1, num_cpus::get()), - current_workers: 0, - importing_blocks: Default::default(), - invalid_block_storage, - log: log.clone(), - } - .spawn_manager(beacon_processor_receive, None); - // generate the Message handler let mut handler = Router { network_globals, chain: beacon_chain, sync_send, network: HandlerNetworkContext::new(network_send, log.clone()), - beacon_processor_send: BeaconProcessorSend(beacon_processor_send), + network_beacon_processor, log: message_handler_log, logger_debounce: TimeLatch::default(), }; @@ -197,14 +198,17 @@ impl Router { Request::Status(status_message) => { self.on_status_request(peer_id, request_id, status_message) } - Request::BlocksByRange(request) => self.send_beacon_processor_work( - BeaconWorkEvent::blocks_by_range_request(peer_id, request_id, request), + Request::BlocksByRange(request) => self.handle_beacon_processor_send_result( + self.network_beacon_processor + .send_blocks_by_range_request(peer_id, request_id, request), ), - Request::BlocksByRoot(request) => self.send_beacon_processor_work( - BeaconWorkEvent::blocks_by_roots_request(peer_id, request_id, request), + Request::BlocksByRoot(request) => self.handle_beacon_processor_send_result( + self.network_beacon_processor + .send_blocks_by_roots_request(peer_id, request_id, request), ), - Request::LightClientBootstrap(request) => self.send_beacon_processor_work( - BeaconWorkEvent::lightclient_bootstrap_request(peer_id, request_id, request), + Request::LightClientBootstrap(request) => self.handle_beacon_processor_send_result( + self.network_beacon_processor + .send_lightclient_bootstrap_request(peer_id, request_id, request), ), } } @@ -219,10 +223,10 @@ impl Router { match response { Response::Status(status_message) => { debug!(self.log, "Received Status Response"; "peer_id" => %peer_id, &status_message); - self.send_beacon_processor_work(BeaconWorkEvent::status_message( - peer_id, - status_message, - )) + self.handle_beacon_processor_send_result( + self.network_beacon_processor + .send_status_message(peer_id, status_message), + ) } Response::BlocksByRange(beacon_block) => { self.on_blocks_by_range_response(peer_id, request_id, beacon_block); @@ -247,36 +251,40 @@ impl Router { ) { match gossip_message { PubsubMessage::AggregateAndProofAttestation(aggregate_and_proof) => self - .send_beacon_processor_work(BeaconWorkEvent::aggregated_attestation( - message_id, - peer_id, - *aggregate_and_proof, - timestamp_now(), - )), - PubsubMessage::Attestation(subnet_attestation) => { - self.send_beacon_processor_work(BeaconWorkEvent::unaggregated_attestation( - message_id, - peer_id, - subnet_attestation.1, - subnet_attestation.0, - should_process, - timestamp_now(), - )) - } - PubsubMessage::BeaconBlock(block) => { - self.send_beacon_processor_work(BeaconWorkEvent::gossip_beacon_block( + .handle_beacon_processor_send_result( + self.network_beacon_processor.send_aggregated_attestation( + message_id, + peer_id, + *aggregate_and_proof, + timestamp_now(), + ), + ), + PubsubMessage::Attestation(subnet_attestation) => self + .handle_beacon_processor_send_result( + self.network_beacon_processor.send_unaggregated_attestation( + message_id, + peer_id, + subnet_attestation.1, + subnet_attestation.0, + should_process, + timestamp_now(), + ), + ), + PubsubMessage::BeaconBlock(block) => self.handle_beacon_processor_send_result( + self.network_beacon_processor.send_gossip_beacon_block( message_id, peer_id, self.network_globals.client(&peer_id), block, timestamp_now(), - )) - } + ), + ), PubsubMessage::VoluntaryExit(exit) => { debug!(self.log, "Received a voluntary exit"; "peer_id" => %peer_id); - self.send_beacon_processor_work(BeaconWorkEvent::gossip_voluntary_exit( - message_id, peer_id, exit, - )) + self.handle_beacon_processor_send_result( + self.network_beacon_processor + .send_gossip_voluntary_exit(message_id, peer_id, exit), + ) } PubsubMessage::ProposerSlashing(proposer_slashing) => { debug!( @@ -284,11 +292,13 @@ impl Router { "Received a proposer slashing"; "peer_id" => %peer_id ); - self.send_beacon_processor_work(BeaconWorkEvent::gossip_proposer_slashing( - message_id, - peer_id, - proposer_slashing, - )) + self.handle_beacon_processor_send_result( + self.network_beacon_processor.send_gossip_proposer_slashing( + message_id, + peer_id, + proposer_slashing, + ), + ) } PubsubMessage::AttesterSlashing(attester_slashing) => { debug!( @@ -296,11 +306,13 @@ impl Router { "Received a attester slashing"; "peer_id" => %peer_id ); - self.send_beacon_processor_work(BeaconWorkEvent::gossip_attester_slashing( - message_id, - peer_id, - attester_slashing, - )) + self.handle_beacon_processor_send_result( + self.network_beacon_processor.send_gossip_attester_slashing( + message_id, + peer_id, + attester_slashing, + ), + ) } PubsubMessage::SignedContributionAndProof(contribution_and_proof) => { trace!( @@ -308,12 +320,14 @@ impl Router { "Received sync committee aggregate"; "peer_id" => %peer_id ); - self.send_beacon_processor_work(BeaconWorkEvent::gossip_sync_contribution( - message_id, - peer_id, - *contribution_and_proof, - timestamp_now(), - )) + self.handle_beacon_processor_send_result( + self.network_beacon_processor.send_gossip_sync_contribution( + message_id, + peer_id, + *contribution_and_proof, + timestamp_now(), + ), + ) } PubsubMessage::SyncCommitteeMessage(sync_committtee_msg) => { trace!( @@ -321,13 +335,15 @@ impl Router { "Received sync committee signature"; "peer_id" => %peer_id ); - self.send_beacon_processor_work(BeaconWorkEvent::gossip_sync_signature( - message_id, - peer_id, - sync_committtee_msg.1, - sync_committtee_msg.0, - timestamp_now(), - )) + self.handle_beacon_processor_send_result( + self.network_beacon_processor.send_gossip_sync_signature( + message_id, + peer_id, + sync_committtee_msg.1, + sync_committtee_msg.0, + timestamp_now(), + ), + ) } PubsubMessage::LightClientFinalityUpdate(light_client_finality_update) => { trace!( @@ -335,13 +351,14 @@ impl Router { "Received light client finality update"; "peer_id" => %peer_id ); - self.send_beacon_processor_work( - BeaconWorkEvent::gossip_light_client_finality_update( - message_id, - peer_id, - light_client_finality_update, - timestamp_now(), - ), + self.handle_beacon_processor_send_result( + self.network_beacon_processor + .send_gossip_light_client_finality_update( + message_id, + peer_id, + *light_client_finality_update, + timestamp_now(), + ), ) } PubsubMessage::LightClientOptimisticUpdate(light_client_optimistic_update) => { @@ -350,21 +367,25 @@ impl Router { "Received light client optimistic update"; "peer_id" => %peer_id ); - self.send_beacon_processor_work( - BeaconWorkEvent::gossip_light_client_optimistic_update( - message_id, - peer_id, - light_client_optimistic_update, - timestamp_now(), - ), + self.handle_beacon_processor_send_result( + self.network_beacon_processor + .send_gossip_light_client_optimistic_update( + message_id, + peer_id, + *light_client_optimistic_update, + timestamp_now(), + ), ) } PubsubMessage::BlsToExecutionChange(bls_to_execution_change) => self - .send_beacon_processor_work(BeaconWorkEvent::gossip_bls_to_execution_change( - message_id, - peer_id, - bls_to_execution_change, - )), + .handle_beacon_processor_send_result( + self.network_beacon_processor + .send_gossip_bls_to_execution_change( + message_id, + peer_id, + bls_to_execution_change, + ), + ), } } @@ -415,7 +436,10 @@ impl Router { request_id, ); - self.send_beacon_processor_work(BeaconWorkEvent::status_message(peer_id, status)) + self.handle_beacon_processor_send_result( + self.network_beacon_processor + .send_status_message(peer_id, status), + ) } /// Handle a `BlocksByRange` response from the peer. @@ -480,20 +504,22 @@ impl Router { }); } - fn send_beacon_processor_work(&mut self, work: BeaconWorkEvent) { - self.beacon_processor_send - .try_send(work) - .unwrap_or_else(|e| { - let work_type = match &*e { - mpsc::error::TrySendError::Closed(work) - | mpsc::error::TrySendError::Full(work) => work.work_type(), - }; - - if self.logger_debounce.elapsed() { - error!(&self.log, "Unable to send message to the beacon processor"; - "error" => %e, "type" => work_type) + fn handle_beacon_processor_send_result( + &mut self, + result: Result<(), crate::network_beacon_processor::Error>, + ) { + if let Err(e) = result { + let work_type = match &e { + mpsc::error::TrySendError::Closed(work) | mpsc::error::TrySendError::Full(work) => { + work.work_type() } - }) + }; + + if self.logger_debounce.elapsed() { + error!(&self.log, "Unable to send message to the beacon processor"; + "error" => %e, "type" => work_type) + } + } } } diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index 2c919233fca..c2719477f1f 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -1,5 +1,5 @@ use super::sync::manager::RequestId as SyncId; -use crate::beacon_processor::InvalidBlockStorage; +use crate::network_beacon_processor::InvalidBlockStorage; use crate::persisted_dht::{clear_dht, load_dht, persist_dht}; use crate::router::{Router, RouterMessage}; use crate::subnet_service::SyncCommitteeService; @@ -9,6 +9,7 @@ use crate::{ NetworkConfig, }; use beacon_chain::{BeaconChain, BeaconChainTypes}; +use beacon_processor::{work_reprocessing_queue::ReprocessQueueMessage, BeaconProcessorSend}; use futures::channel::mpsc::Sender; use futures::future::OptionFuture; use futures::prelude::*; @@ -224,6 +225,8 @@ impl NetworkService { config: &NetworkConfig, executor: task_executor::TaskExecutor, gossipsub_registry: Option<&'_ mut Registry>, + beacon_processor_send: BeaconProcessorSend, + beacon_processor_reprocess_tx: mpsc::Sender, ) -> error::Result<(Arc>, NetworkSenders)> { let network_log = executor.log().clone(); // build the channels for external comms @@ -311,6 +314,8 @@ impl NetworkService { network_senders.network_send(), executor.clone(), invalid_block_storage, + beacon_processor_send, + beacon_processor_reprocess_tx, network_log.clone(), )?; diff --git a/beacon_node/network/src/service/tests.rs b/beacon_node/network/src/service/tests.rs index 83fcc8c9ac8..9943da15a0d 100644 --- a/beacon_node/network/src/service/tests.rs +++ b/beacon_node/network/src/service/tests.rs @@ -4,12 +4,15 @@ mod tests { use crate::persisted_dht::load_dht; use crate::{NetworkConfig, NetworkService}; use beacon_chain::test_utils::BeaconChainHarness; + use beacon_processor::{ + BeaconProcessorSend, MAX_SCHEDULED_WORK_QUEUE_LEN, MAX_WORK_EVENT_QUEUE_LEN, + }; use lighthouse_network::Enr; use slog::{o, Drain, Level, Logger}; use sloggers::{null::NullLoggerBuilder, Build}; use std::str::FromStr; use std::sync::Arc; - use tokio::runtime::Runtime; + use tokio::{runtime::Runtime, sync::mpsc}; use types::MinimalEthSpec; fn get_logger(actual_log: bool) -> Logger { @@ -67,10 +70,20 @@ mod tests { // Create a new network service which implicitly gets dropped at the // end of the block. - let _network_service = - NetworkService::start(beacon_chain.clone(), &config, executor, None) - .await - .unwrap(); + let (beacon_processor_send, _beacon_processor_receive) = + mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN); + let (beacon_processor_reprocess_tx, _beacon_processor_reprocess_rx) = + mpsc::channel(MAX_SCHEDULED_WORK_QUEUE_LEN); + let _network_service = NetworkService::start( + beacon_chain.clone(), + &config, + executor, + None, + BeaconProcessorSend(beacon_processor_send), + beacon_processor_reprocess_tx, + ) + .await + .unwrap(); drop(signal); }); diff --git a/beacon_node/network/src/sync/backfill_sync/mod.rs b/beacon_node/network/src/sync/backfill_sync/mod.rs index 9f676ba017d..a1c2404e5ed 100644 --- a/beacon_node/network/src/sync/backfill_sync/mod.rs +++ b/beacon_node/network/src/sync/backfill_sync/mod.rs @@ -8,7 +8,7 @@ //! If a batch fails, the backfill sync cannot progress. In this scenario, we mark the backfill //! sync as failed, log an error and attempt to retry once a new peer joins the node. -use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent as BeaconWorkEvent}; +use crate::network_beacon_processor::ChainSegmentProcessId; use crate::sync::manager::{BatchProcessResult, Id}; use crate::sync::network_context::SyncNetworkContext; use crate::sync::range_sync::{ @@ -537,8 +537,8 @@ impl BackFillSync { self.current_processing_batch = Some(batch_id); if let Err(e) = network - .processor_channel() - .try_send(BeaconWorkEvent::chain_segment(process_id, blocks)) + .beacon_processor() + .send_chain_segment(process_id, blocks) { crit!(self.log, "Failed to send backfill segment to processor."; "msg" => "process_batch", "error" => %e, "batch" => self.processing_target); diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index aa2694769c2..4340aa41d8b 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -2,6 +2,7 @@ use std::collections::hash_map::Entry; use std::collections::HashMap; use std::time::Duration; +use crate::network_beacon_processor::ChainSegmentProcessId; use beacon_chain::{BeaconChainTypes, BlockError}; use fnv::FnvHashMap; use lighthouse_network::{PeerAction, PeerId}; @@ -11,7 +12,6 @@ use smallvec::SmallVec; use std::sync::Arc; use store::{Hash256, SignedBeaconBlock}; -use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent}; use crate::metrics; use self::parent_lookup::PARENT_FAIL_TOLERANCE; @@ -542,8 +542,8 @@ impl BlockLookups { BlockProcessResult::Ok | BlockProcessResult::Err(BlockError::BlockIsAlreadyKnown { .. }) => { // Check if the beacon processor is available - let beacon_processor_send = match cx.processor_channel_if_enabled() { - Some(channel) => channel, + let beacon_processor = match cx.beacon_processor_if_enabled() { + Some(beacon_processor) => beacon_processor, None => { return trace!( self.log, @@ -555,7 +555,7 @@ impl BlockLookups { let (chain_hash, blocks, hashes, request) = parent_lookup.parts_for_processing(); let process_id = ChainSegmentProcessId::ParentLookup(chain_hash); - match beacon_processor_send.try_send(WorkEvent::chain_segment(process_id, blocks)) { + match beacon_processor.send_chain_segment(process_id, blocks) { Ok(_) => { self.processing_parent_lookups .insert(chain_hash, (hashes, request)); @@ -664,11 +664,15 @@ impl BlockLookups { process_type: BlockProcessType, cx: &mut SyncNetworkContext, ) -> Result<(), ()> { - match cx.processor_channel_if_enabled() { - Some(beacon_processor_send) => { + match cx.beacon_processor_if_enabled() { + Some(beacon_processor) => { trace!(self.log, "Sending block for processing"; "block" => ?block_root, "process" => ?process_type); - let event = WorkEvent::rpc_beacon_block(block_root, block, duration, process_type); - if let Err(e) = beacon_processor_send.try_send(event) { + if let Err(e) = beacon_processor.send_rpc_beacon_block( + block_root, + block, + duration, + process_type, + ) { error!( self.log, "Failed to send sync block to processor"; diff --git a/beacon_node/network/src/sync/block_lookups/tests.rs b/beacon_node/network/src/sync/block_lookups/tests.rs index 82334db0f8e..c588f867bd9 100644 --- a/beacon_node/network/src/sync/block_lookups/tests.rs +++ b/beacon_node/network/src/sync/block_lookups/tests.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use crate::beacon_processor::BeaconProcessorSend; +use crate::network_beacon_processor::NetworkBeaconProcessor; use crate::service::RequestId; use crate::sync::manager::RequestId as SyncId; use crate::NetworkMessage; @@ -9,18 +9,19 @@ use super::*; use beacon_chain::builder::Witness; use beacon_chain::eth1_chain::CachingEth1Backend; +use beacon_processor::WorkEvent; use lighthouse_network::{NetworkGlobals, Request}; use slog::{Drain, Level}; -use slot_clock::SystemTimeSlotClock; +use slot_clock::ManualSlotClock; use store::MemoryStore; use tokio::sync::mpsc; use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; use types::MinimalEthSpec as E; -type T = Witness, E, MemoryStore, MemoryStore>; +type T = Witness, E, MemoryStore, MemoryStore>; struct TestRig { - beacon_processor_rx: mpsc::Receiver>, + beacon_processor_rx: mpsc::Receiver>, network_rx: mpsc::UnboundedReceiver>, rng: XorShiftRng, } @@ -41,8 +42,10 @@ impl TestRig { } }; - let (beacon_processor_tx, beacon_processor_rx) = mpsc::channel(100); let (network_tx, network_rx) = mpsc::unbounded_channel(); + let globals = Arc::new(NetworkGlobals::new_test_globals(Vec::new(), &log)); + let (network_beacon_processor, beacon_processor_rx) = + NetworkBeaconProcessor::null_for_testing(globals); let rng = XorShiftRng::from_seed([42; 16]); let rig = TestRig { beacon_processor_rx, @@ -51,11 +54,9 @@ impl TestRig { }; let bl = BlockLookups::new(log.new(slog::o!("component" => "block_lookups"))); let cx = { - let globals = Arc::new(NetworkGlobals::new_test_globals(Vec::new(), &log)); SyncNetworkContext::new( network_tx, - globals, - BeaconProcessorSend(beacon_processor_tx), + Arc::new(network_beacon_processor), log.new(slog::o!("component" => "network_context")), ) }; @@ -102,7 +103,7 @@ impl TestRig { fn expect_block_process(&mut self) { match self.beacon_processor_rx.try_recv() { Ok(work) => { - assert_eq!(work.work_type(), crate::beacon_processor::RPC_BLOCK); + assert_eq!(work.work_type(), beacon_processor::RPC_BLOCK); } other => panic!("Expected block process, found {:?}", other), } @@ -112,7 +113,7 @@ impl TestRig { fn expect_parent_chain_process(&mut self) { match self.beacon_processor_rx.try_recv() { Ok(work) => { - assert_eq!(work.work_type(), crate::beacon_processor::CHAIN_SEGMENT); + assert_eq!(work.work_type(), beacon_processor::CHAIN_SEGMENT); } other => panic!("Expected chain segment process, found {:?}", other), } diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index c24d4c192b1..72542752c51 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -38,7 +38,7 @@ use super::block_lookups::BlockLookups; use super::network_context::SyncNetworkContext; use super::peer_sync_info::{remote_sync_type, PeerSyncType}; use super::range_sync::{RangeSync, RangeSyncType, EPOCHS_PER_BATCH}; -use crate::beacon_processor::{BeaconProcessorSend, ChainSegmentProcessId}; +use crate::network_beacon_processor::{ChainSegmentProcessId, NetworkBeaconProcessor}; use crate::service::NetworkMessage; use crate::status::ToStatusMessage; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError, EngineState}; @@ -159,9 +159,6 @@ pub struct SyncManager { /// A reference to the underlying beacon chain. chain: Arc>, - /// A reference to the network globals and peer-db. - network_globals: Arc>, - /// A receiving channel sent by the message processor thread. input_channel: mpsc::UnboundedReceiver>, @@ -186,29 +183,22 @@ pub struct SyncManager { pub fn spawn( executor: task_executor::TaskExecutor, beacon_chain: Arc>, - network_globals: Arc>, network_send: mpsc::UnboundedSender>, - beacon_processor_send: BeaconProcessorSend, + beacon_processor: Arc>, + sync_recv: mpsc::UnboundedReceiver>, log: slog::Logger, -) -> mpsc::UnboundedSender> { +) { assert!( MAX_REQUEST_BLOCKS >= T::EthSpec::slots_per_epoch() * EPOCHS_PER_BATCH, "Max blocks that can be requested in a single batch greater than max allowed blocks in a single request" ); - // generate the message channel - let (sync_send, sync_recv) = mpsc::unbounded_channel::>(); // create an instance of the SyncManager + let network_globals = beacon_processor.network_globals.clone(); let mut sync_manager = SyncManager { chain: beacon_chain.clone(), - network_globals: network_globals.clone(), input_channel: sync_recv, - network: SyncNetworkContext::new( - network_send, - network_globals.clone(), - beacon_processor_send, - log.clone(), - ), + network: SyncNetworkContext::new(network_send, beacon_processor, log.clone()), range_sync: RangeSync::new(beacon_chain.clone(), log.clone()), backfill_sync: BackFillSync::new(beacon_chain, network_globals, log.clone()), block_lookups: BlockLookups::new(log.clone()), @@ -218,10 +208,13 @@ pub fn spawn( // spawn the sync manager thread debug!(log, "Sync Manager started"); executor.spawn(async move { Box::pin(sync_manager.main()).await }, "sync"); - sync_send } impl SyncManager { + fn network_globals(&self) -> &NetworkGlobals { + self.network.network_globals() + } + /* Input Handling Functions */ /// A peer has connected which has blocks that are unknown to us. @@ -322,12 +315,12 @@ impl SyncManager { let rpr = new_state.as_str(); // Drop the write lock let update_sync_status = self - .network_globals + .network_globals() .peers .write() .update_sync_status(peer_id, new_state.clone()); if let Some(was_updated) = update_sync_status { - let is_connected = self.network_globals.peers.read().is_connected(peer_id); + let is_connected = self.network_globals().peers.read().is_connected(peer_id); if was_updated { debug!( self.log, @@ -383,7 +376,7 @@ impl SyncManager { let head = self.chain.best_slot(); let current_slot = self.chain.slot().unwrap_or_else(|_| Slot::new(0)); - let peers = self.network_globals.peers.read(); + let peers = self.network_globals().peers.read(); if current_slot >= head && current_slot.sub(head) <= (SLOT_IMPORT_TOLERANCE as u64) && head > 0 @@ -445,8 +438,8 @@ impl SyncManager { }, }; - let old_state = self.network_globals.set_sync_state(new_state); - let new_state = self.network_globals.sync_state.read(); + let old_state = self.network_globals().set_sync_state(new_state); + let new_state = self.network_globals().sync_state.read().clone(); if !new_state.eq(&old_state) { info!(self.log, "Sync state updated"; "old_state" => %old_state, "new_state" => %new_state); // If we have become synced - Subscribe to all the core subnet topics @@ -505,7 +498,7 @@ impl SyncManager { } SyncMessage::UnknownBlock(peer_id, block, block_root) => { // If we are not synced or within SLOT_IMPORT_TOLERANCE of the block, ignore - if !self.network_globals.sync_state.read().is_synced() { + if !self.network_globals().sync_state.read().is_synced() { let head_slot = self.chain.canonical_head.cached_head().head_slot(); let unknown_block_slot = block.slot(); @@ -519,7 +512,7 @@ impl SyncManager { return; } } - if self.network_globals.peers.read().is_connected(&peer_id) + if self.network_globals().peers.read().is_connected(&peer_id) && self.network.is_execution_engine_online() { self.block_lookups @@ -528,8 +521,8 @@ impl SyncManager { } SyncMessage::UnknownBlockHash(peer_id, block_hash) => { // If we are not synced, ignore this block. - if self.network_globals.sync_state.read().is_synced() - && self.network_globals.peers.read().is_connected(&peer_id) + if self.network_globals().sync_state.read().is_synced() + && self.network_globals().peers.read().is_connected(&peer_id) && self.network.is_execution_engine_online() { self.block_lookups diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 03c466eecea..adc235130b0 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -3,7 +3,7 @@ use super::manager::{Id, RequestId as SyncRequestId}; use super::range_sync::{BatchId, ChainId}; -use crate::beacon_processor::BeaconProcessorSend; +use crate::network_beacon_processor::NetworkBeaconProcessor; use crate::service::{NetworkMessage, RequestId}; use crate::status::ToStatusMessage; use beacon_chain::{BeaconChainTypes, EngineState}; @@ -20,9 +20,6 @@ pub struct SyncNetworkContext { /// The network channel to relay messages to the Network service. network_send: mpsc::UnboundedSender>, - /// Access to the network global vars. - network_globals: Arc>, - /// A sequential ID for all RPC requests. request_id: Id, @@ -36,8 +33,8 @@ pub struct SyncNetworkContext { /// `beacon_processor_send`. execution_engine_state: EngineState, - /// Channel to send work to the beacon processor. - beacon_processor_send: BeaconProcessorSend, + /// Sends work to the beacon processor via a channel. + network_beacon_processor: Arc>, /// Logger for the `SyncNetworkContext`. log: slog::Logger, @@ -46,25 +43,27 @@ pub struct SyncNetworkContext { impl SyncNetworkContext { pub fn new( network_send: mpsc::UnboundedSender>, - network_globals: Arc>, - beacon_processor_send: BeaconProcessorSend, + network_beacon_processor: Arc>, log: slog::Logger, ) -> Self { Self { network_send, execution_engine_state: EngineState::Online, // always assume `Online` at the start - network_globals, request_id: 1, range_requests: FnvHashMap::default(), backfill_requests: FnvHashMap::default(), - beacon_processor_send, + network_beacon_processor, log, } } + pub fn network_globals(&self) -> &NetworkGlobals { + &self.network_beacon_processor.network_globals + } + /// Returns the Client type of the peer if known pub fn client_type(&self, peer_id: &PeerId) -> Client { - self.network_globals + self.network_globals() .peers .read() .peer_info(peer_id) @@ -278,13 +277,13 @@ impl SyncNetworkContext { }) } - pub fn processor_channel_if_enabled(&self) -> Option<&BeaconProcessorSend> { + pub fn beacon_processor_if_enabled(&self) -> Option<&Arc>> { self.is_execution_engine_online() - .then_some(&self.beacon_processor_send) + .then_some(&self.network_beacon_processor) } - pub fn processor_channel(&self) -> &BeaconProcessorSend { - &self.beacon_processor_send + pub fn beacon_processor(&self) -> &Arc> { + &self.network_beacon_processor } fn next_id(&mut self) -> Id { diff --git a/beacon_node/network/src/sync/range_sync/chain.rs b/beacon_node/network/src/sync/range_sync/chain.rs index 51ca9e2b071..af547885dca 100644 --- a/beacon_node/network/src/sync/range_sync/chain.rs +++ b/beacon_node/network/src/sync/range_sync/chain.rs @@ -1,5 +1,5 @@ use super::batch::{BatchInfo, BatchProcessingResult, BatchState}; -use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent as BeaconWorkEvent}; +use crate::network_beacon_processor::ChainSegmentProcessId; use crate::sync::{ manager::Id, network_context::SyncNetworkContext, BatchOperationOutcome, BatchProcessResult, }; @@ -294,8 +294,8 @@ impl SyncingChain { return Ok(KeepChain); } - let beacon_processor_send = match network.processor_channel_if_enabled() { - Some(channel) => channel, + let beacon_processor = match network.beacon_processor_if_enabled() { + Some(beacon_processor) => beacon_processor, None => return Ok(KeepChain), }; @@ -317,9 +317,7 @@ impl SyncingChain { let process_id = ChainSegmentProcessId::RangeBatchId(self.id, batch_id); self.current_processing_batch = Some(batch_id); - if let Err(e) = - beacon_processor_send.try_send(BeaconWorkEvent::chain_segment(process_id, blocks)) - { + if let Err(e) = beacon_processor.send_chain_segment(process_id, blocks) { crit!(self.log, "Failed to send chain segment to processor."; "msg" => "process_batch", "error" => %e, "batch" => self.processing_target); // This is unlikely to happen but it would stall syncing since the batch now has no diff --git a/beacon_node/network/src/sync/range_sync/range.rs b/beacon_node/network/src/sync/range_sync/range.rs index 2c35c57d9e4..05ad5204b9e 100644 --- a/beacon_node/network/src/sync/range_sync/range.rs +++ b/beacon_node/network/src/sync/range_sync/range.rs @@ -371,22 +371,23 @@ where #[cfg(test)] mod tests { + use crate::network_beacon_processor::NetworkBeaconProcessor; use crate::service::RequestId; use crate::NetworkMessage; use super::*; - use crate::beacon_processor::{BeaconProcessorSend, WorkEvent as BeaconWorkEvent}; use beacon_chain::builder::Witness; use beacon_chain::eth1_chain::CachingEth1Backend; use beacon_chain::parking_lot::RwLock; use beacon_chain::EngineState; + use beacon_processor::WorkEvent as BeaconWorkEvent; use lighthouse_network::rpc::BlocksByRangeRequest; use lighthouse_network::Request; use lighthouse_network::{rpc::StatusMessage, NetworkGlobals}; use slog::{o, Drain}; use tokio::sync::mpsc; - use slot_clock::SystemTimeSlotClock; + use slot_clock::ManualSlotClock; use std::collections::HashSet; use std::sync::Arc; use store::MemoryStore; @@ -437,7 +438,7 @@ mod tests { } type TestBeaconChainType = - Witness, E, MemoryStore, MemoryStore>; + Witness, E, MemoryStore, MemoryStore>; fn build_log(level: slog::Level, enabled: bool) -> slog::Logger { let decorator = slog_term::TermDecorator::new().build(); @@ -455,7 +456,7 @@ mod tests { struct TestRig { log: slog::Logger, /// To check what does sync send to the beacon processor. - beacon_processor_rx: mpsc::Receiver>, + beacon_processor_rx: mpsc::Receiver>, /// To set up different scenarios where sync is told about known/unkown blocks. chain: Arc, /// Needed by range to handle communication with the network. @@ -583,7 +584,7 @@ mod tests { fn expect_chain_segment(&mut self) { match self.beacon_processor_rx.try_recv() { Ok(work) => { - assert_eq!(work.work_type(), crate::beacon_processor::CHAIN_SEGMENT); + assert_eq!(work.work_type(), beacon_processor::CHAIN_SEGMENT); } other => panic!("Expected chain segment process, found {:?}", other), } @@ -593,17 +594,17 @@ mod tests { fn range(log_enabled: bool) -> (TestRig, RangeSync) { let chain = Arc::new(FakeStorage::default()); let log = build_log(slog::Level::Trace, log_enabled); - let (beacon_processor_tx, beacon_processor_rx) = mpsc::channel(10); let range_sync = RangeSync::::new( chain.clone(), log.new(o!("component" => "range")), ); let (network_tx, network_rx) = mpsc::unbounded_channel(); let globals = Arc::new(NetworkGlobals::new_test_globals(Vec::new(), &log)); + let (network_beacon_processor, beacon_processor_rx) = + NetworkBeaconProcessor::null_for_testing(globals.clone()); let cx = SyncNetworkContext::new( network_tx, - globals.clone(), - BeaconProcessorSend(beacon_processor_tx), + Arc::new(network_beacon_processor), log.new(o!("component" => "network_context")), ); let test_rig = TestRig { From 6fd2ef49e4258a1fb6f1eb7093460c5abb43cbd9 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 10 Jul 2023 10:17:31 -0400 Subject: [PATCH 18/51] Revert "remove into gossip verified block" This reverts commit 246d52d209ef1ddf5659450e37b4462d1c0555d7. --- .../beacon_chain/src/block_verification.rs | 34 ++++++++++++++++ beacon_node/beacon_chain/src/lib.rs | 2 +- beacon_node/http_api/src/publish_blocks.rs | 39 ++++++++++--------- 3 files changed, 55 insertions(+), 20 deletions(-) diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 0f8f03e5fd9..819b3e68cc8 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -796,6 +796,40 @@ pub struct BlockImportData { pub consensus_context: ConsensusContext, } +pub trait IntoGossipVerifiedBlock: Sized { + fn into_gossip_verified_block( + self, + chain: &BeaconChain, + ) -> Result, BlockError>; + fn inner(&self) -> Arc>; +} + +impl IntoGossipVerifiedBlock for GossipVerifiedBlock { + fn into_gossip_verified_block( + self, + _chain: &BeaconChain, + ) -> Result, BlockError> { + Ok(self) + } + + fn inner(&self) -> Arc> { + self.block.clone() + } +} + +impl IntoGossipVerifiedBlock for Arc> { + fn into_gossip_verified_block( + self, + chain: &BeaconChain, + ) -> Result, BlockError> { + GossipVerifiedBlock::new(self, chain) + } + + fn inner(&self) -> Arc> { + self.clone() + } +} + /// Implemented on types that can be converted into a `ExecutionPendingBlock`. /// /// Used to allow functions to accept blocks at various stages of verification. diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 94fb8c85516..9b4519f6fe1 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -71,7 +71,7 @@ pub use beacon_fork_choice_store::{BeaconForkChoiceStore, Error as ForkChoiceSto pub use block_verification::{ get_block_root, AvailabilityPendingExecutedBlock, BlockError, ExecutedBlock, ExecutionPayloadError, ExecutionPendingBlock, GossipVerifiedBlock, IntoExecutionPendingBlock, - PayloadVerificationOutcome, PayloadVerificationStatus, + PayloadVerificationOutcome, PayloadVerificationStatus,IntoGossipVerifiedBlock, }; pub use canonical_head::{CachedHead, CanonicalHead, CanonicalHeadRwLock}; pub use eth1_chain::{Eth1Chain, Eth1ChainBackend}; diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 0ddd72a7263..569497aadc5 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -3,7 +3,7 @@ use crate::metrics; use beacon_chain::blob_verification::BlockWrapper; use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now}; use beacon_chain::{ - AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, + AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes, BlockError,IntoGossipVerifiedBlock, GossipVerifiedBlock, NotifyExecutionLayer, }; use eth2::types::BroadcastValidation; @@ -13,6 +13,7 @@ use lighthouse_network::PubsubMessage; use network::NetworkMessage; use slog::{debug, error, info, warn, Logger}; use slot_clock::SlotClock; +use std::marker::PhantomData; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; @@ -23,28 +24,28 @@ use types::{ }; use warp::Rejection; -pub enum ProvenancedBlock { +pub enum ProvenancedBlock> { /// The payload was built using a local EE. - Local(SignedBlockContents>), + Local(B, PhantomData), /// The payload was build using a remote builder (e.g., via a mev-boost /// compatible relay). - Builder(SignedBlockContents>), + Builder(B, PhantomData), } -impl ProvenancedBlock { - pub fn local(block: SignedBlockContents) -> Self { - Self::Local(block) +impl> ProvenancedBlock { + pub fn local(block: B) -> Self { + Self::Local(block, PhantomData) } - pub fn builder(block: SignedBlockContents) -> Self { - Self::Builder(block) + pub fn builder(block: B) -> Self { + Self::Builder(block, PhantomData) } } /// Handles a request from the HTTP API for full blocks. -pub async fn publish_block( +pub async fn publish_block>( block_root: Option, - provenanced_block: ProvenancedBlock, + provenanced_block: ProvenancedBlock, chain: Arc>, network_tx: &UnboundedSender>, log: Logger, @@ -53,11 +54,11 @@ pub async fn publish_block( let seen_timestamp = timestamp_now(); let (block, blobs_opt, is_locally_built_block) = match provenanced_block { - ProvenancedBlock::Local(block_contents) => { - let (block, maybe_blobs) = block_contents.deconstruct(); + ProvenancedBlock::Local(block_contents, _) => { + let (block, maybe_blobs, ) = block_contents.deconstruct(); (Arc::new(block), maybe_blobs, true) } - ProvenancedBlock::Builder(block_contents) => { + ProvenancedBlock::Builder(block_contents, _) => { let (block, maybe_blobs) = block_contents.deconstruct(); (Arc::new(block), maybe_blobs, false) } @@ -124,8 +125,8 @@ pub async fn publish_block( let log_clone = log.clone(); /* if we can form a `GossipVerifiedBlock`, we've passed our basic gossip checks */ - let gossip_verified_block = GossipVerifiedBlock::new( - BlockWrapper::new(block.clone(), mapped_blobs), + let gossip_verified_block = + BlockWrapper::new(block.clone(), mapped_blobs).into_gossip_verified_block( &chain, ) .map_err(|e| { @@ -270,8 +271,8 @@ pub async fn publish_blinded_block( validation_level: BroadcastValidation, ) -> Result<(), Rejection> { let block_root = block.signed_block().canonical_root(); - let full_block = reconstruct_block(chain.clone(), block_root, block, log.clone()).await?; - publish_block::( + let full_block: ProvenancedBlock> = reconstruct_block(chain.clone(), block_root, block, log.clone()).await?; + publish_block::( Some(block_root), full_block, chain, @@ -290,7 +291,7 @@ pub async fn reconstruct_block( block_root: Hash256, block: SignedBlockContents>, log: Logger, -) -> Result, Rejection> { +) -> Result>, Rejection> { let full_payload_opt = if let Ok(payload_header) = block.signed_block().message().body().execution_payload() { From 782a53ad9d0748f957cdc67a6d522a9bea22bd72 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 11 Jul 2023 16:05:05 -0400 Subject: [PATCH 19/51] fix compile --- .../beacon_chain/src/blob_verification.rs | 1 - .../beacon_chain/src/block_verification.rs | 44 ++++-- .../src/data_availability_checker.rs | 11 ++ beacon_node/beacon_chain/src/lib.rs | 2 +- beacon_node/beacon_chain/src/test_utils.rs | 48 ++++++- beacon_node/http_api/src/publish_blocks.rs | 45 +++---- .../tests/broadcast_validation_tests.rs | 125 +++++++++++------- beacon_node/http_api/tests/tests.rs | 2 +- common/eth2/src/lib.rs | 7 +- common/eth2/src/types.rs | 17 +++ .../src/per_block_processing/tests.rs | 10 +- testing/state_transition_vectors/src/exit.rs | 2 +- 12 files changed, 212 insertions(+), 102 deletions(-) diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index 8e7c4dc0746..501e0ae59df 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -511,7 +511,6 @@ pub fn verify_kzg_for_blob_list( let (blobs, (commitments, proofs)): (Vec<_>, (Vec<_>, Vec<_>)) = blob_list .clone() .into_iter() - //TODO(sean) remove clone .map(|blob| (blob.blob.clone(), (blob.kzg_commitment, blob.kzg_proof))) .unzip(); if validate_blobs::( diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 819b3e68cc8..5881ee7d96b 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -69,7 +69,7 @@ use crate::{ metrics, BeaconChain, BeaconChainError, BeaconChainTypes, }; use derivative::Derivative; -use eth2::types::EventKind; +use eth2::types::{ArcBlockContentsTuple, EventKind, SignedBlockContents}; use execution_layer::PayloadStatus; pub use fork_choice::{AttestationFromBlock, PayloadVerificationStatus}; use parking_lot::RwLockReadGuard; @@ -92,7 +92,7 @@ use std::fs; use std::io::Write; use std::sync::Arc; use std::time::Duration; -use store::{Error as DBError, HotStateSummary, KeyValueStore, StoreOp}; +use store::{Error as DBError, HotStateSummary, KeyValueStore, SignedBlobSidecarList, StoreOp}; use task_executor::JoinHandle; use tree_hash::TreeHash; use types::blob_sidecar::BlobIdentifier; @@ -627,6 +627,13 @@ pub struct GossipVerifiedBlock { consensus_context: ConsensusContext, } +impl GossipVerifiedBlock { + /// Useful for publishing after gossip verification. + pub fn into_block_wrapper(self) -> BlockWrapper { + self.block.into_block_wrapper() + } +} + /// A wrapper around a `SignedBeaconBlock` that indicates that all signatures (except the deposit /// signatures) have been verified. pub struct SignatureVerifiedBlock { @@ -801,32 +808,45 @@ pub trait IntoGossipVerifiedBlock: Sized { self, chain: &BeaconChain, ) -> Result, BlockError>; - fn inner(&self) -> Arc>; + fn inner(&self) -> &SignedBeaconBlock; + fn parts(self) -> ArcBlockContentsTuple; } -impl IntoGossipVerifiedBlock for GossipVerifiedBlock { +impl IntoGossipVerifiedBlock + for ( + GossipVerifiedBlock, + Option>, + ) +{ fn into_gossip_verified_block( self, _chain: &BeaconChain, ) -> Result, BlockError> { - Ok(self) + Ok(self.0) } - - fn inner(&self) -> Arc> { - self.block.clone() + fn inner(&self) -> &SignedBeaconBlock { + self.0.block.as_block() + } + fn parts(self) -> ArcBlockContentsTuple { + (self.0.block.block_cloned(), self.1) } } -impl IntoGossipVerifiedBlock for Arc> { +impl IntoGossipVerifiedBlock for SignedBlockContents { fn into_gossip_verified_block( self, chain: &BeaconChain, ) -> Result, BlockError> { - GossipVerifiedBlock::new(self, chain) + GossipVerifiedBlock::new(self.deconstruct().into(), chain) + } + + fn inner(&self) -> &SignedBeaconBlock { + self.signed_block() } - fn inner(&self) -> Arc> { - self.clone() + fn parts(self) -> ArcBlockContentsTuple { + let (block, blobs) = self.deconstruct(); + (Arc::new(block), blobs) } } diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 4d2d23f9e24..ed5e04e0730 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -531,6 +531,17 @@ pub enum VerifiedBlobs { PreDeneb, } +impl VerifiedBlobs { + pub fn to_blobs(self) -> Option> { + match self { + Self::Available(blobs) => Some(blobs), + Self::NotRequired => None, + Self::EmptyBlobs => None, + Self::PreDeneb => None, + } + } +} + /// A fully available block that is ready to be imported into fork choice. #[derive(Clone, Debug, PartialEq)] pub struct AvailableBlock { diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 9b4519f6fe1..09981de8dc9 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -71,7 +71,7 @@ pub use beacon_fork_choice_store::{BeaconForkChoiceStore, Error as ForkChoiceSto pub use block_verification::{ get_block_root, AvailabilityPendingExecutedBlock, BlockError, ExecutedBlock, ExecutionPayloadError, ExecutionPendingBlock, GossipVerifiedBlock, IntoExecutionPendingBlock, - PayloadVerificationOutcome, PayloadVerificationStatus,IntoGossipVerifiedBlock, + IntoGossipVerifiedBlock, PayloadVerificationOutcome, PayloadVerificationStatus, }; pub use canonical_head::{CachedHead, CanonicalHead, CanonicalHeadRwLock}; pub use eth1_chain::{Eth1Chain, Eth1ChainBackend}; diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 125ff471d4b..15331313b95 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -906,7 +906,7 @@ where &self, mut state: BeaconState, slot: Slot, - ) -> (SignedBeaconBlock, BeaconState) { + ) -> (BlockContentsTuple>, BeaconState) { assert_ne!(slot, 0, "can't produce a block at slot 0"); assert!(slot >= state.slot()); @@ -946,7 +946,44 @@ where &self.spec, ); - (signed_block, pre_state) + let block_contents: BlockContentsTuple> = match &signed_block { + SignedBeaconBlock::Base(_) + | SignedBeaconBlock::Altair(_) + | SignedBeaconBlock::Merge(_) + | SignedBeaconBlock::Capella(_) => (signed_block, None), + SignedBeaconBlock::Deneb(_) => { + if let Some(blobs) = self + .chain + .proposal_blob_cache + .pop(&signed_block.canonical_root()) + { + let signed_blobs: SignedBlobSidecarList = Vec::from(blobs) + .into_iter() + .map(|blob| { + blob.sign( + &self.validator_keypairs[proposer_index].sk, + &state.fork(), + state.genesis_validators_root(), + &self.spec, + ) + }) + .collect::>() + .into(); + let mut guard = self.blob_signature_cache.write(); + for blob in &signed_blobs { + guard.insert( + BlobSignatureKey::new(blob.message.block_root, blob.message.index), + blob.signature.clone(), + ); + } + (signed_block, Some(signed_blobs)) + } else { + (signed_block, None) + } + } + }; + + (block_contents, pre_state) } /// Create a randao reveal for a block at `slot`. @@ -1737,11 +1774,12 @@ where state: BeaconState, slot: Slot, block_modifier: impl FnOnce(&mut BeaconBlock), - ) -> (SignedBeaconBlock, BeaconState) { + ) -> (BlockContentsTuple>, BeaconState) { assert_ne!(slot, 0, "can't produce a block at slot 0"); assert!(slot >= state.slot()); - let (block, state) = self.make_block_return_pre_state(state, slot).await; + let ((block, blobs), state) = self.make_block_return_pre_state(state, slot).await; + let (mut block, _) = block.deconstruct(); block_modifier(&mut block); @@ -1754,7 +1792,7 @@ where state.genesis_validators_root(), &self.spec, ); - (signed_block, state) + ((signed_block, blobs), state) } pub fn make_deposits<'a>( diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 569497aadc5..ed472e0bff5 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -3,8 +3,8 @@ use crate::metrics; use beacon_chain::blob_verification::BlockWrapper; use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now}; use beacon_chain::{ - AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes, BlockError,IntoGossipVerifiedBlock, - GossipVerifiedBlock, NotifyExecutionLayer, + AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, + GossipVerifiedBlock, IntoGossipVerifiedBlock, NotifyExecutionLayer, }; use eth2::types::BroadcastValidation; use eth2::types::SignedBlockContents; @@ -53,16 +53,11 @@ pub async fn publish_block>( ) -> Result<(), Rejection> { let seen_timestamp = timestamp_now(); - let (block, blobs_opt, is_locally_built_block) = match provenanced_block { - ProvenancedBlock::Local(block_contents, _) => { - let (block, maybe_blobs, ) = block_contents.deconstruct(); - (Arc::new(block), maybe_blobs, true) - } - ProvenancedBlock::Builder(block_contents, _) => { - let (block, maybe_blobs) = block_contents.deconstruct(); - (Arc::new(block), maybe_blobs, false) - } + let (block_contents, is_locally_built_block) = match provenanced_block { + ProvenancedBlock::Local(block_contents, _) => (block_contents, true), + ProvenancedBlock::Builder(block_contents, _) => (block_contents, false), }; + let block = block_contents.inner(); let delay = get_block_delay_ms(seen_timestamp, block.message(), &chain.slot_clock); debug!(log, "Signed block received in HTTP API"; "slot" => block.slot()); @@ -107,6 +102,15 @@ pub async fn publish_block>( Ok(()) }; + /* only publish if gossip- and consensus-valid and equivocation-free */ + let chain_clone = chain.clone(); + let slot = block.message().slot(); + let proposer_index = block.message().proposer_index(); + let sender_clone = network_tx.clone(); + let log_clone = log.clone(); + + let (block, blobs_opt) = block_contents.parts(); + let mapped_blobs = blobs_opt.clone().map(|blobs| { VariableList::from( blobs @@ -116,17 +120,9 @@ pub async fn publish_block>( ) }); - /* only publish if gossip- and consensus-valid and equivocation-free */ - let chain_clone = chain.clone(); - let slot = block.message().slot(); - let block_clone = block.clone(); - let proposer_index = block.message().proposer_index(); - let sender_clone = network_tx.clone(); - let log_clone = log.clone(); - /* if we can form a `GossipVerifiedBlock`, we've passed our basic gossip checks */ - let gossip_verified_block = - BlockWrapper::new(block.clone(), mapped_blobs).into_gossip_verified_block( + let gossip_verified_block = GossipVerifiedBlock::new( + BlockWrapper::new(block.clone(), mapped_blobs), &chain, ) .map_err(|e| { @@ -147,6 +143,8 @@ pub async fn publish_block>( .map_err(|_| warp_utils::reject::custom_server_error("unable to publish".into()))?; } + let block_clone = block.clone(); + let publish_fn = move || match validation_level { BroadcastValidation::Gossip => Ok(()), BroadcastValidation::Consensus => publish_block( @@ -271,7 +269,8 @@ pub async fn publish_blinded_block( validation_level: BroadcastValidation, ) -> Result<(), Rejection> { let block_root = block.signed_block().canonical_root(); - let full_block: ProvenancedBlock> = reconstruct_block(chain.clone(), block_root, block, log.clone()).await?; + let full_block: ProvenancedBlock> = + reconstruct_block(chain.clone(), block_root, block, log.clone()).await?; publish_block::( Some(block_root), full_block, @@ -291,7 +290,7 @@ pub async fn reconstruct_block( block_root: Hash256, block: SignedBlockContents>, log: Logger, -) -> Result>, Rejection> { +) -> Result>, Rejection> { let full_payload_opt = if let Ok(payload_header) = block.signed_block().message().body().execution_payload() { diff --git a/beacon_node/http_api/tests/broadcast_validation_tests.rs b/beacon_node/http_api/tests/broadcast_validation_tests.rs index 4819dd99e7a..ccf7439b179 100644 --- a/beacon_node/http_api/tests/broadcast_validation_tests.rs +++ b/beacon_node/http_api/tests/broadcast_validation_tests.rs @@ -2,7 +2,9 @@ use beacon_chain::{ test_utils::{AttestationStrategy, BlockStrategy}, GossipVerifiedBlock, }; -use eth2::types::{BroadcastValidation, SignedBeaconBlock, SignedBlindedBeaconBlock}; +use eth2::types::{ + BroadcastValidation, SignedBeaconBlock, SignedBlindedBeaconBlock, SignedBlockContents, +}; use http_api::test_utils::InteractiveTester; use http_api::{publish_blinded_block, publish_block, reconstruct_block, ProvenancedBlock}; use tree_hash::TreeHash; @@ -63,7 +65,7 @@ pub async fn gossip_invalid() { tester.harness.advance_slot(); - let (block, _): (SignedBeaconBlock, _) = tester + let ((block, blobs), _): ((SignedBeaconBlock, _), _) = tester .harness .make_block_with_modifier(chain_state_before, slot, |b| { *b.state_root_mut() = Hash256::zero(); @@ -73,7 +75,7 @@ pub async fn gossip_invalid() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&block, validation_level) + .post_beacon_blocks_v2(&SignedBlockContents::new(block, blobs), validation_level) .await; assert!(response.is_err()); @@ -115,7 +117,7 @@ pub async fn gossip_partial_pass() { tester.harness.advance_slot(); - let (block, _): (SignedBeaconBlock, _) = tester + let ((block, blobs), _): ((SignedBeaconBlock, _), _) = tester .harness .make_block_with_modifier(chain_state_before, slot, |b| { *b.state_root_mut() = Hash256::random() @@ -124,7 +126,7 @@ pub async fn gossip_partial_pass() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&block, validation_level) + .post_beacon_blocks_v2(&SignedBlockContents::new(block, blobs), validation_level) .await; assert!(response.is_err()); @@ -161,11 +163,15 @@ pub async fn gossip_full_pass() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let (block, _): (SignedBeaconBlock, _) = tester.harness.make_block(state_a, slot_b).await; + let ((block, blobs), _): ((SignedBeaconBlock, _), _) = + tester.harness.make_block(state_a, slot_b).await; let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&block, validation_level) + .post_beacon_blocks_v2( + &SignedBlockContents::new(block.clone(), blobs), + validation_level, + ) .await; assert!(response.is_ok()); @@ -202,7 +208,7 @@ pub async fn consensus_invalid() { tester.harness.advance_slot(); - let (block, _): (SignedBeaconBlock, _) = tester + let ((block, blobs), _): ((SignedBeaconBlock, _), _) = tester .harness .make_block_with_modifier(chain_state_before, slot, |b| { *b.state_root_mut() = Hash256::zero(); @@ -212,7 +218,7 @@ pub async fn consensus_invalid() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&block, validation_level) + .post_beacon_blocks_v2(&SignedBlockContents::new(block, blobs), validation_level) .await; assert!(response.is_err()); @@ -254,14 +260,14 @@ pub async fn consensus_gossip() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let (block, _): (SignedBeaconBlock, _) = tester + let ((block, blobs), _): ((SignedBeaconBlock, _), _) = tester .harness .make_block_with_modifier(state_a, slot_b, |b| *b.state_root_mut() = Hash256::zero()) .await; let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&block, validation_level) + .post_beacon_blocks_v2(&SignedBlockContents::new(block, blobs), validation_level) .await; assert!(response.is_err()); @@ -304,9 +310,9 @@ pub async fn consensus_partial_pass_only_consensus() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let (block_a, state_after_a): (SignedBeaconBlock, _) = + let ((block_a, _), state_after_a): ((SignedBeaconBlock, _), _) = tester.harness.make_block(state_a.clone(), slot_b).await; - let (block_b, state_after_b): (SignedBeaconBlock, _) = + let ((block_b, blobs_b), state_after_b): ((SignedBeaconBlock, _), _) = tester.harness.make_block(state_a, slot_b).await; /* check for `make_block` curios */ @@ -324,7 +330,7 @@ pub async fn consensus_partial_pass_only_consensus() { let publication_result: Result<(), Rejection> = publish_block( None, - ProvenancedBlock::local(gossip_block_b.unwrap()), + ProvenancedBlock::local((gossip_block_b.unwrap(), blobs_b)), tester.harness.chain.clone(), &channel.0, test_logger, @@ -367,11 +373,15 @@ pub async fn consensus_full_pass() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let (block, _): (SignedBeaconBlock, _) = tester.harness.make_block(state_a, slot_b).await; + let ((block, blobs), _): ((SignedBeaconBlock, _), _) = + tester.harness.make_block(state_a, slot_b).await; let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&block, validation_level) + .post_beacon_blocks_v2( + &SignedBlockContents::new(block.clone(), blobs), + validation_level, + ) .await; assert!(response.is_ok()); @@ -410,7 +420,7 @@ pub async fn equivocation_invalid() { tester.harness.advance_slot(); - let (block, _): (SignedBeaconBlock, _) = tester + let ((block, blobs), _): ((SignedBeaconBlock, _), _) = tester .harness .make_block_with_modifier(chain_state_before, slot, |b| { *b.state_root_mut() = Hash256::zero(); @@ -420,7 +430,7 @@ pub async fn equivocation_invalid() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&block, validation_level) + .post_beacon_blocks_v2(&SignedBlockContents::new(block, blobs), validation_level) .await; assert!(response.is_err()); @@ -463,9 +473,9 @@ pub async fn equivocation_consensus_early_equivocation() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let (block_a, state_after_a): (SignedBeaconBlock, _) = + let ((block_a, blobs_a), state_after_a): ((SignedBeaconBlock, _), _) = tester.harness.make_block(state_a.clone(), slot_b).await; - let (block_b, state_after_b): (SignedBeaconBlock, _) = + let ((block_b, blobs_b), state_after_b): ((SignedBeaconBlock, _), _) = tester.harness.make_block(state_a, slot_b).await; /* check for `make_block` curios */ @@ -476,7 +486,10 @@ pub async fn equivocation_consensus_early_equivocation() { /* submit `block_a` as valid */ assert!(tester .client - .post_beacon_blocks_v2(&block_a, validation_level) + .post_beacon_blocks_v2( + &SignedBlockContents::new(block_a.clone(), blobs_a), + validation_level + ) .await .is_ok()); assert!(tester @@ -487,7 +500,10 @@ pub async fn equivocation_consensus_early_equivocation() { /* submit `block_b` which should induce equivocation */ let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&block_b, validation_level) + .post_beacon_blocks_v2( + &SignedBlockContents::new(block_b.clone(), blobs_b), + validation_level, + ) .await; assert!(response.is_err()); @@ -529,14 +545,14 @@ pub async fn equivocation_gossip() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let (block, _): (SignedBeaconBlock, _) = tester + let ((block, blobs), _): ((SignedBeaconBlock, _), _) = tester .harness .make_block_with_modifier(state_a, slot_b, |b| *b.state_root_mut() = Hash256::zero()) .await; let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&block, validation_level) + .post_beacon_blocks_v2(&SignedBlockContents::new(block, blobs), validation_level) .await; assert!(response.is_err()); @@ -582,9 +598,9 @@ pub async fn equivocation_consensus_late_equivocation() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let (block_a, state_after_a): (SignedBeaconBlock, _) = + let ((block_a, _), state_after_a): ((SignedBeaconBlock, _), _) = tester.harness.make_block(state_a.clone(), slot_b).await; - let (block_b, state_after_b): (SignedBeaconBlock, _) = + let ((block_b, blobs_b), state_after_b): ((SignedBeaconBlock, _), _) = tester.harness.make_block(state_a, slot_b).await; /* check for `make_block` curios */ @@ -601,7 +617,7 @@ pub async fn equivocation_consensus_late_equivocation() { let publication_result: Result<(), Rejection> = publish_block( None, - ProvenancedBlock::local(gossip_block_b.unwrap()), + ProvenancedBlock::local((gossip_block_b.unwrap(), blobs_b)), tester.harness.chain, &channel.0, test_logger, @@ -650,11 +666,15 @@ pub async fn equivocation_full_pass() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let (block, _): (SignedBeaconBlock, _) = tester.harness.make_block(state_a, slot_b).await; + let ((block, blobs), _): ((SignedBeaconBlock, _), _) = + tester.harness.make_block(state_a, slot_b).await; let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&block, validation_level) + .post_beacon_blocks_v2( + &SignedBlockContents::new(block.clone(), blobs), + validation_level, + ) .await; assert!(response.is_ok()); @@ -692,7 +712,7 @@ pub async fn blinded_gossip_invalid() { tester.harness.advance_slot(); - let (block, _): (SignedBeaconBlock, _) = tester + let ((block, _), _): ((SignedBeaconBlock, _), _) = tester .harness .make_block_with_modifier(chain_state_before, slot, |b| { *b.state_root_mut() = Hash256::zero(); @@ -746,7 +766,7 @@ pub async fn blinded_gossip_partial_pass() { tester.harness.advance_slot(); - let (block, _): (SignedBeaconBlock, _) = tester + let ((block, _), _): ((SignedBeaconBlock, _), _) = tester .harness .make_block_with_modifier(chain_state_before, slot, |b| { *b.state_root_mut() = Hash256::zero() @@ -794,7 +814,7 @@ pub async fn blinded_gossip_full_pass() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let (block, _): (SignedBlindedBeaconBlock, _) = + let ((block, _), _): ((SignedBlindedBeaconBlock, _), _) = tester.harness.make_blinded_block(state_a, slot_b).await; let response: Result<(), eth2::Error> = tester @@ -837,7 +857,7 @@ pub async fn blinded_consensus_invalid() { tester.harness.advance_slot(); - let (block, _): (SignedBeaconBlock, _) = tester + let ((block, _), _): ((SignedBeaconBlock, _), _) = tester .harness .make_block_with_modifier(chain_state_before, slot, |b| { *b.state_root_mut() = Hash256::zero(); @@ -891,7 +911,7 @@ pub async fn blinded_consensus_gossip() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let (block, _): (SignedBeaconBlock, _) = tester + let ((block, _), _): ((SignedBeaconBlock, _), _) = tester .harness .make_block_with_modifier(state_a, slot_b, |b| *b.state_root_mut() = Hash256::zero()) .await; @@ -942,7 +962,7 @@ pub async fn blinded_consensus_full_pass() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let (block, _): (SignedBlindedBeaconBlock, _) = + let ((block, _), _): ((SignedBlindedBeaconBlock, _), _) = tester.harness.make_blinded_block(state_a, slot_b).await; let response: Result<(), eth2::Error> = tester @@ -986,7 +1006,7 @@ pub async fn blinded_equivocation_invalid() { tester.harness.advance_slot(); - let (block, _): (SignedBeaconBlock, _) = tester + let ((block, _), _): ((SignedBeaconBlock, _), _) = tester .harness .make_block_with_modifier(chain_state_before, slot, |b| { *b.state_root_mut() = Hash256::zero(); @@ -1041,11 +1061,11 @@ pub async fn blinded_equivocation_consensus_early_equivocation() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let (block_a, state_after_a): (SignedBlindedBeaconBlock, _) = tester + let ((block_a, _), state_after_a): ((SignedBlindedBeaconBlock, _), _) = tester .harness .make_blinded_block(state_a.clone(), slot_b) .await; - let (block_b, state_after_b): (SignedBlindedBeaconBlock, _) = + let ((block_b, _), state_after_b): ((SignedBlindedBeaconBlock, _), _) = tester.harness.make_blinded_block(state_a, slot_b).await; /* check for `make_blinded_block` curios */ @@ -1109,7 +1129,7 @@ pub async fn blinded_equivocation_gossip() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let (block, _): (SignedBeaconBlock, _) = tester + let ((block, _), _): ((SignedBeaconBlock, _), _) = tester .harness .make_block_with_modifier(state_a, slot_b, |b| *b.state_root_mut() = Hash256::zero()) .await; @@ -1164,11 +1184,11 @@ pub async fn blinded_equivocation_consensus_late_equivocation() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let (block_a, state_after_a): (SignedBlindedBeaconBlock, _) = tester + let ((block_a, blobs_a), state_after_a): ((SignedBlindedBeaconBlock, _), _) = tester .harness .make_blinded_block(state_a.clone(), slot_b) .await; - let (block_b, state_after_b): (SignedBlindedBeaconBlock, _) = + let ((block_b, blobs_b), state_after_b): ((SignedBlindedBeaconBlock, _), _) = tester.harness.make_blinded_block(state_a, slot_b).await; /* check for `make_blinded_block` curios */ @@ -1178,16 +1198,16 @@ pub async fn blinded_equivocation_consensus_late_equivocation() { let unblinded_block_a = reconstruct_block( tester.harness.chain.clone(), - block_a.state_root(), - block_a, + block_a.canonical_root(), + SignedBlockContents::new(block_a, blobs_a), test_logger.clone(), ) .await .unwrap(); let unblinded_block_b = reconstruct_block( tester.harness.chain.clone(), - block_b.clone().state_root(), - block_b.clone(), + block_b.canonical_root(), + SignedBlockContents::new(block_b.clone(), blobs_b.clone()), test_logger.clone(), ) .await @@ -1202,15 +1222,17 @@ pub async fn blinded_equivocation_consensus_late_equivocation() { ProvenancedBlock::Builder(b, _) => b, }; - let gossip_block_b = GossipVerifiedBlock::new(inner_block_b, &tester.harness.chain); + let gossip_block_b = + GossipVerifiedBlock::new(inner_block_b.deconstruct().into(), &tester.harness.chain); assert!(gossip_block_b.is_ok()); - let gossip_block_a = GossipVerifiedBlock::new(inner_block_a, &tester.harness.chain); + let gossip_block_a = + GossipVerifiedBlock::new(inner_block_a.deconstruct().into(), &tester.harness.chain); assert!(gossip_block_a.is_err()); let channel = tokio::sync::mpsc::unbounded_channel(); let publication_result: Result<(), Rejection> = publish_blinded_block( - block_b, + SignedBlockContents::new(block_b, blobs_b), tester.harness.chain, &channel.0, test_logger, @@ -1254,12 +1276,15 @@ pub async fn blinded_equivocation_full_pass() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let (block, _): (SignedBlindedBeaconBlock, _) = + let ((block, blobs), _): ((SignedBlindedBeaconBlock, _), _) = tester.harness.make_blinded_block(state_a, slot_b).await; let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&block, validation_level) + .post_beacon_blocks_v2( + &SignedBlockContents::new(block.clone(), blobs), + validation_level, + ) .await; assert!(response.is_ok()); diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 2bcb1313395..f5a5d74e29b 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -4157,7 +4157,7 @@ impl ApiTester { .unwrap(); let expected_reorg = EventKind::ChainReorg(SseChainReorg { - slot: self.reorg_block.slot(), + slot: self.reorg_block.signed_block().slot(), depth: 1, old_head_block: self.next_block.signed_block().canonical_root(), old_head_state: self.next_block.signed_block().state_root(), diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index ba5f6016be8..1a961769e9b 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -713,14 +713,14 @@ impl BeaconNodeHttpClient { /// `POST v2/beacon/blocks` pub async fn post_beacon_blocks_v2>( &self, - block: &SignedBeaconBlock, + block_contents: &SignedBlockContents, validation_level: Option, ) -> Result<(), Error> { self.post_generic_with_consensus_version( self.post_beacon_blocks_v2_path(validation_level)?, - block, + block_contents, Some(self.timeouts.proposal), - block.message().body().fork_name(), + block_contents.signed_block().message().body().fork_name(), ) .await?; @@ -728,6 +728,7 @@ impl BeaconNodeHttpClient { } /// `POST v2/beacon/blinded_blocks` + //TODO(sean) update this along with builder updates pub async fn post_beacon_blinded_blocks_v2( &self, block: &SignedBlindedBeaconBlock, diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index b7fc67304c1..9caf827edf0 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -9,6 +9,7 @@ use ssz_derive::Encode; use std::convert::TryFrom; use std::fmt::{self, Display}; use std::str::{from_utf8, FromStr}; +use std::sync::Arc; use std::time::Duration; pub use types::*; @@ -1409,6 +1410,8 @@ pub type BlockContentsTuple = ( Option>, ); +pub type ArcBlockContentsTuple = (Arc>, Option>); + /// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobSidecars`]. #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(untagged)] @@ -1419,6 +1422,20 @@ pub enum SignedBlockContents = FullP } impl> SignedBlockContents { + pub fn new( + block: SignedBeaconBlock, + blobs: Option>, + ) -> Self { + if let Some(blobs) = blobs { + Self::BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars { + signed_block: block, + signed_blob_sidecars: blobs, + }) + } else { + Self::Block(block) + } + } + pub fn signed_block(&self) -> &SignedBeaconBlock { match self { SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => { diff --git a/consensus/state_processing/src/per_block_processing/tests.rs b/consensus/state_processing/src/per_block_processing/tests.rs index 81641b59069..011d4d0ccab 100644 --- a/consensus/state_processing/src/per_block_processing/tests.rs +++ b/consensus/state_processing/src/per_block_processing/tests.rs @@ -63,7 +63,7 @@ async fn valid_block_ok() { let state = harness.get_current_state(); let slot = state.slot(); - let (block, mut state) = harness + let ((block, _), mut state) = harness .make_block_return_pre_state(state, slot + Slot::new(1)) .await; @@ -89,7 +89,7 @@ async fn invalid_block_header_state_slot() { let state = harness.get_current_state(); let slot = state.slot() + Slot::new(1); - let (signed_block, mut state) = harness.make_block_return_pre_state(state, slot).await; + let ((signed_block, _), mut state) = harness.make_block_return_pre_state(state, slot).await; let (mut block, signature) = signed_block.deconstruct(); *block.slot_mut() = slot + Slot::new(1); @@ -120,7 +120,7 @@ async fn invalid_parent_block_root() { let state = harness.get_current_state(); let slot = state.slot(); - let (signed_block, mut state) = harness + let ((signed_block, _), mut state) = harness .make_block_return_pre_state(state, slot + Slot::new(1)) .await; let (mut block, signature) = signed_block.deconstruct(); @@ -155,7 +155,7 @@ async fn invalid_block_signature() { let state = harness.get_current_state(); let slot = state.slot(); - let (signed_block, mut state) = harness + let ((signed_block, _), mut state) = harness .make_block_return_pre_state(state, slot + Slot::new(1)) .await; let (block, _) = signed_block.deconstruct(); @@ -188,7 +188,7 @@ async fn invalid_randao_reveal_signature() { let state = harness.get_current_state(); let slot = state.slot(); - let (signed_block, mut state) = harness + let ((signed_block, _), mut state) = harness .make_block_with_modifier(state, slot + 1, |block| { *block.body_mut().randao_reveal_mut() = Signature::empty(); }) diff --git a/testing/state_transition_vectors/src/exit.rs b/testing/state_transition_vectors/src/exit.rs index 7e7fd23e0d0..3b9235cc4e0 100644 --- a/testing/state_transition_vectors/src/exit.rs +++ b/testing/state_transition_vectors/src/exit.rs @@ -57,7 +57,7 @@ impl ExitTest { block_modifier(&harness, block); }) .await; - (signed_block, state) + (signed_block.0, state) } fn process( From 57bb1d931e9dac016153e37aa33714e7b48c8967 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 11 Jul 2023 17:07:52 -0400 Subject: [PATCH 20/51] fix progressive balance slashing tests --- consensus/fork_choice/tests/tests.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/consensus/fork_choice/tests/tests.rs b/consensus/fork_choice/tests/tests.rs index 06fad8727d4..88fb7d8965e 100644 --- a/consensus/fork_choice/tests/tests.rs +++ b/consensus/fork_choice/tests/tests.rs @@ -1353,6 +1353,14 @@ async fn progressive_balances_cache_attester_slashing() { .apply_blocks_while(|_, state| state.finalized_checkpoint().epoch == 0) .await .unwrap() + // Note: This test may fail if the shuffling used changes, right now it re-runs with + // deterministic shuffling. A shuffling change my cause the slashed proposer to propose + // again in the next epoch, which results in a block processing failure + // (`HeaderInvalid::ProposerSlashed`). The harness should be re-worked to successfully skip + // the slot in this scenario rather than panic-ing. The same applies to + // `progressive_balances_cache_proposer_slashing`. + .apply_blocks(1) + .await .add_previous_epoch_attester_slashing() .await // expect fork choice to import blocks successfully after a previous epoch attester is @@ -1376,6 +1384,14 @@ async fn progressive_balances_cache_proposer_slashing() { .apply_blocks_while(|_, state| state.finalized_checkpoint().epoch == 0) .await .unwrap() + // Note: This test may fail if the shuffling used changes, right now it re-runs with + // deterministic shuffling. A shuffling change my cause the slashed proposer to propose + // again in the next epoch, which results in a block processing failure + // (`HeaderInvalid::ProposerSlashed`). The harness should be re-worked to successfully skip + // the slot in this scenario rather than panic-ing. The same applies to + // `progressive_balances_cache_attester_slashing`. + .apply_blocks(1) + .await .add_previous_epoch_proposer_slashing(MainnetEthSpec::slots_per_epoch()) .await // expect fork choice to import blocks successfully after a previous epoch proposer is From 420e9c490e4bb9f67ca6983e861d5f7f3cc2be73 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 12 Jul 2023 07:05:58 +0000 Subject: [PATCH 21/51] Add state-root command and network support to lcli (#4492) ## Proposed Changes * Add `lcli state-root` command for computing the hash tree root of a `BeaconState`. * Add a `--network` flag which can be used instead of `--testnet-dir` to set the network, e.g. Mainnet, Goerli, Gnosis. * Use the new network flag in `transition-blocks`, `skip-slots`, and `block-root`, which previously only supported mainnet. * **BREAKING CHANGE** Remove the default value of `~/.lighthouse/testnet` from `--testnet-dir`. This may have made sense in previous versions where `lcli` was more testnet focussed, but IMO it is an unnecessary complication and foot-gun today. --- lcli/src/block_root.rs | 9 ++- lcli/src/main.rs | 136 ++++++++++++++++++++++++++++------ lcli/src/skip_slots.rs | 9 ++- lcli/src/state_root.rs | 76 +++++++++++++++++++ lcli/src/transition_blocks.rs | 9 ++- 5 files changed, 211 insertions(+), 28 deletions(-) create mode 100644 lcli/src/state_root.rs diff --git a/lcli/src/block_root.rs b/lcli/src/block_root.rs index a47b48a30ad..a4237d855b5 100644 --- a/lcli/src/block_root.rs +++ b/lcli/src/block_root.rs @@ -31,14 +31,19 @@ use clap::ArgMatches; use clap_utils::{parse_optional, parse_required}; use environment::Environment; use eth2::{types::BlockId, BeaconNodeHttpClient, SensitiveUrl, Timeouts}; +use eth2_network_config::Eth2NetworkConfig; use std::path::PathBuf; use std::time::{Duration, Instant}; use types::{EthSpec, FullPayload, SignedBeaconBlock}; const HTTP_TIMEOUT: Duration = Duration::from_secs(5); -pub fn run(env: Environment, matches: &ArgMatches) -> Result<(), String> { - let spec = &T::default_spec(); +pub fn run( + env: Environment, + network_config: Eth2NetworkConfig, + matches: &ArgMatches, +) -> Result<(), String> { + let spec = &network_config.chain_spec::()?; let executor = env.core_context().executor; /* diff --git a/lcli/src/main.rs b/lcli/src/main.rs index d072beaa4e1..565a5325f0d 100644 --- a/lcli/src/main.rs +++ b/lcli/src/main.rs @@ -15,11 +15,13 @@ mod new_testnet; mod parse_ssz; mod replace_state_pubkeys; mod skip_slots; +mod state_root; mod transition_blocks; use clap::{App, Arg, ArgMatches, SubCommand}; -use clap_utils::parse_path_with_default_in_home_dir; +use clap_utils::parse_optional; use environment::{EnvironmentBuilder, LoggerConfig}; +use eth2_network_config::Eth2NetworkConfig; use parse_ssz::run_parse_ssz; use std::path::PathBuf; use std::process; @@ -50,7 +52,16 @@ fn main() { .value_name("PATH") .takes_value(true) .global(true) - .help("The testnet dir. Defaults to ~/.lighthouse/testnet"), + .help("The testnet dir."), + ) + .arg( + Arg::with_name("network") + .long("network") + .value_name("NAME") + .takes_value(true) + .global(true) + .help("The network to use. Defaults to mainnet.") + .conflicts_with("testnet-dir") ) .subcommand( SubCommand::with_name("skip-slots") @@ -126,7 +137,7 @@ fn main() { .takes_value(true) .conflicts_with("beacon-url") .requires("block-path") - .help("Path to load a BeaconState from file as SSZ."), + .help("Path to load a BeaconState from as SSZ."), ) .arg( Arg::with_name("block-path") @@ -135,7 +146,7 @@ fn main() { .takes_value(true) .conflicts_with("beacon-url") .requires("pre-state-path") - .help("Path to load a SignedBeaconBlock from file as SSZ."), + .help("Path to load a SignedBeaconBlock from as SSZ."), ) .arg( Arg::with_name("post-state-output-path") @@ -808,14 +819,14 @@ fn main() { ) .subcommand( SubCommand::with_name("block-root") - .about("Computes the block root of some block") + .about("Computes the block root of some block.") .arg( Arg::with_name("block-path") .long("block-path") .value_name("PATH") .takes_value(true) .conflicts_with("beacon-url") - .help("Path to load a SignedBeaconBlock from file as SSZ."), + .help("Path to load a SignedBeaconBlock from as SSZ."), ) .arg( Arg::with_name("beacon-url") @@ -841,6 +852,41 @@ fn main() { .help("Number of repeat runs, useful for benchmarking."), ) ) + .subcommand( + SubCommand::with_name("state-root") + .about("Computes the state root of some state.") + .arg( + Arg::with_name("state-path") + .long("state-path") + .value_name("PATH") + .takes_value(true) + .conflicts_with("beacon-url") + .help("Path to load a BeaconState from as SSZ."), + ) + .arg( + Arg::with_name("beacon-url") + .long("beacon-url") + .value_name("URL") + .takes_value(true) + .help("URL to a beacon-API provider."), + ) + .arg( + Arg::with_name("state-id") + .long("state-id") + .value_name("BLOCK_ID") + .takes_value(true) + .requires("beacon-url") + .help("Identifier for a state as per beacon-API standards (slot, root, etc.)"), + ) + .arg( + Arg::with_name("runs") + .long("runs") + .value_name("INTEGER") + .takes_value(true) + .default_value("1") + .help("Number of repeat runs, useful for benchmarking."), + ) + ) .get_matches(); let result = matches @@ -887,17 +933,44 @@ fn run( .build() .map_err(|e| format!("should build env: {:?}", e))?; - let testnet_dir = parse_path_with_default_in_home_dir( - matches, - "testnet-dir", - PathBuf::from(directory::DEFAULT_ROOT_DIR).join("testnet"), - )?; + // Determine testnet-dir path or network name depending on CLI flags. + let (testnet_dir, network_name) = + if let Some(testnet_dir) = parse_optional::(matches, "testnet-dir")? { + (Some(testnet_dir), None) + } else { + let network_name = + parse_optional(matches, "network")?.unwrap_or_else(|| "mainnet".to_string()); + (None, Some(network_name)) + }; + + // Lazily load either the testnet dir or the network config, as required. + // Some subcommands like new-testnet need the testnet dir but not the network config. + let get_testnet_dir = || testnet_dir.clone().ok_or("testnet-dir is required"); + let get_network_config = || { + if let Some(testnet_dir) = &testnet_dir { + Eth2NetworkConfig::load(testnet_dir.clone()).map_err(|e| { + format!( + "Unable to open testnet dir at {}: {}", + testnet_dir.display(), + e + ) + }) + } else { + let network_name = network_name.ok_or("no network name or testnet-dir provided")?; + Eth2NetworkConfig::constant(&network_name)?.ok_or("invalid network name".into()) + } + }; match matches.subcommand() { - ("transition-blocks", Some(matches)) => transition_blocks::run::(env, matches) - .map_err(|e| format!("Failed to transition blocks: {}", e)), + ("transition-blocks", Some(matches)) => { + let network_config = get_network_config()?; + transition_blocks::run::(env, network_config, matches) + .map_err(|e| format!("Failed to transition blocks: {}", e)) + } ("skip-slots", Some(matches)) => { - skip_slots::run::(env, matches).map_err(|e| format!("Failed to skip slots: {}", e)) + let network_config = get_network_config()?; + skip_slots::run::(env, network_config, matches) + .map_err(|e| format!("Failed to skip slots: {}", e)) } ("pretty-ssz", Some(matches)) => { run_parse_ssz::(matches).map_err(|e| format!("Failed to pretty print hex: {}", e)) @@ -906,22 +979,33 @@ fn run( deploy_deposit_contract::run::(env, matches) .map_err(|e| format!("Failed to run deploy-deposit-contract command: {}", e)) } - ("eth1-genesis", Some(matches)) => eth1_genesis::run::(env, testnet_dir, matches) - .map_err(|e| format!("Failed to run eth1-genesis command: {}", e)), - ("interop-genesis", Some(matches)) => interop_genesis::run::(testnet_dir, matches) - .map_err(|e| format!("Failed to run interop-genesis command: {}", e)), + ("eth1-genesis", Some(matches)) => { + let testnet_dir = get_testnet_dir()?; + eth1_genesis::run::(env, testnet_dir, matches) + .map_err(|e| format!("Failed to run eth1-genesis command: {}", e)) + } + ("interop-genesis", Some(matches)) => { + let testnet_dir = get_testnet_dir()?; + interop_genesis::run::(testnet_dir, matches) + .map_err(|e| format!("Failed to run interop-genesis command: {}", e)) + } ("change-genesis-time", Some(matches)) => { + let testnet_dir = get_testnet_dir()?; change_genesis_time::run::(testnet_dir, matches) .map_err(|e| format!("Failed to run change-genesis-time command: {}", e)) } ("create-payload-header", Some(matches)) => create_payload_header::run::(matches) .map_err(|e| format!("Failed to run create-payload-header command: {}", e)), ("replace-state-pubkeys", Some(matches)) => { + let testnet_dir = get_testnet_dir()?; replace_state_pubkeys::run::(testnet_dir, matches) .map_err(|e| format!("Failed to run replace-state-pubkeys command: {}", e)) } - ("new-testnet", Some(matches)) => new_testnet::run::(testnet_dir, matches) - .map_err(|e| format!("Failed to run new_testnet command: {}", e)), + ("new-testnet", Some(matches)) => { + let testnet_dir = get_testnet_dir()?; + new_testnet::run::(testnet_dir, matches) + .map_err(|e| format!("Failed to run new_testnet command: {}", e)) + } ("check-deposit-data", Some(matches)) => check_deposit_data::run(matches) .map_err(|e| format!("Failed to run check-deposit-data command: {}", e)), ("generate-bootnode-enr", Some(matches)) => generate_bootnode_enr::run::(matches) @@ -932,8 +1016,16 @@ fn run( .map_err(|e| format!("Failed to run mnemonic-validators command: {}", e)), ("indexed-attestations", Some(matches)) => indexed_attestations::run::(matches) .map_err(|e| format!("Failed to run indexed-attestations command: {}", e)), - ("block-root", Some(matches)) => block_root::run::(env, matches) - .map_err(|e| format!("Failed to run block-root command: {}", e)), + ("block-root", Some(matches)) => { + let network_config = get_network_config()?; + block_root::run::(env, network_config, matches) + .map_err(|e| format!("Failed to run block-root command: {}", e)) + } + ("state-root", Some(matches)) => { + let network_config = get_network_config()?; + state_root::run::(env, network_config, matches) + .map_err(|e| format!("Failed to run state-root command: {}", e)) + } (other, _) => Err(format!("Unknown subcommand {}. See --help.", other)), } } diff --git a/lcli/src/skip_slots.rs b/lcli/src/skip_slots.rs index e3b2a5acbfd..31fe9fe6480 100644 --- a/lcli/src/skip_slots.rs +++ b/lcli/src/skip_slots.rs @@ -49,6 +49,7 @@ use clap::ArgMatches; use clap_utils::{parse_optional, parse_required}; use environment::Environment; use eth2::{types::StateId, BeaconNodeHttpClient, SensitiveUrl, Timeouts}; +use eth2_network_config::Eth2NetworkConfig; use ssz::Encode; use state_processing::state_advance::{complete_state_advance, partial_state_advance}; use std::fs::File; @@ -59,8 +60,12 @@ use types::{BeaconState, CloneConfig, EthSpec, Hash256}; const HTTP_TIMEOUT: Duration = Duration::from_secs(10); -pub fn run(env: Environment, matches: &ArgMatches) -> Result<(), String> { - let spec = &T::default_spec(); +pub fn run( + env: Environment, + network_config: Eth2NetworkConfig, + matches: &ArgMatches, +) -> Result<(), String> { + let spec = &network_config.chain_spec::()?; let executor = env.core_context().executor; let output_path: Option = parse_optional(matches, "output-path")?; diff --git a/lcli/src/state_root.rs b/lcli/src/state_root.rs new file mode 100644 index 00000000000..efcee2827ab --- /dev/null +++ b/lcli/src/state_root.rs @@ -0,0 +1,76 @@ +use crate::transition_blocks::load_from_ssz_with; +use clap::ArgMatches; +use clap_utils::{parse_optional, parse_required}; +use environment::Environment; +use eth2::{types::StateId, BeaconNodeHttpClient, SensitiveUrl, Timeouts}; +use eth2_network_config::Eth2NetworkConfig; +use std::path::PathBuf; +use std::time::{Duration, Instant}; +use types::{BeaconState, EthSpec}; + +const HTTP_TIMEOUT: Duration = Duration::from_secs(10); + +pub fn run( + env: Environment, + network_config: Eth2NetworkConfig, + matches: &ArgMatches, +) -> Result<(), String> { + let executor = env.core_context().executor; + + let spec = &network_config.chain_spec::()?; + + let state_path: Option = parse_optional(matches, "state-path")?; + let beacon_url: Option = parse_optional(matches, "beacon-url")?; + let runs: usize = parse_required(matches, "runs")?; + + info!( + "Using {} network ({} spec)", + spec.config_name.as_deref().unwrap_or("unknown"), + T::spec_name() + ); + info!("Doing {} runs", runs); + + let state = match (state_path, beacon_url) { + (Some(state_path), None) => { + info!("State path: {:?}", state_path); + load_from_ssz_with(&state_path, spec, BeaconState::from_ssz_bytes)? + } + (None, Some(beacon_url)) => { + let state_id: StateId = parse_required(matches, "state-id")?; + let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT)); + executor + .handle() + .ok_or("shutdown in progress")? + .block_on(async move { + client + .get_debug_beacon_states::(state_id) + .await + .map_err(|e| format!("Failed to download state: {:?}", e)) + }) + .map_err(|e| format!("Failed to complete task: {:?}", e))? + .ok_or_else(|| format!("Unable to locate state at {:?}", state_id))? + .data + } + _ => return Err("must supply either --state-path or --beacon-url".into()), + }; + + /* + * Perform the core "runs". + */ + let mut state_root = None; + for i in 0..runs { + let mut state = state.clone(); + let timer = Instant::now(); + state_root = Some( + state + .update_tree_hash_cache() + .map_err(|e| format!("error computing state root: {e:?}"))?, + ); + info!("Run {}: {:?}", i, timer.elapsed()); + } + + if let Some(state_root) = state_root { + info!("State root is {:?}", state_root); + } + Ok(()) +} diff --git a/lcli/src/transition_blocks.rs b/lcli/src/transition_blocks.rs index 34a45607610..85705177dcb 100644 --- a/lcli/src/transition_blocks.rs +++ b/lcli/src/transition_blocks.rs @@ -71,6 +71,7 @@ use eth2::{ types::{BlockId, StateId}, BeaconNodeHttpClient, SensitiveUrl, Timeouts, }; +use eth2_network_config::Eth2NetworkConfig; use ssz::Encode; use state_processing::{ block_signature_verifier::BlockSignatureVerifier, per_block_processing, per_slot_processing, @@ -94,8 +95,12 @@ struct Config { exclude_post_block_thc: bool, } -pub fn run(env: Environment, matches: &ArgMatches) -> Result<(), String> { - let spec = &T::default_spec(); +pub fn run( + env: Environment, + network_config: Eth2NetworkConfig, + matches: &ArgMatches, +) -> Result<(), String> { + let spec = &network_config.chain_spec::()?; let executor = env.core_context().executor; /* From 62c9170755c43395d228fb2f40e08e947d3f2ca8 Mon Sep 17 00:00:00 2001 From: Jack McPherson Date: Wed, 12 Jul 2023 07:06:00 +0000 Subject: [PATCH 22/51] Remove hidden re-exports to appease Rust 1.73 (#4495) ## Issue Addressed #4494 ## Proposed Changes - Remove explicit re-exports of various types to appease the new compiler lint ## Additional Info It seems `warn(hidden_glob_reexports)` is the main culprit. --- beacon_node/execution_layer/src/lib.rs | 8 ++++---- beacon_node/http_api/tests/fork_tests.rs | 7 ++----- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 0e9df7a50d3..2720569c8d8 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -38,11 +38,11 @@ use tokio::{ }; use tokio_stream::wrappers::WatchStream; use tree_hash::TreeHash; -use types::{AbstractExecPayload, BeaconStateError, ExecPayload, Withdrawals}; +use types::{AbstractExecPayload, BeaconStateError, ExecPayload}; use types::{ - BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionBlockHash, ExecutionPayload, - ExecutionPayloadCapella, ExecutionPayloadMerge, ForkName, ForkVersionedResponse, - ProposerPreparationData, PublicKeyBytes, Signature, SignedBeaconBlock, Slot, Uint256, + BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionPayloadCapella, ExecutionPayloadMerge, + ForkVersionedResponse, ProposerPreparationData, PublicKeyBytes, Signature, SignedBeaconBlock, + Slot, }; mod block_hash; diff --git a/beacon_node/http_api/tests/fork_tests.rs b/beacon_node/http_api/tests/fork_tests.rs index 8a3ba887b39..0ab3c706e2c 100644 --- a/beacon_node/http_api/tests/fork_tests.rs +++ b/beacon_node/http_api/tests/fork_tests.rs @@ -326,11 +326,8 @@ async fn sync_committee_indices_across_fork() { /// Assert that an HTTP API error has the given status code and indexed errors for the given indices. fn assert_server_indexed_error(error: eth2::Error, status_code: u16, indices: Vec) { - let eth2::Error::ServerIndexedMessage(IndexedErrorMessage { - code, - failures, - .. - }) = error else { + let eth2::Error::ServerIndexedMessage(IndexedErrorMessage { code, failures, .. }) = error + else { panic!("wrong error, expected ServerIndexedMessage, got: {error:?}") }; assert_eq!(code, status_code); From 1599487933f9bbde541364e38d8270f6e10fb729 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 12 Jul 2023 10:13:25 -0400 Subject: [PATCH 23/51] fix tests --- .../beacon_chain/src/block_verification.rs | 17 +++++---- beacon_node/http_api/src/publish_blocks.rs | 37 ++++++++----------- .../tests/broadcast_validation_tests.rs | 6 ++- common/eth2/src/types.rs | 12 ++++-- 4 files changed, 38 insertions(+), 34 deletions(-) diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 5881ee7d96b..f82fd7e2887 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -69,7 +69,7 @@ use crate::{ metrics, BeaconChain, BeaconChainError, BeaconChainTypes, }; use derivative::Derivative; -use eth2::types::{ArcBlockContentsTuple, EventKind, SignedBlockContents}; +use eth2::types::{EventKind, SignedBlockContents}; use execution_layer::PayloadStatus; pub use fork_choice::{AttestationFromBlock, PayloadVerificationStatus}; use parking_lot::RwLockReadGuard; @@ -809,7 +809,7 @@ pub trait IntoGossipVerifiedBlock: Sized { chain: &BeaconChain, ) -> Result, BlockError>; fn inner(&self) -> &SignedBeaconBlock; - fn parts(self) -> ArcBlockContentsTuple; + fn blobs(&self) -> Option>; } impl IntoGossipVerifiedBlock @@ -827,8 +827,8 @@ impl IntoGossipVerifiedBlock fn inner(&self) -> &SignedBeaconBlock { self.0.block.as_block() } - fn parts(self) -> ArcBlockContentsTuple { - (self.0.block.block_cloned(), self.1) + fn blobs(&self) -> Option> { + self.1.clone() } } @@ -844,9 +844,8 @@ impl IntoGossipVerifiedBlock for SignedBlockContents ArcBlockContentsTuple { - let (block, blobs) = self.deconstruct(); - (Arc::new(block), blobs) + fn blobs(&self) -> Option> { + self.blobs_cloned() } } @@ -1069,7 +1068,9 @@ impl GossipVerifiedBlock { .observe_proposal(block_root, block.message()) .map_err(|e| BlockError::BeaconChainError(e.into()))? { - SeenBlock::Slashable => return Err(BlockError::Slashable), + SeenBlock::Slashable => { + return Err(BlockError::Slashable); + } SeenBlock::Duplicate => return Err(BlockError::BlockIsAlreadyKnown), SeenBlock::UniqueNonSlashable => {} }; diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index ed472e0bff5..719c1ebcb1d 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -1,10 +1,10 @@ use crate::metrics; -use beacon_chain::blob_verification::BlockWrapper; +use beacon_chain::blob_verification::AsBlock; use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now}; use beacon_chain::{ AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, - GossipVerifiedBlock, IntoGossipVerifiedBlock, NotifyExecutionLayer, + IntoGossipVerifiedBlock, NotifyExecutionLayer, }; use eth2::types::BroadcastValidation; use eth2::types::SignedBlockContents; @@ -20,7 +20,7 @@ use tokio::sync::mpsc::UnboundedSender; use tree_hash::TreeHash; use types::{ AbstractExecPayload, BeaconBlockRef, BlindedPayload, EthSpec, ExecPayload, ExecutionBlockHash, - FullPayload, Hash256, SignedBeaconBlock, SignedBlobSidecarList, VariableList, + FullPayload, Hash256, SignedBeaconBlock, SignedBlobSidecarList, }; use warp::Rejection; @@ -109,26 +109,21 @@ pub async fn publish_block>( let sender_clone = network_tx.clone(); let log_clone = log.clone(); - let (block, blobs_opt) = block_contents.parts(); - - let mapped_blobs = blobs_opt.clone().map(|blobs| { - VariableList::from( - blobs - .into_iter() - .map(|blob| blob.message) - .collect::>(), - ) - }); + // We can clone this because the blobs are `Arc`'d in `BlockContents`, but the block is not, + // so we avoid cloning the block at this point. + let blobs_opt = block_contents.blobs(); /* if we can form a `GossipVerifiedBlock`, we've passed our basic gossip checks */ - let gossip_verified_block = GossipVerifiedBlock::new( - BlockWrapper::new(block.clone(), mapped_blobs), - &chain, - ) - .map_err(|e| { - warn!(log, "Not publishing block, not gossip verified"; "slot" => slot, "error" => ?e); - warp_utils::reject::custom_bad_request(e.to_string()) - })?; + let gossip_verified_block = block_contents + .into_gossip_verified_block(&chain) + .map_err(|e| { + warn!(log, "Not publishing block, not gossip verified"; "slot" => slot, "error" => ?e); + warp_utils::reject::custom_bad_request(e.to_string()) + })?; + + // Clone here, so we can take advantage of the `Arc`. The block in `BlockContents` is not, + // `Arc`'d but blobs are. + let block = gossip_verified_block.block.block_cloned(); let block_root = block_root.unwrap_or(gossip_verified_block.block_root); diff --git a/beacon_node/http_api/tests/broadcast_validation_tests.rs b/beacon_node/http_api/tests/broadcast_validation_tests.rs index ccf7439b179..e8c97a77aa6 100644 --- a/beacon_node/http_api/tests/broadcast_validation_tests.rs +++ b/beacon_node/http_api/tests/broadcast_validation_tests.rs @@ -566,9 +566,11 @@ pub async fn equivocation_gossip() { ); } -/// This test checks that a block that is valid from both a gossip and consensus perspective but that equivocates **late** is rejected when using `broadcast_validation=consensus_and_equivocation`. +/// This test checks that a block that is valid from both a gossip and consensus perspective but +/// that equivocates **late** is rejected when using `broadcast_validation=consensus_and_equivocation`. /// -/// This test is unique in that we can't actually test the HTTP API directly, but instead have to hook into the `publish_blocks` code manually. This is in order to handle the late equivocation case. +/// This test is unique in that we can't actually test the HTTP API directly, but instead have to +/// hook into the `publish_blocks` code manually. This is in order to handle the late equivocation case. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] pub async fn equivocation_consensus_late_equivocation() { /* this test targets gossip-level validation */ diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 9caf827edf0..7bb43d8a9d3 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -9,7 +9,6 @@ use ssz_derive::Encode; use std::convert::TryFrom; use std::fmt::{self, Display}; use std::str::{from_utf8, FromStr}; -use std::sync::Arc; use std::time::Duration; pub use types::*; @@ -1410,8 +1409,6 @@ pub type BlockContentsTuple = ( Option>, ); -pub type ArcBlockContentsTuple = (Arc>, Option>); - /// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobSidecars`]. #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(untagged)] @@ -1445,6 +1442,15 @@ impl> SignedBlockContents Option> { + match self { + SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => { + Some(block_and_sidecars.signed_blob_sidecars.clone()) + } + SignedBlockContents::Block(_block) => None, + } + } + pub fn deconstruct(self) -> BlockContentsTuple { match self { SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => ( From c016f5d7879febfb0f6ecb733cad64bb22e88746 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 12 Jul 2023 12:32:14 -0400 Subject: [PATCH 24/51] gossip validate blobs prior to publish --- beacon_node/beacon_chain/src/beacon_chain.rs | 6 +- .../beacon_chain/src/blob_verification.rs | 41 +++++++---- .../beacon_chain/src/block_verification.rs | 69 ++++++++++++------- .../src/data_availability_checker.rs | 2 +- .../overflow_lru_cache.rs | 2 +- beacon_node/beacon_chain/src/lib.rs | 2 +- .../beacon_chain/tests/block_verification.rs | 14 ++-- beacon_node/http_api/src/publish_blocks.rs | 24 +++++-- .../tests/broadcast_validation_tests.rs | 39 +++++++---- .../network/src/beacon_processor/mod.rs | 2 +- .../beacon_processor/worker/gossip_methods.rs | 16 ++--- 11 files changed, 135 insertions(+), 82 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 13fd7e9108e..84039e42268 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1981,7 +1981,7 @@ impl BeaconChain { self: &Arc, blob_sidecar: SignedBlobSidecar, subnet_id: u64, - ) -> Result, BlobError> { + ) -> Result, BlobError> { blob_verification::validate_blob_sidecar_for_gossip(blob_sidecar, subnet_id, self) } @@ -2711,7 +2711,7 @@ impl BeaconChain { /// Returns an `Err` if the given block was invalid, or an error was encountered during pub async fn verify_block_for_gossip( self: &Arc, - block: BlockWrapper, + block: Arc>, ) -> Result, BlockError> { let chain = self.clone(); self.task_executor @@ -2755,7 +2755,7 @@ impl BeaconChain { pub async fn process_blob( self: &Arc, - blob: GossipVerifiedBlob, + blob: GossipVerifiedBlob, ) -> Result> { self.check_availability_and_maybe_import(blob.slot(), |chain| { chain.data_availability_checker.put_gossip_blob(blob) diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index 501e0ae59df..da3ae2c93b0 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -16,7 +16,7 @@ use eth2::types::BlockContentsTuple; use kzg::Kzg; use slog::{debug, warn}; use ssz_derive::{Decode, Encode}; -use ssz_types::FixedVector; +use ssz_types::{FixedVector, VariableList}; use std::borrow::Cow; use types::blob_sidecar::{BlobIdentifier, FixedBlobSidecarList}; use types::{ @@ -125,25 +125,40 @@ impl From for BlobError { } } +pub type GossipVerifiedBlobList = VariableList< + GossipVerifiedBlob, + <::EthSpec as EthSpec>::MaxBlobsPerBlock, +>; + /// A wrapper around a `BlobSidecar` that indicates it has been approved for re-gossiping on /// the p2p network. -#[derive(Debug, Clone)] -pub struct GossipVerifiedBlob { - blob: Arc>, +#[derive(Debug)] +pub struct GossipVerifiedBlob { + blob: SignedBlobSidecar, } -impl GossipVerifiedBlob { +impl GossipVerifiedBlob { + pub fn new( + blob: SignedBlobSidecar, + chain: &BeaconChain, + ) -> Result> { + let blob_index = blob.message.index; + validate_blob_sidecar_for_gossip(blob, blob_index, chain) + } pub fn id(&self) -> BlobIdentifier { - self.blob.id() + self.blob.message.id() } pub fn block_root(&self) -> Hash256 { - self.blob.block_root + self.blob.message.block_root } - pub fn to_blob(self) -> Arc> { - self.blob + pub fn to_blob(self) -> Arc> { + self.blob.message + } + pub fn signed_blob(&self) -> SignedBlobSidecar { + self.blob.clone() } pub fn slot(&self) -> Slot { - self.blob.slot + self.blob.message.slot } } @@ -151,7 +166,7 @@ pub fn validate_blob_sidecar_for_gossip( signed_blob_sidecar: SignedBlobSidecar, subnet: u64, chain: &BeaconChain, -) -> Result, BlobError> { +) -> Result, BlobError> { let blob_slot = signed_blob_sidecar.message.slot; let blob_index = signed_blob_sidecar.message.index; let block_parent_root = signed_blob_sidecar.message.block_parent_root; @@ -366,7 +381,7 @@ pub fn validate_blob_sidecar_for_gossip( // Note: If this BlobSidecar goes on to fail full verification, we do not evict it from the seen_cache // as alternate blob_sidecars for the same identifier can still be retrieved // over rpc. Evicting them from this cache would allow faster propagation over gossip. So we allow - // retreieval of potentially valid blocks over rpc, but try to punish the proposer for signing + // retrieval of potentially valid blocks over rpc, but try to punish the proposer for signing // invalid messages. Issue for more background // https://github.com/ethereum/consensus-specs/issues/3261 if chain @@ -383,7 +398,7 @@ pub fn validate_blob_sidecar_for_gossip( } Ok(GossipVerifiedBlob { - blob: signed_blob_sidecar.message, + blob: signed_blob_sidecar, }) } diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index f82fd7e2887..796712ed203 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -48,7 +48,10 @@ // returned alongside. #![allow(clippy::result_large_err)] -use crate::blob_verification::{AsBlock, BlobError, BlockWrapper, MaybeAvailableBlock}; +use crate::blob_verification::{ + AsBlock, BlobError, BlockWrapper, GossipVerifiedBlob, GossipVerifiedBlobList, + MaybeAvailableBlock, +}; use crate::data_availability_checker::{ AvailabilityCheckError, AvailabilityPendingBlock, AvailableBlock, }; @@ -79,6 +82,7 @@ use slog::{debug, error, warn, Logger}; use slot_clock::SlotClock; use ssz::Encode; use ssz_derive::{Decode, Encode}; +use ssz_types::VariableList; use state_processing::per_block_processing::{errors::IntoWithIndex, is_merge_transition_block}; use state_processing::{ block_signature_verifier::{BlockSignatureVerifier, Error as BlockSignatureVerifierError}, @@ -803,48 +807,65 @@ pub struct BlockImportData { pub consensus_context: ConsensusContext, } -pub trait IntoGossipVerifiedBlock: Sized { +pub type GossipVerifiedBlockContents = + (GossipVerifiedBlock, Option>); + +pub trait IntoGossipVerifiedBlockContents: Sized { fn into_gossip_verified_block( self, chain: &BeaconChain, - ) -> Result, BlockError>; - fn inner(&self) -> &SignedBeaconBlock; - fn blobs(&self) -> Option>; + ) -> Result, BlockError>; + fn inner_block(&self) -> &SignedBeaconBlock; + fn inner_blobs(&self) -> Option>; } -impl IntoGossipVerifiedBlock - for ( - GossipVerifiedBlock, - Option>, - ) -{ +impl IntoGossipVerifiedBlockContents for GossipVerifiedBlockContents { fn into_gossip_verified_block( self, _chain: &BeaconChain, - ) -> Result, BlockError> { - Ok(self.0) + ) -> Result, BlockError> { + Ok(self) } - fn inner(&self) -> &SignedBeaconBlock { + fn inner_block(&self) -> &SignedBeaconBlock { self.0.block.as_block() } - fn blobs(&self) -> Option> { - self.1.clone() + fn inner_blobs(&self) -> Option> { + self.1.as_ref().map(|blobs| { + VariableList::from( + blobs + .into_iter() + .map(GossipVerifiedBlob::signed_blob) + .collect::>(), + ) + }) } } -impl IntoGossipVerifiedBlock for SignedBlockContents { +impl IntoGossipVerifiedBlockContents for SignedBlockContents { fn into_gossip_verified_block( self, chain: &BeaconChain, - ) -> Result, BlockError> { - GossipVerifiedBlock::new(self.deconstruct().into(), chain) + ) -> Result, BlockError> { + let (block, blobs) = self.deconstruct(); + let gossip_verified_block = GossipVerifiedBlock::new(Arc::new(block), chain)?; + let gossip_verified_blobs = blobs + .map(|blobs| { + Ok::<_, BlobError>(VariableList::from( + blobs + .into_iter() + .map(|blob| GossipVerifiedBlob::new(blob, chain)) + .collect::, BlobError>>()?, + )) + }) + .transpose()?; + Ok((gossip_verified_block, gossip_verified_blobs)) } - fn inner(&self) -> &SignedBeaconBlock { + fn inner_block(&self) -> &SignedBeaconBlock { self.signed_block() } - fn blobs(&self) -> Option> { + fn inner_blobs(&self) -> Option> { self.blobs_cloned() } } @@ -887,10 +908,12 @@ impl GossipVerifiedBlock { /// /// Returns an error if the block is invalid, or i8f the block was unable to be verified. pub fn new( - block: BlockWrapper, + block: Arc>, chain: &BeaconChain, ) -> Result> { - let maybe_available = chain.data_availability_checker.check_availability(block)?; + let maybe_available = chain + .data_availability_checker + .check_availability(block.into())?; // If the block is valid for gossip we don't supply it to the slasher here because // we assume it will be transformed into a fully verified block. We *do* need to supply // it to the slasher if an error occurs, because that's the end of this block's journey, diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index ed5e04e0730..fc48ba47ed8 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -205,7 +205,7 @@ impl DataAvailabilityChecker { /// This should only accept gossip verified blobs, so we should not have to worry about dupes. pub fn put_gossip_blob( &self, - gossip_blob: GossipVerifiedBlob, + gossip_blob: GossipVerifiedBlob, ) -> Result, AvailabilityCheckError> { // Verify the KZG commitments. let kzg_verified_blob = if let Some(kzg) = self.kzg.as_ref() { diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index e969c283da7..6b99e62dea2 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -1075,7 +1075,7 @@ mod test { log: Logger, ) -> ( AvailabilityPendingExecutedBlock, - Vec>, + Vec>>, ) where E: EthSpec, diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 09981de8dc9..f394cabe050 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -71,7 +71,7 @@ pub use beacon_fork_choice_store::{BeaconForkChoiceStore, Error as ForkChoiceSto pub use block_verification::{ get_block_root, AvailabilityPendingExecutedBlock, BlockError, ExecutedBlock, ExecutionPayloadError, ExecutionPendingBlock, GossipVerifiedBlock, IntoExecutionPendingBlock, - IntoGossipVerifiedBlock, PayloadVerificationOutcome, PayloadVerificationStatus, + IntoGossipVerifiedBlockContents, PayloadVerificationOutcome, PayloadVerificationStatus, }; pub use canonical_head::{CachedHead, CanonicalHead, CanonicalHeadRwLock}; pub use eth1_chain::{Eth1Chain, Eth1ChainBackend}; diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 48bd8f2dd60..37479236459 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -851,7 +851,7 @@ async fn block_gossip_verification() { { let gossip_verified = harness .chain - .verify_block_for_gossip(snapshot.beacon_block.clone().into()) + .verify_block_for_gossip(snapshot.beacon_block.clone()) .await .expect("should obtain gossip verified block"); @@ -1079,11 +1079,7 @@ async fn block_gossip_verification() { let block = chain_segment[block_index].beacon_block.clone(); assert!( - harness - .chain - .verify_block_for_gossip(block.into()) - .await - .is_ok(), + harness.chain.verify_block_for_gossip(block).await.is_ok(), "the valid block should be processed" ); @@ -1134,7 +1130,7 @@ async fn verify_block_for_gossip_slashing_detection() { let verified_block = harness .chain - .verify_block_for_gossip(Arc::new(block1).into()) + .verify_block_for_gossip(Arc::new(block1)) .await .unwrap(); @@ -1161,7 +1157,7 @@ async fn verify_block_for_gossip_slashing_detection() { unwrap_err( harness .chain - .verify_block_for_gossip(Arc::new(block2).into()) + .verify_block_for_gossip(Arc::new(block2)) .await, ); @@ -1184,7 +1180,7 @@ async fn verify_block_for_gossip_doppelganger_detection() { let verified_block = harness .chain - .verify_block_for_gossip(Arc::new(block).into()) + .verify_block_for_gossip(Arc::new(block)) .await .unwrap(); let attestations = verified_block.block.message().body().attestations().clone(); diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 719c1ebcb1d..54251608105 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -4,7 +4,7 @@ use beacon_chain::blob_verification::AsBlock; use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now}; use beacon_chain::{ AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, - IntoGossipVerifiedBlock, NotifyExecutionLayer, + IntoGossipVerifiedBlockContents, NotifyExecutionLayer, }; use eth2::types::BroadcastValidation; use eth2::types::SignedBlockContents; @@ -24,7 +24,7 @@ use types::{ }; use warp::Rejection; -pub enum ProvenancedBlock> { +pub enum ProvenancedBlock> { /// The payload was built using a local EE. Local(B, PhantomData), /// The payload was build using a remote builder (e.g., via a mev-boost @@ -32,7 +32,7 @@ pub enum ProvenancedBlock> { Builder(B, PhantomData), } -impl> ProvenancedBlock { +impl> ProvenancedBlock { pub fn local(block: B) -> Self { Self::Local(block, PhantomData) } @@ -43,7 +43,7 @@ impl> ProvenancedBlock } /// Handles a request from the HTTP API for full blocks. -pub async fn publish_block>( +pub async fn publish_block>( block_root: Option, provenanced_block: ProvenancedBlock, chain: Arc>, @@ -57,7 +57,7 @@ pub async fn publish_block>( ProvenancedBlock::Local(block_contents, _) => (block_contents, true), ProvenancedBlock::Builder(block_contents, _) => (block_contents, false), }; - let block = block_contents.inner(); + let block = block_contents.inner_block(); let delay = get_block_delay_ms(seen_timestamp, block.message(), &chain.slot_clock); debug!(log, "Signed block received in HTTP API"; "slot" => block.slot()); @@ -111,10 +111,10 @@ pub async fn publish_block>( // We can clone this because the blobs are `Arc`'d in `BlockContents`, but the block is not, // so we avoid cloning the block at this point. - let blobs_opt = block_contents.blobs(); + let blobs_opt = block_contents.inner_blobs(); /* if we can form a `GossipVerifiedBlock`, we've passed our basic gossip checks */ - let gossip_verified_block = block_contents + let (gossip_verified_block, gossip_verified_blobs) = block_contents .into_gossip_verified_block(&chain) .map_err(|e| { warn!(log, "Not publishing block, not gossip verified"; "slot" => slot, "error" => ?e); @@ -175,6 +175,16 @@ pub async fn publish_block>( } }; + if let Some(gossip_verified_blobs) = gossip_verified_blobs { + for blob in gossip_verified_blobs { + if let Err(e) = chain.process_blob(blob).await { + return Err(warp_utils::reject::custom_bad_request(format!( + "Invalid blob: {e}" + ))); + } + } + } + match chain .process_block( block_root, diff --git a/beacon_node/http_api/tests/broadcast_validation_tests.rs b/beacon_node/http_api/tests/broadcast_validation_tests.rs index e8c97a77aa6..b525c23a5f3 100644 --- a/beacon_node/http_api/tests/broadcast_validation_tests.rs +++ b/beacon_node/http_api/tests/broadcast_validation_tests.rs @@ -1,12 +1,13 @@ use beacon_chain::{ test_utils::{AttestationStrategy, BlockStrategy}, - GossipVerifiedBlock, + GossipVerifiedBlock, IntoGossipVerifiedBlockContents, }; use eth2::types::{ BroadcastValidation, SignedBeaconBlock, SignedBlindedBeaconBlock, SignedBlockContents, }; use http_api::test_utils::InteractiveTester; use http_api::{publish_blinded_block, publish_block, reconstruct_block, ProvenancedBlock}; +use std::sync::Arc; use tree_hash::TreeHash; use types::{Hash256, MainnetEthSpec, Slot}; use warp::Rejection; @@ -314,14 +315,16 @@ pub async fn consensus_partial_pass_only_consensus() { tester.harness.make_block(state_a.clone(), slot_b).await; let ((block_b, blobs_b), state_after_b): ((SignedBeaconBlock, _), _) = tester.harness.make_block(state_a, slot_b).await; + let block_b_root = block_b.canonical_root(); /* check for `make_block` curios */ assert_eq!(block_a.state_root(), state_after_a.tree_hash_root()); assert_eq!(block_b.state_root(), state_after_b.tree_hash_root()); assert_ne!(block_a.state_root(), block_b.state_root()); - let gossip_block_b = GossipVerifiedBlock::new(block_b.clone().into(), &tester.harness.chain); - assert!(gossip_block_b.is_ok()); + let gossip_block_contents_b = SignedBlockContents::new(block_b, blobs_b) + .into_gossip_verified_block(&tester.harness.chain); + assert!(gossip_block_contents_b.is_ok()); let gossip_block_a = GossipVerifiedBlock::new(block_a.clone().into(), &tester.harness.chain); assert!(gossip_block_a.is_err()); @@ -330,7 +333,7 @@ pub async fn consensus_partial_pass_only_consensus() { let publication_result: Result<(), Rejection> = publish_block( None, - ProvenancedBlock::local((gossip_block_b.unwrap(), blobs_b)), + ProvenancedBlock::local(gossip_block_contents_b.unwrap()), tester.harness.chain.clone(), &channel.0, test_logger, @@ -342,7 +345,7 @@ pub async fn consensus_partial_pass_only_consensus() { assert!(tester .harness .chain - .block_is_known_to_fork_choice(&block_b.canonical_root())); + .block_is_known_to_fork_choice(&block_b_root)); } /// This test checks that a block that is valid from both a gossip and consensus perspective is accepted when using `broadcast_validation=consensus`. @@ -600,7 +603,7 @@ pub async fn equivocation_consensus_late_equivocation() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let ((block_a, _), state_after_a): ((SignedBeaconBlock, _), _) = + let ((block_a, blobs_a), state_after_a): ((SignedBeaconBlock, _), _) = tester.harness.make_block(state_a.clone(), slot_b).await; let ((block_b, blobs_b), state_after_b): ((SignedBeaconBlock, _), _) = tester.harness.make_block(state_a, slot_b).await; @@ -610,16 +613,18 @@ pub async fn equivocation_consensus_late_equivocation() { assert_eq!(block_b.state_root(), state_after_b.tree_hash_root()); assert_ne!(block_a.state_root(), block_b.state_root()); - let gossip_block_b = GossipVerifiedBlock::new(block_b.clone().into(), &tester.harness.chain); - assert!(gossip_block_b.is_ok()); - let gossip_block_a = GossipVerifiedBlock::new(block_a.clone().into(), &tester.harness.chain); - assert!(gossip_block_a.is_err()); + let gossip_block_contents_b = SignedBlockContents::new(block_b, blobs_b) + .into_gossip_verified_block(&tester.harness.chain); + assert!(gossip_block_contents_b.is_ok()); + let gossip_block_contents_a = SignedBlockContents::new(block_a, blobs_a) + .into_gossip_verified_block(&tester.harness.chain); + assert!(gossip_block_contents_a.is_err()); let channel = tokio::sync::mpsc::unbounded_channel(); let publication_result: Result<(), Rejection> = publish_block( None, - ProvenancedBlock::local((gossip_block_b.unwrap(), blobs_b)), + ProvenancedBlock::local(gossip_block_contents_b.unwrap()), tester.harness.chain, &channel.0, test_logger, @@ -1224,11 +1229,15 @@ pub async fn blinded_equivocation_consensus_late_equivocation() { ProvenancedBlock::Builder(b, _) => b, }; - let gossip_block_b = - GossipVerifiedBlock::new(inner_block_b.deconstruct().into(), &tester.harness.chain); + let gossip_block_b = GossipVerifiedBlock::new( + Arc::new(inner_block_b.clone().deconstruct().0), + &tester.harness.chain, + ); assert!(gossip_block_b.is_ok()); - let gossip_block_a = - GossipVerifiedBlock::new(inner_block_a.deconstruct().into(), &tester.harness.chain); + let gossip_block_a = GossipVerifiedBlock::new( + Arc::new(inner_block_a.clone().deconstruct().0), + &tester.harness.chain, + ); assert!(gossip_block_a.is_err()); let channel = tokio::sync::mpsc::unbounded_channel(); diff --git a/beacon_node/network/src/beacon_processor/mod.rs b/beacon_node/network/src/beacon_processor/mod.rs index e051b1330e6..a01265db4f9 100644 --- a/beacon_node/network/src/beacon_processor/mod.rs +++ b/beacon_node/network/src/beacon_processor/mod.rs @@ -1852,7 +1852,7 @@ impl BeaconProcessor { message_id, peer_id, peer_client, - block.into(), + block, work_reprocessing_tx, duplicate_cache, invalid_block_storage, diff --git a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs index bd663be9602..3d5f3f4f44d 100644 --- a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs @@ -1,6 +1,6 @@ use crate::{metrics, service::NetworkMessage, sync::SyncMessage}; -use beacon_chain::blob_verification::{AsBlock, BlobError, BlockWrapper, GossipVerifiedBlob}; +use beacon_chain::blob_verification::{AsBlock, BlobError, GossipVerifiedBlob}; use beacon_chain::store::Error; use beacon_chain::{ attestation_verification::{self, Error as AttnError, VerifiedAttestation}, @@ -20,6 +20,7 @@ use ssz::Encode; use std::fs; use std::io::Write; use std::path::PathBuf; +use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use store::hot_cold_store::HotColdDBError; use tokio::sync::mpsc; @@ -754,13 +755,13 @@ impl Worker { pub async fn process_gossip_verified_blob( self, peer_id: PeerId, - verified_blob: GossipVerifiedBlob, + verified_blob: GossipVerifiedBlob, // This value is not used presently, but it might come in handy for debugging. _seen_duration: Duration, ) { let blob_root = verified_blob.block_root(); let blob_slot = verified_blob.slot(); - let blob_clone = verified_blob.clone().to_blob(); + let blob_index = verified_blob.id().index; match self.chain.process_blob(verified_blob).await { Ok(AvailabilityProcessingStatus::Imported(_hash)) => { //TODO(sean) add metrics and logging @@ -778,7 +779,7 @@ impl Worker { "outcome" => ?err, "block root" => ?blob_root, "block slot" => blob_slot, - "blob index" => blob_clone.index, + "blob index" => blob_index, ); self.gossip_penalize_peer( peer_id, @@ -788,7 +789,6 @@ impl Worker { trace!( self.log, "Invalid gossip blob ssz"; - "ssz" => format_args!("0x{}", hex::encode(blob_clone.as_ssz_bytes())), ); } } @@ -807,7 +807,7 @@ impl Worker { message_id: MessageId, peer_id: PeerId, peer_client: Client, - block: BlockWrapper, + block: Arc>, reprocess_tx: mpsc::Sender>, duplicate_cache: DuplicateCache, invalid_block_storage: InvalidBlockStorage, @@ -856,7 +856,7 @@ impl Worker { message_id: MessageId, peer_id: PeerId, peer_client: Client, - block: BlockWrapper, + block: Arc>, reprocess_tx: mpsc::Sender>, seen_duration: Duration, ) -> Option> { @@ -881,7 +881,7 @@ impl Worker { let block_root = if let Ok(verified_block) = &verification_result { verified_block.block_root } else { - block.as_block().canonical_root() + block.canonical_root() }; // Write the time the block was observed into delay cache. From 0c7eed5e580b0f083042860761809282f2f7362b Mon Sep 17 00:00:00 2001 From: Tyler Date: Wed, 12 Jul 2023 22:02:26 +0000 Subject: [PATCH 25/51] bump proc-macro2 (#4464) ## Issue Addressed Addresses issue: #4459 ## Proposed Changes `cargo update -p proc-macro2` proc-macro2 v1.0.58 -> v1.0.63 ## Additional Info See: #4459 / https://github.com/rust-lang/rust/issues/113152 --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 532f7ff2042..3eaa645e6dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6323,9 +6323,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.58" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8" +checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" dependencies = [ "unicode-ident", ] From 2b93c0eb0d02f381c22c6a8eadd122f98018a061 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 14 Jul 2023 15:59:15 -0400 Subject: [PATCH 26/51] remove spec minimal feature gating in tests (#4468) * remove spec minimal feature gating in tests * do merge transition in overflow cache test --- .github/workflows/test-suite.yml | 8 +++--- .gitignore | 3 +- Makefile | 13 +++++---- beacon_node/Cargo.toml | 1 - beacon_node/beacon_chain/Cargo.toml | 1 - .../overflow_lru_cache.rs | 28 ++++++------------- .../execution_layer/src/test_utils/mod.rs | 4 +-- beacon_node/network/Cargo.toml | 1 - .../network/src/sync/block_lookups/tests.rs | 9 +++--- lighthouse/Cargo.toml | 2 +- 10 files changed, 28 insertions(+), 42 deletions(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 0285d63e507..939d548b300 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -119,8 +119,8 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Run operation_pool tests for all known forks run: make test-op-pool - network-minimal-tests: - name: network-minimal-tests + network-tests: + name: network-tests runs-on: ubuntu-latest needs: cargo-fmt steps: @@ -131,8 +131,8 @@ jobs: uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Run network tests for all known forks using the minimal spec - run: make test-network-minimal + - name: Run network tests for all known forks + run: make test-network slasher-tests: name: slasher-tests runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 1e943347d43..bbae3145414 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ genesis.ssz # IntelliJ /*.iml +.idea + # VSCode /.vscode -.idea diff --git a/Makefile b/Makefile index ad125f32442..bd49a9f90e9 100644 --- a/Makefile +++ b/Makefile @@ -106,12 +106,12 @@ build-release-tarballs: # Runs the full workspace tests in **release**, without downloading any additional # test vectors. test-release: - cargo test --workspace --release --exclude ef_tests --exclude beacon_chain --exclude slasher + cargo test --workspace --release --exclude ef_tests --exclude beacon_chain --exclude slasher --exclude network # Runs the full workspace tests in **debug**, without downloading any additional test # vectors. test-debug: - cargo test --workspace --exclude ef_tests --exclude beacon_chain + cargo test --workspace --exclude ef_tests --exclude beacon_chain --exclude network # Runs cargo-fmt (linter). cargo-fmt: @@ -143,13 +143,14 @@ test-op-pool-%: --features 'beacon_chain/fork_from_env'\ -p operation_pool -test-network-minimal: $(patsubst %,test-network-minimal-%,$(FORKS)) +# Run the tests in the `network` crate for all known forks. +test-network: $(patsubst %,test-network-%,$(FORKS)) -test-network-minimal-%: +test-network-%: env FORK_NAME=$* cargo test --release \ - --features 'fork_from_env,spec-minimal'\ + --features 'fork_from_env' \ -p network - + # Run the tests in the `slasher` crate for all supported database backends. test-slasher: cargo test --release -p slasher --features lmdb diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index d3f43fad705..e25fad89ac4 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -13,7 +13,6 @@ node_test_rig = { path = "../testing/node_test_rig" } [features] write_ssz_files = ["beacon_chain/write_ssz_files"] # Writes debugging .ssz files to /tmp during block processing. -spec-minimal = ["beacon_chain/spec-minimal"] [dependencies] eth2_config = { path = "../common/eth2_config" } diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index 77bb514cd8a..f9c8be20f20 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -10,7 +10,6 @@ default = ["participation_metrics"] write_ssz_files = [] # Writes debugging .ssz files to /tmp during block processing. participation_metrics = [] # Exposes validator participation metrics to Prometheus. fork_from_env = [] # Initialise the harness chain spec from the FORK_NAME env variable -spec-minimal = ["kzg/minimal-spec"] [dev-dependencies] maplit = "1.0.2" diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index e969c283da7..ad00dc32be4 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -849,7 +849,6 @@ impl ssz::Decode for OverflowKey { #[cfg(test)] mod test { use super::*; - #[cfg(feature = "spec-minimal")] use crate::{ blob_verification::{ validate_blob_sidecar_for_gossip, verify_kzg_for_blob, GossipVerifiedBlob, @@ -859,31 +858,20 @@ mod test { eth1_finalization_cache::Eth1FinalizationData, test_utils::{BaseHarnessType, BeaconChainHarness, DiskHarnessType}, }; - #[cfg(feature = "spec-minimal")] + use execution_layer::test_utils::DEFAULT_TERMINAL_BLOCK; use fork_choice::PayloadVerificationStatus; - #[cfg(feature = "spec-minimal")] use logging::test_logger; - #[cfg(feature = "spec-minimal")] use slog::{info, Logger}; - #[cfg(feature = "spec-minimal")] use state_processing::ConsensusContext; - #[cfg(feature = "spec-minimal")] use std::collections::{BTreeMap, HashMap, VecDeque}; - #[cfg(feature = "spec-minimal")] use std::ops::AddAssign; - #[cfg(feature = "spec-minimal")] use store::{HotColdDB, ItemStore, LevelDB, StoreConfig}; - #[cfg(feature = "spec-minimal")] use tempfile::{tempdir, TempDir}; - #[cfg(feature = "spec-minimal")] use types::beacon_state::ssz_tagged_beacon_state; - #[cfg(feature = "spec-minimal")] use types::{ChainSpec, ExecPayload, MinimalEthSpec}; - #[cfg(feature = "spec-minimal")] const LOW_VALIDATOR_COUNT: usize = 32; - #[cfg(feature = "spec-minimal")] fn get_store_with_spec( db_path: &TempDir, spec: ChainSpec, @@ -906,7 +894,6 @@ mod test { } // get a beacon chain harness advanced to just before deneb fork - #[cfg(feature = "spec-minimal")] async fn get_deneb_chain( log: Logger, db_path: &TempDir, @@ -994,7 +981,6 @@ mod test { } #[tokio::test] - #[cfg(feature = "spec-minimal")] async fn ssz_tagged_beacon_state_encode_decode_equality() { type E = MinimalEthSpec; let altair_fork_epoch = Epoch::new(1); @@ -1011,6 +997,13 @@ mod test { spec.bellatrix_fork_epoch = Some(bellatrix_fork_epoch); spec.capella_fork_epoch = Some(capella_fork_epoch); spec.deneb_fork_epoch = Some(deneb_fork_epoch); + let genesis_block = execution_layer::test_utils::generate_genesis_block( + spec.terminal_total_difficulty, + DEFAULT_TERMINAL_BLOCK, + ) + .unwrap(); + spec.terminal_block_hash = genesis_block.block_hash; + spec.terminal_block_hash_activation_epoch = bellatrix_fork_epoch; let harness = BeaconChainHarness::builder(E::default()) .spec(spec) @@ -1069,7 +1062,6 @@ mod test { assert_eq!(state, decoded, "Encoded and decoded states should be equal"); } - #[cfg(feature = "spec-minimal")] async fn availability_pending_block( harness: &BeaconChainHarness>, log: Logger, @@ -1166,7 +1158,6 @@ mod test { } #[tokio::test] - #[cfg(feature = "spec-minimal")] async fn overflow_cache_test_insert_components() { type E = MinimalEthSpec; type T = DiskHarnessType; @@ -1287,7 +1278,6 @@ mod test { } #[tokio::test] - #[cfg(feature = "spec-minimal")] async fn overflow_cache_test_overflow() { type E = MinimalEthSpec; type T = DiskHarnessType; @@ -1447,7 +1437,6 @@ mod test { } #[tokio::test] - #[cfg(feature = "spec-minimal")] async fn overflow_cache_test_maintenance() { type E = MinimalEthSpec; type T = DiskHarnessType; @@ -1599,7 +1588,6 @@ mod test { } #[tokio::test] - #[cfg(feature = "spec-minimal")] async fn overflow_cache_test_persist_recover() { type E = MinimalEthSpec; type T = DiskHarnessType; diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index 98a385c6ca7..7940cf9f71b 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -25,8 +25,8 @@ use warp::{http::StatusCode, Filter, Rejection}; use crate::EngineCapabilities; pub use execution_block_generator::{ - generate_genesis_header, generate_pow_block, generate_random_blobs, Block, - ExecutionBlockGenerator, + generate_genesis_block, generate_genesis_header, generate_pow_block, generate_random_blobs, + Block, ExecutionBlockGenerator, }; pub use hook::Hook; pub use mock_builder::{Context as MockBuilderContext, MockBuilder, Operation, TestingBuilder}; diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index 3420e646843..fb0f7281d34 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -49,5 +49,4 @@ operation_pool = { path = "../operation_pool" } execution_layer = { path = "../execution_layer" } [features] -spec-minimal = ["beacon_chain/spec-minimal"] fork_from_env = ["beacon_chain/fork_from_env"] diff --git a/beacon_node/network/src/sync/block_lookups/tests.rs b/beacon_node/network/src/sync/block_lookups/tests.rs index ea6fee0af5a..a061a98d32c 100644 --- a/beacon_node/network/src/sync/block_lookups/tests.rs +++ b/beacon_node/network/src/sync/block_lookups/tests.rs @@ -1,4 +1,3 @@ -#![cfg(feature = "spec-minimal")] use std::sync::Arc; use crate::service::RequestId; @@ -117,7 +116,7 @@ impl TestRig { }; let (bundle, transactions) = execution_layer::test_utils::generate_random_blobs::( num_blobs, - &self.harness.chain.kzg.as_ref().unwrap(), + self.harness.chain.kzg.as_ref().unwrap(), ) .unwrap(); @@ -148,8 +147,8 @@ impl TestRig { block_parent_root: block.parent_root(), proposer_index: block.message().proposer_index(), blob: blob.clone(), - kzg_commitment: kzg_commitment.clone(), - kzg_proof: kzg_proof.clone(), + kzg_commitment, + kzg_proof, }); } } @@ -1393,7 +1392,7 @@ mod deneb_only { fn blobs_response_was_valid(mut self) -> Self { self.rig.expect_empty_network(); - if self.blobs.len() > 0 { + if !self.blobs.is_empty() { self.rig.expect_block_process(ResponseType::Blob); } self diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index ae79972e966..bbde006efc5 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -17,7 +17,7 @@ modern = ["bls/supranational-force-adx"] # Uses the slower Milagro BLS library, which is written in native Rust. milagro = ["bls/milagro"] # Support minimal spec (used for testing only). -spec-minimal = ["beacon_node/spec-minimal"] +spec-minimal = [] # Support Gnosis spec and Gnosis Beacon Chain. gnosis = [] # Support slasher MDBX backend. From 42f54ee561519884e5dbfd7d40bc62e5a5a30a87 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 14 Jul 2023 16:01:57 -0400 Subject: [PATCH 27/51] fix merge conflict issues --- beacon_node/beacon_processor/src/lib.rs | 49 +++----- beacon_node/beacon_processor/src/metrics.rs | 10 ++ .../src/work_reprocessing_queue.rs | 4 +- beacon_node/execution_layer/src/lib.rs | 8 +- beacon_node/network/src/metrics.rs | 10 -- .../gossip_methods.rs | 7 +- .../src/network_beacon_processor/mod.rs | 108 +++++++++++++++++- .../network_beacon_processor/rpc_methods.rs | 6 +- .../network_beacon_processor/sync_methods.rs | 52 ++++++--- .../src/network_beacon_processor/tests.rs | 47 ++++---- beacon_node/network/src/router.rs | 39 ++++--- beacon_node/network/src/service/tests.rs | 4 +- .../src/sync/block_lookups/delayed_lookup.rs | 19 ++- .../network/src/sync/block_lookups/mod.rs | 76 ++++++------ .../network/src/sync/block_lookups/tests.rs | 11 +- beacon_node/network/src/sync/manager.rs | 22 ++-- .../network/src/sync/network_context.rs | 5 +- .../network/src/sync/range_sync/range.rs | 5 +- 18 files changed, 304 insertions(+), 178 deletions(-) diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index f8360dd376d..2e0299cad28 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -38,26 +38,12 @@ //! checks the queues to see if there are more parcels of work that can be spawned in a new worker //! task. -use crate::sync::manager::BlockProcessType; -use crate::{metrics, service::NetworkMessage, sync::SyncMessage}; -use beacon_chain::blob_verification::BlockWrapper; -use beacon_chain::parking_lot::Mutex; -use beacon_chain::{BeaconChain, BeaconChainTypes, GossipVerifiedBlock, NotifyExecutionLayer}; -use derivative::Derivative; use crate::work_reprocessing_queue::{ - spawn_reprocess_scheduler, QueuedAggregate, QueuedBackfillBatch, QueuedGossipBlock, - QueuedLightClientUpdate, QueuedRpcBlock, QueuedUnaggregate, ReadyWork, ReprocessQueueMessage, + QueuedBackfillBatch, QueuedGossipBlock, ReprocessQueueMessage, }; use futures::stream::{Stream, StreamExt}; use futures::task::Poll; -use lighthouse_network::NetworkGlobals; -use lighthouse_network::{MessageId, PeerId}; -use lighthouse_network::rpc::methods::{BlobsByRangeRequest, BlobsByRootRequest}; -use lighthouse_network::rpc::LightClientBootstrapRequest; -use lighthouse_network::{ - rpc::{BlocksByRangeRequest, BlocksByRootRequest, StatusMessage}, - Client, MessageId, NetworkGlobals, PeerId, PeerRequestId, -}; +use lighthouse_network::{MessageId, NetworkGlobals, PeerId}; use logging::TimeLatch; use parking_lot::Mutex; use slog::{crit, debug, error, trace, warn, Logger}; @@ -73,22 +59,14 @@ use std::time::Duration; use task_executor::TaskExecutor; use tokio::sync::mpsc; use tokio::sync::mpsc::error::TrySendError; -use types::{Attestation, EthSpec, Hash256, SignedAggregateAndProof, Slot, SubnetId}; +use types::{Attestation, Hash256, SignedAggregateAndProof, SubnetId}; +use types::{EthSpec, Slot}; use work_reprocessing_queue::IgnoredRpcBlock; -use types::blob_sidecar::FixedBlobSidecarList; -use types::{ - Attestation, AttesterSlashing, Hash256, LightClientFinalityUpdate, LightClientOptimisticUpdate, - ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBlobSidecar, - SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, SubnetId, - SyncCommitteeMessage, SyncSubnetId, -}; use work_reprocessing_queue::{ spawn_reprocess_scheduler, QueuedAggregate, QueuedLightClientUpdate, QueuedRpcBlock, QueuedUnaggregate, ReadyWork, }; -use worker::{Toolbox, Worker}; - mod metrics; pub mod work_reprocessing_queue; @@ -240,7 +218,7 @@ pub const GOSSIP_LIGHT_CLIENT_FINALITY_UPDATE: &str = "light_client_finality_upd pub const GOSSIP_LIGHT_CLIENT_OPTIMISTIC_UPDATE: &str = "light_client_optimistic_update"; pub const RPC_BLOCK: &str = "rpc_block"; pub const IGNORED_RPC_BLOCK: &str = "ignored_rpc_block"; -pub const RPC_BLOB: &str = "rpc_blob"; +pub const RPC_BLOBS: &str = "rpc_blob"; pub const CHAIN_SEGMENT: &str = "chain_segment"; pub const CHAIN_SEGMENT_BACKFILL: &str = "chain_segment_backfill"; pub const STATUS_PROCESSING: &str = "status_processing"; @@ -594,6 +572,7 @@ impl Work { Work::GossipLightClientFinalityUpdate(_) => GOSSIP_LIGHT_CLIENT_FINALITY_UPDATE, Work::GossipLightClientOptimisticUpdate(_) => GOSSIP_LIGHT_CLIENT_OPTIMISTIC_UPDATE, Work::RpcBlock { .. } => RPC_BLOCK, + Work::RpcBlobs { .. } => RPC_BLOBS, Work::IgnoredRpcBlock { .. } => IGNORED_RPC_BLOCK, Work::ChainSegment { .. } => CHAIN_SEGMENT, Work::ChainSegmentBackfill(_) => CHAIN_SEGMENT_BACKFILL, @@ -1359,12 +1338,18 @@ impl BeaconProcessor { beacon_block_root: _, process_fn, } => task_spawner.spawn_async(process_fn), - Work::RpcBlock { process_fn } | Work::RpcBlob { process_fn } => task_spawner.spawn_async(process_fn), + Work::RpcBlock { process_fn } | Work::RpcBlobs { process_fn } => { + task_spawner.spawn_async(process_fn) + } Work::IgnoredRpcBlock { process_fn } => task_spawner.spawn_blocking(process_fn), - Work::GossipBlock(work)| Work::GossipBlob(work) => task_spawner.spawn_async(async move { - work.await; - }), - Work::BlobsByRangeRequest(work) | Work::BlobsByRootsRequest(work)| Work::BlocksByRangeRequest(work) | Work::BlocksByRootsRequest(work) => { + Work::GossipBlock(work) | Work::GossipSignedBlobSidecar(work) => task_spawner + .spawn_async(async move { + work.await; + }), + Work::BlobsByRangeRequest(work) + | Work::BlobsByRootsRequest(work) + | Work::BlocksByRangeRequest(work) + | Work::BlocksByRootsRequest(work) => { task_spawner.spawn_blocking_with_manual_send_idle(work) } Work::ChainSegmentBackfill(process_fn) => task_spawner.spawn_async(process_fn), diff --git a/beacon_node/beacon_processor/src/metrics.rs b/beacon_node/beacon_processor/src/metrics.rs index 65ab0bd8fc5..dbe6d59ee20 100644 --- a/beacon_node/beacon_processor/src/metrics.rs +++ b/beacon_node/beacon_processor/src/metrics.rs @@ -46,6 +46,11 @@ lazy_static::lazy_static! { "beacon_processor_gossip_block_queue_total", "Count of blocks from gossip waiting to be verified." ); + // Gossip blobs. + pub static ref BEACON_PROCESSOR_GOSSIP_BLOB_QUEUE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_gossip_blob_queue_total", + "Count of blocks from gossip waiting to be verified." + ); // Gossip Exits. pub static ref BEACON_PROCESSOR_EXIT_QUEUE_TOTAL: Result = try_create_int_gauge( "beacon_processor_exit_queue_total", @@ -71,6 +76,11 @@ lazy_static::lazy_static! { "beacon_processor_rpc_block_queue_total", "Count of blocks from the rpc waiting to be verified." ); + // Rpc blobs. + pub static ref BEACON_PROCESSOR_RPC_BLOB_QUEUE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_rpc_blob_queue_total", + "Count of blobs from the rpc waiting to be verified." + ); // Chain segments. pub static ref BEACON_PROCESSOR_CHAIN_SEGMENT_QUEUE_TOTAL: Result = try_create_int_gauge( "beacon_processor_chain_segment_queue_total", diff --git a/beacon_node/beacon_processor/src/work_reprocessing_queue.rs b/beacon_node/beacon_processor/src/work_reprocessing_queue.rs index 358efa65766..608f634d537 100644 --- a/beacon_node/beacon_processor/src/work_reprocessing_queue.rs +++ b/beacon_node/beacon_processor/src/work_reprocessing_queue.rs @@ -454,7 +454,7 @@ impl ReprocessQueue { if block_slot <= now && self .ready_work_tx - .try_send(ReadyWork::GossipBlock(early_block)) + .try_send(ReadyWork::Block(early_block)) .is_err() { error!( @@ -757,7 +757,7 @@ impl ReprocessQueue { if self .ready_work_tx - .try_send(ReadyWork::GossipBlock(ready_block)) + .try_send(ReadyWork::Block(ready_block)) .is_err() { error!( diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index e790da45fd9..10734d2e361 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -40,11 +40,13 @@ use tokio::{ }; use tokio_stream::wrappers::WatchStream; use tree_hash::TreeHash; -use types::{AbstractExecPayload, BeaconStateError, ExecPayload}; +use types::beacon_block_body::KzgCommitments; +use types::blob_sidecar::Blobs; +use types::{ + AbstractExecPayload, BeaconStateError, ExecPayload, ExecutionPayloadDeneb, VersionedHash, +}; use types::{ BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionPayloadCapella, ExecutionPayloadMerge, - ForkVersionedResponse, ProposerPreparationData, PublicKeyBytes, Signature, SignedBeaconBlock, - Slot, }; use types::{KzgProofs, Withdrawals}; use types::{ProposerPreparationData, PublicKeyBytes, Signature, Slot, Transaction, Uint256}; diff --git a/beacon_node/network/src/metrics.rs b/beacon_node/network/src/metrics.rs index 17d0915e686..73b98e210fe 100644 --- a/beacon_node/network/src/metrics.rs +++ b/beacon_node/network/src/metrics.rs @@ -67,11 +67,6 @@ lazy_static! { "beacon_processor_gossip_block_early_seconds", "Whenever a gossip block is received early this metrics is set to how early that block was." ); - // Gossip blobs. - pub static ref BEACON_PROCESSOR_GOSSIP_BLOB_QUEUE_TOTAL: Result = try_create_int_gauge( - "beacon_processor_gossip_blob_queue_total", - "Count of blocks from gossip waiting to be verified." - ); pub static ref BEACON_PROCESSOR_GOSSIP_BLOB_VERIFIED_TOTAL: Result = try_create_int_counter( "beacon_processor_gossip_blob_verified_total", "Total number of gossip blob verified for propagation." @@ -125,11 +120,6 @@ lazy_static! { "beacon_processor_rpc_block_imported_total", "Total number of gossip blocks imported to fork choice, etc." ); - // Rpc blobs. - pub static ref BEACON_PROCESSOR_RPC_BLOB_QUEUE_TOTAL: Result = try_create_int_gauge( - "beacon_processor_rpc_blob_queue_total", - "Count of blobs from the rpc waiting to be verified." - ); pub static ref BEACON_PROCESSOR_RPC_BLOB_IMPORTED_TOTAL: Result = try_create_int_counter( "beacon_processor_rpc_blob_imported_total", "Total number of gossip blobs imported." diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index cfaed247753..2255b401703 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -5,7 +5,8 @@ use crate::{ sync::SyncMessage, }; -use beacon_chain::blob_verification::{AsBlock, BlobError, GossipVerifiedBlob}; +use beacon_chain::blob_verification::{BlobError, GossipVerifiedBlob}; +use beacon_chain::block_verification_types::AsBlock; use beacon_chain::store::Error; use beacon_chain::{ attestation_verification::{self, Error as AttnError, VerifiedAttestation}, @@ -600,7 +601,7 @@ impl NetworkBeaconProcessor { // TODO: docs #[allow(clippy::too_many_arguments)] pub async fn process_gossip_blob( - self, + self: &Arc, message_id: MessageId, peer_id: PeerId, _peer_client: Client, @@ -699,7 +700,7 @@ impl NetworkBeaconProcessor { } pub async fn process_gossip_verified_blob( - self, + self: &Arc, peer_id: PeerId, verified_blob: GossipVerifiedBlob, // This value is not used presently, but it might come in handy for debugging. diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 7f0ef1fb817..166417ba93c 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -2,6 +2,7 @@ use crate::{ service::NetworkMessage, sync::{manager::BlockProcessType, SyncMessage}, }; +use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::{ builder::Witness, eth1_chain::CachingEth1Backend, test_utils::BeaconChainHarness, BeaconChain, }; @@ -25,9 +26,11 @@ use store::MemoryStore; use task_executor::test_utils::TestRuntime; use task_executor::TaskExecutor; use tokio::sync::mpsc::{self, error::TrySendError}; +use lighthouse_network::rpc::methods::{BlobsByRangeRequest, BlobsByRootRequest}; use types::*; pub use sync_methods::ChainSegmentProcessId; +use types::blob_sidecar::FixedBlobSidecarList; pub type Error = TrySendError>; @@ -196,6 +199,40 @@ impl NetworkBeaconProcessor { }) } + /// Create a new `Work` event for some blob sidecar. + pub fn send_gossip_blob_sidecar( + self: &Arc, + message_id: MessageId, + peer_id: PeerId, + peer_client: Client, + blob_index: u64, + blob: SignedBlobSidecar, + seen_timestamp: Duration, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = async move { + processor + .process_gossip_blob( + message_id, + peer_id, + peer_client, + blob_index, + blob, + seen_timestamp, + ) + .await + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::GossipSignedBlobSidecar(Box::pin(process_fn)), + }) + } + + pub fn send_banana(){ + + } + /// Create a new `Work` event for some sync committee signature. pub fn send_gossip_sync_signature( self: &Arc, @@ -376,7 +413,7 @@ impl NetworkBeaconProcessor { pub fn send_rpc_beacon_block( self: &Arc, block_root: Hash256, - block: Arc>, + block: RpcBlock, seen_timestamp: Duration, process_type: BlockProcessType, ) -> Result<(), Error> { @@ -392,11 +429,32 @@ impl NetworkBeaconProcessor { }) } + /// Create a new `Work` event for some block, where the result from computation (if any) is + /// sent to the other side of `result_tx`. + pub fn send_rpc_blobs( + self: &Arc, + block_root: Hash256, + blobs: FixedBlobSidecarList, + seen_timestamp: Duration, + process_type: BlockProcessType, + ) -> Result<(), Error> { + let process_fn = self.clone().generate_rpc_blobs_process_fn( + block_root, + blobs, + seen_timestamp, + process_type, + ); + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::RpcBlobs { process_fn }, + }) + } + /// Create a new work event to import `blocks` as a beacon chain segment. pub fn send_chain_segment( self: &Arc, process_id: ChainSegmentProcessId, - blocks: Vec>>, + blocks: Vec>, ) -> Result<(), Error> { let is_backfill = matches!(&process_id, ChainSegmentProcessId::BackSyncBatchId { .. }); let processor = self.clone(); @@ -496,6 +554,52 @@ impl NetworkBeaconProcessor { }) } + /// Create a new work event to process `BlobsByRangeRequest`s from the RPC network. + pub fn send_blobs_by_range_request( + self: &Arc, + peer_id: PeerId, + request_id: PeerRequestId, + request: BlobsByRangeRequest, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = move |send_idle_on_drop| { + processor.handle_blobs_by_range_request( + send_idle_on_drop, + peer_id, + request_id, + request, + ) + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::BlobsByRangeRequest(Box::new(process_fn)), + }) + } + + /// Create a new work event to process `BlobsByRootRequest`s from the RPC network. + pub fn send_blobs_by_roots_request( + self: &Arc, + peer_id: PeerId, + request_id: PeerRequestId, + request: BlobsByRootRequest, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = move |send_idle_on_drop| { + processor.handle_blobs_by_root_request( + send_idle_on_drop, + peer_id, + request_id, + request, + ) + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::BlobsByRootsRequest(Box::new(process_fn)), + }) + } + /// Create a new work event to process `LightClientBootstrap`s from the RPC network. pub fn send_lightclient_bootstrap_request( self: &Arc, diff --git a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs index d55d4bb6260..c653305c537 100644 --- a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs @@ -13,8 +13,8 @@ use lighthouse_network::rpc::*; use lighthouse_network::{PeerId, PeerRequestId, ReportSource, Response, SyncInfo}; use slog::{debug, error, trace, warn}; use slot_clock::SlotClock; -use std::sync::Arc; use std::collections::{hash_map::Entry, HashMap}; +use std::sync::Arc; use task_executor::TaskExecutor; use tokio_stream::StreamExt; use types::blob_sidecar::BlobIdentifier; @@ -217,7 +217,7 @@ impl NetworkBeaconProcessor { } /// Handle a `BlobsByRoot` request from the peer. pub fn handle_blobs_by_root_request( - self, + self: Arc, send_on_drop: SendOnDrop, peer_id: PeerId, request_id: PeerRequestId, @@ -616,7 +616,7 @@ impl NetworkBeaconProcessor { /// Handle a `BlobsByRange` request from the peer. pub fn handle_blobs_by_range_request( - self, + self: Arc, send_on_drop: SendOnDrop, peer_id: PeerId, request_id: PeerRequestId, diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index 7ea551ae057..3a09373c56f 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -1,16 +1,18 @@ -use std::time::Duration; - use crate::metrics; use crate::network_beacon_processor::{NetworkBeaconProcessor, FUTURE_SLOT_TOLERANCE}; +use crate::sync::manager::ResponseType; use crate::sync::BatchProcessResult; use crate::sync::{ manager::{BlockProcessType, SyncMessage}, ChainId, }; +use beacon_chain::data_availability_checker::MaybeAvailableBlock; +use beacon_chain::block_verification_types::{AsBlock, RpcBlock}; +use beacon_chain::data_availability_checker::AvailabilityCheckError; use beacon_chain::{ observed_block_producers::Error as ObserveError, validator_monitor::get_block_delay_ms, - BeaconChainError, BeaconChainTypes, BlockError, ChainSegmentResult, HistoricalBlockError, - NotifyExecutionLayer, + AvailabilityProcessingStatus, BeaconChainError, BeaconChainTypes, BlockError, + ChainSegmentResult, HistoricalBlockError, NotifyExecutionLayer, }; use beacon_processor::{ work_reprocessing_queue::{QueuedRpcBlock, ReprocessQueueMessage}, @@ -19,6 +21,8 @@ use beacon_processor::{ use lighthouse_network::PeerAction; use slog::{debug, error, info, warn}; use slot_clock::SlotClock; +use std::sync::Arc; +use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::sync::mpsc; use types::blob_sidecar::FixedBlobSidecarList; @@ -44,14 +48,14 @@ struct ChainSegmentFailed { } impl NetworkBeaconProcessor { - /// Returns an async closure which processes a beacon block recieved via RPC. + /// Returns an async closure which processes a beacon block received via RPC. /// /// This separate function was required to prevent a cycle during compiler /// type checking. pub fn generate_rpc_beacon_block_process_fn( self: Arc, block_root: Hash256, - block: Arc>, + block: RpcBlock, seen_timestamp: Duration, process_type: BlockProcessType, ) -> AsyncFn { @@ -75,7 +79,7 @@ impl NetworkBeaconProcessor { pub fn generate_rpc_beacon_block_fns( self: Arc, block_root: Hash256, - block: BlockWrapper, + block: RpcBlock, seen_timestamp: Duration, process_type: BlockProcessType, ) -> (AsyncFn, BlockingFn) { @@ -103,7 +107,7 @@ impl NetworkBeaconProcessor { pub async fn process_rpc_block( self: Arc>, block_root: Hash256, - block: Arc>, + block: RpcBlock, seen_timestamp: Duration, process_type: BlockProcessType, reprocess_tx: mpsc::Sender, @@ -254,8 +258,26 @@ impl NetworkBeaconProcessor { drop(handle); } + /// Returns an async closure which processes a list of blobs received via RPC. + /// + /// This separate function was required to prevent a cycle during compiler + /// type checking. + pub fn generate_rpc_blobs_process_fn( + self: Arc, + block_root: Hash256, + block: FixedBlobSidecarList, + seen_timestamp: Duration, + process_type: BlockProcessType, + ) -> AsyncFn { + let process_fn = async move { + self.clone().process_rpc_blobs(block_root, block, seen_timestamp, process_type) + .await; + }; + Box::pin(process_fn) + } + pub async fn process_rpc_blobs( - self, + self: Arc>, block_root: Hash256, blobs: FixedBlobSidecarList, _seen_timestamp: Duration, @@ -284,12 +306,16 @@ impl NetworkBeaconProcessor { }); } + pub fn send_delayed_lookup(&self, block_root: Hash256){ + self.send_sync_message(SyncMessage::MissingGossipBlockComponentsDelayed(block_root)) + } + /// Attempt to import the chain segment (`blocks`) to the beacon chain, informing the sync /// thread if more blocks are needed to process it. pub async fn process_chain_segment( &self, sync_type: ChainSegmentProcessId, - downloaded_blocks: Vec>, + downloaded_blocks: Vec>, notify_execution_layer: NotifyExecutionLayer, ) { let result = match sync_type { @@ -414,7 +440,7 @@ impl NetworkBeaconProcessor { /// Helper function to process blocks batches which only consumes the chain and blocks to process. async fn process_blocks<'a>( &self, - downloaded_blocks: impl Iterator>, + downloaded_blocks: impl Iterator>, notify_execution_layer: NotifyExecutionLayer, ) -> (usize, Result<(), ChainSegmentFailed>) { let blocks: Vec<_> = downloaded_blocks.cloned().collect(); @@ -447,7 +473,7 @@ impl NetworkBeaconProcessor { /// Helper function to process backfill block batches which only consumes the chain and blocks to process. fn process_backfill_blocks( &self, - downloaded_blocks: Vec>, + downloaded_blocks: Vec>, ) -> (usize, Result<(), ChainSegmentFailed>) { let total_blocks = downloaded_blocks.len(); let available_blocks = match downloaded_blocks @@ -455,7 +481,7 @@ impl NetworkBeaconProcessor { .map(|block| { self.chain .data_availability_checker - .check_availability(block) + .check_rpc_block_availability(block) }) .collect::, _>>() { diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index 1273929bd8a..39d3575d6e3 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -10,13 +10,16 @@ use crate::{ use beacon_chain::test_utils::{ test_spec, AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType, }; -use beacon_chain::{BeaconChain, ChainConfig, MAXIMUM_GOSSIP_CLOCK_DISPARITY}; +use beacon_chain::{BeaconChain, ChainConfig, WhenSlotSkipped, MAXIMUM_GOSSIP_CLOCK_DISPARITY}; use beacon_processor::{work_reprocessing_queue::*, *}; +use lighthouse_network::discovery::ConnectionId; +use lighthouse_network::rpc::methods::BlobsByRangeRequest; +use lighthouse_network::rpc::SubstreamId; use lighthouse_network::{ discv5::enr::{CombinedKey, EnrBuilder}, rpc::methods::{MetaData, MetaDataV2}, types::{EnrAttestationBitfield, EnrSyncCommitteeBitfield}, - Client, MessageId, NetworkGlobals, PeerId, + Client, MessageId, NetworkGlobals, PeerId, Response, }; use slot_clock::SlotClock; use std::cmp; @@ -24,9 +27,11 @@ use std::iter::Iterator; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; +use types::blob_sidecar::FixedBlobSidecarList; use types::{ - Attestation, AttesterSlashing, Epoch, EthSpec, Hash256, MainnetEthSpec, ProposerSlashing, - SignedAggregateAndProof, SignedBeaconBlock, SignedVoluntaryExit, SubnetId, + Attestation, AttesterSlashing, Epoch, Hash256, MainnetEthSpec, ProposerSlashing, + SignedAggregateAndProof, SignedBeaconBlock, SignedBlobSidecarList, SignedVoluntaryExit, Slot, + SubnetId, }; type E = MainnetEthSpec; @@ -275,15 +280,15 @@ impl TestRig { pub fn enqueue_gossip_blob(&self, blob_index: usize) { if let Some(blobs) = self.next_blobs.as_ref() { let blob = blobs.get(blob_index).unwrap(); - self.beacon_processor_tx - .try_send(WorkEvent::gossip_signed_blob_sidecar( + self.network_beacon_processor + .send_gossip_blob_sidecar( junk_message_id(), junk_peer_id(), Client::default(), - blob_index as u64, + blob.message.index, blob.clone(), Duration::from_secs(0), - )) + ) .unwrap(); } } @@ -319,26 +324,26 @@ impl TestRig { .map(|b| Some(b.message)) .collect::>(), ); - let event = WorkEvent::rpc_blobs( - self.next_block.canonical_root(), - blobs, - std::time::Duration::default(), - BlockProcessType::SingleBlock { id: 1 }, - ); - self.beacon_processor_tx.try_send(event).unwrap(); + self.network_beacon_processor + .send_rpc_blobs( + self.next_block.canonical_root(), + blobs, + std::time::Duration::default(), + BlockProcessType::SingleBlock { id: 1 }, + ) + .unwrap(); } } pub fn enqueue_blobs_by_range_request(&self, count: u64) { - let event = WorkEvent::blobs_by_range_request( + self.network_beacon_processor.send_blobs_by_range_request( PeerId::random(), (ConnectionId::new(42), SubstreamId::new(24)), BlobsByRangeRequest { start_slot: 0, count, }, - ); - self.beacon_processor_tx.try_send(event).unwrap(); + ).unwrap(); } pub fn enqueue_backfill_batch(&self) { @@ -733,7 +738,7 @@ async fn attestation_to_unknown_block_processed(import_method: BlockImportMethod events.push(RPC_BLOCK); if num_blobs > 0 { rig.enqueue_single_lookup_rpc_blobs(); - events.push(RPC_BLOB); + events.push(RPC_BLOBS); } } }; @@ -816,7 +821,7 @@ async fn aggregate_attestation_to_unknown_block(import_method: BlockImportMethod events.push(RPC_BLOCK); if num_blobs > 0 { rig.enqueue_single_lookup_rpc_blobs(); - events.push(RPC_BLOB); + events.push(RPC_BLOBS); } } }; @@ -996,7 +1001,7 @@ async fn test_rpc_block_reprocessing() { rig.enqueue_single_lookup_rpc_blobs(); if rig.next_blobs.as_ref().map(|b| b.len()).unwrap_or(0) > 0 { - rig.assert_event_journal(&[RPC_BLOB, WORKER_FREED, NOTHING_TO_DO]) + rig.assert_event_journal(&[RPC_BLOBS, WORKER_FREED, NOTHING_TO_DO]) .await; } diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 5dcdb592e6b..5a954d05a40 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -208,11 +208,19 @@ impl Router { self.network_beacon_processor .send_blocks_by_roots_request(peer_id, request_id, request), ), - Request::BlobsByRange(request) => self.send_beacon_processor_work( - BeaconWorkEvent::blobs_by_range_request(peer_id, request_id, request), + Request::BlobsByRange(request) => self.handle_beacon_processor_send_result( + self.network_beacon_processor.send_blobs_by_range_request( + peer_id, + request_id, + request, + ), ), - Request::BlobsByRoot(request) => self.send_beacon_processor_work( - BeaconWorkEvent::blobs_by_root_request(peer_id, request_id, request), + Request::BlobsByRoot(request) => self.handle_beacon_processor_send_result( + self.network_beacon_processor.send_blobs_by_roots_request( + peer_id, + request_id, + request, + ), ), Request::LightClientBootstrap(request) => self.handle_beacon_processor_send_result( self.network_beacon_processor @@ -291,19 +299,20 @@ impl Router { self.network_globals.client(&peer_id), block, timestamp_now(), - )) - } + ), + ), PubsubMessage::BlobSidecar(data) => { let (blob_index, signed_blob) = *data; - let peer_client = self.network_globals.client(&peer_id); - self.send_beacon_processor_work(BeaconWorkEvent::gossip_signed_blob_sidecar( - message_id, - peer_id, - peer_client, - blob_index, - signed_blob, - timestamp_now(), - )) + self.handle_beacon_processor_send_result( + self.network_beacon_processor.send_gossip_blob_sidecar( + message_id, + peer_id, + self.network_globals.client(&peer_id), + blob_index, + signed_blob, + timestamp_now(), + ) + ) } PubsubMessage::VoluntaryExit(exit) => { debug!(self.log, "Received a voluntary exit"; "peer_id" => %peer_id); diff --git a/beacon_node/network/src/service/tests.rs b/beacon_node/network/src/service/tests.rs index 7f697e6144a..10110aa891b 100644 --- a/beacon_node/network/src/service/tests.rs +++ b/beacon_node/network/src/service/tests.rs @@ -3,7 +3,6 @@ mod tests { use crate::persisted_dht::load_dht; use crate::{NetworkConfig, NetworkService}; - use beacon_chain::test_utils::BeaconChainHarness; use beacon_processor::{ BeaconProcessorSend, MAX_SCHEDULED_WORK_QUEUE_LEN, MAX_WORK_EVENT_QUEUE_LEN, }; @@ -13,8 +12,7 @@ mod tests { use std::str::FromStr; use std::sync::Arc; use tokio::{runtime::Runtime, sync::mpsc}; - use types::MinimalEthSpec; - use tokio::runtime::Runtime; + use beacon_chain::test_utils::EphemeralHarnessType; use types::MinimalEthSpec as E; type BeaconChainHarness = beacon_chain::test_utils::BeaconChainHarness>; diff --git a/beacon_node/network/src/sync/block_lookups/delayed_lookup.rs b/beacon_node/network/src/sync/block_lookups/delayed_lookup.rs index a2a44ecdd5d..c492470b4ae 100644 --- a/beacon_node/network/src/sync/block_lookups/delayed_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/delayed_lookup.rs @@ -1,12 +1,12 @@ -use crate::sync::SyncMessage; use beacon_chain::{BeaconChain, BeaconChainTypes}; -use slog::{crit, warn}; +use slog::{crit, }; use slot_clock::SlotClock; use std::sync::Arc; use tokio::sync::mpsc; use tokio::time::interval_at; use tokio::time::Instant; -use types::Hash256; +use types::{ Hash256}; +use crate::network_beacon_processor::NetworkBeaconProcessor; #[derive(Debug)] pub enum DelayedLookupMessage { @@ -35,7 +35,7 @@ pub fn spawn_delayed_lookup_service( executor: &task_executor::TaskExecutor, beacon_chain: Arc>, mut delayed_lookups_recv: mpsc::Receiver, - sync_send: mpsc::UnboundedSender>, + beacon_processor: Arc>, log: slog::Logger, ) { executor.spawn( @@ -52,8 +52,8 @@ pub fn spawn_delayed_lookup_service( } else { delay - seconds_from_current_slot_start }; - tokio::time::Instant::now() + duration_until_start - } + Instant::now() + duration_until_start + } _ => { crit!(log, "Failed to read slot clock, delayed lookup service timing will be inaccurate.\ @@ -69,11 +69,8 @@ pub fn spawn_delayed_lookup_service( while let Ok(msg) = delayed_lookups_recv.try_recv() { match msg { DelayedLookupMessage::MissingComponents(block_root) => { - if let Err(e) = sync_send - .send(SyncMessage::MissingGossipBlockComponentsDelayed(block_root)) - { - warn!(log, "Failed to send delayed lookup message"; "error" => ?e); - } + beacon_processor + .send_delayed_lookup(block_root) } } } diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index 21b7a8dbc24..ff095c719ea 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -1,21 +1,3 @@ -use std::collections::hash_map::Entry; -use std::collections::HashMap; -use std::time::Duration; - -use crate::network_beacon_processor::ChainSegmentProcessId; -use beacon_chain::{BeaconChainTypes, BlockError}; -use fnv::FnvHashMap; -use lighthouse_network::{PeerAction, PeerId}; -use lru_cache::LRUTimeCache; -use slog::{debug, error, trace, warn, Logger}; -use smallvec::SmallVec; -use std::collections::HashMap; -use std::fmt::Debug; -use std::sync::Arc; -use store::{Hash256, SignedBeaconBlock}; - -use crate::metrics; - use self::parent_lookup::PARENT_FAIL_TOLERANCE; use self::parent_lookup::{ParentLookup, ParentVerifyError}; use self::single_block_lookup::{LookupVerifyError, SingleBlockLookup}; @@ -25,10 +7,26 @@ use super::{ manager::{BlockProcessType, Id}, network_context::SyncNetworkContext, }; -use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent}; use crate::metrics; +use crate::network_beacon_processor::ChainSegmentProcessId; use crate::sync::block_lookups::single_block_lookup::LookupId; +use beacon_chain::block_verification_types::{AsBlock, RpcBlock}; +use beacon_chain::data_availability_checker::{AvailabilityCheckError, DataAvailabilityChecker}; +use beacon_chain::{AvailabilityProcessingStatus, BeaconChainTypes, BlockError}; +use lighthouse_network::rpc::RPCError; +use lighthouse_network::{PeerAction, PeerId}; +use lru_cache::LRUTimeCache; pub use single_block_lookup::UnknownParentComponents; +use slog::{debug, error, trace, warn, Logger}; +use smallvec::SmallVec; +use std::collections::HashMap; +use std::fmt::Debug; +use std::sync::Arc; +use std::time::Duration; +use store::{Hash256, SignedBeaconBlock}; +use strum::Display; +use types::blob_sidecar::FixedBlobSidecarList; +use types::{BlobSidecar, Slot}; pub(crate) mod delayed_lookup; mod parent_lookup; @@ -36,7 +34,7 @@ mod single_block_lookup; #[cfg(test)] mod tests; -pub type DownloadedBlocks = (Hash256, BlockWrapper); +pub type DownloadedBlocks = (Hash256, RpcBlock); pub type RootBlockTuple = (Hash256, Arc>); pub type RootBlobsTuple = (Hash256, FixedBlobSidecarList); @@ -381,13 +379,13 @@ impl BlockLookups { }; if !has_pending_parent_request { - let block_wrapper = request_ref + let rpc_block = request_ref .get_downloaded_block() - .unwrap_or(BlockWrapper::Block(block)); + .unwrap_or(RpcBlock::new_without_blobs(block)); // This is the correct block, send it for processing match self.send_block_for_processing( block_root, - block_wrapper, + rpc_block, seen_timestamp, BlockProcessType::SingleBlock { id }, cx, @@ -563,14 +561,13 @@ impl BlockLookups { match parent_lookup.verify_block(block, &mut self.failed_chains) { Ok(Some((block_root, block))) => { parent_lookup.add_current_request_block(block); - if let Some(block_wrapper) = - parent_lookup.current_parent_request.get_downloaded_block() + if let Some(rpc_block) = parent_lookup.current_parent_request.get_downloaded_block() { let chain_hash = parent_lookup.chain_hash(); if self .send_block_for_processing( block_root, - block_wrapper, + rpc_block, seen_timestamp, BlockProcessType::ParentLookup { chain_hash }, cx, @@ -644,13 +641,12 @@ impl BlockLookups { Ok(Some((block_root, blobs))) => { parent_lookup.add_current_request_blobs(blobs); let chain_hash = parent_lookup.chain_hash(); - if let Some(block_wrapper) = - parent_lookup.current_parent_request.get_downloaded_block() + if let Some(rpc_block) = parent_lookup.current_parent_request.get_downloaded_block() { if self .send_block_for_processing( block_root, - block_wrapper, + rpc_block, seen_timestamp, BlockProcessType::ParentLookup { chain_hash }, cx, @@ -914,11 +910,7 @@ impl BlockLookups { BlockError::ParentUnknown(block) => { let slot = block.slot(); let parent_root = block.parent_root(); - let (block, blobs) = block.deconstruct(); - request_ref.add_unknown_parent_block(block); - if let Some(blobs) = blobs { - request_ref.add_unknown_parent_blobs(blobs); - } + request_ref.add_unknown_parent_components(block.into()); self.search_parent(slot, root, parent_root, peer_id.to_peer_id(), cx); ShouldRemoveLookup::False } @@ -1077,7 +1069,6 @@ impl BlockLookups { blocks.push(child_block); }; let process_id = ChainSegmentProcessId::ParentLookup(chain_hash); - let work = WorkEvent::chain_segment(process_id, blocks); match beacon_processor.send_chain_segment(process_id, blocks) { Ok(_) => { @@ -1171,7 +1162,7 @@ impl BlockLookups { .enumerate() .find(|(_, req)| req.block_request_state.requested_block_root == chain_hash) { - if let Some((lookup_id, block_wrapper)) = + if let Some((lookup_id, rpc_block)) = self.single_block_lookups.get_mut(index).and_then(|lookup| { lookup .get_downloaded_block() @@ -1191,7 +1182,7 @@ impl BlockLookups { if self .send_block_for_processing( chain_hash, - block_wrapper, + rpc_block, Duration::from_secs(0), //TODO(sean) pipe this through BlockProcessType::SingleBlock { id }, cx, @@ -1231,7 +1222,7 @@ impl BlockLookups { fn send_block_for_processing( &mut self, block_root: Hash256, - block: BlockWrapper, + block: RpcBlock, duration: Duration, process_type: BlockProcessType, cx: &mut SyncNetworkContext, @@ -1274,11 +1265,12 @@ impl BlockLookups { if blob_count == 0 { return Ok(()); } - match cx.processor_channel_if_enabled() { - Some(beacon_processor_send) => { + match cx.beacon_processor_if_enabled() { + Some(beacon_processor) => { trace!(self.log, "Sending blobs for processing"; "block" => ?block_root, "process_type" => ?process_type); - let event = WorkEvent::rpc_blobs(block_root, blobs, duration, process_type); - if let Err(e) = beacon_processor_send.try_send(event) { + if let Err(e) = + beacon_processor.send_rpc_blobs(block_root, blobs, duration, process_type) + { error!( self.log, "Failed to send sync blobs to processor"; diff --git a/beacon_node/network/src/sync/block_lookups/tests.rs b/beacon_node/network/src/sync/block_lookups/tests.rs index 8d2dc58be91..c8f19b16981 100644 --- a/beacon_node/network/src/sync/block_lookups/tests.rs +++ b/beacon_node/network/src/sync/block_lookups/tests.rs @@ -1,19 +1,20 @@ #![cfg(feature = "spec-minimal")] -use std::sync::Arc; - use crate::network_beacon_processor::NetworkBeaconProcessor; use crate::service::RequestId; use crate::sync::manager::RequestId as SyncId; use crate::NetworkMessage; +use std::sync::Arc; use super::*; use beacon_chain::builder::Witness; use beacon_chain::eth1_chain::CachingEth1Backend; +use beacon_chain::test_utils::{build_log, BeaconChainHarness, EphemeralHarnessType}; use beacon_processor::WorkEvent; +use execution_layer::BlobsBundleV1; +use lighthouse_network::rpc::RPCResponseErrorCode; use lighthouse_network::{NetworkGlobals, Request}; -use slog::{Drain, Level}; -use slot_clock::ManualSlotClock; +use slot_clock::{ManualSlotClock, SlotClock, TestingSlotClock}; use store::MemoryStore; use tokio::sync::mpsc; use types::{ @@ -212,7 +213,7 @@ impl TestRig { }, ResponseType::Blob => match self.beacon_processor_rx.try_recv() { Ok(work) => { - assert_eq!(work.work_type(), beacon_processor::RPC_BLOB); + assert_eq!(work.work_type(), beacon_processor::RPC_BLOBS); } other => panic!("Expected blob process, found {:?}", other), }, diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 405f24b6ee0..8bdf57e2a41 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -127,7 +127,7 @@ pub enum SyncMessage { }, /// A block with an unknown parent has been received. - UnknownParentBlock(PeerId, BlockWrapper, Hash256), + UnknownParentBlock(PeerId, RpcBlock, Hash256), /// A blob with an unknown parent has been received. UnknownParentBlob(PeerId, Arc>), @@ -242,13 +242,20 @@ pub fn spawn( MAX_REQUEST_BLOCKS >= T::EthSpec::slots_per_epoch() * EPOCHS_PER_BATCH, "Max blocks that can be requested in a single batch greater than max allowed blocks in a single request" ); + let (delayed_lookups_send, delayed_lookups_recv) = + mpsc::channel::(DELAY_QUEUE_CHANNEL_SIZE); // create an instance of the SyncManager let network_globals = beacon_processor.network_globals.clone(); let mut sync_manager = SyncManager { chain: beacon_chain.clone(), input_channel: sync_recv, - network: SyncNetworkContext::new(network_send, beacon_processor, log.clone()), + network: SyncNetworkContext::new( + network_send, + beacon_processor.clone(), + beacon_chain.clone(), + log.clone(), + ), range_sync: RangeSync::new(beacon_chain.clone(), log.clone()), backfill_sync: BackFillSync::new(beacon_chain.clone(), network_globals, log.clone()), block_lookups: BlockLookups::new( @@ -260,12 +267,11 @@ pub fn spawn( }; let log_clone = log.clone(); - let sync_send_clone = sync_send.clone(); delayed_lookup::spawn_delayed_lookup_service( &executor, beacon_chain, delayed_lookups_recv, - sync_send, + beacon_processor, log, ); @@ -792,7 +798,7 @@ impl SyncManager { } fn should_search_for_block(&mut self, block_slot: Slot, peer_id: &PeerId) -> bool { - if !self.network_globals.sync_state.read().is_synced() { + if !self.network_globals().sync_state.read().is_synced() { let head_slot = self.chain.canonical_head.cached_head().head_slot(); // if the block is far in the future, ignore it. If its within the slot tolerance of @@ -806,13 +812,13 @@ impl SyncManager { } } - self.network_globals.peers.read().is_connected(peer_id) + self.network_globals().peers.read().is_connected(peer_id) && self.network.is_execution_engine_online() } fn synced_and_connected(&mut self, peer_id: &PeerId) -> bool { - self.network_globals.sync_state.read().is_synced() - && self.network_globals.peers.read().is_connected(peer_id) + self.network_globals().sync_state.read().is_synced() + && self.network_globals().peers.read().is_connected(peer_id) && self.network.is_execution_engine_online() } diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 963871e1f35..7c162f478c5 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -4,8 +4,7 @@ use super::block_sidecar_coupling::BlocksAndBlobsRequestInfo; use super::manager::{Id, RequestId as SyncRequestId}; use super::range_sync::{BatchId, ByRangeRequestType, ChainId}; -use crate::beacon_processor::BeaconProcessorSend; -duse crate::network_beacon_processor::NetworkBeaconProcessor; +use crate::network_beacon_processor::NetworkBeaconProcessor; use crate::service::{NetworkMessage, RequestId}; use crate::status::ToStatusMessage; use crate::sync::block_lookups::{BlobRequestId, BlockRequestId}; @@ -97,6 +96,8 @@ impl SyncNetworkContext { request_id: 1, range_requests: FnvHashMap::default(), backfill_requests: FnvHashMap::default(), + range_blocks_and_blobs_requests: FnvHashMap::default(), + backfill_blocks_and_blobs_requests: FnvHashMap::default(), network_beacon_processor, chain, log, diff --git a/beacon_node/network/src/sync/range_sync/range.rs b/beacon_node/network/src/sync/range_sync/range.rs index ca40b80acde..09a85208d96 100644 --- a/beacon_node/network/src/sync/range_sync/range.rs +++ b/beacon_node/network/src/sync/range_sync/range.rs @@ -395,11 +395,11 @@ mod tests { use slog::{o, Drain}; use tokio::sync::mpsc; - use slot_clock::ManualSlotClock; + use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType}; + use slot_clock::{TestingSlotClock, }; use std::collections::HashSet; use std::sync::Arc; use store::MemoryStore; - use tokio::sync::mpsc; use types::{Hash256, MinimalEthSpec as E}; #[derive(Debug)] @@ -612,7 +612,6 @@ mod tests { let chain = harness.chain; let fake_store = Arc::new(FakeStorage::default()); - let (beacon_processor_tx, beacon_processor_rx) = mpsc::channel(10); let range_sync = RangeSync::::new( fake_store.clone(), log.new(o!("component" => "range")), From 405e95b0ce15409f06504f45c8d93071523e9539 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 14 Jul 2023 16:15:28 -0400 Subject: [PATCH 28/51] fix merge --- .../gossip_methods.rs | 2 +- .../src/network_beacon_processor/mod.rs | 26 +++++-------------- .../network_beacon_processor/sync_methods.rs | 22 ++++++++-------- .../src/network_beacon_processor/tests.rs | 18 +++++++------ beacon_node/network/src/router.rs | 16 ++++-------- beacon_node/network/src/service/tests.rs | 2 +- .../src/sync/block_lookups/delayed_lookup.rs | 6 ++--- .../network/src/sync/block_lookups/mod.rs | 14 ++++++---- beacon_node/network/src/sync/manager.rs | 2 +- .../network/src/sync/range_sync/range.rs | 2 +- 10 files changed, 49 insertions(+), 61 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 2255b401703..000c4d85dc1 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -5,8 +5,8 @@ use crate::{ sync::SyncMessage, }; +use beacon_chain::blob_verification::AsBlock; use beacon_chain::blob_verification::{BlobError, GossipVerifiedBlob}; -use beacon_chain::block_verification_types::AsBlock; use beacon_chain::store::Error; use beacon_chain::{ attestation_verification::{self, Error as AttnError, VerifiedAttestation}, diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 166417ba93c..ef4b3daae7a 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -2,7 +2,7 @@ use crate::{ service::NetworkMessage, sync::{manager::BlockProcessType, SyncMessage}, }; -use beacon_chain::block_verification_types::RpcBlock; +use beacon_chain::blob_verification::BlockWrapper; use beacon_chain::{ builder::Witness, eth1_chain::CachingEth1Backend, test_utils::BeaconChainHarness, BeaconChain, }; @@ -13,6 +13,7 @@ use beacon_processor::{ MAX_SCHEDULED_WORK_QUEUE_LEN, MAX_WORK_EVENT_QUEUE_LEN, }; use environment::null_logger; +use lighthouse_network::rpc::methods::{BlobsByRangeRequest, BlobsByRootRequest}; use lighthouse_network::{ rpc::{BlocksByRangeRequest, BlocksByRootRequest, LightClientBootstrapRequest, StatusMessage}, Client, MessageId, NetworkGlobals, PeerId, PeerRequestId, @@ -26,7 +27,6 @@ use store::MemoryStore; use task_executor::test_utils::TestRuntime; use task_executor::TaskExecutor; use tokio::sync::mpsc::{self, error::TrySendError}; -use lighthouse_network::rpc::methods::{BlobsByRangeRequest, BlobsByRootRequest}; use types::*; pub use sync_methods::ChainSegmentProcessId; @@ -229,9 +229,7 @@ impl NetworkBeaconProcessor { }) } - pub fn send_banana(){ - - } + pub fn send_banana() {} /// Create a new `Work` event for some sync committee signature. pub fn send_gossip_sync_signature( @@ -413,7 +411,7 @@ impl NetworkBeaconProcessor { pub fn send_rpc_beacon_block( self: &Arc, block_root: Hash256, - block: RpcBlock, + block: BlockWrapper, seen_timestamp: Duration, process_type: BlockProcessType, ) -> Result<(), Error> { @@ -454,7 +452,7 @@ impl NetworkBeaconProcessor { pub fn send_chain_segment( self: &Arc, process_id: ChainSegmentProcessId, - blocks: Vec>, + blocks: Vec>, ) -> Result<(), Error> { let is_backfill = matches!(&process_id, ChainSegmentProcessId::BackSyncBatchId { .. }); let processor = self.clone(); @@ -563,12 +561,7 @@ impl NetworkBeaconProcessor { ) -> Result<(), Error> { let processor = self.clone(); let process_fn = move |send_idle_on_drop| { - processor.handle_blobs_by_range_request( - send_idle_on_drop, - peer_id, - request_id, - request, - ) + processor.handle_blobs_by_range_request(send_idle_on_drop, peer_id, request_id, request) }; self.try_send(BeaconWorkEvent { @@ -586,12 +579,7 @@ impl NetworkBeaconProcessor { ) -> Result<(), Error> { let processor = self.clone(); let process_fn = move |send_idle_on_drop| { - processor.handle_blobs_by_root_request( - send_idle_on_drop, - peer_id, - request_id, - request, - ) + processor.handle_blobs_by_root_request(send_idle_on_drop, peer_id, request_id, request) }; self.try_send(BeaconWorkEvent { diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index 3a09373c56f..8d9146e6888 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -6,8 +6,7 @@ use crate::sync::{ manager::{BlockProcessType, SyncMessage}, ChainId, }; -use beacon_chain::data_availability_checker::MaybeAvailableBlock; -use beacon_chain::block_verification_types::{AsBlock, RpcBlock}; +use beacon_chain::blob_verification::{AsBlock, BlockWrapper, MaybeAvailableBlock}; use beacon_chain::data_availability_checker::AvailabilityCheckError; use beacon_chain::{ observed_block_producers::Error as ObserveError, validator_monitor::get_block_delay_ms, @@ -55,7 +54,7 @@ impl NetworkBeaconProcessor { pub fn generate_rpc_beacon_block_process_fn( self: Arc, block_root: Hash256, - block: RpcBlock, + block: BlockWrapper, seen_timestamp: Duration, process_type: BlockProcessType, ) -> AsyncFn { @@ -79,7 +78,7 @@ impl NetworkBeaconProcessor { pub fn generate_rpc_beacon_block_fns( self: Arc, block_root: Hash256, - block: RpcBlock, + block: BlockWrapper, seen_timestamp: Duration, process_type: BlockProcessType, ) -> (AsyncFn, BlockingFn) { @@ -107,7 +106,7 @@ impl NetworkBeaconProcessor { pub async fn process_rpc_block( self: Arc>, block_root: Hash256, - block: RpcBlock, + block: BlockWrapper, seen_timestamp: Duration, process_type: BlockProcessType, reprocess_tx: mpsc::Sender, @@ -270,7 +269,8 @@ impl NetworkBeaconProcessor { process_type: BlockProcessType, ) -> AsyncFn { let process_fn = async move { - self.clone().process_rpc_blobs(block_root, block, seen_timestamp, process_type) + self.clone() + .process_rpc_blobs(block_root, block, seen_timestamp, process_type) .await; }; Box::pin(process_fn) @@ -306,7 +306,7 @@ impl NetworkBeaconProcessor { }); } - pub fn send_delayed_lookup(&self, block_root: Hash256){ + pub fn send_delayed_lookup(&self, block_root: Hash256) { self.send_sync_message(SyncMessage::MissingGossipBlockComponentsDelayed(block_root)) } @@ -315,7 +315,7 @@ impl NetworkBeaconProcessor { pub async fn process_chain_segment( &self, sync_type: ChainSegmentProcessId, - downloaded_blocks: Vec>, + downloaded_blocks: Vec>, notify_execution_layer: NotifyExecutionLayer, ) { let result = match sync_type { @@ -440,7 +440,7 @@ impl NetworkBeaconProcessor { /// Helper function to process blocks batches which only consumes the chain and blocks to process. async fn process_blocks<'a>( &self, - downloaded_blocks: impl Iterator>, + downloaded_blocks: impl Iterator>, notify_execution_layer: NotifyExecutionLayer, ) -> (usize, Result<(), ChainSegmentFailed>) { let blocks: Vec<_> = downloaded_blocks.cloned().collect(); @@ -473,7 +473,7 @@ impl NetworkBeaconProcessor { /// Helper function to process backfill block batches which only consumes the chain and blocks to process. fn process_backfill_blocks( &self, - downloaded_blocks: Vec>, + downloaded_blocks: Vec>, ) -> (usize, Result<(), ChainSegmentFailed>) { let total_blocks = downloaded_blocks.len(); let available_blocks = match downloaded_blocks @@ -481,7 +481,7 @@ impl NetworkBeaconProcessor { .map(|block| { self.chain .data_availability_checker - .check_rpc_block_availability(block) + .check_availability(block) }) .collect::, _>>() { diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index 39d3575d6e3..2c37d177aab 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -336,14 +336,16 @@ impl TestRig { } pub fn enqueue_blobs_by_range_request(&self, count: u64) { - self.network_beacon_processor.send_blobs_by_range_request( - PeerId::random(), - (ConnectionId::new(42), SubstreamId::new(24)), - BlobsByRangeRequest { - start_slot: 0, - count, - }, - ).unwrap(); + self.network_beacon_processor + .send_blobs_by_range_request( + PeerId::random(), + (ConnectionId::new(42), SubstreamId::new(24)), + BlobsByRangeRequest { + start_slot: 0, + count, + }, + ) + .unwrap(); } pub fn enqueue_backfill_batch(&self) { diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 5a954d05a40..30a75a91052 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -209,18 +209,12 @@ impl Router { .send_blocks_by_roots_request(peer_id, request_id, request), ), Request::BlobsByRange(request) => self.handle_beacon_processor_send_result( - self.network_beacon_processor.send_blobs_by_range_request( - peer_id, - request_id, - request, - ), + self.network_beacon_processor + .send_blobs_by_range_request(peer_id, request_id, request), ), Request::BlobsByRoot(request) => self.handle_beacon_processor_send_result( - self.network_beacon_processor.send_blobs_by_roots_request( - peer_id, - request_id, - request, - ), + self.network_beacon_processor + .send_blobs_by_roots_request(peer_id, request_id, request), ), Request::LightClientBootstrap(request) => self.handle_beacon_processor_send_result( self.network_beacon_processor @@ -311,7 +305,7 @@ impl Router { blob_index, signed_blob, timestamp_now(), - ) + ), ) } PubsubMessage::VoluntaryExit(exit) => { diff --git a/beacon_node/network/src/service/tests.rs b/beacon_node/network/src/service/tests.rs index 10110aa891b..544c5dd9c7f 100644 --- a/beacon_node/network/src/service/tests.rs +++ b/beacon_node/network/src/service/tests.rs @@ -3,6 +3,7 @@ mod tests { use crate::persisted_dht::load_dht; use crate::{NetworkConfig, NetworkService}; + use beacon_chain::test_utils::EphemeralHarnessType; use beacon_processor::{ BeaconProcessorSend, MAX_SCHEDULED_WORK_QUEUE_LEN, MAX_WORK_EVENT_QUEUE_LEN, }; @@ -12,7 +13,6 @@ mod tests { use std::str::FromStr; use std::sync::Arc; use tokio::{runtime::Runtime, sync::mpsc}; - use beacon_chain::test_utils::EphemeralHarnessType; use types::MinimalEthSpec as E; type BeaconChainHarness = beacon_chain::test_utils::BeaconChainHarness>; diff --git a/beacon_node/network/src/sync/block_lookups/delayed_lookup.rs b/beacon_node/network/src/sync/block_lookups/delayed_lookup.rs index c492470b4ae..55e9e49db30 100644 --- a/beacon_node/network/src/sync/block_lookups/delayed_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/delayed_lookup.rs @@ -1,12 +1,12 @@ +use crate::network_beacon_processor::NetworkBeaconProcessor; use beacon_chain::{BeaconChain, BeaconChainTypes}; -use slog::{crit, }; +use slog::crit; use slot_clock::SlotClock; use std::sync::Arc; use tokio::sync::mpsc; use tokio::time::interval_at; use tokio::time::Instant; -use types::{ Hash256}; -use crate::network_beacon_processor::NetworkBeaconProcessor; +use types::Hash256; #[derive(Debug)] pub enum DelayedLookupMessage { diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index ff095c719ea..dfe960832a5 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -10,7 +10,7 @@ use super::{ use crate::metrics; use crate::network_beacon_processor::ChainSegmentProcessId; use crate::sync::block_lookups::single_block_lookup::LookupId; -use beacon_chain::block_verification_types::{AsBlock, RpcBlock}; +use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; use beacon_chain::data_availability_checker::{AvailabilityCheckError, DataAvailabilityChecker}; use beacon_chain::{AvailabilityProcessingStatus, BeaconChainTypes, BlockError}; use lighthouse_network::rpc::RPCError; @@ -34,7 +34,7 @@ mod single_block_lookup; #[cfg(test)] mod tests; -pub type DownloadedBlocks = (Hash256, RpcBlock); +pub type DownloadedBlocks = (Hash256, BlockWrapper); pub type RootBlockTuple = (Hash256, Arc>); pub type RootBlobsTuple = (Hash256, FixedBlobSidecarList); @@ -381,7 +381,7 @@ impl BlockLookups { if !has_pending_parent_request { let rpc_block = request_ref .get_downloaded_block() - .unwrap_or(RpcBlock::new_without_blobs(block)); + .unwrap_or(BlockWrapper::Block(block)); // This is the correct block, send it for processing match self.send_block_for_processing( block_root, @@ -910,7 +910,11 @@ impl BlockLookups { BlockError::ParentUnknown(block) => { let slot = block.slot(); let parent_root = block.parent_root(); - request_ref.add_unknown_parent_components(block.into()); + let (block, blobs) = block.deconstruct(); + request_ref.add_unknown_parent_components(UnknownParentComponents::new( + Some(block), + blobs, + )); self.search_parent(slot, root, parent_root, peer_id.to_peer_id(), cx); ShouldRemoveLookup::False } @@ -1222,7 +1226,7 @@ impl BlockLookups { fn send_block_for_processing( &mut self, block_root: Hash256, - block: RpcBlock, + block: BlockWrapper, duration: Duration, process_type: BlockProcessType, cx: &mut SyncNetworkContext, diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 8bdf57e2a41..93b7c9af5bf 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -127,7 +127,7 @@ pub enum SyncMessage { }, /// A block with an unknown parent has been received. - UnknownParentBlock(PeerId, RpcBlock, Hash256), + UnknownParentBlock(PeerId, BlockWrapper, Hash256), /// A blob with an unknown parent has been received. UnknownParentBlob(PeerId, Arc>), diff --git a/beacon_node/network/src/sync/range_sync/range.rs b/beacon_node/network/src/sync/range_sync/range.rs index 09a85208d96..733cbcc9eae 100644 --- a/beacon_node/network/src/sync/range_sync/range.rs +++ b/beacon_node/network/src/sync/range_sync/range.rs @@ -396,7 +396,7 @@ mod tests { use tokio::sync::mpsc; use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType}; - use slot_clock::{TestingSlotClock, }; + use slot_clock::TestingSlotClock; use std::collections::HashSet; use std::sync::Arc; use store::MemoryStore; From 18760822fe57868bde24b9e155814be2ce797af8 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Fri, 14 Jul 2023 13:16:48 -0700 Subject: [PATCH 29/51] Fix beta compiler warnings --- beacon_node/execution_layer/src/lib.rs | 8 ++++---- beacon_node/store/src/lib.rs | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 7de53214122..5d5a27eb855 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -42,13 +42,13 @@ use tokio_stream::wrappers::WatchStream; use tree_hash::TreeHash; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::Blobs; +use types::KzgProofs; use types::{AbstractExecPayload, BeaconStateError, ExecPayload, VersionedHash}; use types::{ - BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionBlockHash, ExecutionPayload, - ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, ForkName, + BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionPayloadCapella, ExecutionPayloadDeneb, + ExecutionPayloadMerge, }; -use types::{KzgProofs, Withdrawals}; -use types::{ProposerPreparationData, PublicKeyBytes, Signature, Slot, Transaction, Uint256}; +use types::{ProposerPreparationData, PublicKeyBytes, Signature, Slot, Transaction}; mod block_hash; mod engine_api; diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index cd2f2da2b95..e6b6e730e44 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -43,7 +43,6 @@ pub use metrics::scrape_for_metrics; use parking_lot::MutexGuard; use std::sync::Arc; use strum::{EnumString, IntoStaticStr}; -use types::blob_sidecar::BlobSidecarList; pub use types::*; pub type ColumnIter<'a> = Box), Error>> + 'a>; From 6c375205fb4ae78b5b532352dea1e75584f6fa44 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 17 Jul 2023 00:14:12 +0000 Subject: [PATCH 30/51] Fix HTTP state API bug and add `--epochs-per-migration` (#4236) ## Issue Addressed Fix an issue observed by `@zlan` on Discord where Lighthouse would sometimes return this error when looking up states via the API: > {"code":500,"message":"UNHANDLED_ERROR: ForkChoiceError(MissingProtoArrayBlock(0xc9cf1495421b6ef3215d82253b388d77321176a1dcef0db0e71a0cd0ffc8cdb7))","stacktraces":[]} ## Proposed Changes The error stems from a faulty assumption in the HTTP API logic: that any state in the hot database must have its block in fork choice. This isn't true because the state's hot database may update much less frequently than the fork choice store, e.g. if reconstructing states (where freezer migration pauses), or if the freezer migration runs slowly. There could also be a race between loading the hot state and checking fork choice, e.g. even if the finalization migration of DB+fork choice were atomic, the update could happen between the 1st and 2nd calls. To address this I've changed the HTTP API logic to use the finalized block's execution status as a fallback where it is safe to do so. In the case where a block is non-canonical and prior to finalization (permanently orphaned) we default `execution_optimistic` to `true`. ## Additional Info I've also added a new CLI flag to reduce the frequency of the finalization migration as this is useful for several purposes: - Spacing out database writes (less frequent, larger batches) - Keeping a limited chain history with high availability, e.g. the last month in the hot database. This new flag made it _substantially_ easier to test this change. It was extracted from `tree-states` (where it's called `--db-migration-period`), which is why this PR also carries the `tree-states` label. --- beacon_node/beacon_chain/src/beacon_chain.rs | 35 ++++++++- beacon_node/beacon_chain/src/chain_config.rs | 3 + beacon_node/beacon_chain/src/lib.rs | 1 + beacon_node/beacon_chain/src/migrate.rs | 61 ++++++++++++++- beacon_node/beacon_chain/src/test_utils.rs | 9 ++- beacon_node/client/src/builder.rs | 5 +- beacon_node/http_api/src/state_id.rs | 33 ++++++-- .../http_api/tests/interactive_tests.rs | 75 ++++++++++++++++++- beacon_node/src/cli.rs | 10 +++ beacon_node/src/config.rs | 6 ++ lighthouse/tests/beacon_node.rs | 18 +++++ 11 files changed, 240 insertions(+), 16 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 01343ff3b15..78f2c3f03b4 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -197,6 +197,17 @@ pub struct PrePayloadAttributes { pub parent_block_number: u64, } +/// Information about a state/block at a specific slot. +#[derive(Debug, Clone, Copy)] +pub struct FinalizationAndCanonicity { + /// True if the slot of the state or block is finalized. + /// + /// This alone DOES NOT imply that the state/block is finalized, use `self.is_finalized()`. + pub slot_is_finalized: bool, + /// True if the state or block is canonical at its slot. + pub canonical: bool, +} + /// Define whether a forkchoiceUpdate needs to be checked for an override (`Yes`) or has already /// been checked (`AlreadyApplied`). It is safe to specify `Yes` even if re-orgs are disabled. #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] @@ -426,6 +437,12 @@ pub struct BeaconChain { type BeaconBlockAndState = (BeaconBlock, BeaconState); +impl FinalizationAndCanonicity { + pub fn is_finalized(self) -> bool { + self.slot_is_finalized && self.canonical + } +} + impl BeaconChain { /// Checks if a block is finalized. /// The finalization check is done with the block slot. The block root is used to verify that @@ -455,16 +472,30 @@ impl BeaconChain { state_root: &Hash256, state_slot: Slot, ) -> Result { + self.state_finalization_and_canonicity(state_root, state_slot) + .map(FinalizationAndCanonicity::is_finalized) + } + + /// Fetch the finalization and canonicity status of the state with `state_root`. + pub fn state_finalization_and_canonicity( + &self, + state_root: &Hash256, + state_slot: Slot, + ) -> Result { let finalized_slot = self .canonical_head .cached_head() .finalized_checkpoint() .epoch .start_slot(T::EthSpec::slots_per_epoch()); - let is_canonical = self + let slot_is_finalized = state_slot <= finalized_slot; + let canonical = self .state_root_at_slot(state_slot)? .map_or(false, |canonical_root| state_root == &canonical_root); - Ok(state_slot <= finalized_slot && is_canonical) + Ok(FinalizationAndCanonicity { + slot_is_finalized, + canonical, + }) } /// Persists the head tracker and fork choice. diff --git a/beacon_node/beacon_chain/src/chain_config.rs b/beacon_node/beacon_chain/src/chain_config.rs index cc7a957ecc8..efbc9905b7a 100644 --- a/beacon_node/beacon_chain/src/chain_config.rs +++ b/beacon_node/beacon_chain/src/chain_config.rs @@ -83,6 +83,8 @@ pub struct ChainConfig { pub enable_backfill_rate_limiting: bool, /// Whether to use `ProgressiveBalancesCache` in unrealized FFG progression calculation. pub progressive_balances_mode: ProgressiveBalancesMode, + /// Number of epochs between each migration of data from the hot database to the freezer. + pub epochs_per_migration: u64, } impl Default for ChainConfig { @@ -114,6 +116,7 @@ impl Default for ChainConfig { always_prepare_payload: false, enable_backfill_rate_limiting: true, progressive_balances_mode: ProgressiveBalancesMode::Checked, + epochs_per_migration: crate::migrate::DEFAULT_EPOCHS_PER_MIGRATION, } } } diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index c5cf74e179c..85ff0f20a01 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -73,6 +73,7 @@ pub use execution_layer::EngineState; pub use execution_payload::NotifyExecutionLayer; pub use fork_choice::{ExecutionStatus, ForkchoiceUpdateParameters}; pub use metrics::scrape_for_metrics; +pub use migrate::MigratorConfig; pub use parking_lot; pub use slot_clock; pub use state_processing::per_block_processing::errors::{ diff --git a/beacon_node/beacon_chain/src/migrate.rs b/beacon_node/beacon_chain/src/migrate.rs index 66f082742eb..8306b66d7b5 100644 --- a/beacon_node/beacon_chain/src/migrate.rs +++ b/beacon_node/beacon_chain/src/migrate.rs @@ -25,10 +25,15 @@ const MIN_COMPACTION_PERIOD_SECONDS: u64 = 7200; /// Compact after a large finality gap, if we respect `MIN_COMPACTION_PERIOD_SECONDS`. const COMPACTION_FINALITY_DISTANCE: u64 = 1024; +/// Default number of epochs to wait between finalization migrations. +pub const DEFAULT_EPOCHS_PER_MIGRATION: u64 = 1; + /// The background migrator runs a thread to perform pruning and migrate state from the hot /// to the cold database. pub struct BackgroundMigrator, Cold: ItemStore> { db: Arc>, + /// Record of when the last migration ran, for enforcing `epochs_per_migration`. + prev_migration: Arc>, #[allow(clippy::type_complexity)] tx_thread: Option, thread::JoinHandle<()>)>>, /// Genesis block root, for persisting the `PersistedBeaconChain`. @@ -36,9 +41,22 @@ pub struct BackgroundMigrator, Cold: ItemStore> log: Logger, } -#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct MigratorConfig { pub blocking: bool, + /// Run migrations at most once per `epochs_per_migration`. + /// + /// If set to 0 or 1, then run every finalization. + pub epochs_per_migration: u64, +} + +impl Default for MigratorConfig { + fn default() -> Self { + Self { + blocking: false, + epochs_per_migration: DEFAULT_EPOCHS_PER_MIGRATION, + } + } } impl MigratorConfig { @@ -46,6 +64,19 @@ impl MigratorConfig { self.blocking = true; self } + + pub fn epochs_per_migration(mut self, epochs_per_migration: u64) -> Self { + self.epochs_per_migration = epochs_per_migration; + self + } +} + +/// Record of when the last migration ran. +pub struct PrevMigration { + /// The epoch at which the last finalization migration ran. + epoch: Epoch, + /// The number of epochs to wait between runs. + epochs_per_migration: u64, } /// Pruning can be successful, or in rare cases deferred to a later point. @@ -92,6 +123,7 @@ pub struct FinalizationNotification { finalized_state_root: BeaconStateHash, finalized_checkpoint: Checkpoint, head_tracker: Arc, + prev_migration: Arc>, genesis_block_root: Hash256, } @@ -103,6 +135,11 @@ impl, Cold: ItemStore> BackgroundMigrator Self { + // Estimate last migration run from DB split slot. + let prev_migration = Arc::new(Mutex::new(PrevMigration { + epoch: db.get_split_slot().epoch(E::slots_per_epoch()), + epochs_per_migration: config.epochs_per_migration, + })); let tx_thread = if config.blocking { None } else { @@ -111,6 +148,7 @@ impl, Cold: ItemStore> BackgroundMigrator, Cold: ItemStore> BackgroundMigrator, Cold: ItemStore> BackgroundMigrator prev_migration.epoch, + "new_finalized_epoch" => epoch, + "epochs_per_migration" => prev_migration.epochs_per_migration, + ); + return; + } + + // Update the previous migration epoch immediately to avoid holding the lock. If the + // migration doesn't succeed then the next migration will be retried at the next scheduled + // run. + prev_migration.epoch = epoch; + drop(prev_migration); + debug!(log, "Database consolidation started"); let finalized_state_root = notif.finalized_state_root; diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 6520c9ba9c8..5fc05c5551f 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -516,18 +516,23 @@ where let validator_keypairs = self .validator_keypairs .expect("cannot build without validator keypairs"); + let chain_config = self.chain_config.unwrap_or_default(); let mut builder = BeaconChainBuilder::new(self.eth_spec_instance) .logger(log.clone()) .custom_spec(spec) .store(self.store.expect("cannot build without store")) - .store_migrator_config(MigratorConfig::default().blocking()) + .store_migrator_config( + MigratorConfig::default() + .blocking() + .epochs_per_migration(chain_config.epochs_per_migration), + ) .task_executor(self.runtime.task_executor.clone()) .execution_layer(self.execution_layer) .dummy_eth1_backend() .expect("should build dummy backend") .shutdown_sender(shutdown_tx) - .chain_config(self.chain_config.unwrap_or_default()) + .chain_config(chain_config) .event_handler(Some(ServerSentEventHandler::new_with_capacity( log.clone(), 5, diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index f3ed7f65a90..b1a507eaa59 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -11,7 +11,7 @@ use beacon_chain::{ slot_clock::{SlotClock, SystemTimeSlotClock}, state_advance_timer::spawn_state_advance_timer, store::{HotColdDB, ItemStore, LevelDB, StoreConfig}, - BeaconChain, BeaconChainTypes, Eth1ChainBackend, ServerSentEventHandler, + BeaconChain, BeaconChainTypes, Eth1ChainBackend, MigratorConfig, ServerSentEventHandler, }; use beacon_processor::{ work_reprocessing_queue::ReprocessQueueMessage, BeaconProcessor, BeaconProcessorSend, @@ -184,6 +184,9 @@ where .store(store) .task_executor(context.executor.clone()) .custom_spec(spec.clone()) + .store_migrator_config( + MigratorConfig::default().epochs_per_migration(chain_config.epochs_per_migration), + ) .chain_config(chain_config) .graffiti(graffiti) .event_handler(event_handler) diff --git a/beacon_node/http_api/src/state_id.rs b/beacon_node/http_api/src/state_id.rs index 9e4aadef17e..5e86053771e 100644 --- a/beacon_node/http_api/src/state_id.rs +++ b/beacon_node/http_api/src/state_id.rs @@ -70,15 +70,32 @@ impl StateId { .map_err(BeaconChainError::DBError) .map_err(warp_utils::reject::beacon_chain_error)? { - let execution_optimistic = chain - .canonical_head - .fork_choice_read_lock() - .is_optimistic_or_invalid_block_no_fallback(&hot_summary.latest_block_root) - .map_err(BeaconChainError::ForkChoiceError) - .map_err(warp_utils::reject::beacon_chain_error)?; - let finalized = chain - .is_finalized_state(root, hot_summary.slot) + let finalization_status = chain + .state_finalization_and_canonicity(root, hot_summary.slot) .map_err(warp_utils::reject::beacon_chain_error)?; + let finalized = finalization_status.is_finalized(); + let fork_choice = chain.canonical_head.fork_choice_read_lock(); + let execution_optimistic = if finalization_status.slot_is_finalized + && !finalization_status.canonical + { + // This block is permanently orphaned and has likely been pruned from fork + // choice. If it isn't found in fork choice, mark it optimistic to be on the + // safe side. + fork_choice + .is_optimistic_or_invalid_block_no_fallback( + &hot_summary.latest_block_root, + ) + .unwrap_or(true) + } else { + // This block is either old and finalized, or recent and unfinalized, so + // it's safe to fallback to the optimistic status of the finalized block. + chain + .canonical_head + .fork_choice_read_lock() + .is_optimistic_or_invalid_block(&hot_summary.latest_block_root) + .map_err(BeaconChainError::ForkChoiceError) + .map_err(warp_utils::reject::beacon_chain_error)? + }; return Ok((*root, execution_optimistic, finalized)); } else if let Some(_cold_state_slot) = chain .store diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index da92419744e..d7ea7c26284 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -2,8 +2,9 @@ use beacon_chain::{ chain_config::{DisallowedReOrgOffsets, ReOrgThreshold}, test_utils::{AttestationStrategy, BlockStrategy, SyncCommitteeStrategy}, + ChainConfig, }; -use eth2::types::DepositContractData; +use eth2::types::{DepositContractData, StateId}; use execution_layer::{ForkchoiceState, PayloadAttributes}; use http_api::test_utils::InteractiveTester; use parking_lot::Mutex; @@ -17,7 +18,7 @@ use std::time::Duration; use tree_hash::TreeHash; use types::{ Address, Epoch, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, FullPayload, - MainnetEthSpec, ProposerPreparationData, Slot, + MainnetEthSpec, MinimalEthSpec, ProposerPreparationData, Slot, }; type E = MainnetEthSpec; @@ -48,6 +49,76 @@ async fn deposit_contract_custom_network() { assert_eq!(result, expected); } +// Test that state lookups by root function correctly for states that are finalized but still +// present in the hot database, and have had their block pruned from fork choice. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn state_by_root_pruned_from_fork_choice() { + type E = MinimalEthSpec; + + let validator_count = 24; + let spec = ForkName::latest().make_genesis_spec(E::default_spec()); + + let tester = InteractiveTester::::new_with_initializer_and_mutator( + Some(spec.clone()), + validator_count, + Some(Box::new(move |builder| { + builder + .deterministic_keypairs(validator_count) + .fresh_ephemeral_store() + .chain_config(ChainConfig { + epochs_per_migration: 1024, + ..ChainConfig::default() + }) + })), + None, + ) + .await; + + let client = &tester.client; + let harness = &tester.harness; + + // Create some chain depth and finalize beyond fork choice's pruning depth. + let num_epochs = 8_u64; + let num_initial = num_epochs * E::slots_per_epoch(); + harness.advance_slot(); + harness + .extend_chain_with_sync( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + SyncCommitteeStrategy::NoValidators, + ) + .await; + + // Should now be finalized. + let finalized_epoch = harness.finalized_checkpoint().epoch; + assert_eq!(finalized_epoch, num_epochs - 2); + + // The split slot should still be at 0. + assert_eq!(harness.chain.store.get_split_slot(), 0); + + // States that are between the split and the finalized slot should be able to be looked up by + // state root. + for slot in 0..finalized_epoch.start_slot(E::slots_per_epoch()).as_u64() { + let state_root = harness + .chain + .state_root_at_slot(Slot::new(slot)) + .unwrap() + .unwrap(); + let response = client + .get_debug_beacon_states::(StateId::Root(state_root)) + .await + .unwrap() + .unwrap(); + + assert!(response.finalized.unwrap()); + assert!(!response.execution_optimistic.unwrap()); + + let mut state = response.data; + assert_eq!(state.update_tree_hash_cache().unwrap(), state_root); + } +} + /// Data structure for tracking fork choice updates received by the mock execution layer. #[derive(Debug, Default)] struct ForkChoiceUpdates { diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index fc53f578880..e46c3d8ca11 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -533,6 +533,16 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { [default: 8192 (mainnet) or 64 (minimal)]") .takes_value(true) ) + .arg( + Arg::with_name("epochs-per-migration") + .long("epochs-per-migration") + .value_name("N") + .help("The number of epochs to wait between running the migration of data from the \ + hot DB to the cold DB. Less frequent runs can be useful for minimizing disk \ + writes") + .default_value("1") + .takes_value(true) + ) .arg( Arg::with_name("block-cache-size") .long("block-cache-size") diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 948c70dd41f..4abf649bfbb 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -400,6 +400,12 @@ pub fn get_config( client_config.store.prune_payloads = prune_payloads; } + if let Some(epochs_per_migration) = + clap_utils::parse_optional(cli_args, "epochs-per-migration")? + { + client_config.chain.epochs_per_migration = epochs_per_migration; + } + /* * Zero-ports * diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 4badcef3cf2..bc5f610881b 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -1733,6 +1733,24 @@ fn no_reconstruct_historic_states_flag() { .run_with_zero_port() .with_config(|config| assert!(!config.chain.reconstruct_historic_states)); } +#[test] +fn epochs_per_migration_default() { + CommandLineTest::new() + .run_with_zero_port() + .with_config(|config| { + assert_eq!( + config.chain.epochs_per_migration, + beacon_node::beacon_chain::migrate::DEFAULT_EPOCHS_PER_MIGRATION + ) + }); +} +#[test] +fn epochs_per_migration_override() { + CommandLineTest::new() + .flag("epochs-per-migration", Some("128")) + .run_with_zero_port() + .with_config(|config| assert_eq!(config.chain.epochs_per_migration, 128)); +} // Tests for Slasher flags. // Using `--slasher-max-db-size` to work around https://github.com/sigp/lighthouse/issues/2342 From 5cd738c882e9fdd625492c52976c838e8c0a89a1 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Mon, 17 Jul 2023 00:14:13 +0000 Subject: [PATCH 31/51] Use unique arg names for eth1-sim (#4463) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Issue Addressed When trying to run `eth1-sim` locally, the simulator doesn't start for me, and panicked due to duplicate arg names for `proposer-nodes` (using same arg names as `nodes`). Not sure why this isn't failing on CI but failing on mine 🤔 ``` thread 'main' panicked at 'Argument short must be unique thread 'main' panicked at 'Argument long must be unique ``` --- testing/simulator/src/cli.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/testing/simulator/src/cli.rs b/testing/simulator/src/cli.rs index 5dc2d5ec84a..ff80201051f 100644 --- a/testing/simulator/src/cli.rs +++ b/testing/simulator/src/cli.rs @@ -25,8 +25,8 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .default_value("4") .help("Number of beacon nodes")) .arg(Arg::with_name("proposer-nodes") - .short("n") - .long("nodes") + .short("p") + .long("proposer_nodes") .takes_value(true) .default_value("2") .help("Number of proposer-only beacon nodes")) @@ -64,8 +64,8 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .default_value("4") .help("Number of beacon nodes")) .arg(Arg::with_name("proposer-nodes") - .short("n") - .long("nodes") + .short("p") + .long("proposer_nodes") .takes_value(true) .default_value("2") .help("Number of proposer-only beacon nodes")) From 1a5de8b0f0ccb147e0df410ffcc9d9e2a57da863 Mon Sep 17 00:00:00 2001 From: Jack McPherson Date: Mon, 17 Jul 2023 00:14:14 +0000 Subject: [PATCH 32/51] Remove instances of required arguments with default values (#4489) ## Issue Addressed #4488 ## Proposed Changes - Remove all instances of the `required` modifier where we have a default value specified for a subcommand ## Additional Info N/A --- lcli/src/main.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lcli/src/main.rs b/lcli/src/main.rs index 565a5325f0d..8fbf5638b4c 100644 --- a/lcli/src/main.rs +++ b/lcli/src/main.rs @@ -40,7 +40,6 @@ fn main() { .long("spec") .value_name("STRING") .takes_value(true) - .required(true) .possible_values(&["minimal", "mainnet", "gnosis"]) .default_value("mainnet") .global(true), @@ -372,7 +371,6 @@ fn main() { .index(2) .value_name("BIP39_MNENMONIC") .takes_value(true) - .required(true) .default_value( "replace nephew blur decorate waste convince soup column \ orient excite play baby", @@ -393,7 +391,6 @@ fn main() { .help("The block hash used when generating an execution payload. This \ value is used for `execution_payload_header.block_hash` as well as \ `execution_payload_header.random`") - .required(true) .default_value( "0x0000000000000000000000000000000000000000000000000000000000000000", ), @@ -411,7 +408,6 @@ fn main() { .value_name("INTEGER") .takes_value(true) .help("The base fee per gas field in the execution payload generated.") - .required(true) .default_value("1000000000"), ) .arg( @@ -420,7 +416,6 @@ fn main() { .value_name("INTEGER") .takes_value(true) .help("The gas limit field in the execution payload generated.") - .required(true) .default_value("30000000"), ) .arg( From 03674c719992d600b7af111e5f6fa890d77a97bd Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 17 Jul 2023 00:14:15 +0000 Subject: [PATCH 33/51] Update mev-rs and remove patches (#4496) ## Issue Addressed Fixes occasional compilation errors with mev-rs (see #4456). ## Proposed Changes - Update `mev-rs` to the latest version, which allows us to remove hacky `[patch]` sections - Update the `axum` version used in `watch` so LH only uses a single version --- Cargo.lock | 96 ++++++++++--------- Cargo.toml | 7 -- beacon_node/execution_layer/Cargo.toml | 6 +- .../src/test_utils/mock_builder.rs | 72 +++++++------- watch/Cargo.toml | 2 +- watch/src/server/mod.rs | 3 +- 6 files changed, 88 insertions(+), 98 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3eaa645e6dd..a1f5c7e2af1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -245,6 +245,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "anvil-rpc" +version = "0.1.0" +source = "git+https://github.com/foundry-rs/foundry?rev=b45456717ffae1af65acdc71099f8cb95e6683a0#b45456717ffae1af65acdc71099f8cb95e6683a0" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "anyhow" version = "1.0.71" @@ -494,9 +503,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.5.17" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acee9fd5073ab6b045a275b3e709c163dd36c90685219cb21804a147b58dba43" +checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39" dependencies = [ "async-trait", "axum-core", @@ -512,22 +521,23 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite 0.2.9", + "rustversion", "serde", "serde_json", + "serde_path_to_error", "serde_urlencoded", "sync_wrapper", "tokio", "tower", - "tower-http", "tower-layer", "tower-service", ] [[package]] name = "axum-core" -version = "0.2.9" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e5939e02c56fecd5c017c37df4238c0a839fa76b7f97acdd7efb804fd181cc" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" dependencies = [ "async-trait", "bytes", @@ -535,6 +545,7 @@ dependencies = [ "http", "http-body", "mime", + "rustversion", "tower-layer", "tower-service", ] @@ -593,7 +604,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon-api-client" version = "0.1.0" -source = "git+https://github.com/ralexstokes/beacon-api-client#30679e9e25d61731cde54e14cd8a3688a39d8e5b" +source = "git+https://github.com/ralexstokes/beacon-api-client?rev=93d7e8c#93d7e8c38fe9782c4862909663e7b57c44f805a9" dependencies = [ "ethereum-consensus", "http", @@ -2492,7 +2503,7 @@ dependencies = [ [[package]] name = "ethereum-consensus" version = "0.1.1" -source = "git+https://github.com/ralexstokes//ethereum-consensus?rev=9b0ee0a8a45b968c8df5e7e64ea1c094e16f053d#9b0ee0a8a45b968c8df5e7e64ea1c094e16f053d" +source = "git+https://github.com/ralexstokes/ethereum-consensus?rev=e380108#e380108d15fcc40349927fdf3d11c71f9edb67c2" dependencies = [ "async-stream", "blst", @@ -2505,8 +2516,9 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", + "serde_yaml", "sha2 0.9.9", - "ssz-rs", + "ssz_rs", "thiserror", "tokio", "tokio-stream", @@ -2779,7 +2791,7 @@ dependencies = [ "serde_json", "slog", "slot_clock", - "ssz-rs", + "ssz_rs", "ssz_types", "state_processing", "strum", @@ -3503,12 +3515,6 @@ dependencies = [ "pin-project-lite 0.2.9", ] -[[package]] -name = "http-range-header" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" - [[package]] name = "http_api" version = "0.1.0" @@ -4969,9 +4975,9 @@ checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "matchit" -version = "0.5.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" +checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" [[package]] name = "md-5" @@ -5060,16 +5066,20 @@ dependencies = [ [[package]] name = "mev-rs" -version = "0.2.1" -source = "git+https://github.com/ralexstokes//mev-rs?rev=7813d4a4a564e0754e9aaab2d95520ba437c3889#7813d4a4a564e0754e9aaab2d95520ba437c3889" +version = "0.3.0" +source = "git+https://github.com/ralexstokes/mev-rs?rev=216657016d5c0889b505857c89ae42c7aa2764af#216657016d5c0889b505857c89ae42c7aa2764af" dependencies = [ + "anvil-rpc", "async-trait", "axum", "beacon-api-client", "ethereum-consensus", "hyper", + "parking_lot 0.12.1", + "reqwest", "serde", - "ssz-rs", + "serde_json", + "ssz_rs", "thiserror", "tokio", "tracing", @@ -7368,6 +7378,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b1b6471d7496b051e03f1958802a73f88b947866f5146f329e47e36554f4e55" +dependencies = [ + "itoa", + "serde", +] + [[package]] name = "serde_repr" version = "0.1.12" @@ -7865,23 +7885,24 @@ dependencies = [ ] [[package]] -name = "ssz-rs" -version = "0.8.0" -source = "git+https://github.com/ralexstokes//ssz-rs?rev=adf1a0b14cef90b9536f28ef89da1fab316465e1#adf1a0b14cef90b9536f28ef89da1fab316465e1" +name = "ssz_rs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "057291e5631f280978fa9c8009390663ca4613359fc1318e36a8c24c392f6d1f" dependencies = [ "bitvec 1.0.1", "hex", "num-bigint", "serde", "sha2 0.9.9", - "ssz-rs-derive", - "thiserror", + "ssz_rs_derive", ] [[package]] -name = "ssz-rs-derive" -version = "0.8.0" -source = "git+https://github.com/ralexstokes//ssz-rs?rev=adf1a0b14cef90b9536f28ef89da1fab316465e1#adf1a0b14cef90b9536f28ef89da1fab316465e1" +name = "ssz_rs_derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f07d54c4d01a1713eb363b55ba51595da15f6f1211435b71466460da022aa140" dependencies = [ "proc-macro2", "quote", @@ -8640,25 +8661,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tower-http" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" -dependencies = [ - "bitflags", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-range-header", - "pin-project-lite 0.2.9", - "tower", - "tower-layer", - "tower-service", -] - [[package]] name = "tower-layer" version = "0.3.2" diff --git a/Cargo.toml b/Cargo.toml index 775842a8c7a..8b820d2a215 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,13 +92,6 @@ resolver = "2" warp = { git = "https://github.com/macladson/warp", rev="7e75acc368229a46a236a8c991bf251fe7fe50ef" } arbitrary = { git = "https://github.com/michaelsproul/arbitrary", rev="f002b99989b561ddce62e4cf2887b0f8860ae991" } -[patch."https://github.com/ralexstokes/mev-rs"] -mev-rs = { git = "https://github.com/ralexstokes//mev-rs", rev = "7813d4a4a564e0754e9aaab2d95520ba437c3889" } -[patch."https://github.com/ralexstokes/ethereum-consensus"] -ethereum-consensus = { git = "https://github.com/ralexstokes//ethereum-consensus", rev = "9b0ee0a8a45b968c8df5e7e64ea1c094e16f053d" } -[patch."https://github.com/ralexstokes/ssz-rs"] -ssz-rs = { git = "https://github.com/ralexstokes//ssz-rs", rev = "adf1a0b14cef90b9536f28ef89da1fab316465e1" } - [profile.maxperf] inherits = "release" lto = "fat" diff --git a/beacon_node/execution_layer/Cargo.toml b/beacon_node/execution_layer/Cargo.toml index 2cb28346f54..8492a5159e2 100644 --- a/beacon_node/execution_layer/Cargo.toml +++ b/beacon_node/execution_layer/Cargo.toml @@ -41,9 +41,9 @@ lazy_static = "1.4.0" ethers-core = "1.0.2" builder_client = { path = "../builder_client" } fork_choice = { path = "../../consensus/fork_choice" } -mev-rs = { git = "https://github.com/ralexstokes/mev-rs" } -ethereum-consensus = { git = "https://github.com/ralexstokes/ethereum-consensus" } -ssz-rs = { git = "https://github.com/ralexstokes/ssz-rs" } +mev-rs = { git = "https://github.com/ralexstokes/mev-rs", rev = "216657016d5c0889b505857c89ae42c7aa2764af" } +ethereum-consensus = { git = "https://github.com/ralexstokes/ethereum-consensus", rev = "e380108" } +ssz_rs = "0.9.0" tokio-stream = { version = "0.1.9", features = [ "sync" ] } strum = "0.24.0" keccak-hash = "0.10.0" diff --git a/beacon_node/execution_layer/src/test_utils/mock_builder.rs b/beacon_node/execution_layer/src/test_utils/mock_builder.rs index 668d1fb3b1c..9471be636dc 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_builder.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_builder.rs @@ -11,11 +11,17 @@ use ethereum_consensus::{ }; use fork_choice::ForkchoiceUpdateParameters; use mev_rs::{ - bellatrix::{BuilderBid as BuilderBidBellatrix, SignedBuilderBid as SignedBuilderBidBellatrix}, - capella::{BuilderBid as BuilderBidCapella, SignedBuilderBid as SignedBuilderBidCapella}, - sign_builder_message, verify_signed_builder_message, BidRequest, BlindedBlockProviderError, - BlindedBlockProviderServer, BuilderBid, ExecutionPayload as ServerPayload, - SignedBlindedBeaconBlock, SignedBuilderBid, SignedValidatorRegistration, + blinded_block_provider::Server as BlindedBlockProviderServer, + signing::{sign_builder_message, verify_signed_builder_message}, + types::{ + bellatrix::{ + BuilderBid as BuilderBidBellatrix, SignedBuilderBid as SignedBuilderBidBellatrix, + }, + capella::{BuilderBid as BuilderBidCapella, SignedBuilderBid as SignedBuilderBidCapella}, + BidRequest, BuilderBid, ExecutionPayload as ServerPayload, SignedBlindedBeaconBlock, + SignedBuilderBid, SignedValidatorRegistration, + }, + Error as MevError, }; use parking_lot::RwLock; use sensitive_url::SensitiveUrl; @@ -47,7 +53,7 @@ pub enum Operation { } impl Operation { - fn apply(self, bid: &mut B) -> Result<(), BlindedBlockProviderError> { + fn apply(self, bid: &mut B) -> Result<(), MevError> { match self { Operation::FeeRecipient(fee_recipient) => { *bid.fee_recipient_mut() = to_ssz_rs(&fee_recipient)? @@ -73,7 +79,7 @@ pub trait BidStuff { fn prev_randao_mut(&mut self) -> &mut Hash32; fn block_number_mut(&mut self) -> &mut u64; fn timestamp_mut(&mut self) -> &mut u64; - fn withdrawals_root_mut(&mut self) -> Result<&mut Root, BlindedBlockProviderError>; + fn withdrawals_root_mut(&mut self) -> Result<&mut Root, MevError>; fn sign_builder_message( &mut self, @@ -134,11 +140,9 @@ impl BidStuff for BuilderBid { } } - fn withdrawals_root_mut(&mut self) -> Result<&mut Root, BlindedBlockProviderError> { + fn withdrawals_root_mut(&mut self) -> Result<&mut Root, MevError> { match self { - Self::Bellatrix(_) => Err(BlindedBlockProviderError::Custom( - "withdrawals_root called on bellatrix bid".to_string(), - )), + Self::Bellatrix(_) => Err(MevError::InvalidFork), Self::Capella(bid) => Ok(&mut bid.header.withdrawals_root), } } @@ -274,7 +278,7 @@ impl MockBuilder { *self.invalidate_signatures.write() = false; } - fn apply_operations(&self, bid: &mut B) -> Result<(), BlindedBlockProviderError> { + fn apply_operations(&self, bid: &mut B) -> Result<(), MevError> { let mut guard = self.operations.write(); while let Some(op) = guard.pop() { op.apply(bid)?; @@ -288,7 +292,7 @@ impl mev_rs::BlindedBlockProvider for MockBuilder { async fn register_validators( &self, registrations: &mut [SignedValidatorRegistration], - ) -> Result<(), BlindedBlockProviderError> { + ) -> Result<(), MevError> { for registration in registrations { let pubkey = registration.message.public_key.clone(); let message = &mut registration.message; @@ -307,10 +311,7 @@ impl mev_rs::BlindedBlockProvider for MockBuilder { Ok(()) } - async fn fetch_best_bid( - &self, - bid_request: &BidRequest, - ) -> Result { + async fn fetch_best_bid(&self, bid_request: &BidRequest) -> Result { let slot = Slot::new(bid_request.slot); let fork = self.spec.fork_name_at_slot::(slot); let signed_cached_data = self @@ -336,7 +337,7 @@ impl mev_rs::BlindedBlockProvider for MockBuilder { .map_err(convert_err)? .block_hash(); if head_execution_hash != from_ssz_rs(&bid_request.parent_hash)? { - return Err(BlindedBlockProviderError::Custom(format!( + return Err(custom_err(format!( "head mismatch: {} {}", head_execution_hash, bid_request.parent_hash ))); @@ -396,7 +397,7 @@ impl mev_rs::BlindedBlockProvider for MockBuilder { .get_debug_beacon_states(StateId::Head) .await .map_err(convert_err)? - .ok_or_else(|| BlindedBlockProviderError::Custom("missing head state".to_string()))? + .ok_or_else(|| custom_err("missing head state".to_string()))? .data; let prev_randao = head_state .get_randao_mix(head_state.current_epoch()) @@ -409,10 +410,7 @@ impl mev_rs::BlindedBlockProvider for MockBuilder { PayloadAttributes::new(timestamp, *prev_randao, fee_recipient, Some(vec![])) } ForkName::Base | ForkName::Altair => { - return Err(BlindedBlockProviderError::Custom(format!( - "Unsupported fork: {}", - fork - ))); + return Err(MevError::InvalidFork); } }; @@ -452,12 +450,7 @@ impl mev_rs::BlindedBlockProvider for MockBuilder { value: to_ssz_rs(&Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI))?, public_key: self.builder_sk.public_key(), }), - ForkName::Base | ForkName::Altair => { - return Err(BlindedBlockProviderError::Custom(format!( - "Unsupported fork: {}", - fork - ))) - } + ForkName::Base | ForkName::Altair => return Err(MevError::InvalidFork), }; *message.gas_limit_mut() = cached_data.gas_limit; @@ -475,7 +468,7 @@ impl mev_rs::BlindedBlockProvider for MockBuilder { async fn open_bid( &self, signed_block: &mut SignedBlindedBeaconBlock, - ) -> Result { + ) -> Result { let node = match signed_block { SignedBlindedBeaconBlock::Bellatrix(block) => { block.message.body.execution_payload_header.hash_tree_root() @@ -496,9 +489,7 @@ impl mev_rs::BlindedBlockProvider for MockBuilder { } } -pub fn from_ssz_rs( - ssz_rs_data: &T, -) -> Result { +pub fn from_ssz_rs(ssz_rs_data: &T) -> Result { U::from_ssz_bytes( ssz_rs::serialize(ssz_rs_data) .map_err(convert_err)? @@ -507,12 +498,17 @@ pub fn from_ssz_rs( .map_err(convert_err) } -pub fn to_ssz_rs( - ssz_data: &T, -) -> Result { +pub fn to_ssz_rs(ssz_data: &T) -> Result { ssz_rs::deserialize::(&ssz_data.as_ssz_bytes()).map_err(convert_err) } -fn convert_err(e: E) -> BlindedBlockProviderError { - BlindedBlockProviderError::Custom(format!("{e:?}")) +fn convert_err(e: E) -> MevError { + custom_err(format!("{e:?}")) +} + +// This is a bit of a hack since the `Custom` variant was removed from `mev_rs::Error`. +fn custom_err(s: String) -> MevError { + MevError::Consensus(ethereum_consensus::state_transition::Error::Io( + std::io::Error::new(std::io::ErrorKind::Other, s), + )) } diff --git a/watch/Cargo.toml b/watch/Cargo.toml index d1793a9d068..1a003167dc8 100644 --- a/watch/Cargo.toml +++ b/watch/Cargo.toml @@ -21,7 +21,7 @@ types = { path = "../consensus/types" } eth2 = { path = "../common/eth2" } beacon_node = { path = "../beacon_node"} tokio = { version = "1.14.0", features = ["time"] } -axum = "0.5.15" +axum = "0.6.18" hyper = "0.14.20" serde = "1.0.116" serde_json = "1.0.58" diff --git a/watch/src/server/mod.rs b/watch/src/server/mod.rs index 09d5ec6aac5..d8ae0eb6c62 100644 --- a/watch/src/server/mod.rs +++ b/watch/src/server/mod.rs @@ -5,7 +5,6 @@ use crate::config::Config as FullConfig; use crate::database::{self, PgPool}; use crate::suboptimal_attestations::{attestation_routes, blockprint_attestation_routes}; use axum::{ - handler::Handler, http::{StatusCode, Uri}, routing::get, Extension, Json, Router, @@ -104,7 +103,7 @@ pub fn start_server( } let app = routes - .fallback(route_not_found.into_service()) + .fallback(route_not_found) .layer(Extension(pool)) .layer(Extension(slots_per_epoch)); From 5569d97a070b3fbd68f2515cd02bc527a9b5cf2d Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 17 Jul 2023 00:14:16 +0000 Subject: [PATCH 34/51] Remove wget dependency (#4497) ## Proposed Changes Replace `wget` in the EF-tests makefile with `curl`. On macOS `curl` is pre-installed, and I found myself making this change to avoid installing `wget`. The `-L` flag is used to follow redirects which is useful if a repo gets renamed, and more similar to `wget`'s default behaviour. --- testing/ef_tests/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/ef_tests/Makefile b/testing/ef_tests/Makefile index f7562f477a2..81a1739eb12 100644 --- a/testing/ef_tests/Makefile +++ b/testing/ef_tests/Makefile @@ -13,7 +13,7 @@ BLS_TARBALL = $(patsubst %,%-$(BLS_TEST_TAG).tar.gz,$(BLS_TEST)) BLS_OUTPUT_DIR := $(OUTPUT_DIR)/$(BLS_TEST_REPO_NAME) BLS_BASE_URL := https://github.com/ethereum/$(BLS_TEST_REPO_NAME)/releases/download/$(BLS_TEST_TAG) -WGET := $(if $(LIGHTHOUSE_GITHUB_TOKEN),wget --header="Authorization: $(LIGHTHOUSE_GITHUB_TOKEN)",wget) +CURL := $(if $(LIGHTHOUSE_GITHUB_TOKEN),curl -L --header "Authorization: $(LIGHTHOUSE_GITHUB_TOKEN)",curl -L) all: make $(OUTPUT_DIR) @@ -27,11 +27,11 @@ $(OUTPUT_DIR): $(TARBALLS) $(BLS_OUTPUT_DIR): mkdir $(BLS_OUTPUT_DIR) - $(WGET) $(BLS_BASE_URL)/$(BLS_TEST).tar.gz -O $(BLS_TARBALL) + $(CURL) $(BLS_BASE_URL)/$(BLS_TEST).tar.gz -o $(BLS_TARBALL) tar -xzf $(BLS_TARBALL) -C $(BLS_OUTPUT_DIR) %-$(TESTS_TAG).tar.gz: - $(WGET) $(BASE_URL)/$*.tar.gz -O $@ + $(CURL) $(BASE_URL)/$*.tar.gz -o $@ clean-test-files: rm -rf $(OUTPUT_DIR) $(BLS_OUTPUT_DIR) From 68d5a6cf993fc1f8d09eb190db71c0df735a90b2 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Mon, 17 Jul 2023 00:14:17 +0000 Subject: [PATCH 35/51] Clean up local testnet files without prompting (#4498) ## Issue Addressed Addresses an issue where CI could fail due to an nonexisting file error: ``` Run ./clean.sh rm: cannot remove '/home/runner/.lighthouse/local-testnet/geth_datadir4/geth/fastcache.tmp.1549331618': No such file or directory Error: Process completed with exit code 1. ``` This seems to happen quite frequently now, I'm not sure exactly why but perhaps worth trying suppressing the prompt? https://github.com/sigp/lighthouse/actions/runs/5455027574/jobs/9925916159?pr=4463 --- scripts/local_testnet/clean.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/local_testnet/clean.sh b/scripts/local_testnet/clean.sh index 6db8753d02b..cd915e470d6 100755 --- a/scripts/local_testnet/clean.sh +++ b/scripts/local_testnet/clean.sh @@ -9,5 +9,5 @@ set -Eeuo pipefail source ./vars.env if [ -d $DATADIR ]; then - rm -r $DATADIR + rm -rf $DATADIR fi From d4a61756caba0e52242fddfff095fb077afdb831 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Mon, 17 Jul 2023 00:14:18 +0000 Subject: [PATCH 36/51] CI fix: add retries to eth1 sim tests (#4501) ## Issue Addressed This PR attempts to workaround the recent frequent eth1 simulator failures caused by missing eth logs from Anvil. > FailedToInsertDeposit(NonConsecutive { log_index: 1, expected: 0 }) This usually occurs at the beginning of the tests, and it guarantees a timeout after a few hours if this log shows up, and this is currently causing our CIs to fail quite frequently. Example failure here: https://github.com/sigp/lighthouse/actions/runs/5525760195/jobs/10079736914 ## Proposed Changes The quick fix applied here adds a timeout to node startup and restarts the node again. - Add a 60 seconds timeout to beacon node startup in eth1 simulator tests. It takes ~10 seconds on my machine, but could take longer on CI runners. - Wrap the startup code in a retry function, that allows for 3 retries before returning an error. ## Additional Info We should probably raise an issue under the Anvil GitHub repo there so this can be further investigated. --- Cargo.lock | 1 + testing/node_test_rig/Cargo.toml | 1 + testing/node_test_rig/src/lib.rs | 19 ++-- testing/simulator/src/eth1_sim.rs | 168 ++++++++++++++++++------------ testing/simulator/src/main.rs | 1 + testing/simulator/src/retry.rs | 63 +++++++++++ 6 files changed, 183 insertions(+), 70 deletions(-) create mode 100644 testing/simulator/src/retry.rs diff --git a/Cargo.lock b/Cargo.lock index a1f5c7e2af1..4bb3ab99e28 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5531,6 +5531,7 @@ dependencies = [ "execution_layer", "sensitive_url", "tempfile", + "tokio", "types", "validator_client", "validator_dir", diff --git a/testing/node_test_rig/Cargo.toml b/testing/node_test_rig/Cargo.toml index ea5d005c16a..ac77349c58e 100644 --- a/testing/node_test_rig/Cargo.toml +++ b/testing/node_test_rig/Cargo.toml @@ -14,3 +14,4 @@ validator_client = { path = "../../validator_client" } validator_dir = { path = "../../common/validator_dir", features = ["insecure_keys"] } sensitive_url = { path = "../../common/sensitive_url" } execution_layer = { path = "../../beacon_node/execution_layer" } +tokio = { version = "1.14.0", features = ["time"] } diff --git a/testing/node_test_rig/src/lib.rs b/testing/node_test_rig/src/lib.rs index d4fd115bec3..62db67b8c57 100644 --- a/testing/node_test_rig/src/lib.rs +++ b/testing/node_test_rig/src/lib.rs @@ -10,6 +10,7 @@ use std::path::PathBuf; use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; use tempfile::{Builder as TempBuilder, TempDir}; +use tokio::time::timeout; use types::EthSpec; use validator_client::ProductionValidatorClient; use validator_dir::insecure_keys::build_deterministic_validator_dirs; @@ -24,6 +25,8 @@ pub use validator_client::Config as ValidatorConfig; /// The global timeout for HTTP requests to the beacon node. const HTTP_TIMEOUT: Duration = Duration::from_secs(4); +/// The timeout for a beacon node to start up. +const STARTUP_TIMEOUT: Duration = Duration::from_secs(60); /// Provides a beacon node that is running in the current process on a given tokio executor (it /// is _local_ to this process). @@ -51,12 +54,16 @@ impl LocalBeaconNode { client_config.set_data_dir(datadir.path().into()); client_config.network.network_dir = PathBuf::from(datadir.path()).join("network"); - ProductionBeaconNode::new(context, client_config) - .await - .map(move |client| Self { - client: client.into_inner(), - datadir, - }) + timeout( + STARTUP_TIMEOUT, + ProductionBeaconNode::new(context, client_config), + ) + .await + .map_err(|_| format!("Beacon node startup timed out after {:?}", STARTUP_TIMEOUT))? + .map(move |client| Self { + client: client.into_inner(), + datadir, + }) } } diff --git a/testing/simulator/src/eth1_sim.rs b/testing/simulator/src/eth1_sim.rs index 3e764d27d02..57c944cf1a7 100644 --- a/testing/simulator/src/eth1_sim.rs +++ b/testing/simulator/src/eth1_sim.rs @@ -1,14 +1,16 @@ use crate::local_network::{EXECUTION_PORT, TERMINAL_BLOCK, TERMINAL_DIFFICULTY}; -use crate::{checks, LocalNetwork, E}; +use crate::{checks, LocalNetwork}; use clap::ArgMatches; use eth1::{Eth1Endpoint, DEFAULT_CHAIN_ID}; use eth1_test_rig::AnvilEth1Instance; +use crate::retry::with_retry; use execution_layer::http::deposit_methods::Eth1Id; use futures::prelude::*; +use node_test_rig::environment::RuntimeContext; use node_test_rig::{ environment::{EnvironmentBuilder, LoggerConfig}, - testing_client_config, testing_validator_config, ClientGenesis, ValidatorFiles, + testing_client_config, testing_validator_config, ClientConfig, ClientGenesis, ValidatorFiles, }; use rayon::prelude::*; use sensitive_url::SensitiveUrl; @@ -107,71 +109,24 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { let context = env.core_context(); let main_future = async { - /* - * Deploy the deposit contract, spawn tasks to keep creating new blocks and deposit - * validators. - */ - let anvil_eth1_instance = AnvilEth1Instance::new(DEFAULT_CHAIN_ID.into()).await?; - let deposit_contract = anvil_eth1_instance.deposit_contract; - let chain_id = anvil_eth1_instance.anvil.chain_id(); - let anvil = anvil_eth1_instance.anvil; - let eth1_endpoint = SensitiveUrl::parse(anvil.endpoint().as_str()) - .expect("Unable to parse anvil endpoint."); - let deposit_contract_address = deposit_contract.address(); - - // Start a timer that produces eth1 blocks on an interval. - tokio::spawn(async move { - let mut interval = tokio::time::interval(eth1_block_time); - loop { - interval.tick().await; - let _ = anvil.evm_mine().await; - } - }); - - // Submit deposits to the deposit contract. - tokio::spawn(async move { - for i in 0..total_validator_count { - println!("Submitting deposit for validator {}...", i); - let _ = deposit_contract - .deposit_deterministic_async::(i, deposit_amount) - .await; - } - }); - - let mut beacon_config = testing_client_config(); - - beacon_config.genesis = ClientGenesis::DepositContract; - beacon_config.eth1.endpoint = Eth1Endpoint::NoAuth(eth1_endpoint); - beacon_config.eth1.deposit_contract_address = deposit_contract_address; - beacon_config.eth1.deposit_contract_deploy_block = 0; - beacon_config.eth1.lowest_cached_block_number = 0; - beacon_config.eth1.follow_distance = 1; - beacon_config.eth1.node_far_behind_seconds = 20; - beacon_config.dummy_eth1_backend = false; - beacon_config.sync_eth1_chain = true; - beacon_config.eth1.auto_update_interval_millis = eth1_block_time.as_millis() as u64; - beacon_config.eth1.chain_id = Eth1Id::from(chain_id); - beacon_config.network.target_peers = node_count + proposer_nodes - 1; - - beacon_config.network.enr_address = (Some(Ipv4Addr::LOCALHOST), None); - - if post_merge_sim { - let el_config = execution_layer::Config { - execution_endpoints: vec![SensitiveUrl::parse(&format!( - "http://localhost:{}", - EXECUTION_PORT - )) - .unwrap()], - ..Default::default() - }; - - beacon_config.execution_layer = Some(el_config); - } - /* * Create a new `LocalNetwork` with one beacon node. */ - let network = LocalNetwork::new(context.clone(), beacon_config.clone()).await?; + let max_retries = 3; + let (network, beacon_config) = with_retry(max_retries, || { + Box::pin(create_local_network( + LocalNetworkParams { + eth1_block_time, + total_validator_count, + deposit_amount, + node_count, + proposer_nodes, + post_merge_sim, + }, + context.clone(), + )) + }) + .await?; /* * One by one, add beacon nodes to the network. @@ -341,3 +296,88 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { Ok(()) } + +struct LocalNetworkParams { + eth1_block_time: Duration, + total_validator_count: usize, + deposit_amount: u64, + node_count: usize, + proposer_nodes: usize, + post_merge_sim: bool, +} + +async fn create_local_network( + LocalNetworkParams { + eth1_block_time, + total_validator_count, + deposit_amount, + node_count, + proposer_nodes, + post_merge_sim, + }: LocalNetworkParams, + context: RuntimeContext, +) -> Result<(LocalNetwork, ClientConfig), String> { + /* + * Deploy the deposit contract, spawn tasks to keep creating new blocks and deposit + * validators. + */ + let anvil_eth1_instance = AnvilEth1Instance::new(DEFAULT_CHAIN_ID.into()).await?; + let deposit_contract = anvil_eth1_instance.deposit_contract; + let chain_id = anvil_eth1_instance.anvil.chain_id(); + let anvil = anvil_eth1_instance.anvil; + let eth1_endpoint = + SensitiveUrl::parse(anvil.endpoint().as_str()).expect("Unable to parse anvil endpoint."); + let deposit_contract_address = deposit_contract.address(); + + // Start a timer that produces eth1 blocks on an interval. + tokio::spawn(async move { + let mut interval = tokio::time::interval(eth1_block_time); + loop { + interval.tick().await; + let _ = anvil.evm_mine().await; + } + }); + + // Submit deposits to the deposit contract. + tokio::spawn(async move { + for i in 0..total_validator_count { + println!("Submitting deposit for validator {}...", i); + let _ = deposit_contract + .deposit_deterministic_async::(i, deposit_amount) + .await; + } + }); + + let mut beacon_config = testing_client_config(); + + beacon_config.genesis = ClientGenesis::DepositContract; + beacon_config.eth1.endpoint = Eth1Endpoint::NoAuth(eth1_endpoint); + beacon_config.eth1.deposit_contract_address = deposit_contract_address; + beacon_config.eth1.deposit_contract_deploy_block = 0; + beacon_config.eth1.lowest_cached_block_number = 0; + beacon_config.eth1.follow_distance = 1; + beacon_config.eth1.node_far_behind_seconds = 20; + beacon_config.dummy_eth1_backend = false; + beacon_config.sync_eth1_chain = true; + beacon_config.eth1.auto_update_interval_millis = eth1_block_time.as_millis() as u64; + beacon_config.eth1.chain_id = Eth1Id::from(chain_id); + beacon_config.network.target_peers = node_count + proposer_nodes - 1; + + beacon_config.network.enr_address = (Some(Ipv4Addr::LOCALHOST), None); + + if post_merge_sim { + let el_config = execution_layer::Config { + execution_endpoints: vec![SensitiveUrl::parse(&format!( + "http://localhost:{}", + EXECUTION_PORT + )) + .unwrap()], + ..Default::default() + }; + + beacon_config.execution_layer = Some(el_config); + } + + let network = LocalNetwork::new(context, beacon_config.clone()).await?; + Ok((network, beacon_config)) +} diff --git a/testing/simulator/src/main.rs b/testing/simulator/src/main.rs index a19777c5abc..e8af9c18067 100644 --- a/testing/simulator/src/main.rs +++ b/testing/simulator/src/main.rs @@ -21,6 +21,7 @@ mod cli; mod eth1_sim; mod local_network; mod no_eth1_sim; +mod retry; mod sync_sim; use cli::cli_app; diff --git a/testing/simulator/src/retry.rs b/testing/simulator/src/retry.rs new file mode 100644 index 00000000000..a4eb52cea1f --- /dev/null +++ b/testing/simulator/src/retry.rs @@ -0,0 +1,63 @@ +use std::fmt::Debug; +use std::future::Future; +use std::pin::Pin; + +/// Executes the function with a specified number of retries if the function returns an error. +/// Once it exceeds `max_retries` and still fails, the error is returned. +pub async fn with_retry(max_retries: usize, mut func: F) -> Result +where + F: FnMut() -> Pin>>>, + E: Debug, +{ + let mut retry_count = 0; + loop { + let result = Box::pin(func()).await; + if result.is_ok() || retry_count >= max_retries { + break result; + } + retry_count += 1; + + if let Err(e) = result { + eprintln!( + "Operation failed with error {:?}, retrying {} of {}", + e, retry_count, max_retries + ); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::VecDeque; + + async fn my_async_func(is_ok: bool) -> Result<(), ()> { + if is_ok { + Ok(()) + } else { + Err(()) + } + } + + #[tokio::test] + async fn test_with_retry_ok() { + let res = with_retry(3, || Box::pin(my_async_func(true))).await; + assert!(res.is_ok()); + } + + #[tokio::test] + async fn test_with_retry_2nd_ok() { + let mut mock_results = VecDeque::from([false, true]); + let res = with_retry(3, || { + Box::pin(my_async_func(mock_results.pop_front().unwrap())) + }) + .await; + assert!(res.is_ok()); + } + + #[tokio::test] + async fn test_with_retry_fail() { + let res = with_retry(3, || Box::pin(my_async_func(false))).await; + assert!(res.is_err()); + } +} From f2223feb21bb10dbdebce097c5cda384b1ff44a3 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Mon, 17 Jul 2023 00:14:19 +0000 Subject: [PATCH 37/51] Rust 1.71 lints (#4503) ## Issue Addressed N/A ## Proposed Changes Add lints for rust 1.71 [3789134](https://github.com/sigp/lighthouse/pull/4503/commits/3789134ae2b06cedbe8becf1db49b2d23689ef94) is probably the one that needs most attention as it changes beacon state code. I changed the `is_in_inactivity_leak ` function to return a `ArithError` as not all consumers of that function work well with a `BeaconState::Error`. --- .../beacon_chain/src/attestation_rewards.rs | 2 +- beacon_node/execution_layer/src/lib.rs | 42 +++++++++---------- .../http_api/src/block_packing_efficiency.rs | 2 +- consensus/state_processing/src/lib.rs | 2 +- .../block_signature_verifier.rs | 2 +- .../src/per_block_processing/tests.rs | 2 +- .../altair/inactivity_updates.rs | 2 +- .../altair/rewards_and_penalties.rs | 2 +- .../state_processing/src/verify_operation.rs | 8 ++-- consensus/types/src/beacon_state.rs | 20 +++++---- .../types/src/beacon_state/committee_cache.rs | 2 +- .../types/src/beacon_state/tree_hash_cache.rs | 2 +- consensus/types/src/chain_spec.rs | 2 +- consensus/types/src/execution_payload.rs | 4 +- consensus/types/src/lib.rs | 2 +- consensus/types/src/participation_list.rs | 2 +- consensus/types/src/subnet_id.rs | 2 +- consensus/types/src/test_utils/mod.rs | 2 +- consensus/types/src/test_utils/test_random.rs | 2 +- slasher/tests/backend.rs | 2 +- 20 files changed, 56 insertions(+), 50 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_rewards.rs b/beacon_node/beacon_chain/src/attestation_rewards.rs index a4a661197f7..460bf18bcfd 100644 --- a/beacon_node/beacon_chain/src/attestation_rewards.rs +++ b/beacon_node/beacon_chain/src/attestation_rewards.rs @@ -86,7 +86,7 @@ impl BeaconChain { let ideal_reward = reward_numerator .safe_div(active_increments)? .safe_div(WEIGHT_DENOMINATOR)?; - if !state.is_in_inactivity_leak(previous_epoch, spec) { + if !state.is_in_inactivity_leak(previous_epoch, spec)? { ideal_rewards_hashmap .insert((flag_index, effective_balance), (ideal_reward, penalty)); } else { diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 2720569c8d8..d72686baf55 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -163,7 +163,7 @@ impl> BlockProposalContents ExecutionLayer { BlockProposalContents::Payload { payload: relay.data.message.header, block_value: relay.data.message.value, - _phantom: PhantomData::default(), + _phantom: PhantomData, }, )), Err(reason) if !reason.payload_invalid() => { @@ -913,7 +913,7 @@ impl ExecutionLayer { BlockProposalContents::Payload { payload: relay.data.message.header, block_value: relay.data.message.value, - _phantom: PhantomData::default(), + _phantom: PhantomData, }, )), // If the payload is valid then use it. The local EE failed @@ -922,7 +922,7 @@ impl ExecutionLayer { BlockProposalContents::Payload { payload: relay.data.message.header, block_value: relay.data.message.value, - _phantom: PhantomData::default(), + _phantom: PhantomData, }, )), Err(reason) => { @@ -1129,7 +1129,7 @@ impl ExecutionLayer { Ok(BlockProposalContents::Payload { payload: execution_payload.into(), block_value, - _phantom: PhantomData::default(), + _phantom: PhantomData, }) }) .await @@ -2018,6 +2018,22 @@ async fn timed_future, T>(metric: &str, future: F) -> (T, (result, duration) } +fn noop( + _: &ExecutionLayer, + _: ExecutionPayloadRef, +) -> Option> { + None +} + +#[cfg(test)] +/// Returns the duration since the unix epoch. +fn timestamp_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_else(|_| Duration::from_secs(0)) + .as_secs() +} + #[cfg(test)] mod test { use super::*; @@ -2164,19 +2180,3 @@ mod test { .await; } } - -fn noop( - _: &ExecutionLayer, - _: ExecutionPayloadRef, -) -> Option> { - None -} - -#[cfg(test)] -/// Returns the duration since the unix epoch. -fn timestamp_now() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_else(|_| Duration::from_secs(0)) - .as_secs() -} diff --git a/beacon_node/http_api/src/block_packing_efficiency.rs b/beacon_node/http_api/src/block_packing_efficiency.rs index 1b924f38288..e099e130a8b 100644 --- a/beacon_node/http_api/src/block_packing_efficiency.rs +++ b/beacon_node/http_api/src/block_packing_efficiency.rs @@ -75,7 +75,7 @@ impl PackingEfficiencyHandler { available_attestations: HashSet::new(), included_attestations: HashMap::new(), committee_store: CommitteeStore::new(), - _phantom: PhantomData::default(), + _phantom: PhantomData, }; handler.compute_epoch(start_epoch, &starting_state, spec)?; diff --git a/consensus/state_processing/src/lib.rs b/consensus/state_processing/src/lib.rs index 7340206a345..a3ee7254062 100644 --- a/consensus/state_processing/src/lib.rs +++ b/consensus/state_processing/src/lib.rs @@ -2,7 +2,7 @@ #![cfg_attr( not(test), deny( - clippy::integer_arithmetic, + clippy::arithmetic_side_effects, clippy::disallowed_methods, clippy::indexing_slicing, clippy::unwrap_used, diff --git a/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs b/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs index 709302eec17..22309334fa3 100644 --- a/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs +++ b/consensus/state_processing/src/per_block_processing/block_signature_verifier.rs @@ -1,4 +1,4 @@ -#![allow(clippy::integer_arithmetic)] +#![allow(clippy::arithmetic_side_effects)] use super::signature_sets::{Error as SignatureSetError, *}; use crate::per_block_processing::errors::{AttestationInvalid, BlockOperationError}; diff --git a/consensus/state_processing/src/per_block_processing/tests.rs b/consensus/state_processing/src/per_block_processing/tests.rs index ddb9ca6ad54..16fa2462f5e 100644 --- a/consensus/state_processing/src/per_block_processing/tests.rs +++ b/consensus/state_processing/src/per_block_processing/tests.rs @@ -960,7 +960,7 @@ async fn fork_spanning_exit() { spec.bellatrix_fork_epoch = Some(Epoch::new(4)); spec.shard_committee_period = 0; - let harness = BeaconChainHarness::builder(MainnetEthSpec::default()) + let harness = BeaconChainHarness::builder(MainnetEthSpec) .spec(spec.clone()) .deterministic_keypairs(VALIDATOR_COUNT) .mock_execution_layer() diff --git a/consensus/state_processing/src/per_epoch_processing/altair/inactivity_updates.rs b/consensus/state_processing/src/per_epoch_processing/altair/inactivity_updates.rs index 967f642e85d..a895567d12c 100644 --- a/consensus/state_processing/src/per_epoch_processing/altair/inactivity_updates.rs +++ b/consensus/state_processing/src/per_epoch_processing/altair/inactivity_updates.rs @@ -34,7 +34,7 @@ pub fn process_inactivity_updates( .safe_add_assign(spec.inactivity_score_bias)?; } // Decrease the score of all validators for forgiveness when not during a leak - if !state.is_in_inactivity_leak(previous_epoch, spec) { + if !state.is_in_inactivity_leak(previous_epoch, spec)? { let inactivity_score = state.get_inactivity_score_mut(index)?; inactivity_score .safe_sub_assign(min(spec.inactivity_score_recovery_rate, *inactivity_score))?; diff --git a/consensus/state_processing/src/per_epoch_processing/altair/rewards_and_penalties.rs b/consensus/state_processing/src/per_epoch_processing/altair/rewards_and_penalties.rs index e2aa67a6193..19d57130c9b 100644 --- a/consensus/state_processing/src/per_epoch_processing/altair/rewards_and_penalties.rs +++ b/consensus/state_processing/src/per_epoch_processing/altair/rewards_and_penalties.rs @@ -77,7 +77,7 @@ pub fn get_flag_index_deltas( let mut delta = Delta::default(); if unslashed_participating_indices.contains(index)? { - if !state.is_in_inactivity_leak(previous_epoch, spec) { + if !state.is_in_inactivity_leak(previous_epoch, spec)? { let reward_numerator = base_reward .safe_mul(weight)? .safe_mul(unslashed_participating_increments)?; diff --git a/consensus/state_processing/src/verify_operation.rs b/consensus/state_processing/src/verify_operation.rs index 864844080fb..b3924cd9732 100644 --- a/consensus/state_processing/src/verify_operation.rs +++ b/consensus/state_processing/src/verify_operation.rs @@ -138,7 +138,7 @@ impl VerifyOperation for SignedVoluntaryExit { Ok(SigVerifiedOp::new(self, state)) } - #[allow(clippy::integer_arithmetic)] + #[allow(clippy::arithmetic_side_effects)] fn verification_epochs(&self) -> SmallVec<[Epoch; MAX_FORKS_VERIFIED_AGAINST]> { smallvec![self.message.epoch] } @@ -156,7 +156,7 @@ impl VerifyOperation for AttesterSlashing { Ok(SigVerifiedOp::new(self, state)) } - #[allow(clippy::integer_arithmetic)] + #[allow(clippy::arithmetic_side_effects)] fn verification_epochs(&self) -> SmallVec<[Epoch; MAX_FORKS_VERIFIED_AGAINST]> { smallvec![ self.attestation_1.data.target.epoch, @@ -177,7 +177,7 @@ impl VerifyOperation for ProposerSlashing { Ok(SigVerifiedOp::new(self, state)) } - #[allow(clippy::integer_arithmetic)] + #[allow(clippy::arithmetic_side_effects)] fn verification_epochs(&self) -> SmallVec<[Epoch; MAX_FORKS_VERIFIED_AGAINST]> { // Only need a single epoch because the slots of the two headers must be equal. smallvec![self @@ -200,7 +200,7 @@ impl VerifyOperation for SignedBlsToExecutionChange { Ok(SigVerifiedOp::new(self, state)) } - #[allow(clippy::integer_arithmetic)] + #[allow(clippy::arithmetic_side_effects)] fn verification_epochs(&self) -> SmallVec<[Epoch; MAX_FORKS_VERIFIED_AGAINST]> { smallvec![] } diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 1fa4dee3a0e..6a205e307ad 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -442,7 +442,7 @@ impl BeaconState { } /// Specialised deserialisation method that uses the `ChainSpec` as context. - #[allow(clippy::integer_arithmetic)] + #[allow(clippy::arithmetic_side_effects)] pub fn from_ssz_bytes(bytes: &[u8], spec: &ChainSpec) -> Result { // Slot is after genesis_time (u64) and genesis_validators_root (Hash256). let slot_start = ::ssz_fixed_len() + ::ssz_fixed_len(); @@ -1734,16 +1734,22 @@ impl BeaconState { previous_epoch: Epoch, val_index: usize, ) -> Result { - self.get_validator(val_index).map(|val| { - val.is_active_at(previous_epoch) - || (val.slashed && previous_epoch + Epoch::new(1) < val.withdrawable_epoch) - }) + let val = self.get_validator(val_index)?; + Ok(val.is_active_at(previous_epoch) + || (val.slashed && previous_epoch.safe_add(Epoch::new(1))? < val.withdrawable_epoch)) } /// Passing `previous_epoch` to this function rather than computing it internally provides /// a tangible speed improvement in state processing. - pub fn is_in_inactivity_leak(&self, previous_epoch: Epoch, spec: &ChainSpec) -> bool { - (previous_epoch - self.finalized_checkpoint().epoch) > spec.min_epochs_to_inactivity_penalty + pub fn is_in_inactivity_leak( + &self, + previous_epoch: Epoch, + spec: &ChainSpec, + ) -> Result { + Ok( + (previous_epoch.safe_sub(self.finalized_checkpoint().epoch)?) + > spec.min_epochs_to_inactivity_penalty, + ) } /// Get the `SyncCommittee` associated with the next slot. Useful because sync committees diff --git a/consensus/types/src/beacon_state/committee_cache.rs b/consensus/types/src/beacon_state/committee_cache.rs index 8afef1183be..2db8fbe7632 100644 --- a/consensus/types/src/beacon_state/committee_cache.rs +++ b/consensus/types/src/beacon_state/committee_cache.rs @@ -1,4 +1,4 @@ -#![allow(clippy::integer_arithmetic)] +#![allow(clippy::arithmetic_side_effects)] use super::BeaconState; use crate::*; diff --git a/consensus/types/src/beacon_state/tree_hash_cache.rs b/consensus/types/src/beacon_state/tree_hash_cache.rs index d1d63e3c806..69cd6fbb87b 100644 --- a/consensus/types/src/beacon_state/tree_hash_cache.rs +++ b/consensus/types/src/beacon_state/tree_hash_cache.rs @@ -1,4 +1,4 @@ -#![allow(clippy::integer_arithmetic)] +#![allow(clippy::arithmetic_side_effects)] #![allow(clippy::disallowed_methods)] #![allow(clippy::indexing_slicing)] diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index 59571822309..fbb6a3d8573 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -451,7 +451,7 @@ impl ChainSpec { Hash256::from(domain) } - #[allow(clippy::integer_arithmetic)] + #[allow(clippy::arithmetic_side_effects)] pub const fn attestation_subnet_prefix_bits(&self) -> u32 { let attestation_subnet_count_bits = self.attestation_subnet_count.ilog2(); self.attestation_subnet_extra_bits as u32 + attestation_subnet_count_bits diff --git a/consensus/types/src/execution_payload.rs b/consensus/types/src/execution_payload.rs index 77ef6407e87..690138da674 100644 --- a/consensus/types/src/execution_payload.rs +++ b/consensus/types/src/execution_payload.rs @@ -106,7 +106,7 @@ impl ExecutionPayload { } } - #[allow(clippy::integer_arithmetic)] + #[allow(clippy::arithmetic_side_effects)] /// Returns the maximum size of an execution payload. pub fn max_execution_payload_merge_size() -> usize { // Fixed part @@ -117,7 +117,7 @@ impl ExecutionPayload { + (T::max_transactions_per_payload() * (ssz::BYTES_PER_LENGTH_OFFSET + T::max_bytes_per_transaction())) } - #[allow(clippy::integer_arithmetic)] + #[allow(clippy::arithmetic_side_effects)] /// Returns the maximum size of an execution payload. pub fn max_execution_payload_capella_size() -> usize { // Fixed part diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index aefb45490a8..874d7cd2bdf 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -3,7 +3,7 @@ #![cfg_attr( not(test), deny( - clippy::integer_arithmetic, + clippy::arithmetic_side_effects, clippy::disallowed_methods, clippy::indexing_slicing ) diff --git a/consensus/types/src/participation_list.rs b/consensus/types/src/participation_list.rs index 89a56cb87d8..be119fbef26 100644 --- a/consensus/types/src/participation_list.rs +++ b/consensus/types/src/participation_list.rs @@ -1,4 +1,4 @@ -#![allow(clippy::integer_arithmetic)] +#![allow(clippy::arithmetic_side_effects)] use crate::{Hash256, ParticipationFlags, Unsigned, VariableList}; use cached_tree_hash::{int_log, CacheArena, CachedTreeHash, Error, TreeHashCache}; diff --git a/consensus/types/src/subnet_id.rs b/consensus/types/src/subnet_id.rs index 6793fe55740..eb25b57b0d7 100644 --- a/consensus/types/src/subnet_id.rs +++ b/consensus/types/src/subnet_id.rs @@ -72,7 +72,7 @@ impl SubnetId { .into()) } - #[allow(clippy::integer_arithmetic)] + #[allow(clippy::arithmetic_side_effects)] /// Computes the set of subnets the node should be subscribed to during the current epoch, /// along with the first epoch in which these subscriptions are no longer valid. pub fn compute_subnets_for_epoch( diff --git a/consensus/types/src/test_utils/mod.rs b/consensus/types/src/test_utils/mod.rs index c0333bcfd66..d172342ee64 100644 --- a/consensus/types/src/test_utils/mod.rs +++ b/consensus/types/src/test_utils/mod.rs @@ -1,4 +1,4 @@ -#![allow(clippy::integer_arithmetic)] +#![allow(clippy::arithmetic_side_effects)] use std::fmt::Debug; diff --git a/consensus/types/src/test_utils/test_random.rs b/consensus/types/src/test_utils/test_random.rs index 43396dedc0d..51b79d8d531 100644 --- a/consensus/types/src/test_utils/test_random.rs +++ b/consensus/types/src/test_utils/test_random.rs @@ -28,7 +28,7 @@ pub trait TestRandom { impl TestRandom for PhantomData { fn random_for_test(_rng: &mut impl RngCore) -> Self { - PhantomData::default() + PhantomData } } diff --git a/slasher/tests/backend.rs b/slasher/tests/backend.rs index 9e68107de74..fd1a6ae14f6 100644 --- a/slasher/tests/backend.rs +++ b/slasher/tests/backend.rs @@ -1,4 +1,4 @@ -#![cfg(all(feature = "lmdb"))] +#![cfg(feature = "lmdb")] use slasher::{config::MDBX_DATA_FILENAME, Config, DatabaseBackend, DatabaseBackendOverride}; use std::fs::File; From a25ec16a6751574c009fa30dbeb62a6ca1ca030d Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Mon, 17 Jul 2023 00:14:20 +0000 Subject: [PATCH 38/51] Speed up CI by installing foundry with Github action (#4505) ## Issue Addressed Speed up CI by installing foundry with Github action instead of building Anvil from source. Building anvil from source on GItHub hosted runners currently takes about 10 mins. Using the `foundry-toolchain` action to install only takes about 2 seconds. --- .github/workflows/test-suite.yml | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index e6b75ea7b17..e3342ac3705 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -58,8 +58,8 @@ jobs: uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install anvil - run: cargo install --git https://github.com/foundry-rs/foundry --locked anvil + - name: Install Foundry (anvil) + uses: foundry-rs/foundry-toolchain@v1 - name: Run tests in release run: make test-release release-tests-windows: @@ -78,9 +78,8 @@ jobs: run: | choco install python protoc visualstudio2019-workload-vctools -y npm config set msvs_version 2019 - - name: Install anvil - # Extra feature to work around https://github.com/foundry-rs/foundry/issues/5115 - run: cargo install --git https://github.com/foundry-rs/foundry --locked anvil --features ethers/ipc + - name: Install Foundry (anvil) + uses: foundry-rs/foundry-toolchain@v1 - name: Install make run: choco install -y make - uses: KyleMayes/install-llvm-action@v1 @@ -141,8 +140,8 @@ jobs: uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install anvil - run: cargo install --git https://github.com/foundry-rs/foundry --locked anvil + - name: Install Foundry (anvil) + uses: foundry-rs/foundry-toolchain@v1 - name: Run tests in debug run: make test-debug state-transition-vectors-ubuntu: @@ -197,8 +196,8 @@ jobs: uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install anvil - run: cargo install --git https://github.com/foundry-rs/foundry --locked anvil + - name: Install Foundry (anvil) + uses: foundry-rs/foundry-toolchain@v1 - name: Run the beacon chain sim that starts from an eth1 contract run: cargo run --release --bin simulator eth1-sim merge-transition-ubuntu: @@ -213,8 +212,8 @@ jobs: uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install anvil - run: cargo install --git https://github.com/foundry-rs/foundry --locked anvil + - name: Install Foundry (anvil) + uses: foundry-rs/foundry-toolchain@v1 - name: Run the beacon chain sim and go through the merge transition run: cargo run --release --bin simulator eth1-sim --post-merge no-eth1-simulator-ubuntu: @@ -243,8 +242,8 @@ jobs: uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install anvil - run: cargo install --git https://github.com/foundry-rs/foundry --locked anvil + - name: Install Foundry (anvil) + uses: foundry-rs/foundry-toolchain@v1 - name: Run the syncing simulator run: cargo run --release --bin simulator syncing-sim doppelganger-protection-test: From 4435a222219888c41e4bfe79c310367c16b73ef0 Mon Sep 17 00:00:00 2001 From: Divma Date: Mon, 17 Jul 2023 05:31:53 +0000 Subject: [PATCH 39/51] Cleanup unreachable code in `lcli::generate_bootnode_enr` and some tests (#4485) ## Issue Addressed n/a Noticed this while working on something else ## Proposed Changes - leverage the appropriate types to avoid a bunch of `unwrap` and errors ## Additional Info n/a --- .../src/discovery/enr_ext.rs | 19 ++++++++++++------- .../lighthouse_network/src/discovery/mod.rs | 10 ++++++---- .../lighthouse_network/src/types/globals.rs | 5 ++--- lcli/src/generate_bootnode_enr.rs | 14 ++++++-------- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs index 3df7f7c16f3..5ce0c55cacd 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs @@ -1,7 +1,10 @@ //! ENR extension trait to support libp2p integration. use crate::{Enr, Multiaddr, PeerId}; use discv5::enr::{CombinedKey, CombinedPublicKey}; -use libp2p::core::{identity::Keypair, identity::PublicKey, multiaddr::Protocol}; +use libp2p::{ + core::{identity::Keypair, identity::PublicKey, multiaddr::Protocol}, + identity::secp256k1, +}; use tiny_keccak::{Hasher, Keccak}; /// Extend ENR for libp2p types. @@ -36,6 +39,8 @@ pub trait CombinedKeyPublicExt { pub trait CombinedKeyExt { /// Converts a libp2p key into an ENR combined key. fn from_libp2p(key: &libp2p::core::identity::Keypair) -> Result; + /// Converts a [`secp256k1::Keypair`] into and Enr [`CombinedKey`]. + fn from_secp256k1(key: &secp256k1::Keypair) -> CombinedKey; } impl EnrExt for Enr { @@ -220,12 +225,7 @@ impl CombinedKeyPublicExt for CombinedPublicKey { impl CombinedKeyExt for CombinedKey { fn from_libp2p(key: &libp2p::core::identity::Keypair) -> Result { match key { - Keypair::Secp256k1(key) => { - let secret = - discv5::enr::k256::ecdsa::SigningKey::from_slice(&key.secret().to_bytes()) - .expect("libp2p key must be valid"); - Ok(CombinedKey::Secp256k1(secret)) - } + Keypair::Secp256k1(key) => Ok(CombinedKey::from_secp256k1(key)), Keypair::Ed25519(key) => { let ed_keypair = discv5::enr::ed25519_dalek::SigningKey::from_bytes( &(key.encode()[..32]) @@ -237,6 +237,11 @@ impl CombinedKeyExt for CombinedKey { Keypair::Ecdsa(_) => Err("Ecdsa keypairs not supported"), } } + fn from_secp256k1(key: &secp256k1::Keypair) -> Self { + let secret = discv5::enr::k256::ecdsa::SigningKey::from_slice(&key.secret().to_bytes()) + .expect("libp2p key must be valid"); + CombinedKey::Secp256k1(secret) + } } // helper function to convert a peer_id to a node_id. This is only possible for secp256k1/ed25519 libp2p diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index 3ee74ebf018..d4d0baef6b7 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -1101,6 +1101,7 @@ mod tests { use super::*; use crate::rpc::methods::{MetaData, MetaDataV2}; use enr::EnrBuilder; + use libp2p::identity::secp256k1; use slog::{o, Drain}; use types::{BitVector, MinimalEthSpec, SubnetId}; @@ -1119,10 +1120,10 @@ mod tests { } async fn build_discovery() -> Discovery { - let keypair = libp2p::identity::Keypair::generate_secp256k1(); + let keypair = secp256k1::Keypair::generate(); let mut config = NetworkConfig::default(); config.set_listening_addr(crate::ListenAddress::unused_v4_ports()); - let enr_key: CombinedKey = CombinedKey::from_libp2p(&keypair).unwrap(); + let enr_key: CombinedKey = CombinedKey::from_secp256k1(&keypair); let enr: Enr = build_enr::(&enr_key, &config, &EnrForkId::default()).unwrap(); let log = build_log(slog::Level::Debug, false); let globals = NetworkGlobals::new( @@ -1138,6 +1139,7 @@ mod tests { false, &log, ); + let keypair = Keypair::Secp256k1(keypair); Discovery::new(&keypair, &config, Arc::new(globals), &log) .await .unwrap() @@ -1184,8 +1186,8 @@ mod tests { fn make_enr(subnet_ids: Vec) -> Enr { let mut builder = EnrBuilder::new("v4"); - let keypair = libp2p::identity::Keypair::generate_secp256k1(); - let enr_key: CombinedKey = CombinedKey::from_libp2p(&keypair).unwrap(); + let keypair = secp256k1::Keypair::generate(); + let enr_key: CombinedKey = CombinedKey::from_secp256k1(&keypair); // set the "attnets" field on our ENR let mut bitfield = BitVector::::new(); diff --git a/beacon_node/lighthouse_network/src/types/globals.rs b/beacon_node/lighthouse_network/src/types/globals.rs index 295616f36ba..97eaaa0051b 100644 --- a/beacon_node/lighthouse_network/src/types/globals.rs +++ b/beacon_node/lighthouse_network/src/types/globals.rs @@ -134,9 +134,8 @@ impl NetworkGlobals { log: &slog::Logger, ) -> NetworkGlobals { use crate::CombinedKeyExt; - let keypair = libp2p::identity::Keypair::generate_secp256k1(); - let enr_key: discv5::enr::CombinedKey = - discv5::enr::CombinedKey::from_libp2p(&keypair).unwrap(); + let keypair = libp2p::identity::secp256k1::Keypair::generate(); + let enr_key: discv5::enr::CombinedKey = discv5::enr::CombinedKey::from_secp256k1(&keypair); let enr = discv5::enr::EnrBuilder::new("v4").build(&enr_key).unwrap(); NetworkGlobals::new( enr, diff --git a/lcli/src/generate_bootnode_enr.rs b/lcli/src/generate_bootnode_enr.rs index 8662a804761..0584cd65496 100644 --- a/lcli/src/generate_bootnode_enr.rs +++ b/lcli/src/generate_bootnode_enr.rs @@ -1,6 +1,7 @@ use clap::ArgMatches; use lighthouse_network::{ - discovery::{build_enr, CombinedKey, CombinedKeyExt, Keypair, ENR_FILENAME}, + discovery::{build_enr, CombinedKey, CombinedKeyExt, ENR_FILENAME}, + libp2p::identity::secp256k1, NetworkConfig, NETWORK_KEY_FILENAME, }; use std::fs::File; @@ -29,8 +30,8 @@ pub fn run(matches: &ArgMatches) -> Result<(), String> { config.enr_udp4_port = Some(udp_port); config.enr_tcp6_port = Some(tcp_port); - let local_keypair = Keypair::generate_secp256k1(); - let enr_key = CombinedKey::from_libp2p(&local_keypair)?; + let secp256k1_keypair = secp256k1::Keypair::generate(); + let enr_key = CombinedKey::from_secp256k1(&secp256k1_keypair); let enr_fork_id = EnrForkId { fork_digest: ChainSpec::compute_fork_digest(genesis_fork_version, Hash256::zero()), next_fork_version: genesis_fork_version, @@ -47,13 +48,10 @@ pub fn run(matches: &ArgMatches) -> Result<(), String> { .write_all(enr.to_base64().as_bytes()) .map_err(|e| format!("Unable to write ENR to {}: {:?}", ENR_FILENAME, e))?; - let secret_bytes = match local_keypair { - Keypair::Secp256k1(key) => key.secret().to_bytes(), - _ => return Err("Key is not a secp256k1 key".into()), - }; - let mut key_file = File::create(output_dir.join(NETWORK_KEY_FILENAME)) .map_err(|e| format!("Unable to create {}: {:?}", NETWORK_KEY_FILENAME, e))?; + + let secret_bytes = secp256k1_keypair.secret().to_bytes(); key_file .write_all(&secret_bytes) .map_err(|e| format!("Unable to write key to {}: {:?}", NETWORK_KEY_FILENAME, e))?; From 0f514cbb368089b6a2064c4de3e825a545f3c860 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 17 Jul 2023 09:50:32 -0400 Subject: [PATCH 40/51] fixes after merge --- beacon_node/execution_layer/src/lib.rs | 65 +++++++------------ .../src/test_utils/mock_builder.rs | 4 +- .../network/src/sync/range_sync/range.rs | 2 +- consensus/types/src/blob_sidecar.rs | 2 +- consensus/types/src/execution_payload.rs | 2 +- 5 files changed, 30 insertions(+), 45 deletions(-) diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index e2ef23f7d1b..2a1f3191738 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -237,7 +237,6 @@ impl> BlockProposalContents BlockProposalContents::PayloadAndBlobs { @@ -939,7 +938,6 @@ impl ExecutionLayer { BlockProposalContents::Payload { payload: relay.data.message.header, block_value: relay.data.message.value, - _phantom: PhantomData, }, )), Err(reason) if !reason.payload_invalid() => { @@ -994,7 +992,6 @@ impl ExecutionLayer { BlockProposalContents::Payload { payload: relay.data.message.header, block_value: relay.data.message.value, - _phantom: PhantomData, }, )), // If the payload is valid then use it. The local EE failed @@ -1003,7 +1000,6 @@ impl ExecutionLayer { BlockProposalContents::Payload { payload: relay.data.message.header, block_value: relay.data.message.value, - _phantom: PhantomData, }, )), Err(reason) => { @@ -1182,36 +1178,30 @@ impl ExecutionLayer { "parent_hash" => ?parent_hash, ); engine.api.get_payload::(current_fork, payload_id).await - }; - let payload_response = payload_fut.await; - let (execution_payload, block_value) = payload_response.map(|payload_response| { - if payload_response.execution_payload_ref().fee_recipient() != payload_attributes.suggested_fee_recipient() { - error!( - self.log(), - "Inconsistent fee recipient"; - "msg" => "The fee recipient returned from the Execution Engine differs \ - from the suggested_fee_recipient set on the beacon node. This could \ - indicate that fees are being diverted to another address. Please \ - ensure that the value of suggested_fee_recipient is set correctly and \ - that the Execution Engine is trusted.", - "fee_recipient" => ?payload_response.execution_payload_ref().fee_recipient(), - "suggested_fee_recipient" => ?payload_attributes.suggested_fee_recipient(), - ); - } - if f(self, payload_response.execution_payload_ref()).is_some() { - warn!( - self.log(), - "Duplicate payload cached, this might indicate redundant proposal \ - attempts." - ); - } - payload_response.into() - })?; - Ok(BlockProposalContents::Payload { - payload: execution_payload.into(), - block_value, - _phantom: PhantomData, - }) + }.await?; + + if payload_response.execution_payload_ref().fee_recipient() != payload_attributes.suggested_fee_recipient() { + error!( + self.log(), + "Inconsistent fee recipient"; + "msg" => "The fee recipient returned from the Execution Engine differs \ + from the suggested_fee_recipient set on the beacon node. This could \ + indicate that fees are being diverted to another address. Please \ + ensure that the value of suggested_fee_recipient is set correctly and \ + that the Execution Engine is trusted.", + "fee_recipient" => ?payload_response.execution_payload_ref().fee_recipient(), + "suggested_fee_recipient" => ?payload_attributes.suggested_fee_recipient(), + ); + } + if f(self, payload_response.execution_payload_ref()).is_some() { + warn!( + self.log(), + "Duplicate payload cached, this might indicate redundant proposal \ + attempts." + ); + } + + Ok(payload_response.into()) }) .await .map_err(Box::new) @@ -2136,13 +2126,6 @@ async fn timed_future, T>(metric: &str, future: F) -> (T, (result, duration) } -fn noop( - _: &ExecutionLayer, - _: ExecutionPayloadRef, -) -> Option> { - None -} - #[cfg(test)] /// Returns the duration since the unix epoch. fn timestamp_now() -> u64 { diff --git a/beacon_node/execution_layer/src/test_utils/mock_builder.rs b/beacon_node/execution_layer/src/test_utils/mock_builder.rs index 278dacb826a..7a28e694297 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_builder.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_builder.rs @@ -450,7 +450,9 @@ impl mev_rs::BlindedBlockProvider for MockBuilder { value: to_ssz_rs(&Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI))?, public_key: self.builder_sk.public_key(), }), - ForkName::Base | ForkName::Altair | ForkName::Deneb => return Err(MevError::InvalidFork), + ForkName::Base | ForkName::Altair | ForkName::Deneb => { + return Err(MevError::InvalidFork) + } }; *message.gas_limit_mut() = cached_data.gas_limit; diff --git a/beacon_node/network/src/sync/range_sync/range.rs b/beacon_node/network/src/sync/range_sync/range.rs index 733cbcc9eae..28cdc7afcf7 100644 --- a/beacon_node/network/src/sync/range_sync/range.rs +++ b/beacon_node/network/src/sync/range_sync/range.rs @@ -603,7 +603,7 @@ mod tests { fn range(log_enabled: bool) -> (TestRig, RangeSync) { let log = build_log(slog::Level::Trace, log_enabled); // Initialise a new beacon chain - let harness = BeaconChainHarness::>::builder(E::default()) + let harness = BeaconChainHarness::>::builder(E) .default_spec() .logger(log.clone()) .deterministic_keypairs(1) diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index be68a79e8b5..214e7676403 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -126,7 +126,7 @@ impl BlobSidecar { }) } - #[allow(clippy::integer_arithmetic)] + #[allow(clippy::arithmetic_side_effects)] pub fn max_size() -> usize { // Fixed part Self::empty().as_ssz_bytes().len() diff --git a/consensus/types/src/execution_payload.rs b/consensus/types/src/execution_payload.rs index 157f6c7a84f..4186cdf198e 100644 --- a/consensus/types/src/execution_payload.rs +++ b/consensus/types/src/execution_payload.rs @@ -139,7 +139,7 @@ impl ExecutionPayload { + (T::max_withdrawals_per_payload() * ::ssz_fixed_len()) } - #[allow(clippy::integer_arithmetic)] + #[allow(clippy::arithmetic_side_effects)] /// Returns the maximum size of an execution payload. pub fn max_execution_payload_deneb_size() -> usize { // Fixed part From a618830f8f92e976aa9042b27be5812a961c84e9 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 17 Jul 2023 10:28:37 -0400 Subject: [PATCH 41/51] fix smol lint --- beacon_node/network/src/sync/block_lookups/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/network/src/sync/block_lookups/tests.rs b/beacon_node/network/src/sync/block_lookups/tests.rs index c8f19b16981..011747c3905 100644 --- a/beacon_node/network/src/sync/block_lookups/tests.rs +++ b/beacon_node/network/src/sync/block_lookups/tests.rs @@ -101,7 +101,7 @@ impl TestRig { let mut blob_sidecars = vec![]; if let Ok(message) = block.message_deneb_mut() { // get random number between 0 and Max Blobs - let mut payload: &mut FullPayloadDeneb = &mut message.body.execution_payload; + let payload: &mut FullPayloadDeneb = &mut message.body.execution_payload; let num_blobs = match num_blobs { NumBlobs::Random => { let mut num_blobs = rand::random::() % E::max_blobs_per_block(); From aeee5beac255cbabbc18a59e1a10094c42fbe7a0 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 17 Jul 2023 10:46:54 -0400 Subject: [PATCH 42/51] smol fixes --- beacon_node/network/src/network_beacon_processor/mod.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index ef4b3daae7a..8e44aba15ab 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -229,8 +229,6 @@ impl NetworkBeaconProcessor { }) } - pub fn send_banana() {} - /// Create a new `Work` event for some sync committee signature. pub fn send_gossip_sync_signature( self: &Arc, @@ -427,7 +425,7 @@ impl NetworkBeaconProcessor { }) } - /// Create a new `Work` event for some block, where the result from computation (if any) is + /// Create a new `Work` event for some blobs, where the result from computation (if any) is /// sent to the other side of `result_tx`. pub fn send_rpc_blobs( self: &Arc, From e1d0724abf1789849fe22ecb0a6f641f2b3487f7 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Mon, 17 Jul 2023 13:29:12 -0700 Subject: [PATCH 43/51] Fix more beta compiler warnings --- beacon_node/execution_layer/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 2a1f3191738..3c754a76c5d 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -48,8 +48,8 @@ use types::{ use types::{ BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionPayloadCapella, ExecutionPayloadMerge, }; -use types::{KzgProofs, Withdrawals}; -use types::{ProposerPreparationData, PublicKeyBytes, Signature, Slot, Transaction, Uint256}; +use types::{KzgProofs}; +use types::{ProposerPreparationData, PublicKeyBytes, Signature, Slot, Transaction}; mod block_hash; mod engine_api; From cffa56238491ea49808189ea6d9c1309526c4a81 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 17 Jul 2023 16:32:09 -0400 Subject: [PATCH 44/51] cargo fmt --- beacon_node/execution_layer/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 3c754a76c5d..1f7023270ff 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -42,13 +42,13 @@ use tokio_stream::wrappers::WatchStream; use tree_hash::TreeHash; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::Blobs; +use types::KzgProofs; use types::{ AbstractExecPayload, BeaconStateError, ExecPayload, ExecutionPayloadDeneb, VersionedHash, }; use types::{ BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionPayloadCapella, ExecutionPayloadMerge, }; -use types::{KzgProofs}; use types::{ProposerPreparationData, PublicKeyBytes, Signature, Slot, Transaction}; mod block_hash; From fc7f1ba6b94ac3300338aad4d5168aaac5df3fac Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Tue, 18 Jul 2023 01:48:40 +0000 Subject: [PATCH 45/51] Phase 0 attestation rewards via Beacon API (#4474) ## Issue Addressed Addresses #4026. Beacon-API spec [here](https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getAttestationsRewards). Endpoint: `POST /eth/v1/beacon/rewards/attestations/{epoch}` This endpoint already supports post-Altair epochs. This PR adds support for phase 0 rewards calculation. ## Proposed Changes - [x] Attestation rewards API to support phase 0 rewards calculation, re-using logic from `state_processing`. Refactored `get_attestation_deltas` slightly to support computing deltas for a subset of validators. - [x] Add `inclusion_delay` to `ideal_rewards` (`beacon-API` spec update to follow) - [x] Add `inactivity` penalties to both `ideal_rewards` and `total_rewards` (`beacon-API` spec update to follow) - [x] Add tests to compute attestation rewards and compare results with beacon states ## Additional Notes - The extra penalty for missing attestations or being slashed during an inactivity leak is currently not included in the API response (for both phase 0 and Altair) in the spec. - I went with adding `inactivity` as a separate component rather than combining them with the 4 rewards, because this is how it was grouped in [the phase 0 spec](https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#get_attestation_deltas). During inactivity leak, all rewards include the optimal reward, and inactivity penalties are calculated separately (see below code snippet from the spec), so it would be quite confusing if we merge them. This would also work better with Altair, because there's no "cancelling" of rewards and inactivity penalties are more separate. - Altair calculation logic (to include inactivity penalties) to be updated in a follow-up PR. ```python def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ Return attestation reward/penalty deltas for each validator. """ source_rewards, source_penalties = get_source_deltas(state) target_rewards, target_penalties = get_target_deltas(state) head_rewards, head_penalties = get_head_deltas(state) inclusion_delay_rewards, _ = get_inclusion_delay_deltas(state) _, inactivity_penalties = get_inactivity_penalty_deltas(state) rewards = [ source_rewards[i] + target_rewards[i] + head_rewards[i] + inclusion_delay_rewards[i] for i in range(len(state.validators)) ] penalties = [ source_penalties[i] + target_penalties[i] + head_penalties[i] + inactivity_penalties[i] for i in range(len(state.validators)) ] return rewards, penalties ``` ## Example API Response
Click me ```json { "ideal_rewards": [ { "effective_balance": "1000000000", "head": "6638", "target": "6638", "source": "6638", "inclusion_delay": "9783", "inactivity": "0" }, { "effective_balance": "2000000000", "head": "13276", "target": "13276", "source": "13276", "inclusion_delay": "19565", "inactivity": "0" }, { "effective_balance": "3000000000", "head": "19914", "target": "19914", "source": "19914", "inclusion_delay": "29349", "inactivity": "0" }, { "effective_balance": "4000000000", "head": "26553", "target": "26553", "source": "26553", "inclusion_delay": "39131", "inactivity": "0" }, { "effective_balance": "5000000000", "head": "33191", "target": "33191", "source": "33191", "inclusion_delay": "48914", "inactivity": "0" }, { "effective_balance": "6000000000", "head": "39829", "target": "39829", "source": "39829", "inclusion_delay": "58697", "inactivity": "0" }, { "effective_balance": "7000000000", "head": "46468", "target": "46468", "source": "46468", "inclusion_delay": "68480", "inactivity": "0" }, { "effective_balance": "8000000000", "head": "53106", "target": "53106", "source": "53106", "inclusion_delay": "78262", "inactivity": "0" }, { "effective_balance": "9000000000", "head": "59744", "target": "59744", "source": "59744", "inclusion_delay": "88046", "inactivity": "0" }, { "effective_balance": "10000000000", "head": "66383", "target": "66383", "source": "66383", "inclusion_delay": "97828", "inactivity": "0" }, { "effective_balance": "11000000000", "head": "73021", "target": "73021", "source": "73021", "inclusion_delay": "107611", "inactivity": "0" }, { "effective_balance": "12000000000", "head": "79659", "target": "79659", "source": "79659", "inclusion_delay": "117394", "inactivity": "0" }, { "effective_balance": "13000000000", "head": "86298", "target": "86298", "source": "86298", "inclusion_delay": "127176", "inactivity": "0" }, { "effective_balance": "14000000000", "head": "92936", "target": "92936", "source": "92936", "inclusion_delay": "136959", "inactivity": "0" }, { "effective_balance": "15000000000", "head": "99574", "target": "99574", "source": "99574", "inclusion_delay": "146742", "inactivity": "0" }, { "effective_balance": "16000000000", "head": "106212", "target": "106212", "source": "106212", "inclusion_delay": "156525", "inactivity": "0" }, { "effective_balance": "17000000000", "head": "112851", "target": "112851", "source": "112851", "inclusion_delay": "166307", "inactivity": "0" }, { "effective_balance": "18000000000", "head": "119489", "target": "119489", "source": "119489", "inclusion_delay": "176091", "inactivity": "0" }, { "effective_balance": "19000000000", "head": "126127", "target": "126127", "source": "126127", "inclusion_delay": "185873", "inactivity": "0" }, { "effective_balance": "20000000000", "head": "132766", "target": "132766", "source": "132766", "inclusion_delay": "195656", "inactivity": "0" }, { "effective_balance": "21000000000", "head": "139404", "target": "139404", "source": "139404", "inclusion_delay": "205439", "inactivity": "0" }, { "effective_balance": "22000000000", "head": "146042", "target": "146042", "source": "146042", "inclusion_delay": "215222", "inactivity": "0" }, { "effective_balance": "23000000000", "head": "152681", "target": "152681", "source": "152681", "inclusion_delay": "225004", "inactivity": "0" }, { "effective_balance": "24000000000", "head": "159319", "target": "159319", "source": "159319", "inclusion_delay": "234787", "inactivity": "0" }, { "effective_balance": "25000000000", "head": "165957", "target": "165957", "source": "165957", "inclusion_delay": "244570", "inactivity": "0" }, { "effective_balance": "26000000000", "head": "172596", "target": "172596", "source": "172596", "inclusion_delay": "254352", "inactivity": "0" }, { "effective_balance": "27000000000", "head": "179234", "target": "179234", "source": "179234", "inclusion_delay": "264136", "inactivity": "0" }, { "effective_balance": "28000000000", "head": "185872", "target": "185872", "source": "185872", "inclusion_delay": "273918", "inactivity": "0" }, { "effective_balance": "29000000000", "head": "192510", "target": "192510", "source": "192510", "inclusion_delay": "283701", "inactivity": "0" }, { "effective_balance": "30000000000", "head": "199149", "target": "199149", "source": "199149", "inclusion_delay": "293484", "inactivity": "0" }, { "effective_balance": "31000000000", "head": "205787", "target": "205787", "source": "205787", "inclusion_delay": "303267", "inactivity": "0" }, { "effective_balance": "32000000000", "head": "212426", "target": "212426", "source": "212426", "inclusion_delay": "313050", "inactivity": "0" } ], "total_rewards": [ { "validator_index": "0", "head": "212426", "target": "212426", "source": "212426", "inclusion_delay": "313050", "inactivity": "0" }, { "validator_index": "32", "head": "212426", "target": "212426", "source": "212426", "inclusion_delay": "313050", "inactivity": "0" }, { "validator_index": "63", "head": "-357771", "target": "-357771", "source": "-357771", "inclusion_delay": "0", "inactivity": "0" } ] } ```
--- Cargo.lock | 1 + beacon_node/beacon_chain/Cargo.toml | 1 + .../beacon_chain/src/attestation_rewards.rs | 238 ++++++++++++++++-- beacon_node/beacon_chain/src/errors.rs | 4 +- beacon_node/beacon_chain/tests/rewards.rs | 192 +++++++++++++- beacon_node/http_api/src/lib.rs | 8 +- .../src/lighthouse/attestation_rewards.rs | 19 +- consensus/state_processing/src/common/base.rs | 12 + .../src/per_epoch_processing/base.rs | 2 +- .../base/rewards_and_penalties.rs | 103 +++++--- .../ef_tests/src/cases/epoch_processing.rs | 2 +- testing/ef_tests/src/cases/rewards.rs | 2 +- 12 files changed, 514 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4bb3ab99e28..fda8cd761f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -630,6 +630,7 @@ dependencies = [ "eth1", "eth2", "ethereum_hashing", + "ethereum_serde_utils", "ethereum_ssz", "ethereum_ssz_derive", "execution_layer", diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index 7f884f561b3..b537327fb31 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -27,6 +27,7 @@ operation_pool = { path = "../operation_pool" } rayon = "1.4.1" serde = "1.0.116" serde_derive = "1.0.116" +ethereum_serde_utils = "0.5.0" slog = { version = "2.5.2", features = ["max_level_trace"] } sloggers = { version = "2.1.1", features = ["json"] } slot_clock = { path = "../../common/slot_clock" } diff --git a/beacon_node/beacon_chain/src/attestation_rewards.rs b/beacon_node/beacon_chain/src/attestation_rewards.rs index 460bf18bcfd..9fc21b668fa 100644 --- a/beacon_node/beacon_chain/src/attestation_rewards.rs +++ b/beacon_node/beacon_chain/src/attestation_rewards.rs @@ -3,7 +3,8 @@ use eth2::lighthouse::attestation_rewards::{IdealAttestationRewards, TotalAttest use eth2::lighthouse::StandardAttestationRewards; use participation_cache::ParticipationCache; use safe_arith::SafeArith; -use slog::{debug, Logger}; +use serde_utils::quoted_u64::Quoted; +use slog::debug; use state_processing::{ common::altair::BaseRewardPerIncrement, per_epoch_processing::altair::{participation_cache, rewards_and_penalties::get_flag_weight}, @@ -15,32 +16,111 @@ use store::consts::altair::{ }; use types::consts::altair::WEIGHT_DENOMINATOR; -use types::{Epoch, EthSpec}; +use types::{BeaconState, Epoch, EthSpec}; use eth2::types::ValidatorId; +use state_processing::common::base::get_base_reward_from_effective_balance; +use state_processing::per_epoch_processing::base::rewards_and_penalties::{ + get_attestation_component_delta, get_attestation_deltas_all, get_attestation_deltas_subset, + get_inactivity_penalty_delta, get_inclusion_delay_delta, +}; +use state_processing::per_epoch_processing::base::validator_statuses::InclusionInfo; +use state_processing::per_epoch_processing::base::{ + TotalBalances, ValidatorStatus, ValidatorStatuses, +}; impl BeaconChain { pub fn compute_attestation_rewards( &self, epoch: Epoch, validators: Vec, - log: Logger, ) -> Result { - debug!(log, "computing attestation rewards"; "epoch" => epoch, "validator_count" => validators.len()); + debug!(self.log, "computing attestation rewards"; "epoch" => epoch, "validator_count" => validators.len()); // Get state - let spec = &self.spec; - let state_slot = (epoch + 1).end_slot(T::EthSpec::slots_per_epoch()); let state_root = self .state_root_at_slot(state_slot)? .ok_or(BeaconChainError::NoStateForSlot(state_slot))?; - let mut state = self + let state = self .get_state(&state_root, Some(state_slot))? .ok_or(BeaconChainError::MissingBeaconState(state_root))?; + match state { + BeaconState::Base(_) => self.compute_attestation_rewards_base(state, validators), + BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) => { + self.compute_attestation_rewards_altair(state, validators) + } + } + } + + fn compute_attestation_rewards_base( + &self, + mut state: BeaconState, + validators: Vec, + ) -> Result { + let spec = &self.spec; + let mut validator_statuses = ValidatorStatuses::new(&state, spec)?; + validator_statuses.process_attestations(&state)?; + + let ideal_rewards = + self.compute_ideal_rewards_base(&state, &validator_statuses.total_balances)?; + + let indices_to_attestation_delta = if validators.is_empty() { + get_attestation_deltas_all(&state, &validator_statuses, spec)? + .into_iter() + .enumerate() + .collect() + } else { + let validator_indices = Self::validators_ids_to_indices(&mut state, validators)?; + get_attestation_deltas_subset(&state, &validator_statuses, &validator_indices, spec)? + }; + + let mut total_rewards = vec![]; + + for (index, delta) in indices_to_attestation_delta.into_iter() { + let head_delta = delta.head_delta; + let head = (head_delta.rewards as i64).safe_sub(head_delta.penalties as i64)?; + + let target_delta = delta.target_delta; + let target = (target_delta.rewards as i64).safe_sub(target_delta.penalties as i64)?; + + let source_delta = delta.source_delta; + let source = (source_delta.rewards as i64).safe_sub(source_delta.penalties as i64)?; + + // No penalties associated with inclusion delay + let inclusion_delay = delta.inclusion_delay_delta.rewards; + let inactivity = delta.inactivity_penalty_delta.penalties.wrapping_neg() as i64; + + let rewards = TotalAttestationRewards { + validator_index: index as u64, + head, + target, + source, + inclusion_delay: Some(Quoted { + value: inclusion_delay, + }), + inactivity, + }; + + total_rewards.push(rewards); + } + + Ok(StandardAttestationRewards { + ideal_rewards, + total_rewards, + }) + } + + fn compute_attestation_rewards_altair( + &self, + mut state: BeaconState, + validators: Vec, + ) -> Result { + let spec = &self.spec; + // Calculate ideal_rewards let participation_cache = ParticipationCache::new(&state, spec)?; @@ -71,7 +151,7 @@ impl BeaconChain { let base_reward_per_increment = BaseRewardPerIncrement::new(total_active_balance, spec)?; - for effective_balance_eth in 0..=32 { + for effective_balance_eth in 1..=self.max_effective_balance_increment_steps()? { let effective_balance = effective_balance_eth.safe_mul(spec.effective_balance_increment)?; let base_reward = @@ -101,20 +181,12 @@ impl BeaconChain { let validators = if validators.is_empty() { participation_cache.eligible_validator_indices().to_vec() } else { - validators - .into_iter() - .map(|validator| match validator { - ValidatorId::Index(i) => Ok(i as usize), - ValidatorId::PublicKey(pubkey) => state - .get_validator_index(&pubkey)? - .ok_or(BeaconChainError::ValidatorPubkeyUnknown(pubkey)), - }) - .collect::, _>>()? + Self::validators_ids_to_indices(&mut state, validators)? }; for validator_index in &validators { let eligible = state.is_eligible_validator(previous_epoch, *validator_index)?; - let mut head_reward = 0u64; + let mut head_reward = 0i64; let mut target_reward = 0i64; let mut source_reward = 0i64; @@ -132,7 +204,7 @@ impl BeaconChain { .map_err(|_| BeaconChainError::AttestationRewardsError)?; if voted_correctly { if flag_index == TIMELY_HEAD_FLAG_INDEX { - head_reward += ideal_reward; + head_reward += *ideal_reward as i64; } else if flag_index == TIMELY_TARGET_FLAG_INDEX { target_reward += *ideal_reward as i64; } else if flag_index == TIMELY_SOURCE_FLAG_INDEX { @@ -152,6 +224,9 @@ impl BeaconChain { head: head_reward, target: target_reward, source: source_reward, + inclusion_delay: None, + // TODO: altair calculation logic needs to be updated to include inactivity penalty + inactivity: 0, }); } @@ -173,6 +248,9 @@ impl BeaconChain { head: 0, target: 0, source: 0, + inclusion_delay: None, + // TODO: altair calculation logic needs to be updated to include inactivity penalty + inactivity: 0, }); match *flag_index { TIMELY_SOURCE_FLAG_INDEX => entry.source += ideal_reward, @@ -192,4 +270,126 @@ impl BeaconChain { total_rewards, }) } + + fn max_effective_balance_increment_steps(&self) -> Result { + let spec = &self.spec; + let max_steps = spec + .max_effective_balance + .safe_div(spec.effective_balance_increment)?; + Ok(max_steps) + } + + fn validators_ids_to_indices( + state: &mut BeaconState, + validators: Vec, + ) -> Result, BeaconChainError> { + let indices = validators + .into_iter() + .map(|validator| match validator { + ValidatorId::Index(i) => Ok(i as usize), + ValidatorId::PublicKey(pubkey) => state + .get_validator_index(&pubkey)? + .ok_or(BeaconChainError::ValidatorPubkeyUnknown(pubkey)), + }) + .collect::, _>>()?; + Ok(indices) + } + + fn compute_ideal_rewards_base( + &self, + state: &BeaconState, + total_balances: &TotalBalances, + ) -> Result, BeaconChainError> { + let spec = &self.spec; + let previous_epoch = state.previous_epoch(); + let finality_delay = previous_epoch + .safe_sub(state.finalized_checkpoint().epoch)? + .as_u64(); + + let ideal_validator_status = ValidatorStatus { + is_previous_epoch_attester: true, + is_slashed: false, + inclusion_info: Some(InclusionInfo { + delay: 1, + ..Default::default() + }), + ..Default::default() + }; + + let mut ideal_attestation_rewards_list = Vec::new(); + + for effective_balance_step in 1..=self.max_effective_balance_increment_steps()? { + let effective_balance = + effective_balance_step.safe_mul(spec.effective_balance_increment)?; + let base_reward = get_base_reward_from_effective_balance::( + effective_balance, + total_balances.current_epoch(), + spec, + )?; + + // compute ideal head rewards + let head = get_attestation_component_delta( + true, + total_balances.previous_epoch_attesters(), + total_balances, + base_reward, + finality_delay, + spec, + )? + .rewards; + + // compute ideal target rewards + let target = get_attestation_component_delta( + true, + total_balances.previous_epoch_target_attesters(), + total_balances, + base_reward, + finality_delay, + spec, + )? + .rewards; + + // compute ideal source rewards + let source = get_attestation_component_delta( + true, + total_balances.previous_epoch_head_attesters(), + total_balances, + base_reward, + finality_delay, + spec, + )? + .rewards; + + // compute ideal inclusion delay rewards + let inclusion_delay = + get_inclusion_delay_delta(&ideal_validator_status, base_reward, spec)? + .0 + .rewards; + + // compute inactivity penalty + let inactivity = get_inactivity_penalty_delta( + &ideal_validator_status, + base_reward, + finality_delay, + spec, + )? + .penalties + .wrapping_neg() as i64; + + let ideal_attestation_rewards = IdealAttestationRewards { + effective_balance, + head, + target, + source, + inclusion_delay: Some(Quoted { + value: inclusion_delay, + }), + inactivity, + }; + + ideal_attestation_rewards_list.push(ideal_attestation_rewards); + } + + Ok(ideal_attestation_rewards_list) + } } diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 50bcf426536..87141311505 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -24,7 +24,7 @@ use state_processing::{ }, signature_sets::Error as SignatureSetError, state_advance::Error as StateAdvanceError, - BlockProcessingError, BlockReplayError, SlotProcessingError, + BlockProcessingError, BlockReplayError, EpochProcessingError, SlotProcessingError, }; use std::time::Duration; use task_executor::ShutdownReason; @@ -60,6 +60,7 @@ pub enum BeaconChainError { MissingBeaconBlock(Hash256), MissingBeaconState(Hash256), SlotProcessingError(SlotProcessingError), + EpochProcessingError(EpochProcessingError), StateAdvanceError(StateAdvanceError), UnableToAdvanceState(String), NoStateForAttestation { @@ -217,6 +218,7 @@ pub enum BeaconChainError { } easy_from_to!(SlotProcessingError, BeaconChainError); +easy_from_to!(EpochProcessingError, BeaconChainError); easy_from_to!(AttestationValidationError, BeaconChainError); easy_from_to!(SyncCommitteeMessageValidationError, BeaconChainError); easy_from_to!(ExitValidationError, BeaconChainError); diff --git a/beacon_node/beacon_chain/tests/rewards.rs b/beacon_node/beacon_chain/tests/rewards.rs index b61bea12429..be271804b99 100644 --- a/beacon_node/beacon_chain/tests/rewards.rs +++ b/beacon_node/beacon_chain/tests/rewards.rs @@ -9,19 +9,22 @@ use beacon_chain::{ test_utils::{AttestationStrategy, BlockStrategy, RelativeSyncCommittee}, types::{Epoch, EthSpec, Keypair, MinimalEthSpec}, }; +use eth2::lighthouse::attestation_rewards::TotalAttestationRewards; +use eth2::lighthouse::StandardAttestationRewards; +use eth2::types::ValidatorId; use lazy_static::lazy_static; +use types::beacon_state::Error as BeaconStateError; +use types::{BeaconState, ChainSpec}; pub const VALIDATOR_COUNT: usize = 64; +type E = MinimalEthSpec; + lazy_static! { static ref KEYPAIRS: Vec = generate_deterministic_keypairs(VALIDATOR_COUNT); } -fn get_harness() -> BeaconChainHarness> { - let mut spec = E::default_spec(); - - spec.altair_fork_epoch = Some(Epoch::new(0)); // We use altair for all tests - +fn get_harness(spec: ChainSpec) -> BeaconChainHarness> { let harness = BeaconChainHarness::builder(E::default()) .spec(spec) .keypairs(KEYPAIRS.to_vec()) @@ -35,8 +38,11 @@ fn get_harness() -> BeaconChainHarness> { #[tokio::test] async fn test_sync_committee_rewards() { - let num_block_produced = MinimalEthSpec::slots_per_epoch(); - let harness = get_harness::(); + let mut spec = E::default_spec(); + spec.altair_fork_epoch = Some(Epoch::new(0)); + + let harness = get_harness(spec); + let num_block_produced = E::slots_per_epoch(); let latest_block_root = harness .extend_chain( @@ -119,3 +125,175 @@ async fn test_sync_committee_rewards() { mismatches.join(",") ); } + +#[tokio::test] +async fn test_verify_attestation_rewards_base() { + let harness = get_harness(E::default_spec()); + + // epoch 0 (N), only two thirds of validators vote. + let two_thirds = (VALIDATOR_COUNT / 3) * 2; + let two_thirds_validators: Vec = (0..two_thirds).collect(); + harness + .extend_chain( + E::slots_per_epoch() as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::SomeValidators(two_thirds_validators), + ) + .await; + + let initial_balances: Vec = harness.get_current_state().balances().clone().into(); + + // extend slots to beginning of epoch N + 2 + harness.extend_slots(E::slots_per_epoch() as usize).await; + + // compute reward deltas for all validators in epoch N + let StandardAttestationRewards { + ideal_rewards, + total_rewards, + } = harness + .chain + .compute_attestation_rewards(Epoch::new(0), vec![]) + .unwrap(); + + // assert no inactivity penalty for both ideal rewards and individual validators + assert!(ideal_rewards.iter().all(|reward| reward.inactivity == 0)); + assert!(total_rewards.iter().all(|reward| reward.inactivity == 0)); + + // apply attestation rewards to initial balances + let expected_balances = apply_attestation_rewards(&initial_balances, total_rewards); + + // verify expected balances against actual balances + let balances: Vec = harness.get_current_state().balances().clone().into(); + assert_eq!(expected_balances, balances); +} + +#[tokio::test] +async fn test_verify_attestation_rewards_base_inactivity_leak() { + let spec = E::default_spec(); + let harness = get_harness(spec.clone()); + + let half = VALIDATOR_COUNT / 2; + let half_validators: Vec = (0..half).collect(); + // target epoch is the epoch where the chain enters inactivity leak + let target_epoch = &spec.min_epochs_to_inactivity_penalty + 1; + + // advance until beginning of epoch N + 1 and get balances + harness + .extend_chain( + (E::slots_per_epoch() * (target_epoch + 1)) as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::SomeValidators(half_validators.clone()), + ) + .await; + let initial_balances: Vec = harness.get_current_state().balances().clone().into(); + + // extend slots to beginning of epoch N + 2 + harness.advance_slot(); + harness + .extend_chain( + E::slots_per_epoch() as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::SomeValidators(half_validators), + ) + .await; + let _slot = harness.get_current_slot(); + + // compute reward deltas for all validators in epoch N + let StandardAttestationRewards { + ideal_rewards, + total_rewards, + } = harness + .chain + .compute_attestation_rewards(Epoch::new(target_epoch), vec![]) + .unwrap(); + + // assert inactivity penalty for both ideal rewards and individual validators + assert!(ideal_rewards.iter().all(|reward| reward.inactivity < 0)); + assert!(total_rewards.iter().all(|reward| reward.inactivity < 0)); + + // apply attestation rewards to initial balances + let expected_balances = apply_attestation_rewards(&initial_balances, total_rewards); + + // verify expected balances against actual balances + let balances: Vec = harness.get_current_state().balances().clone().into(); + assert_eq!(expected_balances, balances); +} + +#[tokio::test] +async fn test_verify_attestation_rewards_base_subset_only() { + let harness = get_harness(E::default_spec()); + + // epoch 0 (N), only two thirds of validators vote. + let two_thirds = (VALIDATOR_COUNT / 3) * 2; + let two_thirds_validators: Vec = (0..two_thirds).collect(); + harness + .extend_chain( + E::slots_per_epoch() as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::SomeValidators(two_thirds_validators), + ) + .await; + + // a small subset of validators to compute attestation rewards for + let validators_subset = [0, VALIDATOR_COUNT / 2, VALIDATOR_COUNT - 1]; + + // capture balances before transitioning to N + 2 + let initial_balances = get_validator_balances(harness.get_current_state(), &validators_subset); + + // extend slots to beginning of epoch N + 2 + harness.extend_slots(E::slots_per_epoch() as usize).await; + + let validators_subset_ids: Vec = validators_subset + .into_iter() + .map(|idx| ValidatorId::Index(idx as u64)) + .collect(); + + // compute reward deltas for the subset of validators in epoch N + let StandardAttestationRewards { + ideal_rewards: _, + total_rewards, + } = harness + .chain + .compute_attestation_rewards(Epoch::new(0), validators_subset_ids) + .unwrap(); + + // apply attestation rewards to initial balances + let expected_balances = apply_attestation_rewards(&initial_balances, total_rewards); + + // verify expected balances against actual balances + let balances = get_validator_balances(harness.get_current_state(), &validators_subset); + assert_eq!(expected_balances, balances); +} + +/// Apply a vec of `TotalAttestationRewards` to initial balances, and return +fn apply_attestation_rewards( + initial_balances: &[u64], + attestation_rewards: Vec, +) -> Vec { + initial_balances + .iter() + .zip(attestation_rewards) + .map(|(&initial_balance, rewards)| { + let expected_balance = initial_balance as i64 + + rewards.head + + rewards.source + + rewards.target + + rewards.inclusion_delay.map(|q| q.value).unwrap_or(0) as i64 + + rewards.inactivity; + expected_balance as u64 + }) + .collect::>() +} + +fn get_validator_balances(state: BeaconState, validators: &[usize]) -> Vec { + validators + .iter() + .flat_map(|&id| { + state + .balances() + .get(id) + .cloned() + .ok_or(BeaconStateError::BalancesOutOfBounds(id)) + }) + .collect() +} diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 27bcc4d8a13..1f93f461108 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -2034,15 +2034,11 @@ pub fn serve( .and(warp::path::param::()) .and(warp::path::end()) .and(warp::body::json()) - .and(log_filter.clone()) .and_then( - |chain: Arc>, - epoch: Epoch, - validators: Vec, - log: Logger| { + |chain: Arc>, epoch: Epoch, validators: Vec| { blocking_json_task(move || { let attestation_rewards = chain - .compute_attestation_rewards(epoch, validators, log) + .compute_attestation_rewards(epoch, validators) .map_err(|e| match e { BeaconChainError::MissingBeaconState(root) => { warp_utils::reject::custom_not_found(format!( diff --git a/common/eth2/src/lighthouse/attestation_rewards.rs b/common/eth2/src/lighthouse/attestation_rewards.rs index bebd1c661b3..fa3f93d06f8 100644 --- a/common/eth2/src/lighthouse/attestation_rewards.rs +++ b/common/eth2/src/lighthouse/attestation_rewards.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Serialize}; +use serde_utils::quoted_u64::Quoted; // Details about the rewards paid for attestations // All rewards in GWei @@ -17,6 +18,12 @@ pub struct IdealAttestationRewards { // Ideal attester's reward for source vote in gwei #[serde(with = "serde_utils::quoted_u64")] pub source: u64, + // Ideal attester's inclusion_delay reward in gwei (phase0 only) + #[serde(skip_serializing_if = "Option::is_none")] + pub inclusion_delay: Option>, + // Ideal attester's inactivity penalty in gwei + #[serde(with = "serde_utils::quoted_i64")] + pub inactivity: i64, } #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] @@ -25,16 +32,20 @@ pub struct TotalAttestationRewards { #[serde(with = "serde_utils::quoted_u64")] pub validator_index: u64, // attester's reward for head vote in gwei - #[serde(with = "serde_utils::quoted_u64")] - pub head: u64, + #[serde(with = "serde_utils::quoted_i64")] + pub head: i64, // attester's reward for target vote in gwei #[serde(with = "serde_utils::quoted_i64")] pub target: i64, // attester's reward for source vote in gwei #[serde(with = "serde_utils::quoted_i64")] pub source: i64, - // TBD attester's inclusion_delay reward in gwei (phase0 only) - // pub inclusion_delay: u64, + // attester's inclusion_delay reward in gwei (phase0 only) + #[serde(skip_serializing_if = "Option::is_none")] + pub inclusion_delay: Option>, + // attester's inactivity penalty in gwei + #[serde(with = "serde_utils::quoted_i64")] + pub inactivity: i64, } #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] diff --git a/consensus/state_processing/src/common/base.rs b/consensus/state_processing/src/common/base.rs index b5cb382721f..a8d04ad6cd4 100644 --- a/consensus/state_processing/src/common/base.rs +++ b/consensus/state_processing/src/common/base.rs @@ -17,3 +17,15 @@ pub fn get_base_reward( .safe_div(spec.base_rewards_per_epoch) .map_err(Into::into) } + +pub fn get_base_reward_from_effective_balance( + effective_balance: u64, + total_active_balance: u64, + spec: &ChainSpec, +) -> Result { + effective_balance + .safe_mul(spec.base_reward_factor)? + .safe_div(total_active_balance.integer_sqrt())? + .safe_div(spec.base_rewards_per_epoch) + .map_err(Into::into) +} diff --git a/consensus/state_processing/src/per_epoch_processing/base.rs b/consensus/state_processing/src/per_epoch_processing/base.rs index 680563ce74c..c5864bd1ef2 100644 --- a/consensus/state_processing/src/per_epoch_processing/base.rs +++ b/consensus/state_processing/src/per_epoch_processing/base.rs @@ -36,7 +36,7 @@ pub fn process_epoch( justification_and_finalization_state.apply_changes_to_state(state); // Rewards and Penalties. - process_rewards_and_penalties(state, &mut validator_statuses, spec)?; + process_rewards_and_penalties(state, &validator_statuses, spec)?; // Registry Updates. process_registry_updates(state, spec)?; diff --git a/consensus/state_processing/src/per_epoch_processing/base/rewards_and_penalties.rs b/consensus/state_processing/src/per_epoch_processing/base/rewards_and_penalties.rs index e7a4d9c4dcd..74c96d8aee5 100644 --- a/consensus/state_processing/src/per_epoch_processing/base/rewards_and_penalties.rs +++ b/consensus/state_processing/src/per_epoch_processing/base/rewards_and_penalties.rs @@ -45,7 +45,7 @@ impl AttestationDelta { /// Apply attester and proposer rewards. pub fn process_rewards_and_penalties( state: &mut BeaconState, - validator_statuses: &mut ValidatorStatuses, + validator_statuses: &ValidatorStatuses, spec: &ChainSpec, ) -> Result<(), Error> { if state.current_epoch() == T::genesis_epoch() { @@ -59,7 +59,7 @@ pub fn process_rewards_and_penalties( return Err(Error::ValidatorStatusesInconsistent); } - let deltas = get_attestation_deltas(state, validator_statuses, spec)?; + let deltas = get_attestation_deltas_all(state, validator_statuses, spec)?; // Apply the deltas, erroring on overflow above but not on overflow below (saturating at 0 // instead). @@ -73,10 +73,41 @@ pub fn process_rewards_and_penalties( } /// Apply rewards for participation in attestations during the previous epoch. -pub fn get_attestation_deltas( +pub fn get_attestation_deltas_all( state: &BeaconState, validator_statuses: &ValidatorStatuses, spec: &ChainSpec, +) -> Result, Error> { + get_attestation_deltas(state, validator_statuses, None, spec) +} + +/// Apply rewards for participation in attestations during the previous epoch, and only compute +/// rewards for a subset of validators. +pub fn get_attestation_deltas_subset( + state: &BeaconState, + validator_statuses: &ValidatorStatuses, + validators_subset: &Vec, + spec: &ChainSpec, +) -> Result, Error> { + get_attestation_deltas(state, validator_statuses, Some(validators_subset), spec).map(|deltas| { + deltas + .into_iter() + .enumerate() + .filter(|(index, _)| validators_subset.contains(index)) + .collect() + }) +} + +/// Apply rewards for participation in attestations during the previous epoch. +/// If `maybe_validators_subset` specified, only the deltas for the specified validator subset is +/// returned, otherwise deltas for all validators are returned. +/// +/// Returns a vec of validator indices to `AttestationDelta`. +fn get_attestation_deltas( + state: &BeaconState, + validator_statuses: &ValidatorStatuses, + maybe_validators_subset: Option<&Vec>, + spec: &ChainSpec, ) -> Result, Error> { let previous_epoch = state.previous_epoch(); let finality_delay = state @@ -88,6 +119,13 @@ pub fn get_attestation_deltas( let total_balances = &validator_statuses.total_balances; + // Ignore validator if a subset is specified and validator is not in the subset + let include_validator_delta = |idx| match maybe_validators_subset.as_ref() { + None => true, + Some(validators_subset) if validators_subset.contains(&idx) => true, + Some(_) => false, + }; + for (index, validator) in validator_statuses.statuses.iter().enumerate() { // Ignore ineligible validators. All sub-functions of the spec do this except for // `get_inclusion_delay_deltas`. It's safe to do so here because any validator that is in @@ -99,41 +137,46 @@ pub fn get_attestation_deltas( let base_reward = get_base_reward(state, index, total_balances.current_epoch(), spec)?; - let source_delta = - get_source_delta(validator, base_reward, total_balances, finality_delay, spec)?; - let target_delta = - get_target_delta(validator, base_reward, total_balances, finality_delay, spec)?; - let head_delta = - get_head_delta(validator, base_reward, total_balances, finality_delay, spec)?; let (inclusion_delay_delta, proposer_delta) = get_inclusion_delay_delta(validator, base_reward, spec)?; - let inactivity_penalty_delta = - get_inactivity_penalty_delta(validator, base_reward, finality_delay, spec)?; - - let delta = deltas - .get_mut(index) - .ok_or(Error::DeltaOutOfBounds(index))?; - delta.source_delta.combine(source_delta)?; - delta.target_delta.combine(target_delta)?; - delta.head_delta.combine(head_delta)?; - delta.inclusion_delay_delta.combine(inclusion_delay_delta)?; - delta - .inactivity_penalty_delta - .combine(inactivity_penalty_delta)?; + + if include_validator_delta(index) { + let source_delta = + get_source_delta(validator, base_reward, total_balances, finality_delay, spec)?; + let target_delta = + get_target_delta(validator, base_reward, total_balances, finality_delay, spec)?; + let head_delta = + get_head_delta(validator, base_reward, total_balances, finality_delay, spec)?; + let inactivity_penalty_delta = + get_inactivity_penalty_delta(validator, base_reward, finality_delay, spec)?; + + let delta = deltas + .get_mut(index) + .ok_or(Error::DeltaOutOfBounds(index))?; + delta.source_delta.combine(source_delta)?; + delta.target_delta.combine(target_delta)?; + delta.head_delta.combine(head_delta)?; + delta.inclusion_delay_delta.combine(inclusion_delay_delta)?; + delta + .inactivity_penalty_delta + .combine(inactivity_penalty_delta)?; + } if let Some((proposer_index, proposer_delta)) = proposer_delta { - deltas - .get_mut(proposer_index) - .ok_or(Error::ValidatorStatusesInconsistent)? - .inclusion_delay_delta - .combine(proposer_delta)?; + if include_validator_delta(proposer_index) { + deltas + .get_mut(proposer_index) + .ok_or(Error::ValidatorStatusesInconsistent)? + .inclusion_delay_delta + .combine(proposer_delta)?; + } } } Ok(deltas) } -fn get_attestation_component_delta( +pub fn get_attestation_component_delta( index_in_unslashed_attesting_indices: bool, attesting_balance: u64, total_balances: &TotalBalances, @@ -216,7 +259,7 @@ fn get_head_delta( ) } -fn get_inclusion_delay_delta( +pub fn get_inclusion_delay_delta( validator: &ValidatorStatus, base_reward: u64, spec: &ChainSpec, @@ -242,7 +285,7 @@ fn get_inclusion_delay_delta( } } -fn get_inactivity_penalty_delta( +pub fn get_inactivity_penalty_delta( validator: &ValidatorStatus, base_reward: u64, finality_delay: u64, diff --git a/testing/ef_tests/src/cases/epoch_processing.rs b/testing/ef_tests/src/cases/epoch_processing.rs index 31542ba4476..5e711871560 100644 --- a/testing/ef_tests/src/cases/epoch_processing.rs +++ b/testing/ef_tests/src/cases/epoch_processing.rs @@ -120,7 +120,7 @@ impl EpochTransition for RewardsAndPenalties { BeaconState::Base(_) => { let mut validator_statuses = base::ValidatorStatuses::new(state, spec)?; validator_statuses.process_attestations(state)?; - base::process_rewards_and_penalties(state, &mut validator_statuses, spec) + base::process_rewards_and_penalties(state, &validator_statuses, spec) } BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) => { altair::process_rewards_and_penalties( diff --git a/testing/ef_tests/src/cases/rewards.rs b/testing/ef_tests/src/cases/rewards.rs index c59ceabe0b1..ee0fc265e11 100644 --- a/testing/ef_tests/src/cases/rewards.rs +++ b/testing/ef_tests/src/cases/rewards.rs @@ -118,7 +118,7 @@ impl Case for RewardsTest { let mut validator_statuses = ValidatorStatuses::new(&state, spec)?; validator_statuses.process_attestations(&state)?; - let deltas = base::rewards_and_penalties::get_attestation_deltas( + let deltas = base::rewards_and_penalties::get_attestation_deltas_all( &state, &validator_statuses, spec, From 071dd4cd9c0ba4ef62f37d52895b83b15adbb36c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 21 Jul 2023 04:46:52 +0000 Subject: [PATCH 46/51] Add self-hosted runners v2 (#4506) ## Issue Addressed NA ## Proposed Changes Carries on from #4115, with the following modifications: 1. Self-hosted runners are only enabled if `github.repository == sigp/lighthouse`. - This allows forks to still have Github-hosted CI. - This gives us a method to switch back to Github-runners if we have extended downtime on self-hosted. 1. Does not remove any existing dependency builds for Github-hosted runners (e.g., installing the latest Rust). 1. Adds the `WATCH_HOST` environment variable which defines where we expect to find the postgres db in the `watch` tests. This should be set to `host.docker.internal` for the tests to pass on self-hosted runners. ## Additional Info NA Co-authored-by: antondlr --- .github/workflows/test-suite.yml | 24 +++++++++++++++++++----- watch/tests/tests.rs | 11 +++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index e3342ac3705..ff7a9cf2f10 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -17,6 +17,10 @@ env: PINNED_NIGHTLY: nightly-2023-04-16 # Prevent Github API rate limiting. LIGHTHOUSE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Enable self-hosted runners for the sigp repo only. + SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/lighthouse' }} + # Self-hosted runners need to reference a different host for `./watch` tests. + WATCH_HOST: ${{ github.repository == 'sigp/lighthouse' && 'host.docker.internal' || 'localhost' }} jobs: target-branch-check: name: target-branch-check @@ -48,11 +52,13 @@ jobs: run: make cargo-fmt release-tests-ubuntu: name: release-tests-ubuntu - runs-on: ubuntu-latest + # Use self-hosted runners only on the sigp repo. + runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "large"]') || 'ubuntu-latest' }} needs: cargo-fmt steps: - uses: actions/checkout@v3 - name: Get latest version of stable Rust + if: env.SELF_HOSTED_RUNNERS == false run: rustup update stable - name: Install Protoc uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 @@ -64,11 +70,12 @@ jobs: run: make test-release release-tests-windows: name: release-tests-windows - runs-on: windows-2019 + runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows"]') || 'windows-2019' }} needs: cargo-fmt steps: - uses: actions/checkout@v3 - name: Get latest version of stable Rust + if: env.SELF_HOSTED_RUNNERS == false run: rustup update stable - name: Use Node.js uses: actions/setup-node@v2 @@ -83,6 +90,7 @@ jobs: - name: Install make run: choco install -y make - uses: KyleMayes/install-llvm-action@v1 + if: env.SELF_HOSTED_RUNNERS == false with: version: "15.0" directory: ${{ runner.temp }}/llvm @@ -92,11 +100,13 @@ jobs: run: make test-release beacon-chain-tests: name: beacon-chain-tests - runs-on: ubuntu-latest + # Use self-hosted runners only on the sigp repo. + runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "large"]') || 'ubuntu-latest' }} needs: cargo-fmt steps: - uses: actions/checkout@v3 - name: Get latest version of stable Rust + if: env.SELF_HOSTED_RUNNERS == false run: rustup update stable - name: Install Protoc uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 @@ -130,11 +140,13 @@ jobs: run: make test-slasher debug-tests-ubuntu: name: debug-tests-ubuntu - runs-on: ubuntu-22.04 + # Use self-hosted runners only on the sigp repo. + runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "large"]') || 'ubuntu-latest' }} needs: cargo-fmt steps: - uses: actions/checkout@v3 - name: Get latest version of stable Rust + if: env.SELF_HOSTED_RUNNERS == false run: rustup update stable - name: Install Protoc uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 @@ -160,11 +172,13 @@ jobs: run: make run-state-transition-tests ef-tests-ubuntu: name: ef-tests-ubuntu - runs-on: ubuntu-latest + # Use self-hosted runners only on the sigp repo. + runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "small"]') || 'ubuntu-latest' }} needs: cargo-fmt steps: - uses: actions/checkout@v3 - name: Get latest version of stable Rust + if: env.SELF_HOSTED_RUNNERS == false run: rustup update stable - name: Install Protoc uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 diff --git a/watch/tests/tests.rs b/watch/tests/tests.rs index acdda8c306a..28700ccdce7 100644 --- a/watch/tests/tests.rs +++ b/watch/tests/tests.rs @@ -22,6 +22,7 @@ use watch::{ }; use log::error; +use std::env; use std::net::SocketAddr; use std::time::Duration; use tokio::{runtime, task::JoinHandle}; @@ -36,6 +37,11 @@ const VALIDATOR_COUNT: usize = 32; const SLOTS_PER_EPOCH: u64 = 32; const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5); +/// Set this environment variable to use a different hostname for connecting to +/// the database. Can be set to `host.docker.internal` for docker-in-docker +/// setups. +const WATCH_HOST_ENV_VARIABLE: &str = "WATCH_HOST"; + fn build_test_config(config: &DatabaseConfig) -> PostgresConfig { let mut postgres_config = PostgresConfig::new(); postgres_config @@ -71,6 +77,10 @@ pub async fn create_test_database(config: &DatabaseConfig) { .expect("Database creation failed"); } +pub fn get_host_from_env() -> String { + env::var(WATCH_HOST_ENV_VARIABLE).unwrap_or_else(|_| "localhost".to_string()) +} + struct TesterBuilder { pub harness: BeaconChainHarness>, pub config: Config, @@ -107,6 +117,7 @@ impl TesterBuilder { database: DatabaseConfig { dbname: random_dbname(), port: database_port, + host: get_host_from_env(), ..Default::default() }, server: ServerConfig { From f1f04bc68a5506c839693d25bc8c626c48ec8669 Mon Sep 17 00:00:00 2001 From: Gua00va <105484243+Gua00va@users.noreply.github.com> Date: Fri, 21 Jul 2023 20:26:57 +0530 Subject: [PATCH 47/51] Add Changes to BlobSidecars Endpoint (#4455) * changed name * Fix sidecars * Added query type and parrameter * added query struct and function * added method * improved filtering method * added blob_sidecar_list_indexed to block_id * minor blobqueryindex fix * function and formatting fix * minor function and naming fix * minor changes --- beacon_node/http_api/src/block_id.rs | 21 +++++++++++++++++++++ beacon_node/http_api/src/lib.rs | 10 ++++++---- common/eth2/src/types.rs | 7 +++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/beacon_node/http_api/src/block_id.rs b/beacon_node/http_api/src/block_id.rs index d5f6ac88645..1a3d852dab2 100644 --- a/beacon_node/http_api/src/block_id.rs +++ b/beacon_node/http_api/src/block_id.rs @@ -1,5 +1,6 @@ use crate::{state_id::checkpoint_slot_and_execution_optimistic, ExecutionOptimistic}; use beacon_chain::{BeaconChain, BeaconChainError, BeaconChainTypes, WhenSlotSkipped}; +use eth2::types::BlobIndicesQuery; use eth2::types::BlockId as CoreBlockId; use std::fmt; use std::str::FromStr; @@ -266,6 +267,26 @@ impl BlockId { Err(e) => Err(warp_utils::reject::beacon_chain_error(e)), } } + + pub async fn blob_sidecar_list_filtered( + &self, + indices: BlobIndicesQuery, + chain: &BeaconChain, + ) -> Result, warp::Rejection> { + let blob_sidecar_list = self.blob_sidecar_list(&chain).await?; + let blob_sidecar_list_filtered = match indices.indices { + Some(vec) => { + let list = blob_sidecar_list + .into_iter() + .filter(|blob_sidecar| vec.contains(&blob_sidecar.index)) + .collect(); + BlobSidecarList::new(list) + .map_err(|e| warp_utils::reject::custom_server_error(format!("{:?}", e)))? + } + None => blob_sidecar_list, + }; + Ok(blob_sidecar_list_filtered) + } } impl FromStr for BlockId { diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 6707719ae23..c9b6af0a48a 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -1487,21 +1487,23 @@ pub fn serve( .and(warp::path("beacon")) .and(warp::path("blob_sidecars")) .and(block_id_or_err) + .and(warp::query::()) .and(warp::path::end()) .and(chain_filter.clone()) .and(warp::header::optional::("accept")) .and_then( |block_id: BlockId, + indices: api_types::BlobIndicesQuery, chain: Arc>, accept_header: Option| { async move { - let blob_sidecar_list = block_id.blob_sidecar_list(&chain).await?; - + let blob_sidecar_list_filtered = + block_id.blob_sidecar_list_filtered(indices, &chain).await?; match accept_header { Some(api_types::Accept::Ssz) => Response::builder() .status(200) .header("Content-Type", "application/octet-stream") - .body(blob_sidecar_list.as_ssz_bytes().into()) + .body(blob_sidecar_list_filtered.as_ssz_bytes().into()) .map_err(|e| { warp_utils::reject::custom_server_error(format!( "failed to create response: {}", @@ -1509,7 +1511,7 @@ pub fn serve( )) }), _ => Ok(warp::reply::json(&api_types::GenericResponse::from( - blob_sidecar_list, + blob_sidecar_list_filtered, )) .into_response()), } diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 7bb43d8a9d3..29eb0ac6163 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -648,6 +648,13 @@ pub struct ValidatorBalancesQuery { pub id: Option>, } +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BlobIndicesQuery { + #[serde(default, deserialize_with = "option_query_vec")] + pub indices: Option>, +} + #[derive(Clone, Serialize, Deserialize)] #[serde(transparent)] pub struct ValidatorIndexData(#[serde(with = "serde_utils::quoted_u64_vec")] pub Vec); From 54c6e1dd3d1b54dfe63f350440a5455fe4e872a7 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Mon, 24 Jul 2023 21:09:07 +1000 Subject: [PATCH 48/51] Fix compilation --- beacon_node/beacon_chain/src/attestation_rewards.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_rewards.rs b/beacon_node/beacon_chain/src/attestation_rewards.rs index 9fc21b668fa..08950d65509 100644 --- a/beacon_node/beacon_chain/src/attestation_rewards.rs +++ b/beacon_node/beacon_chain/src/attestation_rewards.rs @@ -50,9 +50,10 @@ impl BeaconChain { match state { BeaconState::Base(_) => self.compute_attestation_rewards_base(state, validators), - BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) => { - self.compute_attestation_rewards_altair(state, validators) - } + BeaconState::Altair(_) + | BeaconState::Merge(_) + | BeaconState::Capella(_) + | BeaconState::Deneb(_) => self.compute_attestation_rewards_altair(state, validators), } } From fe94a05dd11efc2817b77b880b029914d7feb313 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Mon, 24 Jul 2023 23:02:29 +1000 Subject: [PATCH 49/51] Fix lint --- beacon_node/http_api/src/block_id.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/http_api/src/block_id.rs b/beacon_node/http_api/src/block_id.rs index 1a3d852dab2..fa99ea5b4f7 100644 --- a/beacon_node/http_api/src/block_id.rs +++ b/beacon_node/http_api/src/block_id.rs @@ -273,7 +273,7 @@ impl BlockId { indices: BlobIndicesQuery, chain: &BeaconChain, ) -> Result, warp::Rejection> { - let blob_sidecar_list = self.blob_sidecar_list(&chain).await?; + let blob_sidecar_list = self.blob_sidecar_list(chain).await?; let blob_sidecar_list_filtered = match indices.indices { Some(vec) => { let list = blob_sidecar_list From 33dd13c798f36f5db22211968d1ff4271bc8fd71 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 25 Jul 2023 10:51:10 -0400 Subject: [PATCH 50/51] Refactor deneb block processing (#4511) * Revert "fix merge" This reverts commit 405e95b0ce15409f06504f45c8d93071523e9539. * refactor deneb block processing * cargo fmt * fix ci --- beacon_node/beacon_chain/src/beacon_chain.rs | 17 +- .../beacon_chain/src/blob_verification.rs | 240 +-------- .../beacon_chain/src/block_verification.rs | 247 ++-------- .../src/block_verification_types.rs | 438 +++++++++++++++++ .../src/data_availability_checker.rs | 465 ++++++------------ .../overflow_lru_cache.rs | 29 +- beacon_node/beacon_chain/src/lib.rs | 9 +- beacon_node/beacon_chain/src/test_utils.rs | 63 ++- .../tests/attestation_production.rs | 16 +- .../beacon_chain/tests/block_verification.rs | 84 ++-- .../tests/payload_invalidation.rs | 6 +- beacon_node/beacon_chain/tests/store_tests.rs | 29 +- beacon_node/http_api/src/publish_blocks.rs | 2 +- .../http_api/tests/interactive_tests.rs | 2 +- beacon_node/http_api/tests/status_tests.rs | 4 +- .../gossip_methods.rs | 2 +- .../src/network_beacon_processor/mod.rs | 6 +- .../network_beacon_processor/sync_methods.rs | 17 +- .../network/src/sync/backfill_sync/mod.rs | 6 +- .../network/src/sync/block_lookups/mod.rs | 14 +- .../src/sync/block_lookups/parent_lookup.rs | 8 +- .../sync/block_lookups/single_block_lookup.rs | 26 +- .../network/src/sync/block_lookups/tests.rs | 2 +- .../src/sync/block_sidecar_coupling.rs | 44 +- beacon_node/network/src/sync/manager.rs | 14 +- .../network/src/sync/network_context.rs | 4 +- .../network/src/sync/range_sync/batch.rs | 16 +- .../network/src/sync/range_sync/chain.rs | 4 +- .../network/src/sync/range_sync/range.rs | 9 +- consensus/fork_choice/tests/tests.rs | 18 +- .../state_processing/src/consensus_context.rs | 12 - consensus/types/src/lib.rs | 7 +- consensus/types/src/signed_beacon_block.rs | 21 + 33 files changed, 930 insertions(+), 951 deletions(-) create mode 100644 beacon_node/beacon_chain/src/block_verification_types.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index b28d0b81f8d..d136188b673 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -8,13 +8,16 @@ use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckEarlyAttesterCache} use crate::beacon_proposer_cache::compute_proposer_duties_from_head; use crate::beacon_proposer_cache::BeaconProposerCache; use crate::blob_cache::BlobCache; -use crate::blob_verification::{self, AsBlock, BlobError, BlockWrapper, GossipVerifiedBlob}; +use crate::blob_verification::{self, BlobError, GossipVerifiedBlob}; use crate::block_times_cache::BlockTimesCache; use crate::block_verification::POS_PANDA_BANNER; use crate::block_verification::{ check_block_is_finalized_checkpoint_or_descendant, check_block_relevancy, get_block_root, - signature_verify_chain_segment, AvailableExecutedBlock, BlockError, BlockImportData, - ExecutedBlock, ExecutionPendingBlock, GossipVerifiedBlock, IntoExecutionPendingBlock, + signature_verify_chain_segment, BlockError, ExecutionPendingBlock, GossipVerifiedBlock, + IntoExecutionPendingBlock, +}; +use crate::block_verification_types::{ + AsBlock, AvailableExecutedBlock, BlockImportData, ExecutedBlock, RpcBlock, }; pub use crate::canonical_head::{CanonicalHead, CanonicalHeadRwLock}; use crate::chain_config::ChainConfig; @@ -122,7 +125,7 @@ use types::*; pub type ForkChoiceError = fork_choice::Error; /// Alias to appease clippy. -type HashBlockTuple = (Hash256, BlockWrapper); +type HashBlockTuple = (Hash256, RpcBlock); /// The time-out before failure during an operation to take a read/write RwLock on the block /// processing cache. @@ -2521,7 +2524,7 @@ impl BeaconChain { /// This method is potentially long-running and should not run on the core executor. pub fn filter_chain_segment( self: &Arc, - chain_segment: Vec>, + chain_segment: Vec>, ) -> Result>, ChainSegmentResult> { // This function will never import any blocks. let imported_blocks = 0; @@ -2627,7 +2630,7 @@ impl BeaconChain { /// `Self::process_block`. pub async fn process_chain_segment( self: &Arc, - chain_segment: Vec>, + chain_segment: Vec>, notify_execution_layer: NotifyExecutionLayer, ) -> ChainSegmentResult { let mut imported_blocks = 0; @@ -2804,7 +2807,7 @@ impl BeaconChain { /// /// - `SignedBeaconBlock` /// - `GossipVerifiedBlock` - /// - `BlockWrapper` + /// - `RpcBlock` /// /// ## Errors /// diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index da3ae2c93b0..78e48f0ed65 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -7,22 +7,18 @@ use crate::beacon_chain::{ BeaconChain, BeaconChainTypes, BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT, MAXIMUM_GOSSIP_CLOCK_DISPARITY, VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT, }; -use crate::data_availability_checker::{ - AvailabilityCheckError, AvailabilityPendingBlock, AvailableBlock, -}; +use crate::data_availability_checker::AvailabilityCheckError; use crate::kzg_utils::{validate_blob, validate_blobs}; use crate::BeaconChainError; -use eth2::types::BlockContentsTuple; use kzg::Kzg; use slog::{debug, warn}; use ssz_derive::{Decode, Encode}; -use ssz_types::{FixedVector, VariableList}; +use ssz_types::VariableList; use std::borrow::Cow; -use types::blob_sidecar::{BlobIdentifier, FixedBlobSidecarList}; +use types::blob_sidecar::BlobIdentifier; use types::{ - BeaconBlockRef, BeaconState, BeaconStateError, BlobSidecar, BlobSidecarList, ChainSpec, - CloneConfig, Epoch, EthSpec, FullPayload, Hash256, KzgCommitment, RelativeEpoch, - SignedBeaconBlock, SignedBeaconBlockHeader, SignedBlobSidecar, Slot, + BeaconState, BeaconStateError, BlobSidecar, BlobSidecarList, ChainSpec, CloneConfig, EthSpec, + Hash256, KzgCommitment, RelativeEpoch, SignedBlobSidecar, Slot, }; #[derive(Debug)] @@ -519,13 +515,12 @@ pub fn verify_kzg_for_blob( /// Note: This function should be preferred over calling `verify_kzg_for_blob` /// in a loop since this function kzg verifies a list of blobs more efficiently. pub fn verify_kzg_for_blob_list( - blob_list: Vec>>, + blob_list: &BlobSidecarList, kzg: &Kzg, -) -> Result, AvailabilityCheckError> { +) -> Result<(), AvailabilityCheckError> { let _timer = crate::metrics::start_timer(&crate::metrics::KZG_VERIFICATION_BATCH_TIMES); let (blobs, (commitments, proofs)): (Vec<_>, (Vec<_>, Vec<_>)) = blob_list - .clone() - .into_iter() + .iter() .map(|blob| (blob.blob.clone(), (blob.kzg_commitment, blob.kzg_proof))) .unzip(); if validate_blobs::( @@ -536,225 +531,8 @@ pub fn verify_kzg_for_blob_list( ) .map_err(AvailabilityCheckError::Kzg)? { - Ok(blob_list - .into_iter() - .map(|blob| KzgVerifiedBlob { blob }) - .collect()) + Ok(()) } else { Err(AvailabilityCheckError::KzgVerificationFailed) } } - -pub type KzgVerifiedBlobList = Vec>; - -#[derive(Debug, Clone)] -pub enum MaybeAvailableBlock { - /// This variant is fully available. - /// i.e. for pre-deneb blocks, it contains a (`SignedBeaconBlock`, `Blobs::None`) and for - /// post-4844 blocks, it contains a `SignedBeaconBlock` and a Blobs variant other than `Blobs::None`. - Available(AvailableBlock), - /// This variant is not fully available and requires blobs to become fully available. - AvailabilityPending(AvailabilityPendingBlock), -} - -/// Trait for common block operations. -pub trait AsBlock { - fn slot(&self) -> Slot; - fn epoch(&self) -> Epoch; - fn parent_root(&self) -> Hash256; - fn state_root(&self) -> Hash256; - fn signed_block_header(&self) -> SignedBeaconBlockHeader; - fn message(&self) -> BeaconBlockRef; - fn as_block(&self) -> &SignedBeaconBlock; - fn block_cloned(&self) -> Arc>; - fn canonical_root(&self) -> Hash256; - fn into_block_wrapper(self) -> BlockWrapper; -} - -impl AsBlock for MaybeAvailableBlock { - fn slot(&self) -> Slot { - self.as_block().slot() - } - fn epoch(&self) -> Epoch { - self.as_block().epoch() - } - fn parent_root(&self) -> Hash256 { - self.as_block().parent_root() - } - fn state_root(&self) -> Hash256 { - self.as_block().state_root() - } - fn signed_block_header(&self) -> SignedBeaconBlockHeader { - self.as_block().signed_block_header() - } - fn message(&self) -> BeaconBlockRef { - self.as_block().message() - } - fn as_block(&self) -> &SignedBeaconBlock { - match &self { - MaybeAvailableBlock::Available(block) => block.as_block(), - MaybeAvailableBlock::AvailabilityPending(block) => block.as_block(), - } - } - fn block_cloned(&self) -> Arc> { - match &self { - MaybeAvailableBlock::Available(block) => block.block_cloned(), - MaybeAvailableBlock::AvailabilityPending(block) => block.block_cloned(), - } - } - fn canonical_root(&self) -> Hash256 { - self.as_block().canonical_root() - } - - fn into_block_wrapper(self) -> BlockWrapper { - match self { - MaybeAvailableBlock::Available(available_block) => available_block.into_block_wrapper(), - MaybeAvailableBlock::AvailabilityPending(pending_block) => { - BlockWrapper::Block(pending_block.to_block()) - } - } - } -} - -impl AsBlock for &MaybeAvailableBlock { - fn slot(&self) -> Slot { - self.as_block().slot() - } - fn epoch(&self) -> Epoch { - self.as_block().epoch() - } - fn parent_root(&self) -> Hash256 { - self.as_block().parent_root() - } - fn state_root(&self) -> Hash256 { - self.as_block().state_root() - } - fn signed_block_header(&self) -> SignedBeaconBlockHeader { - self.as_block().signed_block_header() - } - fn message(&self) -> BeaconBlockRef { - self.as_block().message() - } - fn as_block(&self) -> &SignedBeaconBlock { - match &self { - MaybeAvailableBlock::Available(block) => block.as_block(), - MaybeAvailableBlock::AvailabilityPending(block) => block.as_block(), - } - } - fn block_cloned(&self) -> Arc> { - match &self { - MaybeAvailableBlock::Available(block) => block.block_cloned(), - MaybeAvailableBlock::AvailabilityPending(block) => block.block_cloned(), - } - } - fn canonical_root(&self) -> Hash256 { - self.as_block().canonical_root() - } - - fn into_block_wrapper(self) -> BlockWrapper { - self.clone().into_block_wrapper() - } -} - -#[derive(Debug, Clone, Derivative)] -#[derivative(Hash(bound = "E: EthSpec"))] -pub enum BlockWrapper { - Block(Arc>), - BlockAndBlobs(Arc>, FixedBlobSidecarList), -} - -impl BlockWrapper { - pub fn new(block: Arc>, blobs: Option>) -> Self { - match blobs { - Some(blobs) => { - let blobs = FixedVector::from(blobs.into_iter().map(Some).collect::>()); - BlockWrapper::BlockAndBlobs(block, blobs) - } - None => BlockWrapper::Block(block), - } - } - pub fn deconstruct(self) -> (Arc>, Option>) { - match self { - BlockWrapper::Block(block) => (block, None), - BlockWrapper::BlockAndBlobs(block, blobs) => (block, Some(blobs)), - } - } -} - -impl AsBlock for BlockWrapper { - fn slot(&self) -> Slot { - self.as_block().slot() - } - fn epoch(&self) -> Epoch { - self.as_block().epoch() - } - fn parent_root(&self) -> Hash256 { - self.as_block().parent_root() - } - fn state_root(&self) -> Hash256 { - self.as_block().state_root() - } - fn signed_block_header(&self) -> SignedBeaconBlockHeader { - self.as_block().signed_block_header() - } - fn message(&self) -> BeaconBlockRef { - self.as_block().message() - } - fn as_block(&self) -> &SignedBeaconBlock { - match &self { - BlockWrapper::Block(block) => block, - BlockWrapper::BlockAndBlobs(block, _) => block, - } - } - fn block_cloned(&self) -> Arc> { - match &self { - BlockWrapper::Block(block) => block.clone(), - BlockWrapper::BlockAndBlobs(block, _) => block.clone(), - } - } - fn canonical_root(&self) -> Hash256 { - self.as_block().canonical_root() - } - - fn into_block_wrapper(self) -> BlockWrapper { - self - } -} - -impl BlockWrapper { - pub fn n_blobs(&self) -> usize { - match self { - BlockWrapper::Block(_) => 0, - BlockWrapper::BlockAndBlobs(_, blobs) => blobs.len(), - } - } -} - -impl From>> for BlockWrapper { - fn from(value: Arc>) -> Self { - Self::Block(value) - } -} - -impl From> for BlockWrapper { - fn from(value: SignedBeaconBlock) -> Self { - Self::Block(Arc::new(value)) - } -} - -impl From>> for BlockWrapper { - fn from(value: BlockContentsTuple>) -> Self { - match value.1 { - Some(variable_list) => { - let mut blobs = Vec::with_capacity(E::max_blobs_per_block()); - for blob in variable_list { - if blob.message.index < E::max_blobs_per_block() as u64 { - blobs.insert(blob.message.index as usize, Some(blob.message)); - } - } - Self::BlockAndBlobs(Arc::new(value.0), FixedVector::from(blobs)) - } - None => Self::Block(Arc::new(value.0)), - } - } -} diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 796712ed203..c9e0ee4c9d2 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -48,13 +48,11 @@ // returned alongside. #![allow(clippy::result_large_err)] -use crate::blob_verification::{ - AsBlock, BlobError, BlockWrapper, GossipVerifiedBlob, GossipVerifiedBlobList, - MaybeAvailableBlock, -}; -use crate::data_availability_checker::{ - AvailabilityCheckError, AvailabilityPendingBlock, AvailableBlock, +use crate::blob_verification::{BlobError, GossipVerifiedBlob}; +use crate::block_verification_types::{ + AsBlock, BlockImportData, GossipVerifiedBlockContents, RpcBlock, }; +use crate::data_availability_checker::{AvailabilityCheckError, MaybeAvailableBlock}; use crate::eth1_finalization_cache::Eth1FinalizationData; use crate::execution_payload::{ is_optimistic_candidate_block, validate_execution_payload_for_gossip, validate_merge_block, @@ -99,13 +97,11 @@ use std::time::Duration; use store::{Error as DBError, HotStateSummary, KeyValueStore, SignedBlobSidecarList, StoreOp}; use task_executor::JoinHandle; use tree_hash::TreeHash; -use types::blob_sidecar::BlobIdentifier; use types::ExecPayload; -use types::{ssz_tagged_beacon_state, ssz_tagged_signed_beacon_block}; use types::{ - BeaconBlockRef, BeaconState, BeaconStateError, BlindedPayload, ChainSpec, CloneConfig, Epoch, - EthSpec, ExecutionBlockHash, Hash256, InconsistentFork, PublicKey, PublicKeyBytes, - RelativeEpoch, SignedBeaconBlock, SignedBeaconBlockHeader, Slot, + BeaconBlockRef, BeaconState, BeaconStateError, ChainSpec, CloneConfig, Epoch, EthSpec, + ExecutionBlockHash, Hash256, InconsistentFork, PublicKey, PublicKeyBytes, RelativeEpoch, + SignedBeaconBlock, SignedBeaconBlockHeader, Slot, }; pub const POS_PANDA_BANNER: &str = r#" @@ -153,7 +149,7 @@ pub enum BlockError { /// /// It's unclear if this block is valid, but it cannot be processed without already knowing /// its parent. - ParentUnknown(BlockWrapper), + ParentUnknown(RpcBlock), /// The block slot is greater than the present slot. /// /// ## Peer scoring @@ -558,7 +554,7 @@ fn process_block_slash_info( /// The given `chain_segment` must contain only blocks from the same epoch, otherwise an error /// will be returned. pub fn signature_verify_chain_segment( - mut chain_segment: Vec<(Hash256, BlockWrapper)>, + mut chain_segment: Vec<(Hash256, RpcBlock)>, chain: &BeaconChain, ) -> Result>, BlockError> { if chain_segment.is_empty() { @@ -595,7 +591,7 @@ pub fn signature_verify_chain_segment( let maybe_available_block = chain .data_availability_checker - .check_availability(block.clone())?; + .check_rpc_block_availability(block.clone())?; // Save the block and its consensus context. The context will have had its proposer index // and attesting indices filled in, which can be used to accelerate later block processing. @@ -625,19 +621,12 @@ pub fn signature_verify_chain_segment( #[derive(Derivative)] #[derivative(Debug(bound = "T: BeaconChainTypes"))] pub struct GossipVerifiedBlock { - pub block: MaybeAvailableBlock, + pub block: Arc>, pub block_root: Hash256, parent: Option>, consensus_context: ConsensusContext, } -impl GossipVerifiedBlock { - /// Useful for publishing after gossip verification. - pub fn into_block_wrapper(self) -> BlockWrapper { - self.block.into_block_wrapper() - } -} - /// A wrapper around a `SignedBeaconBlock` that indicates that all signatures (except the deposit /// signatures) have been verified. pub struct SignatureVerifiedBlock { @@ -669,147 +658,6 @@ pub struct ExecutionPendingBlock { pub payload_verification_handle: PayloadVerificationHandle, } -pub enum ExecutedBlock { - Available(AvailableExecutedBlock), - AvailabilityPending(AvailabilityPendingExecutedBlock), -} - -impl ExecutedBlock { - pub fn as_block(&self) -> &SignedBeaconBlock { - match self { - Self::Available(available) => available.block.block(), - Self::AvailabilityPending(pending) => pending.block.as_block(), - } - } -} - -impl std::fmt::Debug for ExecutedBlock { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.as_block()) - } -} - -impl ExecutedBlock { - pub fn new( - block: MaybeAvailableBlock, - import_data: BlockImportData, - payload_verification_outcome: PayloadVerificationOutcome, - ) -> Self { - match block { - MaybeAvailableBlock::Available(available_block) => { - Self::Available(AvailableExecutedBlock::new( - available_block, - import_data, - payload_verification_outcome, - )) - } - MaybeAvailableBlock::AvailabilityPending(pending_block) => { - Self::AvailabilityPending(AvailabilityPendingExecutedBlock::new( - pending_block, - import_data, - payload_verification_outcome, - )) - } - } - } -} - -#[derive(Debug, PartialEq)] -pub struct AvailableExecutedBlock { - pub block: AvailableBlock, - pub import_data: BlockImportData, - pub payload_verification_outcome: PayloadVerificationOutcome, -} - -impl AvailableExecutedBlock { - pub fn new( - block: AvailableBlock, - import_data: BlockImportData, - payload_verification_outcome: PayloadVerificationOutcome, - ) -> Self { - Self { - block, - import_data, - payload_verification_outcome, - } - } - - pub fn get_all_blob_ids(&self) -> Vec { - let num_blobs_expected = self - .block - .message() - .body() - .blob_kzg_commitments() - .map_or(0, |commitments| commitments.len()); - let mut blob_ids = Vec::with_capacity(num_blobs_expected); - for i in 0..num_blobs_expected { - blob_ids.push(BlobIdentifier { - block_root: self.import_data.block_root, - index: i as u64, - }); - } - blob_ids - } -} - -#[derive(Encode, Decode, Clone)] -pub struct AvailabilityPendingExecutedBlock { - pub block: AvailabilityPendingBlock, - pub import_data: BlockImportData, - pub payload_verification_outcome: PayloadVerificationOutcome, -} - -impl AvailabilityPendingExecutedBlock { - pub fn new( - block: AvailabilityPendingBlock, - import_data: BlockImportData, - payload_verification_outcome: PayloadVerificationOutcome, - ) -> Self { - Self { - block, - import_data, - payload_verification_outcome, - } - } - - pub fn num_blobs_expected(&self) -> usize { - self.block - .kzg_commitments() - .map_or(0, |commitments| commitments.len()) - } - - pub fn get_all_blob_ids(&self) -> Vec { - let block_root = self.import_data.block_root; - self.block - .get_filtered_blob_ids(Some(block_root), |_, _| true) - } - - pub fn get_filtered_blob_ids( - &self, - filter: impl Fn(usize, Hash256) -> bool, - ) -> Vec { - self.block - .get_filtered_blob_ids(Some(self.import_data.block_root), filter) - } -} - -#[derive(Debug, PartialEq, Encode, Decode, Clone)] -// TODO (mark): investigate using an Arc / Arc -// here to make this cheaper to clone -pub struct BlockImportData { - pub block_root: Hash256, - #[ssz(with = "ssz_tagged_beacon_state")] - pub state: BeaconState, - #[ssz(with = "ssz_tagged_signed_beacon_block")] - pub parent_block: SignedBeaconBlock>, - pub parent_eth1_finalization_data: Eth1FinalizationData, - pub confirmed_state_roots: Vec, - pub consensus_context: ConsensusContext, -} - -pub type GossipVerifiedBlockContents = - (GossipVerifiedBlock, Option>); - pub trait IntoGossipVerifiedBlockContents: Sized { fn into_gossip_verified_block( self, @@ -911,27 +759,23 @@ impl GossipVerifiedBlock { block: Arc>, chain: &BeaconChain, ) -> Result> { - let maybe_available = chain - .data_availability_checker - .check_availability(block.into())?; // If the block is valid for gossip we don't supply it to the slasher here because // we assume it will be transformed into a fully verified block. We *do* need to supply // it to the slasher if an error occurs, because that's the end of this block's journey, // and it could be a repeat proposal (a likely cause for slashing!). - let header = maybe_available.signed_block_header(); - Self::new_without_slasher_checks(maybe_available, chain).map_err(|e| { + let header = block.signed_block_header(); + Self::new_without_slasher_checks(block, chain).map_err(|e| { process_block_slash_info(chain, BlockSlashInfo::from_early_error(header, e)) }) } /// As for new, but doesn't pass the block to the slasher. fn new_without_slasher_checks( - block: MaybeAvailableBlock, + block: Arc>, chain: &BeaconChain, ) -> Result> { // Ensure the block is the correct structure for the fork at `block.slot()`. block - .as_block() .fork_name(&chain.spec) .map_err(BlockError::InconsistentFork)?; @@ -947,7 +791,7 @@ impl GossipVerifiedBlock { }); } - let block_root = get_block_root(block.as_block()); + let block_root = get_block_root(&block); // Disallow blocks that conflict with the anchor (weak subjectivity checkpoint), if any. check_block_against_anchor_slot(block.message(), chain)?; @@ -1067,7 +911,7 @@ impl GossipVerifiedBlock { let pubkey = pubkey_cache .get(block.message().proposer_index() as usize) .ok_or_else(|| BlockError::UnknownValidator(block.message().proposer_index()))?; - block.as_block().verify_signature( + block.verify_signature( Some(block_root), pubkey, &fork, @@ -1111,8 +955,7 @@ impl GossipVerifiedBlock { // Having checked the proposer index and the block root we can cache them. let consensus_context = ConsensusContext::new(block.slot()) .set_current_block_root(block_root) - .set_proposer_index(block.as_block().message().proposer_index()) - .set_kzg_commitments_consistent(true); + .set_proposer_index(block.as_block().message().proposer_index()); Ok(Self { block, @@ -1155,11 +998,10 @@ impl SignatureVerifiedBlock { /// /// Returns an error if the block is invalid, or if the block was unable to be verified. pub fn new( - block: BlockWrapper, + block: MaybeAvailableBlock, block_root: Hash256, chain: &BeaconChain, ) -> Result> { - let block = chain.data_availability_checker.check_availability(block)?; // Ensure the block is the correct structure for the fork at `block.slot()`. block .as_block() @@ -1182,10 +1024,8 @@ impl SignatureVerifiedBlock { let mut signature_verifier = get_signature_verifier(&state, &pubkey_cache, &chain.spec); - let mut consensus_context = ConsensusContext::new(block.slot()) - .set_current_block_root(block_root) - // An `AvailabileBlock is passed in here, so we know this check has been run.` - .set_kzg_commitments_consistent(true); + let mut consensus_context = + ConsensusContext::new(block.slot()).set_current_block_root(block_root); signature_verifier.include_all_signatures(block.as_block(), &mut consensus_context)?; @@ -1203,7 +1043,7 @@ impl SignatureVerifiedBlock { /// As for `new` above but producing `BlockSlashInfo`. pub fn check_slashable( - block: BlockWrapper, + block: MaybeAvailableBlock, block_root: Hash256, chain: &BeaconChain, ) -> Result>> { @@ -1238,11 +1078,11 @@ impl SignatureVerifiedBlock { // signature. let mut consensus_context = from.consensus_context; signature_verifier - .include_all_signatures_except_proposal(block.as_block(), &mut consensus_context)?; + .include_all_signatures_except_proposal(block.as_ref(), &mut consensus_context)?; if signature_verifier.verify().is_ok() { Ok(Self { - block, + block: MaybeAvailableBlock::AvailabilityPending(block), block_root: from.block_root, parent: Some(parent), consensus_context, @@ -1299,6 +1139,7 @@ impl IntoExecutionPendingBlock for SignatureVerifiedBloc } } +//TODO(sean) can this be deleted impl IntoExecutionPendingBlock for Arc> { /// Verifies the `SignedBeaconBlock` by first transforming it into a `SignatureVerifiedBlock` /// and then using that implementation of `IntoExecutionPendingBlock` to complete verification. @@ -1311,8 +1152,16 @@ impl IntoExecutionPendingBlock for Arc IntoExecutionPendingBlock for Arc IntoExecutionPendingBlock for BlockWrapper { +impl IntoExecutionPendingBlock for RpcBlock { /// Verifies the `SignedBeaconBlock` by first transforming it into a `SignatureVerifiedBlock` /// and then using that implementation of `IntoExecutionPendingBlock` to complete verification. fn into_execution_pending_block_slashable( @@ -1333,8 +1182,16 @@ impl IntoExecutionPendingBlock for BlockWrapper ExecutionPendingBlock { // because it will revert finalization. Note that the finalized block is stored in fork // choice, so we will not reject any child of the finalized block (this is relevant during // genesis). - return Err(BlockError::ParentUnknown(block.into_block_wrapper())); + return Err(BlockError::ParentUnknown(block.into_rpc_block())); } /* @@ -1826,7 +1683,7 @@ pub fn check_block_is_finalized_checkpoint_or_descendant< block_parent_root: block.parent_root(), }) } else { - Err(BlockError::ParentUnknown(block.into_block_wrapper())) + Err(BlockError::ParentUnknown(block.into_rpc_block())) } } } @@ -1898,8 +1755,8 @@ pub fn get_block_root(block: &SignedBeaconBlock) -> Hash256 { #[allow(clippy::type_complexity)] fn verify_parent_block_is_known( chain: &BeaconChain, - block: MaybeAvailableBlock, -) -> Result<(ProtoBlock, MaybeAvailableBlock), BlockError> { + block: Arc>, +) -> Result<(ProtoBlock, Arc>), BlockError> { if let Some(proto_block) = chain .canonical_head .fork_choice_read_lock() @@ -1907,7 +1764,9 @@ fn verify_parent_block_is_known( { Ok((proto_block, block)) } else { - Err(BlockError::ParentUnknown(block.into_block_wrapper())) + Err(BlockError::ParentUnknown(RpcBlock::new_without_blobs( + block, + ))) } } @@ -1938,7 +1797,7 @@ fn load_parent>( .fork_choice_read_lock() .contains_block(&block.parent_root()) { - return Err(BlockError::ParentUnknown(block.into_block_wrapper())); + return Err(BlockError::ParentUnknown(block.into_rpc_block())); } let block_delay = chain diff --git a/beacon_node/beacon_chain/src/block_verification_types.rs b/beacon_node/beacon_chain/src/block_verification_types.rs new file mode 100644 index 00000000000..e80372a6da1 --- /dev/null +++ b/beacon_node/beacon_chain/src/block_verification_types.rs @@ -0,0 +1,438 @@ +use crate::blob_verification::GossipVerifiedBlobList; +use crate::data_availability_checker::AvailabilityCheckError; +pub use crate::data_availability_checker::{AvailableBlock, MaybeAvailableBlock}; +use crate::eth1_finalization_cache::Eth1FinalizationData; +use crate::{data_availability_checker, GossipVerifiedBlock, PayloadVerificationOutcome}; +use derivative::Derivative; +use ssz_derive::{Decode, Encode}; +use state_processing::ConsensusContext; +use std::sync::Arc; +use types::{ + blob_sidecar::BlobIdentifier, ssz_tagged_beacon_state, ssz_tagged_signed_beacon_block, + ssz_tagged_signed_beacon_block_arc, +}; +use types::{ + BeaconBlockRef, BeaconState, BlindedPayload, BlobSidecarList, Epoch, EthSpec, Hash256, + SignedBeaconBlock, SignedBeaconBlockHeader, Slot, +}; + +/// A block that has been received over RPC. It has 2 internal variants: +/// +/// 1. `BlockAndBlobs`: A fully available post deneb block with all the blobs available. This variant +/// is only constructed after making consistency checks between blocks and blobs. +/// Hence, it is fully self contained w.r.t verification. i.e. this block has all the required +/// data to get verfied and imported into fork choice. +/// +/// 2. `Block`: This can be a fully available pre-deneb block **or** a post-deneb block that may or may +/// not require blobs to be considered fully available. +/// +/// Note: We make a distinction over blocks received over gossip because +/// in a post-deneb world, the blobs corresponding to a given block that are received +/// over rpc do not contain the proposer signature for dos resistance. +#[derive(Debug, Clone, Derivative)] +#[derivative(Hash(bound = "E: EthSpec"))] +pub struct RpcBlock { + block: RpcBlockInner, +} + +/// Note: This variant is intentionally private because we want to safely construct the +/// internal variants after applying consistency checks to ensure that the block and blobs +/// are consistent with respect to each other. +#[derive(Debug, Clone, Derivative)] +#[derivative(Hash(bound = "E: EthSpec"))] +enum RpcBlockInner { + /// Single block lookup response. This should potentially hit the data availability cache. + Block(Arc>), + /// This variant is used with parent lookups and by-range responses. It should have all blobs + /// ordered, all block roots matching, and the correct number of blobs for this block. + BlockAndBlobs(Arc>, BlobSidecarList), +} + +impl RpcBlock { + /// Constructs a `Block` variant. + pub fn new_without_blobs(block: Arc>) -> Self { + Self { + block: RpcBlockInner::Block(block), + } + } + + /// Constructs a new `BlockAndBlobs` variant after making consistency + /// checks between the provided blocks and blobs. + pub fn new( + block: Arc>, + blobs: Option>, + ) -> Result { + if let Some(blobs) = blobs.as_ref() { + data_availability_checker::consistency_checks(&block, blobs)?; + } + let inner = match blobs { + Some(blobs) => RpcBlockInner::BlockAndBlobs(block, blobs), + None => RpcBlockInner::Block(block), + }; + Ok(Self { block: inner }) + } + + pub fn deconstruct(self) -> (Arc>, Option>) { + match self.block { + RpcBlockInner::Block(block) => (block, None), + RpcBlockInner::BlockAndBlobs(block, blobs) => (block, Some(blobs)), + } + } + pub fn n_blobs(&self) -> usize { + match &self.block { + RpcBlockInner::Block(_) => 0, + RpcBlockInner::BlockAndBlobs(_, blobs) => blobs.len(), + } + } +} + +impl From>> for RpcBlock { + fn from(value: Arc>) -> Self { + Self::new_without_blobs(value) + } +} + +impl From> for RpcBlock { + fn from(value: SignedBeaconBlock) -> Self { + Self::new_without_blobs(Arc::new(value)) + } +} + +/// A block that has gone through all pre-deneb block processing checks including block processing +/// and execution by an EL client. This block hasn't completed data availability checks. +/// +/// +/// It contains 2 variants: +/// 1. `Available`: This block has been executed and also contains all data to consider it a +/// fully available block. i.e. for post-deneb, this implies that this contains all the +/// required blobs. +/// 2. `AvailabilityPending`: This block hasn't received all required blobs to consider it a +/// fully available block. +pub enum ExecutedBlock { + Available(AvailableExecutedBlock), + AvailabilityPending(AvailabilityPendingExecutedBlock), +} + +impl ExecutedBlock { + pub fn new( + block: MaybeAvailableBlock, + import_data: BlockImportData, + payload_verification_outcome: PayloadVerificationOutcome, + ) -> Self { + match block { + MaybeAvailableBlock::Available(available_block) => { + Self::Available(AvailableExecutedBlock::new( + available_block, + import_data, + payload_verification_outcome, + )) + } + MaybeAvailableBlock::AvailabilityPending(pending_block) => { + Self::AvailabilityPending(AvailabilityPendingExecutedBlock::new( + pending_block, + import_data, + payload_verification_outcome, + )) + } + } + } + + pub fn as_block(&self) -> &SignedBeaconBlock { + match self { + Self::Available(available) => available.block.block(), + Self::AvailabilityPending(pending) => &pending.block, + } + } +} + +/// A block that has completed all pre-deneb block processing checks including verification +/// by an EL client **and** has all requisite blob data to be imported into fork choice. +#[derive(PartialEq)] +pub struct AvailableExecutedBlock { + pub block: AvailableBlock, + pub import_data: BlockImportData, + pub payload_verification_outcome: PayloadVerificationOutcome, +} + +impl AvailableExecutedBlock { + pub fn new( + block: AvailableBlock, + import_data: BlockImportData, + payload_verification_outcome: PayloadVerificationOutcome, + ) -> Self { + Self { + block, + import_data, + payload_verification_outcome, + } + } + + pub fn get_all_blob_ids(&self) -> Vec { + let num_blobs_expected = self + .block + .message() + .body() + .blob_kzg_commitments() + .map_or(0, |commitments| commitments.len()); + let mut blob_ids = Vec::with_capacity(num_blobs_expected); + for i in 0..num_blobs_expected { + blob_ids.push(BlobIdentifier { + block_root: self.import_data.block_root, + index: i as u64, + }); + } + blob_ids + } +} + +/// A block that has completed all pre-deneb block processing checks, verification +/// by an EL client but does not have all requisite blob data to get imported into +/// fork choice. +#[derive(Encode, Decode, Clone)] +pub struct AvailabilityPendingExecutedBlock { + #[ssz(with = "ssz_tagged_signed_beacon_block_arc")] + pub block: Arc>, + pub import_data: BlockImportData, + pub payload_verification_outcome: PayloadVerificationOutcome, +} + +impl AvailabilityPendingExecutedBlock { + pub fn new( + block: Arc>, + import_data: BlockImportData, + payload_verification_outcome: PayloadVerificationOutcome, + ) -> Self { + Self { + block, + import_data, + payload_verification_outcome, + } + } + + pub fn num_blobs_expected(&self) -> usize { + self.block + .message() + .body() + .blob_kzg_commitments() + .map_or(0, |commitments| commitments.len()) + } + + pub fn get_all_blob_ids(&self) -> Vec { + let block_root = self.import_data.block_root; + self.block + .get_filtered_blob_ids(Some(block_root), |_, _| true) + } + + pub fn get_filtered_blob_ids( + &self, + filter: impl Fn(usize, Hash256) -> bool, + ) -> Vec { + self.block + .get_filtered_blob_ids(Some(self.import_data.block_root), filter) + } +} + +#[derive(Debug, PartialEq, Encode, Decode, Clone)] +// TODO (mark): investigate using an Arc / Arc +// here to make this cheaper to clone +pub struct BlockImportData { + pub block_root: Hash256, + #[ssz(with = "ssz_tagged_beacon_state")] + pub state: BeaconState, + #[ssz(with = "ssz_tagged_signed_beacon_block")] + pub parent_block: SignedBeaconBlock>, + pub parent_eth1_finalization_data: Eth1FinalizationData, + pub confirmed_state_roots: Vec, + pub consensus_context: ConsensusContext, +} + +pub type GossipVerifiedBlockContents = + (GossipVerifiedBlock, Option>); + +/// Trait for common block operations. +pub trait AsBlock { + fn slot(&self) -> Slot; + fn epoch(&self) -> Epoch; + fn parent_root(&self) -> Hash256; + fn state_root(&self) -> Hash256; + fn signed_block_header(&self) -> SignedBeaconBlockHeader; + fn message(&self) -> BeaconBlockRef; + fn as_block(&self) -> &SignedBeaconBlock; + fn block_cloned(&self) -> Arc>; + fn canonical_root(&self) -> Hash256; + fn into_rpc_block(self) -> RpcBlock; +} + +impl AsBlock for Arc> { + fn slot(&self) -> Slot { + SignedBeaconBlock::slot(self) + } + + fn epoch(&self) -> Epoch { + SignedBeaconBlock::epoch(self) + } + + fn parent_root(&self) -> Hash256 { + SignedBeaconBlock::parent_root(self) + } + + fn state_root(&self) -> Hash256 { + SignedBeaconBlock::state_root(self) + } + + fn signed_block_header(&self) -> SignedBeaconBlockHeader { + SignedBeaconBlock::signed_block_header(self) + } + + fn message(&self) -> BeaconBlockRef { + SignedBeaconBlock::message(self) + } + + fn as_block(&self) -> &SignedBeaconBlock { + self + } + + fn block_cloned(&self) -> Arc> { + Arc::>::clone(self) + } + + fn canonical_root(&self) -> Hash256 { + SignedBeaconBlock::canonical_root(self) + } + + fn into_rpc_block(self) -> RpcBlock { + RpcBlock::new_without_blobs(self) + } +} + +impl AsBlock for MaybeAvailableBlock { + fn slot(&self) -> Slot { + self.as_block().slot() + } + fn epoch(&self) -> Epoch { + self.as_block().epoch() + } + fn parent_root(&self) -> Hash256 { + self.as_block().parent_root() + } + fn state_root(&self) -> Hash256 { + self.as_block().state_root() + } + fn signed_block_header(&self) -> SignedBeaconBlockHeader { + self.as_block().signed_block_header() + } + fn message(&self) -> BeaconBlockRef { + self.as_block().message() + } + fn as_block(&self) -> &SignedBeaconBlock { + match &self { + MaybeAvailableBlock::Available(block) => block.as_block(), + MaybeAvailableBlock::AvailabilityPending(block) => block, + } + } + fn block_cloned(&self) -> Arc> { + match &self { + MaybeAvailableBlock::Available(block) => block.block_cloned(), + MaybeAvailableBlock::AvailabilityPending(block) => block.clone(), + } + } + fn canonical_root(&self) -> Hash256 { + self.as_block().canonical_root() + } + + fn into_rpc_block(self) -> RpcBlock { + match self { + MaybeAvailableBlock::Available(available_block) => available_block.into_rpc_block(), + MaybeAvailableBlock::AvailabilityPending(block) => RpcBlock::new_without_blobs(block), + } + } +} + +impl AsBlock for AvailableBlock { + fn slot(&self) -> Slot { + self.block().slot() + } + + fn epoch(&self) -> Epoch { + self.block().epoch() + } + + fn parent_root(&self) -> Hash256 { + self.block().parent_root() + } + + fn state_root(&self) -> Hash256 { + self.block().state_root() + } + + fn signed_block_header(&self) -> SignedBeaconBlockHeader { + self.block().signed_block_header() + } + + fn message(&self) -> BeaconBlockRef { + self.block().message() + } + + fn as_block(&self) -> &SignedBeaconBlock { + self.block() + } + + fn block_cloned(&self) -> Arc> { + AvailableBlock::block_cloned(self) + } + + fn canonical_root(&self) -> Hash256 { + self.block().canonical_root() + } + + fn into_rpc_block(self) -> RpcBlock { + let (block, blobs_opt) = self.deconstruct(); + // Circumvent the constructor here, because an Available block will have already had + // consistency checks performed. + let inner = match blobs_opt { + None => RpcBlockInner::Block(block), + Some(blobs) => RpcBlockInner::BlockAndBlobs(block, blobs), + }; + RpcBlock { block: inner } + } +} + +impl AsBlock for RpcBlock { + fn slot(&self) -> Slot { + self.as_block().slot() + } + fn epoch(&self) -> Epoch { + self.as_block().epoch() + } + fn parent_root(&self) -> Hash256 { + self.as_block().parent_root() + } + fn state_root(&self) -> Hash256 { + self.as_block().state_root() + } + fn signed_block_header(&self) -> SignedBeaconBlockHeader { + self.as_block().signed_block_header() + } + fn message(&self) -> BeaconBlockRef { + self.as_block().message() + } + fn as_block(&self) -> &SignedBeaconBlock { + match &self.block { + RpcBlockInner::Block(block) => block, + RpcBlockInner::BlockAndBlobs(block, _) => block, + } + } + fn block_cloned(&self) -> Arc> { + match &self.block { + RpcBlockInner::Block(block) => block.clone(), + RpcBlockInner::BlockAndBlobs(block, _) => block.clone(), + } + } + fn canonical_root(&self) -> Hash256 { + self.as_block().canonical_root() + } + + fn into_rpc_block(self) -> RpcBlock { + self + } +} diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index fc48ba47ed8..9a53a7139c7 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -1,28 +1,25 @@ use crate::blob_verification::{ - verify_kzg_for_blob, verify_kzg_for_blob_list, AsBlock, BlockWrapper, GossipVerifiedBlob, - KzgVerifiedBlob, KzgVerifiedBlobList, MaybeAvailableBlock, + verify_kzg_for_blob, verify_kzg_for_blob_list, GossipVerifiedBlob, KzgVerifiedBlob, +}; +use crate::block_verification_types::{ + AvailabilityPendingExecutedBlock, AvailableExecutedBlock, RpcBlock, }; -use crate::block_verification::{AvailabilityPendingExecutedBlock, AvailableExecutedBlock}; - use crate::data_availability_checker::overflow_lru_cache::OverflowLRUCache; use crate::{BeaconChain, BeaconChainTypes, BeaconStore}; use kzg::Error as KzgError; use kzg::Kzg; use slog::{debug, error}; use slot_clock::SlotClock; -use ssz_types::{Error, FixedVector, VariableList}; +use ssz_types::{Error, VariableList}; use std::collections::HashSet; +use std::fmt; +use std::fmt::Debug; use std::sync::Arc; use strum::IntoStaticStr; use task_executor::TaskExecutor; -use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::{BlobIdentifier, BlobSidecar, FixedBlobSidecarList}; use types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; -use types::ssz_tagged_signed_beacon_block; -use types::{ - BeaconBlockRef, BlobSidecarList, ChainSpec, Epoch, EthSpec, FullPayload, Hash256, - SignedBeaconBlock, SignedBeaconBlockHeader, Slot, -}; +use types::{BlobSidecarList, ChainSpec, Epoch, EthSpec, Hash256, SignedBeaconBlock, Slot}; mod overflow_lru_cache; @@ -50,12 +47,20 @@ pub enum AvailabilityCheckError { }, IncorrectFork, BlobIndexInvalid(u64), + UnorderedBlobs { + blob_index: u64, + expected_index: u64, + }, StoreError(store::Error), DecodeError(ssz::DecodeError), BlockBlobRootMismatch { block_root: Hash256, blob_block_root: Hash256, }, + BlockBlobSlotMismatch { + block_slot: Slot, + blob_slot: Slot, + }, } impl From for AvailabilityCheckError { @@ -92,12 +97,23 @@ pub struct DataAvailabilityChecker { /// /// Indicates if the block is fully `Available` or if we need blobs or blocks /// to "complete" the requirements for an `AvailableBlock`. -#[derive(Debug, PartialEq)] +#[derive(PartialEq)] pub enum Availability { MissingComponents(Hash256), Available(Box>), } +impl Debug for Availability { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::MissingComponents(block_root) => { + write!(f, "MissingComponents({})", block_root) + } + Self::Available(block) => write!(f, "Available({:?})", block.import_data.block_root), + } + } +} + impl Availability { /// Returns all the blob identifiers associated with an `AvailableBlock`. /// Returns `None` if avaiability hasn't been fully satisfied yet. @@ -230,86 +246,51 @@ impl DataAvailabilityChecker { /// Checks if a block is available, returns a `MaybeAvailableBlock` that may include the fully /// available block. - pub fn check_availability( + pub fn check_rpc_block_availability( &self, - block: BlockWrapper, + block: RpcBlock, ) -> Result, AvailabilityCheckError> { - match block { - BlockWrapper::Block(block) => self.check_availability_without_blobs(block), - BlockWrapper::BlockAndBlobs(block, blob_list) => { - let kzg = self - .kzg - .as_ref() - .ok_or(AvailabilityCheckError::KzgNotInitialized)?; - let filtered_blobs = blob_list.iter().flatten().cloned().collect(); - let verified_blobs = verify_kzg_for_blob_list(filtered_blobs, kzg)?; - - Ok(MaybeAvailableBlock::Available( - self.check_availability_with_blobs(block, verified_blobs)?, - )) + let (block, blobs) = block.deconstruct(); + match blobs { + None => { + if self.blobs_required_for_block(&block) { + Ok(MaybeAvailableBlock::AvailabilityPending(block)) + } else { + Ok(MaybeAvailableBlock::Available(AvailableBlock { + block, + blobs: None, + })) + } } - } - } - - /// Verifies a block against a set of KZG verified blobs. Returns an AvailableBlock if block's - /// commitments are consistent with the provided verified blob commitments. - pub fn check_availability_with_blobs( - &self, - block: Arc>, - blobs: KzgVerifiedBlobList, - ) -> Result, AvailabilityCheckError> { - match self.check_availability_without_blobs(block)? { - MaybeAvailableBlock::Available(block) => Ok(block), - MaybeAvailableBlock::AvailabilityPending(pending_block) => { - pending_block.make_available(blobs) + Some(blob_list) => { + let verified_blobs = if self.blobs_required_for_block(&block) { + let kzg = self + .kzg + .as_ref() + .ok_or(AvailabilityCheckError::KzgNotInitialized)?; + verify_kzg_for_blob_list(&blob_list, kzg)?; + Some(blob_list) + } else { + None + }; + Ok(MaybeAvailableBlock::Available(AvailableBlock { + block, + blobs: verified_blobs, + })) } } } - /// Verifies a block as much as possible, returning a MaybeAvailableBlock enum that may include - /// an AvailableBlock if no blobs are required. Otherwise this will return an AvailabilityPendingBlock. - pub fn check_availability_without_blobs( - &self, - block: Arc>, - ) -> Result, AvailabilityCheckError> { - let blob_requirements = self.get_blob_requirements(&block)?; - let blobs = match blob_requirements { - BlobRequirements::EmptyBlobs => VerifiedBlobs::EmptyBlobs, - BlobRequirements::NotRequired => VerifiedBlobs::NotRequired, - BlobRequirements::PreDeneb => VerifiedBlobs::PreDeneb, - BlobRequirements::Required => { - return Ok(MaybeAvailableBlock::AvailabilityPending( - AvailabilityPendingBlock { block }, - )) - } - }; - Ok(MaybeAvailableBlock::Available(AvailableBlock { - block, - blobs, - })) - } - /// Determines the blob requirements for a block. Answers the question: "Does this block require /// blobs?". - fn get_blob_requirements( - &self, - block: &Arc>>, - ) -> Result { - let verified_blobs = - if let Ok(block_kzg_commitments) = block.message().body().blob_kzg_commitments() { - if self.da_check_required(block.epoch()) { - if block_kzg_commitments.is_empty() { - BlobRequirements::EmptyBlobs - } else { - BlobRequirements::Required - } - } else { - BlobRequirements::NotRequired - } - } else { - BlobRequirements::PreDeneb - }; - Ok(verified_blobs) + fn blobs_required_for_block(&self, block: &SignedBeaconBlock) -> bool { + let block_within_da_period = self.da_check_required(block.epoch()); + let block_has_kzg_commitments = block + .message() + .body() + .blob_kzg_commitments() + .map_or(false, |commitments| !commitments.is_empty()); + block_within_da_period && block_has_kzg_commitments } /// The epoch at which we require a data availability check in block processing. @@ -340,6 +321,87 @@ impl DataAvailabilityChecker { } } +/// Verifies an `SignedBeaconBlock` against a set of KZG verified blobs. +/// This does not check whether a block *should* have blobs, these checks should have been +/// completed when producing the `AvailabilityPendingBlock`. +pub fn make_available( + block: Arc>, + blobs: Vec>, +) -> Result, AvailabilityCheckError> { + let blobs = VariableList::new(blobs.into_iter().map(|blob| blob.to_blob()).collect())?; + + consistency_checks(&block, &blobs)?; + + Ok(AvailableBlock { + block, + blobs: Some(blobs), + }) +} + +/// Makes the following checks to ensure that the list of blobs correspond block: +/// +/// * Check that a block is post-deneb +/// * Checks that the number of blobs is equal to the length of kzg commitments in the list +/// * Checks that the index, slot, root and kzg_commitment in the block match the blobs in the correct order +/// +/// Returns `Ok(())` if all consistency checks pass and an error otherwise. +pub fn consistency_checks( + block: &SignedBeaconBlock, + blobs: &[Arc>], +) -> Result<(), AvailabilityCheckError> { + let Ok(block_kzg_commitments) = block + .message() + .body() + .blob_kzg_commitments() else { + return Ok(()) + }; + + if blobs.len() != block_kzg_commitments.len() { + return Err(AvailabilityCheckError::NumBlobsMismatch { + num_kzg_commitments: block_kzg_commitments.len(), + num_blobs: blobs.len(), + }); + } + + if block_kzg_commitments.is_empty() { + return Ok(()); + } + + let block_root = blobs + .first() + .map(|blob| blob.block_root) + .unwrap_or(block.canonical_root()); + for (index, (block_commitment, blob)) in + block_kzg_commitments.iter().zip(blobs.iter()).enumerate() + { + let index = index as u64; + if index != blob.index { + return Err(AvailabilityCheckError::UnorderedBlobs { + blob_index: blob.index, + expected_index: index, + }); + } + if block_root != blob.block_root { + return Err(AvailabilityCheckError::BlockBlobRootMismatch { + block_root, + blob_block_root: blob.block_root, + }); + } + if block.slot() != blob.slot { + return Err(AvailabilityCheckError::BlockBlobSlotMismatch { + block_slot: block.slot(), + blob_slot: blob.slot, + }); + } + if *block_commitment != blob.kzg_commitment { + return Err(AvailabilityCheckError::KzgCommitmentMismatch { + blob_index: blob.index, + }); + } + } + Ok(()) +} + pub fn start_availability_cache_maintenance_service( executor: TaskExecutor, chain: Arc>, @@ -425,244 +487,37 @@ async fn availability_cache_maintenance_service( } } -pub enum BlobRequirements { - Required, - /// This block is from outside the data availability boundary so doesn't require - /// a data availability check. - NotRequired, - /// The block's `kzg_commitments` field is empty so it does not contain any blobs. - EmptyBlobs, - /// This is a block prior to the 4844 fork, so doesn't require any blobs - PreDeneb, -} - -/// A wrapper over a `SignedBeaconBlock` where we have not verified availability of -/// corresponding `BlobSidecar`s and hence, is not ready for import into fork choice. -/// -/// Note: This wrapper does not necessarily correspond to a pre-deneb block as a pre-deneb -/// block that is ready for import will be of type `AvailableBlock` with its `blobs` field -/// set to `VerifiedBlobs::PreDeneb`. -#[derive(Clone, Debug, PartialEq)] -pub struct AvailabilityPendingBlock { - block: Arc>, -} - -impl AvailabilityPendingBlock { - pub fn slot(&self) -> Slot { - self.block.slot() - } - pub fn num_blobs_expected(&self) -> usize { - self.block.num_expected_blobs() - } - - pub fn get_all_blob_ids(&self, block_root: Option) -> Vec { - self.block.get_expected_blob_ids(block_root) - } - - pub fn get_filtered_blob_ids( - &self, - block_root: Option, - filter: impl Fn(usize, Hash256) -> bool, - ) -> Vec { - self.block.get_filtered_blob_ids(block_root, filter) - } -} - -impl AvailabilityPendingBlock { - pub fn to_block(self) -> Arc> { - self.block - } - pub fn as_block(&self) -> &SignedBeaconBlock { - &self.block - } - pub fn block_cloned(&self) -> Arc> { - self.block.clone() - } - pub fn kzg_commitments(&self) -> Result<&KzgCommitments, AvailabilityCheckError> { - self.block - .message() - .body() - .blob_kzg_commitments() - .map_err(|_| AvailabilityCheckError::IncorrectFork) - } - - /// Verifies an AvailabilityPendingBlock against a set of KZG verified blobs. - /// This does not check whether a block *should* have blobs, these checks should have been - /// completed when producing the `AvailabilityPendingBlock`. - pub fn make_available( - self, - blobs: Vec>, - ) -> Result, AvailabilityCheckError> { - let block_kzg_commitments = self.kzg_commitments()?; - if blobs.len() != block_kzg_commitments.len() { - return Err(AvailabilityCheckError::NumBlobsMismatch { - num_kzg_commitments: block_kzg_commitments.len(), - num_blobs: blobs.len(), - }); - } - - for (block_commitment, blob) in block_kzg_commitments.iter().zip(blobs.iter()) { - if *block_commitment != blob.kzg_commitment() { - return Err(AvailabilityCheckError::KzgCommitmentMismatch { - blob_index: blob.as_blob().index, - }); - } - } - - let blobs = VariableList::new(blobs.into_iter().map(|blob| blob.to_blob()).collect())?; - - Ok(AvailableBlock { - block: self.block, - blobs: VerifiedBlobs::Available(blobs), - }) - } -} - -#[derive(Clone, Debug, PartialEq)] -pub enum VerifiedBlobs { - /// These blobs are available. - Available(BlobSidecarList), - /// This block is from outside the data availability boundary so doesn't require - /// a data availability check. - NotRequired, - /// The block's `kzg_commitments` field is empty so it does not contain any blobs. - EmptyBlobs, - /// This is a block prior to the 4844 fork, so doesn't require any blobs - PreDeneb, -} - -impl VerifiedBlobs { - pub fn to_blobs(self) -> Option> { - match self { - Self::Available(blobs) => Some(blobs), - Self::NotRequired => None, - Self::EmptyBlobs => None, - Self::PreDeneb => None, - } - } -} - /// A fully available block that is ready to be imported into fork choice. #[derive(Clone, Debug, PartialEq)] pub struct AvailableBlock { block: Arc>, - blobs: VerifiedBlobs, + blobs: Option>, } impl AvailableBlock { pub fn block(&self) -> &SignedBeaconBlock { &self.block } - - pub fn da_check_required(&self) -> bool { - match self.blobs { - VerifiedBlobs::PreDeneb | VerifiedBlobs::NotRequired => false, - VerifiedBlobs::EmptyBlobs | VerifiedBlobs::Available(_) => true, - } - } - - pub fn deconstruct(self) -> (Arc>, Option>) { - match self.blobs { - VerifiedBlobs::EmptyBlobs | VerifiedBlobs::NotRequired | VerifiedBlobs::PreDeneb => { - (self.block, None) - } - VerifiedBlobs::Available(blobs) => (self.block, Some(blobs)), - } - } - - pub fn blobs(&self) -> Option<&BlobSidecarList> { - match &self.blobs { - VerifiedBlobs::Available(blobs) => Some(blobs), - _ => None, - } - } -} - -impl AsBlock for AvailableBlock { - fn slot(&self) -> Slot { - self.block.slot() - } - - fn epoch(&self) -> Epoch { - self.block.epoch() - } - - fn parent_root(&self) -> Hash256 { - self.block.parent_root() - } - - fn state_root(&self) -> Hash256 { - self.block.state_root() - } - - fn signed_block_header(&self) -> SignedBeaconBlockHeader { - self.block.signed_block_header() - } - - fn message(&self) -> BeaconBlockRef { - self.block.message() - } - - fn as_block(&self) -> &SignedBeaconBlock { - &self.block - } - - fn block_cloned(&self) -> Arc> { + pub fn block_cloned(&self) -> Arc> { self.block.clone() } - fn canonical_root(&self) -> Hash256 { - self.block.canonical_root() - } - - fn into_block_wrapper(self) -> BlockWrapper { - let (block, blobs_opt) = self.deconstruct(); - if let Some(blobs) = blobs_opt { - let blobs_vec = blobs.iter().cloned().map(Option::Some).collect::>(); - BlockWrapper::BlockAndBlobs(block, FixedVector::from(blobs_vec)) - } else { - BlockWrapper::Block(block) - } - } -} - -// The standard implementation of Encode for SignedBeaconBlock -// requires us to use ssz(enum_behaviour = "transparent"). This -// prevents us from implementing Decode. We need to use a -// custom Encode and Decode in this wrapper object that essentially -// encodes it as if it were ssz(enum_behaviour = "union") -impl ssz::Encode for AvailabilityPendingBlock { - fn is_ssz_fixed_len() -> bool { - ssz_tagged_signed_beacon_block::encode::is_ssz_fixed_len() - } - - fn ssz_append(&self, buf: &mut Vec) { - ssz_tagged_signed_beacon_block::encode::ssz_append(self.block.as_ref(), buf); - } - - fn ssz_bytes_len(&self) -> usize { - ssz_tagged_signed_beacon_block::encode::ssz_bytes_len(self.block.as_ref()) - } -} - -impl ssz::Decode for AvailabilityPendingBlock { - fn is_ssz_fixed_len() -> bool { - ssz_tagged_signed_beacon_block::decode::is_ssz_fixed_len() + pub fn blobs(&self) -> Option<&BlobSidecarList> { + self.blobs.as_ref() } - fn from_ssz_bytes(bytes: &[u8]) -> Result { - Ok(Self { - block: Arc::new(ssz_tagged_signed_beacon_block::decode::from_ssz_bytes( - bytes, - )?), - }) + pub fn deconstruct(self) -> (Arc>, Option>) { + let AvailableBlock { block, blobs } = self; + (block, blobs) } } -#[cfg(test)] -mod test { - #[test] - fn check_encode_decode_availability_pending_block() { - // todo.. (difficult to create default beacon blocks to test) - } +#[derive(Debug, Clone)] +pub enum MaybeAvailableBlock { + /// This variant is fully available. + /// i.e. for pre-deneb blocks, it contains a (`SignedBeaconBlock`, `Blobs::None`) and for + /// post-4844 blocks, it contains a `SignedBeaconBlock` and a Blobs variant other than `Blobs::None`. + Available(AvailableBlock), + /// This variant is not fully available and requires blobs to become fully available. + AvailabilityPending(Arc>), } diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index cfc2f95760a..e08f1994d11 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -29,8 +29,10 @@ use crate::beacon_chain::BeaconStore; use crate::blob_verification::KzgVerifiedBlob; -use crate::block_verification::{AvailabilityPendingExecutedBlock, AvailableExecutedBlock}; -use crate::data_availability_checker::{Availability, AvailabilityCheckError}; +use crate::block_verification_types::{ + AsBlock, AvailabilityPendingExecutedBlock, AvailableExecutedBlock, +}; +use crate::data_availability_checker::{make_available, Availability, AvailabilityCheckError}; use crate::store::{DBColumn, KeyValueStore}; use crate::BeaconChainTypes; use lru::LruCache; @@ -102,7 +104,7 @@ impl PendingComponents { pub fn epoch(&self) -> Option { self.executed_block .as_ref() - .map(|pending_block| pending_block.block.as_block().epoch()) + .map(|pending_block| pending_block.block.epoch()) .or_else(|| { for maybe_blob in self.verified_blobs.iter() { if maybe_blob.is_some() { @@ -119,7 +121,7 @@ impl PendingComponents { let block_opt = self .executed_block .as_ref() - .map(|block| block.block.block.clone()); + .map(|block| block.block.clone()); let blobs = self .verified_blobs .iter() @@ -538,7 +540,7 @@ impl OverflowLRUCache { import_data, payload_verification_outcome, } = executed_block; - let available_block = block.make_available(vec![])?; + let available_block = make_available(block, vec![])?; return Ok(Availability::Available(Box::new( AvailableExecutedBlock::new( available_block, @@ -588,7 +590,7 @@ impl OverflowLRUCache { return Ok(Availability::MissingComponents(import_data.block_root)) }; - let available_block = block.make_available(verified_blobs)?; + let available_block = make_available(block, verified_blobs)?; Ok(Availability::Available(Box::new( AvailableExecutedBlock::new( available_block, @@ -758,7 +760,6 @@ impl OverflowLRUCache { value_bytes.as_slice(), )? .block - .as_block() .epoch() } OverflowKey::Blob(_, _) => { @@ -853,8 +854,8 @@ mod test { blob_verification::{ validate_blob_sidecar_for_gossip, verify_kzg_for_blob, GossipVerifiedBlob, }, - block_verification::{BlockImportData, PayloadVerificationOutcome}, - data_availability_checker::AvailabilityPendingBlock, + block_verification::PayloadVerificationOutcome, + block_verification_types::BlockImportData, eth1_finalization_cache::Eth1FinalizationData, test_utils::{BaseHarnessType, BeaconChainHarness, DiskHarnessType}, }; @@ -1129,10 +1130,6 @@ mod test { }; let slot = block.slot(); - let apb: AvailabilityPendingBlock = AvailabilityPendingBlock { - block: Arc::new(block), - }; - let consensus_context = ConsensusContext::::new(slot); let import_data: BlockImportData = BlockImportData { block_root, @@ -1149,7 +1146,7 @@ mod test { }; let availability_pending_block = AvailabilityPendingExecutedBlock { - block: apb, + block: Arc::new(block), import_data, payload_verification_outcome, }; @@ -1301,7 +1298,7 @@ mod test { // we need blocks with blobs continue; } - let root = pending_block.block.block.canonical_root(); + let root = pending_block.block.canonical_root(); pending_blocks.push_back(pending_block); pending_blobs.push_back(blobs); roots.push_back(root); @@ -1462,7 +1459,7 @@ mod test { // we need blocks with blobs continue; } - let root = pending_block.block.as_block().canonical_root(); + let root = pending_block.block.canonical_root(); let epoch = pending_block .block .as_block() diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 3f1ad7581ae..3410196c8c4 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -12,6 +12,7 @@ pub mod blob_verification; pub mod block_reward; mod block_times_cache; mod block_verification; +pub mod block_verification_types; pub mod builder; pub mod canonical_head; pub mod capella_readiness; @@ -69,10 +70,12 @@ pub use self::historical_blocks::HistoricalBlockError; pub use attestation_verification::Error as AttestationError; pub use beacon_fork_choice_store::{BeaconForkChoiceStore, Error as ForkChoiceStoreError}; pub use block_verification::{ - get_block_root, AvailabilityPendingExecutedBlock, BlockError, ExecutedBlock, - ExecutionPayloadError, ExecutionPendingBlock, GossipVerifiedBlock, IntoExecutionPendingBlock, - IntoGossipVerifiedBlockContents, PayloadVerificationOutcome, PayloadVerificationStatus, + get_block_root, BlockError, ExecutionPayloadError, ExecutionPendingBlock, GossipVerifiedBlock, + IntoExecutionPendingBlock, IntoGossipVerifiedBlockContents, PayloadVerificationOutcome, + PayloadVerificationStatus, }; +pub use block_verification_types::AvailabilityPendingExecutedBlock; +pub use block_verification_types::ExecutedBlock; pub use canonical_head::{CachedHead, CanonicalHead, CanonicalHeadRwLock}; pub use eth1_chain::{Eth1Chain, Eth1ChainBackend}; pub use events::ServerSentEventHandler; diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 147a0c3baec..286aa0c0a08 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1,4 +1,4 @@ -use crate::blob_verification::{AsBlock, BlockWrapper}; +use crate::block_verification_types::{AsBlock, RpcBlock}; use crate::observed_operations::ObservationOutcome; pub use crate::persisted_beacon_chain::PersistedBeaconChain; pub use crate::{ @@ -694,18 +694,18 @@ where .execution_block_generator() } - pub fn get_head_block(&self) -> BlockWrapper { + pub fn get_head_block(&self) -> RpcBlock { let block = self.chain.head_beacon_block(); let block_root = block.canonical_root(); let blobs = self.chain.get_blobs(&block_root).unwrap(); - BlockWrapper::new(block, blobs) + RpcBlock::new(block, blobs).unwrap() } - pub fn get_full_block(&self, block_root: &Hash256) -> BlockWrapper { + pub fn get_full_block(&self, block_root: &Hash256) -> RpcBlock { let block = self.chain.get_blinded_block(block_root).unwrap().unwrap(); let full_block = self.chain.store.make_full_block(block_root, block).unwrap(); let blobs = self.chain.get_blobs(block_root).unwrap(); - BlockWrapper::new(Arc::new(full_block), blobs) + RpcBlock::new(Arc::new(full_block), blobs).unwrap() } pub fn get_all_validators(&self) -> Vec { @@ -1873,18 +1873,31 @@ where (deposits, state) } - pub async fn process_block>>( + pub async fn process_block( &self, slot: Slot, block_root: Hash256, - block: B, + block_contents: BlockContentsTuple>, ) -> Result> { self.set_current_slot(slot); + let (block, blobs) = block_contents; + // Note: we are just dropping signatures here and skipping signature verification. + let blobs_without_signatures = blobs.as_ref().map(|blobs| { + VariableList::from( + blobs + .into_iter() + .map(|blob| blob.message.clone()) + .collect::>(), + ) + }); let block_hash: SignedBeaconBlockHash = self .chain - .process_block(block_root, block.into(), NotifyExecutionLayer::Yes, || { - Ok(()) - }) + .process_block( + block_root, + RpcBlock::new(Arc::new(block), blobs_without_signatures).unwrap(), + NotifyExecutionLayer::Yes, + || Ok(()), + ) .await? .try_into() .unwrap(); @@ -1892,16 +1905,25 @@ where Ok(block_hash) } - pub async fn process_block_result>>( + pub async fn process_block_result( &self, - block: B, + block_contents: BlockContentsTuple>, ) -> Result> { - let wrapped_block = block.into(); + let (block, blobs) = block_contents; + // Note: we are just dropping signatures here and skipping signature verification. + let blobs_without_signatures = blobs.as_ref().map(|blobs| { + VariableList::from( + blobs + .into_iter() + .map(|blob| blob.message.clone()) + .collect::>(), + ) + }); let block_hash: SignedBeaconBlockHash = self .chain .process_block( - wrapped_block.canonical_root(), - wrapped_block, + block.canonical_root(), + RpcBlock::new(Arc::new(block), blobs_without_signatures).unwrap(), NotifyExecutionLayer::Yes, || Ok(()), ) @@ -1976,11 +1998,16 @@ where BlockError, > { self.set_current_slot(slot); - let (block, new_state) = self.make_block(state, slot).await; + let (block_contents, new_state) = self.make_block(state, slot).await; + let block_hash = self - .process_block(slot, block.0.canonical_root(), block.clone()) + .process_block( + slot, + block_contents.0.canonical_root(), + block_contents.clone(), + ) .await?; - Ok((block_hash, block, new_state)) + Ok((block_hash, block_contents, new_state)) } pub fn attest_block( diff --git a/beacon_node/beacon_chain/tests/attestation_production.rs b/beacon_node/beacon_chain/tests/attestation_production.rs index 97122c00043..907e7a40bb5 100644 --- a/beacon_node/beacon_chain/tests/attestation_production.rs +++ b/beacon_node/beacon_chain/tests/attestation_production.rs @@ -1,6 +1,6 @@ #![cfg(not(debug_assertions))] -use beacon_chain::blob_verification::BlockWrapper; +use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy}; use beacon_chain::{StateSkipConfig, WhenSlotSkipped}; use lazy_static::lazy_static; @@ -133,11 +133,11 @@ async fn produces_attestations() { assert_eq!(data.target.epoch, state.current_epoch(), "bad target epoch"); assert_eq!(data.target.root, target_root, "bad target root"); - let block_wrapper = - BlockWrapper::::new(Arc::new(block.clone()), blobs.clone()); - let beacon_chain::blob_verification::MaybeAvailableBlock::Available(available_block) = chain + let rpc_block = + RpcBlock::::new(Arc::new(block.clone()), blobs.clone()).unwrap(); + let beacon_chain::data_availability_checker::MaybeAvailableBlock::Available(available_block) = chain .data_availability_checker - .check_availability(block_wrapper) + .check_rpc_block_availability(rpc_block) .unwrap() else { panic!("block should be available") @@ -209,10 +209,10 @@ async fn early_attester_cache_old_request() { .get_blobs(&head.beacon_block_root) .expect("should get blobs"); - let block_wrapper = BlockWrapper::::new(head.beacon_block.clone(), head_blobs); - let beacon_chain::blob_verification::MaybeAvailableBlock::Available(available_block) = harness.chain + let rpc_block = RpcBlock::::new(head.beacon_block.clone(), head_blobs).unwrap(); + let beacon_chain::data_availability_checker::MaybeAvailableBlock::Available(available_block) = harness.chain .data_availability_checker - .check_availability(block_wrapper) + .check_rpc_block_availability(rpc_block) .unwrap() else { panic!("block should be available") diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 37479236459..102707a3892 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -1,12 +1,10 @@ #![cfg(not(debug_assertions))] -use beacon_chain::blob_verification::BlockWrapper; +use beacon_chain::block_verification_types::{AsBlock, ExecutedBlock, RpcBlock}; use beacon_chain::test_utils::BlobSignatureKey; use beacon_chain::{ - blob_verification::AsBlock, test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType}, - AvailabilityProcessingStatus, BeaconChain, BeaconChainTypes, ExecutedBlock, - ExecutionPendingBlock, + AvailabilityProcessingStatus, BeaconChain, BeaconChainTypes, ExecutionPendingBlock, }; use beacon_chain::{ BeaconSnapshot, BlockError, ChainSegmentResult, IntoExecutionPendingBlock, NotifyExecutionLayer, @@ -156,11 +154,13 @@ fn get_harness(validator_count: usize) -> BeaconChainHarness], blobs: &[Option>], -) -> Vec> { +) -> Vec> { chain_segment .iter() .zip(blobs.into_iter()) - .map(|(snapshot, blobs)| BlockWrapper::new(snapshot.beacon_block.clone(), blobs.clone())) + .map(|(snapshot, blobs)| { + RpcBlock::new(snapshot.beacon_block.clone(), blobs.clone()).unwrap() + }) .collect() } @@ -217,7 +217,7 @@ fn update_parent_roots(snapshots: &mut [BeaconSnapshot]) { async fn chain_segment_full_segment() { let harness = get_harness(VALIDATOR_COUNT); let (chain_segment, chain_segment_blobs) = get_chain_segment().await; - let blocks: Vec> = chain_segment_blocks(&chain_segment, &chain_segment_blobs) + let blocks: Vec> = chain_segment_blocks(&chain_segment, &chain_segment_blobs) .into_iter() .map(|block| block.into()) .collect(); @@ -256,11 +256,10 @@ async fn chain_segment_varying_chunk_size() { for chunk_size in &[1, 2, 3, 5, 31, 32, 33, 42] { let harness = get_harness(VALIDATOR_COUNT); let (chain_segment, chain_segment_blobs) = get_chain_segment().await; - let blocks: Vec> = - chain_segment_blocks(&chain_segment, &chain_segment_blobs) - .into_iter() - .map(|block| block.into()) - .collect(); + let blocks: Vec> = chain_segment_blocks(&chain_segment, &chain_segment_blobs) + .into_iter() + .map(|block| block.into()) + .collect(); harness .chain @@ -299,11 +298,10 @@ async fn chain_segment_non_linear_parent_roots() { /* * Test with a block removed. */ - let mut blocks: Vec> = - chain_segment_blocks(&chain_segment, &chain_segment_blobs) - .into_iter() - .map(|block| block.into()) - .collect(); + let mut blocks: Vec> = chain_segment_blocks(&chain_segment, &chain_segment_blobs) + .into_iter() + .map(|block| block.into()) + .collect(); blocks.remove(2); assert!( @@ -321,11 +319,10 @@ async fn chain_segment_non_linear_parent_roots() { /* * Test with a modified parent root. */ - let mut blocks: Vec> = - chain_segment_blocks(&chain_segment, &chain_segment_blobs) - .into_iter() - .map(|block| block.into()) - .collect(); + let mut blocks: Vec> = chain_segment_blocks(&chain_segment, &chain_segment_blobs) + .into_iter() + .map(|block| block.into()) + .collect(); let (mut block, signature) = blocks[3].as_block().clone().deconstruct(); *block.parent_root_mut() = Hash256::zero(); @@ -357,11 +354,10 @@ async fn chain_segment_non_linear_slots() { * Test where a child is lower than the parent. */ - let mut blocks: Vec> = - chain_segment_blocks(&chain_segment, &chain_segment_blobs) - .into_iter() - .map(|block| block.into()) - .collect(); + let mut blocks: Vec> = chain_segment_blocks(&chain_segment, &chain_segment_blobs) + .into_iter() + .map(|block| block.into()) + .collect(); let (mut block, signature) = blocks[3].as_block().clone().deconstruct(); *block.slot_mut() = Slot::new(0); blocks[3] = Arc::new(SignedBeaconBlock::from_block(block, signature)).into(); @@ -382,11 +378,10 @@ async fn chain_segment_non_linear_slots() { * Test where a child is equal to the parent. */ - let mut blocks: Vec> = - chain_segment_blocks(&chain_segment, &chain_segment_blobs) - .into_iter() - .map(|block| block.into()) - .collect(); + let mut blocks: Vec> = chain_segment_blocks(&chain_segment, &chain_segment_blobs) + .into_iter() + .map(|block| block.into()) + .collect(); let (mut block, signature) = blocks[3].as_block().clone().deconstruct(); *block.slot_mut() = blocks[2].slot(); blocks[3] = Arc::new(SignedBeaconBlock::from_block(block, signature)).into(); @@ -412,10 +407,12 @@ async fn assert_invalid_signature( snapshots: &[BeaconSnapshot], item: &str, ) { - let blocks: Vec> = snapshots + let blocks: Vec> = snapshots .iter() .zip(chain_segment_blobs.iter()) - .map(|(snapshot, blobs)| BlockWrapper::new(snapshot.beacon_block.clone(), blobs.clone())) + .map(|(snapshot, blobs)| { + RpcBlock::new(snapshot.beacon_block.clone(), blobs.clone()).unwrap() + }) .collect(); // Ensure the block will be rejected if imported in a chain segment. @@ -440,7 +437,9 @@ async fn assert_invalid_signature( .iter() .take(block_index) .zip(chain_segment_blobs.iter()) - .map(|(snapshot, blobs)| BlockWrapper::new(snapshot.beacon_block.clone(), blobs.clone())) + .map(|(snapshot, blobs)| { + RpcBlock::new(snapshot.beacon_block.clone(), blobs.clone()).unwrap() + }) .collect(); // We don't care if this fails, we just call this to ensure that all prior blocks have been // imported prior to this test. @@ -454,10 +453,11 @@ async fn assert_invalid_signature( .chain .process_block( snapshots[block_index].beacon_block.canonical_root(), - BlockWrapper::new( + RpcBlock::new( snapshots[block_index].beacon_block.clone(), chain_segment_blobs[block_index].clone(), - ), + ) + .unwrap(), NotifyExecutionLayer::Yes, || Ok(()), ) @@ -509,7 +509,7 @@ async fn invalid_signature_gossip_block() { .take(block_index) .zip(chain_segment_blobs.iter()) .map(|(snapshot, blobs)| { - BlockWrapper::new(snapshot.beacon_block.clone(), blobs.clone()) + RpcBlock::new(snapshot.beacon_block.clone(), blobs.clone()).unwrap() }) .collect(); harness @@ -552,11 +552,11 @@ async fn invalid_signature_block_proposal() { block.clone(), junk_signature(), )); - let blocks: Vec> = snapshots + let blocks: Vec> = snapshots .iter() .zip(chain_segment_blobs.iter()) .map(|(snapshot, blobs)| { - BlockWrapper::new(snapshot.beacon_block.clone(), blobs.clone()) + RpcBlock::new(snapshot.beacon_block.clone(), blobs.clone()).unwrap() }) .collect::>(); // Ensure the block will be rejected if imported in a chain segment. @@ -765,11 +765,11 @@ async fn invalid_signature_deposit() { Arc::new(SignedBeaconBlock::from_block(block, signature)); update_parent_roots(&mut snapshots); update_proposal_signatures(&mut snapshots, &harness); - let blocks: Vec> = snapshots + let blocks: Vec> = snapshots .iter() .zip(chain_segment_blobs.iter()) .map(|(snapshot, blobs)| { - BlockWrapper::new(snapshot.beacon_block.clone(), blobs.clone()) + RpcBlock::new(snapshot.beacon_block.clone(), blobs.clone()).unwrap() }) .collect(); assert!( diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index 60ba8f288e2..fb7cc516ff3 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -221,7 +221,7 @@ impl InvalidPayloadRig { let head = self.harness.chain.head_snapshot(); let state = head.beacon_state.clone_with_only_committee_caches(); let slot = slot_override.unwrap_or(state.slot() + 1); - let ((block, _), post_state) = self.harness.make_block(state, slot).await; + let ((block, blobs), post_state) = self.harness.make_block(state, slot).await; let block_root = block.canonical_root(); let set_new_payload = |payload: Payload| match payload { @@ -285,7 +285,7 @@ impl InvalidPayloadRig { } let root = self .harness - .process_block(slot, block.canonical_root(), block.clone()) + .process_block(slot, block.canonical_root(), (block.clone(), blobs.clone())) .await .unwrap(); @@ -326,7 +326,7 @@ impl InvalidPayloadRig { match self .harness - .process_block(slot, block.canonical_root(), block) + .process_block(slot, block.canonical_root(), (block, blobs)) .await { Err(error) if evaluate_error(&error) => (), diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index 7a86b5f9382..18f1cbd7c26 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -1,7 +1,7 @@ #![cfg(not(debug_assertions))] use beacon_chain::attestation_verification::Error as AttnError; -use beacon_chain::blob_verification::BlockWrapper; +use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::builder::BeaconChainBuilder; use beacon_chain::schema_change::migrate_schema; use beacon_chain::test_utils::{ @@ -10,7 +10,7 @@ use beacon_chain::test_utils::{ }; use beacon_chain::validator_monitor::DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD; use beacon_chain::{ - blob_verification::MaybeAvailableBlock, historical_blocks::HistoricalBlockError, + data_availability_checker::MaybeAvailableBlock, historical_blocks::HistoricalBlockError, migrate::MigratorConfig, BeaconChain, BeaconChainError, BeaconChainTypes, BeaconSnapshot, ChainConfig, NotifyExecutionLayer, ServerSentEventHandler, WhenSlotSkipped, }; @@ -2039,7 +2039,10 @@ async fn garbage_collect_temp_states_from_failed_block() { // The block should be rejected, but should store a bunch of temporary states. harness.set_current_slot(block_slot); - harness.process_block_result(block).await.unwrap_err(); + harness + .process_block_result((block, None)) + .await + .unwrap_err(); assert_eq!( store.iter_temporary_state_roots().count(), @@ -2176,7 +2179,7 @@ async fn weak_subjectivity_sync() { beacon_chain .process_block( full_block.canonical_root(), - BlockWrapper::new(Arc::new(full_block), blobs), + RpcBlock::new(Arc::new(full_block), blobs).unwrap(), NotifyExecutionLayer::Yes, || Ok(()), ) @@ -2236,7 +2239,7 @@ async fn weak_subjectivity_sync() { if let MaybeAvailableBlock::Available(block) = harness .chain .data_availability_checker - .check_availability(BlockWrapper::new(Arc::new(full_block), blobs)) + .check_rpc_block_availability(RpcBlock::new(Arc::new(full_block), blobs).unwrap()) .expect("should check availability") { available_blocks.push(block); @@ -2456,14 +2459,14 @@ async fn revert_minority_fork_on_resume() { harness1.process_attestations(attestations.clone()); harness2.process_attestations(attestations); - let ((block, _), new_state) = harness1.make_block(state, slot).await; + let ((block, blobs), new_state) = harness1.make_block(state, slot).await; harness1 - .process_block(slot, block.canonical_root(), block.clone()) + .process_block(slot, block.canonical_root(), (block.clone(), blobs.clone())) .await .unwrap(); harness2 - .process_block(slot, block.canonical_root(), block.clone()) + .process_block(slot, block.canonical_root(), (block.clone(), blobs.clone())) .await .unwrap(); @@ -2497,17 +2500,17 @@ async fn revert_minority_fork_on_resume() { harness2.process_attestations(attestations); // Minority chain block (no attesters). - let ((block1, _), new_state1) = harness1.make_block(state1, slot).await; + let ((block1, blobs1), new_state1) = harness1.make_block(state1, slot).await; harness1 - .process_block(slot, block1.canonical_root(), block1) + .process_block(slot, block1.canonical_root(), (block1, blobs1)) .await .unwrap(); state1 = new_state1; // Majority chain block (all attesters). - let ((block2, _), new_state2) = harness2.make_block(state2, slot).await; + let ((block2, blobs2), new_state2) = harness2.make_block(state2, slot).await; harness2 - .process_block(slot, block2.canonical_root(), block2.clone()) + .process_block(slot, block2.canonical_root(), (block2.clone(), blobs2)) .await .unwrap(); @@ -2560,7 +2563,7 @@ async fn revert_minority_fork_on_resume() { let initial_split_slot = resumed_harness.chain.store.get_split_slot(); for block in &majority_blocks { resumed_harness - .process_block_result(block.clone()) + .process_block_result((block.clone(), None)) .await .unwrap(); diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 54251608105..38c40b890db 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -1,6 +1,6 @@ use crate::metrics; -use beacon_chain::blob_verification::AsBlock; +use beacon_chain::block_verification_types::AsBlock; use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now}; use beacon_chain::{ AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 40f21f727a8..f27a4d9519d 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -636,7 +636,7 @@ pub async fn proposer_boost_re_org_test( // Applying block C should cause it to become head regardless (re-org or continuation). let block_root_c = harness - .process_block_result(block_c.clone()) + .process_block_result((block_c.clone(), None)) .await .unwrap() .into(); diff --git a/beacon_node/http_api/tests/status_tests.rs b/beacon_node/http_api/tests/status_tests.rs index 664c5f2ea53..b3d1e9daa85 100644 --- a/beacon_node/http_api/tests/status_tests.rs +++ b/beacon_node/http_api/tests/status_tests.rs @@ -124,7 +124,7 @@ async fn el_error_on_new_payload() { // Attempt to process the block, which should error. harness.advance_slot(); assert!(matches!( - harness.process_block_result(block.clone()).await, + harness.process_block_result((block.clone(), None)).await, Err(BlockError::ExecutionPayloadError(_)) )); @@ -143,7 +143,7 @@ async fn el_error_on_new_payload() { validation_error: None, }, ); - harness.process_block_result(block).await.unwrap(); + harness.process_block_result((block, None)).await.unwrap(); let api_response = tester.client.get_node_syncing().await.unwrap().data; assert_eq!(api_response.el_offline, Some(false)); diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 000c4d85dc1..2255b401703 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -5,8 +5,8 @@ use crate::{ sync::SyncMessage, }; -use beacon_chain::blob_verification::AsBlock; use beacon_chain::blob_verification::{BlobError, GossipVerifiedBlob}; +use beacon_chain::block_verification_types::AsBlock; use beacon_chain::store::Error; use beacon_chain::{ attestation_verification::{self, Error as AttnError, VerifiedAttestation}, diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 8e44aba15ab..4a214c3637a 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -2,7 +2,7 @@ use crate::{ service::NetworkMessage, sync::{manager::BlockProcessType, SyncMessage}, }; -use beacon_chain::blob_verification::BlockWrapper; +use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::{ builder::Witness, eth1_chain::CachingEth1Backend, test_utils::BeaconChainHarness, BeaconChain, }; @@ -409,7 +409,7 @@ impl NetworkBeaconProcessor { pub fn send_rpc_beacon_block( self: &Arc, block_root: Hash256, - block: BlockWrapper, + block: RpcBlock, seen_timestamp: Duration, process_type: BlockProcessType, ) -> Result<(), Error> { @@ -450,7 +450,7 @@ impl NetworkBeaconProcessor { pub fn send_chain_segment( self: &Arc, process_id: ChainSegmentProcessId, - blocks: Vec>, + blocks: Vec>, ) -> Result<(), Error> { let is_backfill = matches!(&process_id, ChainSegmentProcessId::BackSyncBatchId { .. }); let processor = self.clone(); diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index 8d9146e6888..b21bc6abde8 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -6,8 +6,9 @@ use crate::sync::{ manager::{BlockProcessType, SyncMessage}, ChainId, }; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper, MaybeAvailableBlock}; +use beacon_chain::block_verification_types::{AsBlock, RpcBlock}; use beacon_chain::data_availability_checker::AvailabilityCheckError; +use beacon_chain::data_availability_checker::MaybeAvailableBlock; use beacon_chain::{ observed_block_producers::Error as ObserveError, validator_monitor::get_block_delay_ms, AvailabilityProcessingStatus, BeaconChainError, BeaconChainTypes, BlockError, @@ -54,7 +55,7 @@ impl NetworkBeaconProcessor { pub fn generate_rpc_beacon_block_process_fn( self: Arc, block_root: Hash256, - block: BlockWrapper, + block: RpcBlock, seen_timestamp: Duration, process_type: BlockProcessType, ) -> AsyncFn { @@ -78,7 +79,7 @@ impl NetworkBeaconProcessor { pub fn generate_rpc_beacon_block_fns( self: Arc, block_root: Hash256, - block: BlockWrapper, + block: RpcBlock, seen_timestamp: Duration, process_type: BlockProcessType, ) -> (AsyncFn, BlockingFn) { @@ -106,7 +107,7 @@ impl NetworkBeaconProcessor { pub async fn process_rpc_block( self: Arc>, block_root: Hash256, - block: BlockWrapper, + block: RpcBlock, seen_timestamp: Duration, process_type: BlockProcessType, reprocess_tx: mpsc::Sender, @@ -315,7 +316,7 @@ impl NetworkBeaconProcessor { pub async fn process_chain_segment( &self, sync_type: ChainSegmentProcessId, - downloaded_blocks: Vec>, + downloaded_blocks: Vec>, notify_execution_layer: NotifyExecutionLayer, ) { let result = match sync_type { @@ -440,7 +441,7 @@ impl NetworkBeaconProcessor { /// Helper function to process blocks batches which only consumes the chain and blocks to process. async fn process_blocks<'a>( &self, - downloaded_blocks: impl Iterator>, + downloaded_blocks: impl Iterator>, notify_execution_layer: NotifyExecutionLayer, ) -> (usize, Result<(), ChainSegmentFailed>) { let blocks: Vec<_> = downloaded_blocks.cloned().collect(); @@ -473,7 +474,7 @@ impl NetworkBeaconProcessor { /// Helper function to process backfill block batches which only consumes the chain and blocks to process. fn process_backfill_blocks( &self, - downloaded_blocks: Vec>, + downloaded_blocks: Vec>, ) -> (usize, Result<(), ChainSegmentFailed>) { let total_blocks = downloaded_blocks.len(); let available_blocks = match downloaded_blocks @@ -481,7 +482,7 @@ impl NetworkBeaconProcessor { .map(|block| { self.chain .data_availability_checker - .check_availability(block) + .check_rpc_block_availability(block) }) .collect::, _>>() { diff --git a/beacon_node/network/src/sync/backfill_sync/mod.rs b/beacon_node/network/src/sync/backfill_sync/mod.rs index e5a1428005a..0900e034f00 100644 --- a/beacon_node/network/src/sync/backfill_sync/mod.rs +++ b/beacon_node/network/src/sync/backfill_sync/mod.rs @@ -14,7 +14,7 @@ use crate::sync::network_context::SyncNetworkContext; use crate::sync::range_sync::{ BatchConfig, BatchId, BatchInfo, BatchOperationOutcome, BatchProcessingResult, BatchState, }; -use beacon_chain::blob_verification::BlockWrapper; +use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::{BeaconChain, BeaconChainTypes}; use lighthouse_network::types::{BackFillState, NetworkGlobals}; use lighthouse_network::{PeerAction, PeerId}; @@ -55,7 +55,7 @@ impl BatchConfig for BackFillBatchConfig { fn max_batch_processing_attempts() -> u8 { MAX_BATCH_PROCESSING_ATTEMPTS } - fn batch_attempt_hash(blocks: &[BlockWrapper]) -> u64 { + fn batch_attempt_hash(blocks: &[RpcBlock]) -> u64 { use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; let mut hasher = DefaultHasher::new(); @@ -392,7 +392,7 @@ impl BackFillSync { batch_id: BatchId, peer_id: &PeerId, request_id: Id, - beacon_block: Option>, + beacon_block: Option>, ) -> Result { // check if we have this batch let batch = match self.batches.get_mut(&batch_id) { diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index dfe960832a5..ff095c719ea 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -10,7 +10,7 @@ use super::{ use crate::metrics; use crate::network_beacon_processor::ChainSegmentProcessId; use crate::sync::block_lookups::single_block_lookup::LookupId; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; +use beacon_chain::block_verification_types::{AsBlock, RpcBlock}; use beacon_chain::data_availability_checker::{AvailabilityCheckError, DataAvailabilityChecker}; use beacon_chain::{AvailabilityProcessingStatus, BeaconChainTypes, BlockError}; use lighthouse_network::rpc::RPCError; @@ -34,7 +34,7 @@ mod single_block_lookup; #[cfg(test)] mod tests; -pub type DownloadedBlocks = (Hash256, BlockWrapper); +pub type DownloadedBlocks = (Hash256, RpcBlock); pub type RootBlockTuple = (Hash256, Arc>); pub type RootBlobsTuple = (Hash256, FixedBlobSidecarList); @@ -381,7 +381,7 @@ impl BlockLookups { if !has_pending_parent_request { let rpc_block = request_ref .get_downloaded_block() - .unwrap_or(BlockWrapper::Block(block)); + .unwrap_or(RpcBlock::new_without_blobs(block)); // This is the correct block, send it for processing match self.send_block_for_processing( block_root, @@ -910,11 +910,7 @@ impl BlockLookups { BlockError::ParentUnknown(block) => { let slot = block.slot(); let parent_root = block.parent_root(); - let (block, blobs) = block.deconstruct(); - request_ref.add_unknown_parent_components(UnknownParentComponents::new( - Some(block), - blobs, - )); + request_ref.add_unknown_parent_components(block.into()); self.search_parent(slot, root, parent_root, peer_id.to_peer_id(), cx); ShouldRemoveLookup::False } @@ -1226,7 +1222,7 @@ impl BlockLookups { fn send_block_for_processing( &mut self, block_root: Hash256, - block: BlockWrapper, + block: RpcBlock, duration: Duration, process_type: BlockProcessType, cx: &mut SyncNetworkContext, diff --git a/beacon_node/network/src/sync/block_lookups/parent_lookup.rs b/beacon_node/network/src/sync/block_lookups/parent_lookup.rs index 5175450d9e0..6d870b5aba3 100644 --- a/beacon_node/network/src/sync/block_lookups/parent_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/parent_lookup.rs @@ -3,8 +3,8 @@ use super::{BlobRequestId, BlockRequestId, DownloadedBlocks, PeerShouldHave, Res use crate::sync::block_lookups::single_block_lookup::{State, UnknownParentComponents}; use crate::sync::block_lookups::{RootBlobsTuple, RootBlockTuple}; use crate::sync::{manager::SLOT_IMPORT_TOLERANCE, network_context::SyncNetworkContext}; -use beacon_chain::blob_verification::AsBlock; -use beacon_chain::blob_verification::BlockWrapper; +use beacon_chain::block_verification_types::AsBlock; +use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::data_availability_checker::DataAvailabilityChecker; use beacon_chain::BeaconChainTypes; use lighthouse_network::PeerId; @@ -147,7 +147,7 @@ impl ParentLookup { .check_peer_disconnected(peer_id) } - pub fn add_unknown_parent_block(&mut self, block: BlockWrapper) { + pub fn add_unknown_parent_block(&mut self, block: RpcBlock) { let next_parent = block.parent_root(); // Cache the block. @@ -203,7 +203,7 @@ impl ParentLookup { self, ) -> ( Hash256, - Vec>, + Vec>, Vec, SingleBlockLookup, ) { diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs index b6e0cef8365..90829905b8a 100644 --- a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs @@ -1,6 +1,6 @@ use crate::sync::block_lookups::{BlobRequestId, BlockRequestId, RootBlobsTuple, RootBlockTuple}; use crate::sync::network_context::SyncNetworkContext; -use beacon_chain::blob_verification::BlockWrapper; +use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::data_availability_checker::DataAvailabilityChecker; use beacon_chain::{get_block_root, BeaconChainTypes}; use lighthouse_network::rpc::methods::BlobsByRootRequest; @@ -138,6 +138,16 @@ pub struct UnknownParentComponents { pub downloaded_blobs: FixedBlobSidecarList, } +impl From> for UnknownParentComponents { + fn from(value: RpcBlock) -> Self { + let (block, blobs) = value.deconstruct(); + let fixed_blobs = blobs.map(|blobs| { + FixedBlobSidecarList::from(blobs.into_iter().map(Some).collect::>()) + }); + Self::new(Some(block), fixed_blobs) + } +} + impl UnknownParentComponents { pub fn new( block: Option>>, @@ -284,7 +294,7 @@ impl SingleBlockLookup Option> { + pub fn get_downloaded_block(&mut self) -> Option> { self.unknown_parent_components .as_mut() .and_then(|components| { @@ -302,8 +312,16 @@ impl SingleBlockLookup>(); + let blobs = VariableList::from(filtered); + RpcBlock::new(block.clone(), Some(blobs)).ok() }) } else { None diff --git a/beacon_node/network/src/sync/block_lookups/tests.rs b/beacon_node/network/src/sync/block_lookups/tests.rs index 80b0f3c797c..2d523b04852 100644 --- a/beacon_node/network/src/sync/block_lookups/tests.rs +++ b/beacon_node/network/src/sync/block_lookups/tests.rs @@ -1474,7 +1474,7 @@ mod deneb_only { fn parent_block_unknown_parent(mut self) -> Self { self.bl.parent_block_processed( self.block_root, - BlockProcessingResult::Err(BlockError::ParentUnknown(BlockWrapper::Block( + BlockProcessingResult::Err(BlockError::ParentUnknown(RpcBlock::new_without_blobs( self.parent_block.clone().expect("parent block"), ))), ResponseType::Block, diff --git a/beacon_node/network/src/sync/block_sidecar_coupling.rs b/beacon_node/network/src/sync/block_sidecar_coupling.rs index 7e5362a6f0d..fce7a2e30d1 100644 --- a/beacon_node/network/src/sync/block_sidecar_coupling.rs +++ b/beacon_node/network/src/sync/block_sidecar_coupling.rs @@ -1,5 +1,5 @@ -use beacon_chain::blob_verification::BlockWrapper; -use ssz_types::FixedVector; +use beacon_chain::block_verification_types::RpcBlock; +use ssz_types::VariableList; use std::{collections::VecDeque, sync::Arc}; use types::{BlobSidecar, EthSpec, SignedBeaconBlock}; @@ -16,28 +16,28 @@ pub struct BlocksAndBlobsRequestInfo { } impl BlocksAndBlobsRequestInfo { - pub fn add_block_response(&mut self, maybe_block: Option>>) { - match maybe_block { + pub fn add_block_response(&mut self, block_opt: Option>>) { + match block_opt { Some(block) => self.accumulated_blocks.push_back(block), None => self.is_blocks_stream_terminated = true, } } - pub fn add_sidecar_response(&mut self, maybe_sidecar: Option>>) { - match maybe_sidecar { + pub fn add_sidecar_response(&mut self, sidecar_opt: Option>>) { + match sidecar_opt { Some(sidecar) => self.accumulated_sidecars.push_back(sidecar), None => self.is_sidecars_stream_terminated = true, } } - pub fn into_responses(self) -> Result>, &'static str> { + pub fn into_responses(self) -> Result>, &'static str> { let BlocksAndBlobsRequestInfo { accumulated_blocks, accumulated_sidecars, .. } = self; - // ASSUMPTION: There can't be more more blobs than blocks. i.e. sending any blob (empty + // There can't be more more blobs than blocks. i.e. sending any blob (empty // included) for a skipped slot is not permitted. let mut responses = Vec::with_capacity(accumulated_blocks.len()); let mut blob_iter = accumulated_sidecars.into_iter().peekable(); @@ -50,29 +50,23 @@ impl BlocksAndBlobsRequestInfo { .unwrap_or(false); pair_next_blob } { - blob_list.push(blob_iter.next().expect("iterator is not empty")); + blob_list.push(blob_iter.next().ok_or("Missing next blob")?); } - if blob_list.is_empty() { - responses.push(BlockWrapper::Block(block)) - } else { - let mut blobs_fixed = vec![None; T::max_blobs_per_block()]; - for blob in blob_list { - let blob_index = blob.index as usize; - let Some(blob_opt) = blobs_fixed.get_mut(blob_index) else { + let mut blobs_buffer = vec![None; T::max_blobs_per_block()]; + for blob in blob_list { + let blob_index = blob.index as usize; + let Some(blob_opt) = blobs_buffer.get_mut(blob_index) else { return Err("Invalid blob index"); }; - if blob_opt.is_some() { - return Err("Repeat blob index"); - } else { - *blob_opt = Some(blob); - } + if blob_opt.is_some() { + return Err("Repeat blob index"); + } else { + *blob_opt = Some(blob); } - responses.push(BlockWrapper::BlockAndBlobs( - block, - FixedVector::from(blobs_fixed), - )) } + let blobs = VariableList::from(blobs_buffer.into_iter().flatten().collect::>()); + responses.push(RpcBlock::new(block, Some(blobs))?) } // if accumulated sidecars is not empty, throw an error. diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 93b7c9af5bf..5e8fc4a4e9e 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -46,8 +46,8 @@ use crate::sync::block_lookups::delayed_lookup::DelayedLookupMessage; pub use crate::sync::block_lookups::ResponseType; use crate::sync::block_lookups::UnknownParentComponents; use crate::sync::range_sync::ByRangeRequestType; -use beacon_chain::blob_verification::AsBlock; -use beacon_chain::blob_verification::BlockWrapper; +use beacon_chain::block_verification_types::AsBlock; +use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::{ AvailabilityProcessingStatus, BeaconChain, BeaconChainTypes, BlockError, EngineState, MAXIMUM_GOSSIP_CLOCK_DISPARITY, @@ -127,7 +127,7 @@ pub enum SyncMessage { }, /// A block with an unknown parent has been received. - UnknownParentBlock(PeerId, BlockWrapper, Hash256), + UnknownParentBlock(PeerId, RpcBlock, Hash256), /// A blob with an unknown parent has been received. UnknownParentBlob(PeerId, Arc>), @@ -614,15 +614,13 @@ impl SyncManager { } => self.rpc_blob_received(request_id, peer_id, blob_sidecar, seen_timestamp), SyncMessage::UnknownParentBlock(peer_id, block, block_root) => { let block_slot = block.slot(); - let (block, blobs) = block.deconstruct(); let parent_root = block.parent_root(); - let parent_components = UnknownParentComponents::new(Some(block), blobs); self.handle_unknown_parent( peer_id, block_root, parent_root, block_slot, - Some(parent_components), + Some(block.into()), ); } SyncMessage::UnknownParentBlob(peer_id, blob) => { @@ -910,7 +908,7 @@ impl SyncManager { batch_id, &peer_id, id, - block.map(BlockWrapper::Block), + block.map(Into::into), ) { Ok(ProcessResult::SyncCompleted) => self.update_sync_state(), Ok(ProcessResult::Successful) => {} @@ -934,7 +932,7 @@ impl SyncManager { chain_id, batch_id, id, - block.map(BlockWrapper::Block), + block.map(Into::into), ); self.update_sync_state(); } diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 7c162f478c5..d635dd2ea18 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -8,7 +8,7 @@ use crate::network_beacon_processor::NetworkBeaconProcessor; use crate::service::{NetworkMessage, RequestId}; use crate::status::ToStatusMessage; use crate::sync::block_lookups::{BlobRequestId, BlockRequestId}; -use beacon_chain::blob_verification::BlockWrapper; +use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::{BeaconChain, BeaconChainTypes, EngineState}; use fnv::FnvHashMap; use lighthouse_network::rpc::methods::{BlobsByRangeRequest, BlobsByRootRequest}; @@ -22,7 +22,7 @@ use types::{BlobSidecar, EthSpec, SignedBeaconBlock}; pub struct BlocksAndBlobsByRangeResponse { pub batch_id: BatchId, - pub responses: Result>, &'static str>, + pub responses: Result>, &'static str>, } pub struct BlocksAndBlobsByRangeRequest { diff --git a/beacon_node/network/src/sync/range_sync/batch.rs b/beacon_node/network/src/sync/range_sync/batch.rs index 138d320965e..f5c320cb880 100644 --- a/beacon_node/network/src/sync/range_sync/batch.rs +++ b/beacon_node/network/src/sync/range_sync/batch.rs @@ -1,5 +1,5 @@ use crate::sync::manager::Id; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; +use beacon_chain::block_verification_types::{AsBlock, RpcBlock}; use lighthouse_network::rpc::methods::BlocksByRangeRequest; use lighthouse_network::PeerId; use std::collections::HashSet; @@ -56,7 +56,7 @@ pub trait BatchConfig { /// Note that simpler hashing functions considered in the past (hash of first block, hash of last /// block, number of received blocks) are not good enough to differentiate attempts. For this /// reason, we hash the complete set of blocks both in RangeSync and BackFillSync. - fn batch_attempt_hash(blocks: &[BlockWrapper]) -> u64; + fn batch_attempt_hash(blocks: &[RpcBlock]) -> u64; } pub struct RangeSyncBatchConfig {} @@ -68,7 +68,7 @@ impl BatchConfig for RangeSyncBatchConfig { fn max_batch_processing_attempts() -> u8 { MAX_BATCH_PROCESSING_ATTEMPTS } - fn batch_attempt_hash(blocks: &[BlockWrapper]) -> u64 { + fn batch_attempt_hash(blocks: &[RpcBlock]) -> u64 { let mut hasher = std::collections::hash_map::DefaultHasher::new(); blocks.hash(&mut hasher); hasher.finish() @@ -116,9 +116,9 @@ pub enum BatchState { /// The batch has failed either downloading or processing, but can be requested again. AwaitingDownload, /// The batch is being downloaded. - Downloading(PeerId, Vec>, Id), + Downloading(PeerId, Vec>, Id), /// The batch has been completely downloaded and is ready for processing. - AwaitingProcessing(PeerId, Vec>), + AwaitingProcessing(PeerId, Vec>), /// The batch is being processed. Processing(Attempt), /// The batch was successfully processed and is waiting to be validated. @@ -251,7 +251,7 @@ impl BatchInfo { } /// Adds a block to a downloading batch. - pub fn add_block(&mut self, block: BlockWrapper) -> Result<(), WrongState> { + pub fn add_block(&mut self, block: RpcBlock) -> Result<(), WrongState> { match self.state.poison() { BatchState::Downloading(peer, mut blocks, req_id) => { blocks.push(block); @@ -383,7 +383,7 @@ impl BatchInfo { } } - pub fn start_processing(&mut self) -> Result>, WrongState> { + pub fn start_processing(&mut self) -> Result>, WrongState> { match self.state.poison() { BatchState::AwaitingProcessing(peer, blocks) => { self.state = BatchState::Processing(Attempt::new::(peer, &blocks)); @@ -481,7 +481,7 @@ pub struct Attempt { } impl Attempt { - fn new(peer_id: PeerId, blocks: &[BlockWrapper]) -> Self { + fn new(peer_id: PeerId, blocks: &[RpcBlock]) -> Self { let hash = B::batch_attempt_hash(blocks); Attempt { peer_id, hash } } diff --git a/beacon_node/network/src/sync/range_sync/chain.rs b/beacon_node/network/src/sync/range_sync/chain.rs index 3b3cdb6ae3b..4d399b5cb99 100644 --- a/beacon_node/network/src/sync/range_sync/chain.rs +++ b/beacon_node/network/src/sync/range_sync/chain.rs @@ -3,7 +3,7 @@ use crate::network_beacon_processor::ChainSegmentProcessId; use crate::sync::{ manager::Id, network_context::SyncNetworkContext, BatchOperationOutcome, BatchProcessResult, }; -use beacon_chain::blob_verification::BlockWrapper; +use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::BeaconChainTypes; use fnv::FnvHashMap; use lighthouse_network::{PeerAction, PeerId}; @@ -221,7 +221,7 @@ impl SyncingChain { batch_id: BatchId, peer_id: &PeerId, request_id: Id, - beacon_block: Option>, + beacon_block: Option>, ) -> ProcessingResult { // check if we have this batch let batch = match self.batches.get_mut(&batch_id) { diff --git a/beacon_node/network/src/sync/range_sync/range.rs b/beacon_node/network/src/sync/range_sync/range.rs index 28cdc7afcf7..4ca518f9898 100644 --- a/beacon_node/network/src/sync/range_sync/range.rs +++ b/beacon_node/network/src/sync/range_sync/range.rs @@ -47,7 +47,7 @@ use crate::status::ToStatusMessage; use crate::sync::manager::Id; use crate::sync::network_context::SyncNetworkContext; use crate::sync::BatchProcessResult; -use beacon_chain::blob_verification::BlockWrapper; +use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::{BeaconChain, BeaconChainTypes}; use lighthouse_network::rpc::GoodbyeReason; use lighthouse_network::PeerId; @@ -210,7 +210,7 @@ where chain_id: ChainId, batch_id: BatchId, request_id: Id, - beacon_block: Option>, + beacon_block: Option>, ) { // check if this chunk removes the chain match self.chains.call_by_id(chain_id, |chain| { @@ -387,19 +387,18 @@ mod tests { use beacon_chain::builder::Witness; use beacon_chain::eth1_chain::CachingEth1Backend; use beacon_chain::parking_lot::RwLock; + use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType}; use beacon_chain::EngineState; use beacon_processor::WorkEvent as BeaconWorkEvent; use lighthouse_network::rpc::BlocksByRangeRequest; use lighthouse_network::Request; use lighthouse_network::{rpc::StatusMessage, NetworkGlobals}; use slog::{o, Drain}; - use tokio::sync::mpsc; - - use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType}; use slot_clock::TestingSlotClock; use std::collections::HashSet; use std::sync::Arc; use store::MemoryStore; + use tokio::sync::mpsc; use types::{Hash256, MinimalEthSpec as E}; #[derive(Debug)] diff --git a/consensus/fork_choice/tests/tests.rs b/consensus/fork_choice/tests/tests.rs index 88fb7d8965e..9d39eb3e383 100644 --- a/consensus/fork_choice/tests/tests.rs +++ b/consensus/fork_choice/tests/tests.rs @@ -1,9 +1,5 @@ #![cfg(not(debug_assertions))] -use std::fmt; -use std::sync::Mutex; -use std::time::Duration; - use beacon_chain::test_utils::{ AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType, }; @@ -14,6 +10,9 @@ use beacon_chain::{ use fork_choice::{ ForkChoiceStore, InvalidAttestation, InvalidBlock, PayloadVerificationStatus, QueuedAttestation, }; +use std::fmt; +use std::sync::Mutex; +use std::time::Duration; use store::MemoryStore; use types::{ test_utils::generate_deterministic_keypair, BeaconBlockRef, BeaconState, ChainSpec, Checkpoint, @@ -195,17 +194,18 @@ impl ForkChoiceTest { let validators = self.harness.get_all_validators(); loop { let slot = self.harness.get_current_slot(); - let (block, state_) = self.harness.make_block(state, slot).await; + let (block_contents, state_) = self.harness.make_block(state, slot).await; state = state_; - if !predicate(block.0.message(), &state) { + if !predicate(block_contents.0.message(), &state) { break; } - if let Ok(block_hash) = self.harness.process_block_result(block.clone()).await { + let block = block_contents.0.clone(); + if let Ok(block_hash) = self.harness.process_block_result(block_contents).await { self.harness.attest_block( &state, - block.0.state_root(), + block.state_root(), block_hash, - &block.0, + &block, &validators, ); self.harness.advance_slot(); diff --git a/consensus/state_processing/src/consensus_context.rs b/consensus/state_processing/src/consensus_context.rs index 78803ab4eb4..8e49a0d4983 100644 --- a/consensus/state_processing/src/consensus_context.rs +++ b/consensus/state_processing/src/consensus_context.rs @@ -21,8 +21,6 @@ pub struct ConsensusContext { #[ssz(skip_serializing, skip_deserializing)] indexed_attestations: HashMap<(AttestationData, BitList), IndexedAttestation>, - /// Whether `verify_kzg_commitments_against_transactions` has successfully passed. - kzg_commitments_consistent: bool, } #[derive(Debug, PartialEq, Clone)] @@ -45,7 +43,6 @@ impl ConsensusContext { proposer_index: None, current_block_root: None, indexed_attestations: HashMap::new(), - kzg_commitments_consistent: false, } } @@ -161,13 +158,4 @@ impl ConsensusContext { pub fn num_cached_indexed_attestations(&self) -> usize { self.indexed_attestations.len() } - - pub fn set_kzg_commitments_consistent(mut self, kzg_commitments_consistent: bool) -> Self { - self.kzg_commitments_consistent = kzg_commitments_consistent; - self - } - - pub fn kzg_commitments_consistent(&self) -> bool { - self.kzg_commitments_consistent - } } diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 58b7aaeea73..3b067e0ce22 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -169,9 +169,10 @@ pub use crate::selection_proof::SelectionProof; pub use crate::shuffling_id::AttestationShufflingId; pub use crate::signed_aggregate_and_proof::SignedAggregateAndProof; pub use crate::signed_beacon_block::{ - ssz_tagged_signed_beacon_block, SignedBeaconBlock, SignedBeaconBlockAltair, - SignedBeaconBlockBase, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockHash, - SignedBeaconBlockMerge, SignedBlindedBeaconBlock, + ssz_tagged_signed_beacon_block, ssz_tagged_signed_beacon_block_arc, SignedBeaconBlock, + SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockCapella, + SignedBeaconBlockDeneb, SignedBeaconBlockHash, SignedBeaconBlockMerge, + SignedBlindedBeaconBlock, }; pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader; pub use crate::signed_blob::*; diff --git a/consensus/types/src/signed_beacon_block.rs b/consensus/types/src/signed_beacon_block.rs index cf7fd6819ea..c52ba11d278 100644 --- a/consensus/types/src/signed_beacon_block.rs +++ b/consensus/types/src/signed_beacon_block.rs @@ -642,6 +642,27 @@ pub mod ssz_tagged_signed_beacon_block { } } +pub mod ssz_tagged_signed_beacon_block_arc { + use super::*; + pub mod encode { + pub use super::ssz_tagged_signed_beacon_block::encode::*; + } + + pub mod decode { + pub use super::ssz_tagged_signed_beacon_block::decode::{is_ssz_fixed_len, ssz_fixed_len}; + use super::*; + #[allow(unused_imports)] + use ssz::*; + use std::sync::Arc; + + pub fn from_ssz_bytes>( + bytes: &[u8], + ) -> Result>, DecodeError> { + ssz_tagged_signed_beacon_block::decode::from_ssz_bytes(bytes).map(Arc::new) + } + } +} + #[cfg(test)] mod test { use super::*; From 8c341bb9ccfb1a6e0cfb990488152aa91b905be2 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 25 Jul 2023 14:40:04 -0400 Subject: [PATCH 51/51] cargo fmt (#4541) --- .../beacon_chain/src/block_verification_types.rs | 16 ++++++++-------- .../src/data_availability_checker.rs | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/beacon_node/beacon_chain/src/block_verification_types.rs b/beacon_node/beacon_chain/src/block_verification_types.rs index e80372a6da1..c41090c421f 100644 --- a/beacon_node/beacon_chain/src/block_verification_types.rs +++ b/beacon_node/beacon_chain/src/block_verification_types.rs @@ -17,15 +17,15 @@ use types::{ }; /// A block that has been received over RPC. It has 2 internal variants: -/// +/// /// 1. `BlockAndBlobs`: A fully available post deneb block with all the blobs available. This variant /// is only constructed after making consistency checks between blocks and blobs. /// Hence, it is fully self contained w.r.t verification. i.e. this block has all the required /// data to get verfied and imported into fork choice. -/// +/// /// 2. `Block`: This can be a fully available pre-deneb block **or** a post-deneb block that may or may /// not require blobs to be considered fully available. -/// +/// /// Note: We make a distinction over blocks received over gossip because /// in a post-deneb world, the blobs corresponding to a given block that are received /// over rpc do not contain the proposer signature for dos resistance. @@ -98,15 +98,15 @@ impl From> for RpcBlock { } } -/// A block that has gone through all pre-deneb block processing checks including block processing +/// A block that has gone through all pre-deneb block processing checks including block processing /// and execution by an EL client. This block hasn't completed data availability checks. -/// -/// +/// +/// /// It contains 2 variants: /// 1. `Available`: This block has been executed and also contains all data to consider it a /// fully available block. i.e. for post-deneb, this implies that this contains all the /// required blobs. -/// 2. `AvailabilityPending`: This block hasn't received all required blobs to consider it a +/// 2. `AvailabilityPending`: This block hasn't received all required blobs to consider it a /// fully available block. pub enum ExecutedBlock { Available(AvailableExecutedBlock), @@ -186,7 +186,7 @@ impl AvailableExecutedBlock { } /// A block that has completed all pre-deneb block processing checks, verification -/// by an EL client but does not have all requisite blob data to get imported into +/// by an EL client but does not have all requisite blob data to get imported into /// fork choice. #[derive(Encode, Decode, Clone)] pub struct AvailabilityPendingExecutedBlock { diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 9a53a7139c7..b416b378846 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -339,12 +339,12 @@ pub fn make_available( } /// Makes the following checks to ensure that the list of blobs correspond block: -/// +/// /// * Check that a block is post-deneb /// * Checks that the number of blobs is equal to the length of kzg commitments in the list /// * Checks that the index, slot, root and kzg_commitment in the block match the blobs in the correct order -/// -/// Returns `Ok(())` if all consistency checks pass and an error otherwise. +/// +/// Returns `Ok(())` if all consistency checks pass and an error otherwise. pub fn consistency_checks( block: &SignedBeaconBlock, blobs: &[Arc>],