diff --git a/crates/common/fork_choice/beacon/src/handlers.rs b/crates/common/fork_choice/beacon/src/handlers.rs index 84e5fa524..e87a26507 100644 --- a/crates/common/fork_choice/beacon/src/handlers.rs +++ b/crates/common/fork_choice/beacon/src/handlers.rs @@ -28,32 +28,36 @@ pub async fn on_block( verify_blob_availability: bool, ) -> anyhow::Result<()> { let block = &signed_block.message; + let parent_root = block.parent_root; + let block_slot = block.slot; + let block_root = block.tree_hash_root(); // Parent block must be known ensure!( - store.db.state_provider().get(block.parent_root)?.is_some(), - "Missing parent block state for parent_root: {:x}", - block.parent_root + store.db.state_provider().get(parent_root)?.is_some(), + "Missing parent block state for parent_root: {parent_root:x}", ); // Blocks cannot be in the future. If they are, their consideration must be delayed until they // are in the past. + let current_slot = store.get_current_slot()?; ensure!( - store.get_current_slot()? >= block.slot, - "Block slot is ahead of current slot: block.slot = {}, store.get_current_slot() = {}", - block.slot, - store.get_current_slot()? + current_slot >= block_slot, + "Block slot is ahead of current slot: block.slot = {block_slot}, store.get_current_slot() = {current_slot}", ); // Check that block is later than the finalized epoch slot (optimization to reduce calls to // get_ancestor) let finalized_slot = compute_start_slot_at_epoch(store.db.finalized_checkpoint_provider().get()?.epoch); - ensure!(block.slot > finalized_slot); + ensure!( + block_slot > finalized_slot, + "Block slot must be greater than finalized slot: block.slot = {block_slot}, finalized_slot = {finalized_slot}", + ); // Check block is a descendant of the finalized block at the checkpoint finalized slot let finalized_checkpoint_block = store.get_checkpoint_block( - block.parent_root, + parent_root, store.db.finalized_checkpoint_provider().get()?.epoch, )?; ensure!(store.db.finalized_checkpoint_provider().get()?.root == finalized_checkpoint_block); @@ -63,7 +67,10 @@ pub async fn on_block( // available *Note*: Extraneous or invalid data (in addition to the // expected/referenced valid data) received on the p2p network MUST NOT invalidate // a block that is otherwise valid and available - ensure!(store.is_data_available(block.tree_hash_root(), &block.body.blob_kzg_commitments)?); + ensure!( + store.is_data_available(block_root)?, + "Data not available for block root: {block_root:x}", + ); } // Check the block is valid and compute the post-state @@ -71,10 +78,9 @@ pub async fn on_block( let mut state = store .db .state_provider() - .get(block.parent_root)? + .get(parent_root)? .ok_or_else(|| anyhow!("beacon state not found"))? .clone(); - let block_root = block.tree_hash_root(); state .state_transition(signed_block, true, execution_engine) .await?; diff --git a/crates/common/fork_choice/beacon/src/store.rs b/crates/common/fork_choice/beacon/src/store.rs index 2b61aa1ab..975a77869 100644 --- a/crates/common/fork_choice/beacon/src/store.rs +++ b/crates/common/fork_choice/beacon/src/store.rs @@ -6,7 +6,6 @@ use hashbrown::HashMap; use ream_bls::BLSSignature; use ream_consensus_beacon::{ attestation::Attestation, - blob_sidecar::BlobIdentifier, data_column_sidecar::{ColumnIdentifier, DataColumnSidecar, NUMBER_OF_COLUMNS}, electra::{ beacon_block::{BeaconBlock, SignedBeaconBlock}, @@ -19,13 +18,10 @@ use ream_consensus_misc::{ checkpoint::Checkpoint, constants::beacon::{GENESIS_EPOCH, GENESIS_SLOT, INTERVALS_PER_SLOT, SLOTS_PER_EPOCH}, misc::{compute_epoch_at_slot, compute_start_slot_at_epoch, is_shuffling_stable}, - polynomial_commitments::kzg_commitment::KZGCommitment, }; use ream_network_spec::networks::beacon_network_spec; use ream_operation_pool::OperationPool; -use ream_polynomial_commitments::handlers::{ - verify_blob_kzg_proof_batch, verify_data_column_sidecar_kzg_proofs, -}; +use ream_polynomial_commitments::handlers::verify_data_column_sidecar_kzg_proofs; use ream_storage::{ db::beacon::BeaconDB, tables::{ @@ -734,70 +730,25 @@ impl Store { /// Check if data is available for a block. /// /// For Fulu: https://ethereum.github.io/consensus-specs/specs/fulu/fork-choice/#modified-is_data_available - /// For Deneb: https://ethereum.github.io/consensus-specs/specs/deneb/fork-choice/#is_data_available - pub fn is_data_available( - &self, - beacon_block_root: B256, - blob_kzg_commitments: &[KZGCommitment], - ) -> anyhow::Result { + pub fn is_data_available(&self, beacon_block_root: B256) -> anyhow::Result { // `retrieve_column_sidecars` is implementation and context dependent, replacing // `retrieve_blobs_and_proofs`. For the given block root, it returns all column // sidecars to sample, or raises an exception if they are not available. // The p2p network does not guarantee sidecar retrieval outside of // `MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS` epochs. let column_sidecars = self.retrieve_column_sidecars(beacon_block_root)?; + ensure!(!column_sidecars.is_empty(), "No column sidecars available"); - if !column_sidecars.is_empty() { - // Fulu column sidecars validation - for column_sidecar in column_sidecars { - if !column_sidecar.verify() { - return Ok(false); - } - if !verify_data_column_sidecar_kzg_proofs(&column_sidecar)? { - return Ok(false); - } - } - Ok(true) - } else { - // Fallback to Deneb blobs validation - if blob_kzg_commitments.is_empty() { - return Ok(true); + // Fulu column sidecars validation + for column_sidecar in column_sidecars { + if !column_sidecar.verify() { + return Ok(false); } - - self.is_blob_data_available(beacon_block_root, blob_kzg_commitments) - } - } - - /// Check if blob data is available for a Deneb block. - /// - /// Spec: https://ethereum.github.io/consensus-specs/specs/deneb/fork-choice/#is_data_available - fn is_blob_data_available( - &self, - beacon_block_root: B256, - blob_kzg_commitments: &[KZGCommitment], - ) -> anyhow::Result { - let mut blobs = Vec::with_capacity(blob_kzg_commitments.len()); - let mut proofs = Vec::with_capacity(blob_kzg_commitments.len()); - - // Retrieve all blobs and proofs - for index in 0..blob_kzg_commitments.len() { - let blob_id = BlobIdentifier::new(beacon_block_root, index as u64); - if let Some(blob_and_proof) = self.db.blobs_and_proofs_provider().get(blob_id)? { - blobs.push(blob_and_proof.blob); - proofs.push(blob_and_proof.proof); - } else { - // Data not available + if !verify_data_column_sidecar_kzg_proofs(&column_sidecar)? { return Ok(false); } } - - // Verify the number of blobs matches - if blobs.len() != blob_kzg_commitments.len() { - return Ok(false); - } - - // Verify blob KZG proofs - verify_blob_kzg_proof_batch(&blobs, blob_kzg_commitments, &proofs) + Ok(true) } /// Retrieve column sidecars for a block. diff --git a/crates/storage/src/db/lean.rs b/crates/storage/src/db/lean.rs index 95c7ea184..ce1f7901d 100644 --- a/crates/storage/src/db/lean.rs +++ b/crates/storage/src/db/lean.rs @@ -173,7 +173,7 @@ impl LeanDB { return Ok(()); } - table_metrics.sort_by(|a, b| b.1.cmp(&a.1)); + table_metrics.sort_by_key(|b| std::cmp::Reverse(b.1)); let mut report = String::with_capacity(512); let total_mb = total_bytes as f64 / (1024.0 * 1024.0); if total_mb >= 1024.0 { diff --git a/crates/storage/src/db/mod.rs b/crates/storage/src/db/mod.rs index e1fb2b59c..e1b4f974b 100644 --- a/crates/storage/src/db/mod.rs +++ b/crates/storage/src/db/mod.rs @@ -15,7 +15,7 @@ use crate::{ beacon::{ beacon_block::BeaconBlockTable, beacon_state::BeaconStateTable, blobs_and_proofs::BLOB_FOLDER_NAME, block_timeliness::BlockTimelinessTable, - checkpoint_states::CheckpointStatesTable, + checkpoint_states::CheckpointStatesTable, column_sidecars::COLUMN_FOLDER_NAME, equivocating_indices::EQUIVOCATING_INDICES_FIELD, finalized_checkpoint::FinalizedCheckpointField, genesis_time::GenesisTimeField, justified_checkpoint::JustifiedCheckpointField, latest_messages::LatestMessagesTable, @@ -89,6 +89,7 @@ impl ReamDB { write_txn.commit()?; fs::create_dir_all(self.data_dir.join(BLOB_FOLDER_NAME))?; + fs::create_dir_all(self.data_dir.join(COLUMN_FOLDER_NAME))?; Ok(BeaconDB { db: self.db.clone(), diff --git a/testing/ef-tests/src/macros/fork_choice.rs b/testing/ef-tests/src/macros/fork_choice.rs index 1c6cdda32..ff7002734 100644 --- a/testing/ef-tests/src/macros/fork_choice.rs +++ b/testing/ef-tests/src/macros/fork_choice.rs @@ -9,6 +9,7 @@ macro_rules! test_fork_choice { use alloy_primitives::{hex, map::HashMap, B256, hex::FromHex}; use ream_bls::BLSSignature; use ream_consensus_beacon::{ + data_column_sidecar::{ColumnIdentifier, DataColumnSidecar}, attestation::Attestation, attester_slashing::AttesterSlashing, blob_sidecar::BlobIdentifier, electra::{beacon_block::{BeaconBlock, SignedBeaconBlock}, beacon_state::BeaconState}, }; use ream_consensus_misc::{checkpoint::Checkpoint, polynomial_commitments::kzg_proof::KZGProof}; @@ -67,8 +68,7 @@ macro_rules! test_fork_choice { #[derive(Debug, Deserialize)] pub struct Block { pub block: String, - pub blobs: Option, - pub proofs: Option>, + pub columns: Option>, pub valid: Option, } @@ -152,20 +152,17 @@ macro_rules! test_fork_choice { panic!("cannot find test asset (block_{blocks:?}.ssz_snappy)") }); - if let (Some(blobs), Some(proof)) = (blocks.blobs, blocks.proofs) { - let blobs_path = case_dir.join(format!("{}.ssz_snappy", blobs)); - let blobs: VariableList = utils::read_ssz_snappy(&blobs_path).expect("Could not read blob file."); - let proof: Vec = proof - .into_iter() - .map(|proof| KZGProof::from_hex(proof).expect("could not get KZGProof")) - .collect(); - let blobs_and_proofs = blobs.into_iter().zip(proof.into_iter()).map(|(blob, proof)| BlobAndProofV1 { blob, proof } ).collect::>(); - for (index, blob_and_proof) in blobs_and_proofs.into_iter().enumerate() { - store.db.blobs_and_proofs_provider().insert(BlobIdentifier::new(block.message.tree_hash_root(), index as u64), blob_and_proof)?; + let verify_blob_availability = blocks.columns.is_some(); + + if let Some(columns) = blocks.columns { + for (index, column) in columns.into_iter().enumerate() { + let column_path = case_dir.join(format!("{}.ssz_snappy", column)); + let column: DataColumnSidecar = utils::read_ssz_snappy(&column_path).expect("Could not read column file."); + store.db.column_sidecars_provider().insert(ColumnIdentifier::new(block.message.tree_hash_root(), index as u64), column)?; } } - assert_eq!(on_block(&mut store, &block, &mock_engine, true).await.is_ok(), blocks.valid.unwrap_or(true), "Unexpected result on on_block"); + assert_eq!(on_block(&mut store, &block, &mock_engine, verify_blob_availability).await.is_ok(), blocks.valid.unwrap_or(true), "Unexpected result on on_block"); } ForkChoiceStep::Attestation(attestations) => { let attestation_path =