feat: updated on_block so ef tests pass

crates/common/fork_choice/beacon/src/handlers.rs

@@ -28,32 +28,36 @@ pub async fn on_block(
     verify_blob_availability: bool,
 ) -> anyhow::Result<()> {
     let block = &signed_block.message;
+    let parent_root = block.parent_root;
+    let block_slot = block.slot;
+    let block_root = block.tree_hash_root();
 
     // Parent block must be known
     ensure!(
-        store.db.state_provider().get(block.parent_root)?.is_some(),
-        "Missing parent block state for parent_root: {:x}",
-        block.parent_root
+        store.db.state_provider().get(parent_root)?.is_some(),
+        "Missing parent block state for parent_root: {parent_root:x}",
     );
 
     // Blocks cannot be in the future. If they are, their consideration must be delayed until they
     // are in the past.
+    let current_slot = store.get_current_slot()?;
     ensure!(
-        store.get_current_slot()? >= block.slot,
-        "Block slot is ahead of current slot: block.slot = {}, store.get_current_slot() = {}",
-        block.slot,
-        store.get_current_slot()?
+        current_slot >= block_slot,
+        "Block slot is ahead of current slot: block.slot = {block_slot}, store.get_current_slot() = {current_slot}",
     );
 
     // Check that block is later than the finalized epoch slot (optimization to reduce calls to
     // get_ancestor)
     let finalized_slot =
         compute_start_slot_at_epoch(store.db.finalized_checkpoint_provider().get()?.epoch);
-    ensure!(block.slot > finalized_slot);
+    ensure!(
+        block_slot > finalized_slot,
+        "Block slot must be greater than finalized slot: block.slot = {block_slot}, finalized_slot = {finalized_slot}",
+    );
 
     // Check block is a descendant of the finalized block at the checkpoint finalized slot
     let finalized_checkpoint_block = store.get_checkpoint_block(
-        block.parent_root,
+        parent_root,
         store.db.finalized_checkpoint_provider().get()?.epoch,
     )?;
     ensure!(store.db.finalized_checkpoint_provider().get()?.root == finalized_checkpoint_block);
@@ -63,18 +67,20 @@ pub async fn on_block(
         // available *Note*: Extraneous or invalid data (in addition to the
         // expected/referenced valid data) received on the p2p network MUST NOT invalidate
         // a block that is otherwise valid and available
-        ensure!(store.is_data_available(block.tree_hash_root(), &block.body.blob_kzg_commitments)?);
+        ensure!(
+            store.is_data_available(block_root)?,
+            "Data not available for block root: {block_root:x}",
+        );
     }
 
     // Check the block is valid and compute the post-state
     // Make a copy of the state to avoid mutability issues
     let mut state = store
         .db
         .state_provider()
-        .get(block.parent_root)?
+        .get(parent_root)?
         .ok_or_else(|| anyhow!("beacon state not found"))?
         .clone();
-    let block_root = block.tree_hash_root();
     state
         .state_transition(signed_block, true, execution_engine)
         .await?;

crates/common/fork_choice/beacon/src/store.rs

@@ -6,7 +6,6 @@ use hashbrown::HashMap;
 use ream_bls::BLSSignature;
 use ream_consensus_beacon::{
     attestation::Attestation,
-    blob_sidecar::BlobIdentifier,
     data_column_sidecar::{ColumnIdentifier, DataColumnSidecar, NUMBER_OF_COLUMNS},
     electra::{
         beacon_block::{BeaconBlock, SignedBeaconBlock},
@@ -19,13 +18,10 @@ use ream_consensus_misc::{
     checkpoint::Checkpoint,
     constants::beacon::{GENESIS_EPOCH, GENESIS_SLOT, INTERVALS_PER_SLOT, SLOTS_PER_EPOCH},
     misc::{compute_epoch_at_slot, compute_start_slot_at_epoch, is_shuffling_stable},
-    polynomial_commitments::kzg_commitment::KZGCommitment,
 };
 use ream_network_spec::networks::beacon_network_spec;
 use ream_operation_pool::OperationPool;
-use ream_polynomial_commitments::handlers::{
-    verify_blob_kzg_proof_batch, verify_data_column_sidecar_kzg_proofs,
-};
+use ream_polynomial_commitments::handlers::verify_data_column_sidecar_kzg_proofs;
 use ream_storage::{
     db::beacon::BeaconDB,
     tables::{
@@ -734,70 +730,25 @@ impl Store {
     /// Check if data is available for a block.
     ///
     /// For Fulu: https://ethereum.github.io/consensus-specs/specs/fulu/fork-choice/#modified-is_data_available
-    /// For Deneb: https://ethereum.github.io/consensus-specs/specs/deneb/fork-choice/#is_data_available
-    pub fn is_data_available(
-        &self,
-        beacon_block_root: B256,
-        blob_kzg_commitments: &[KZGCommitment],
-    ) -> anyhow::Result<bool> {
+    pub fn is_data_available(&self, beacon_block_root: B256) -> anyhow::Result<bool> {
         // `retrieve_column_sidecars` is implementation and context dependent, replacing
         // `retrieve_blobs_and_proofs`. For the given block root, it returns all column
         // sidecars to sample, or raises an exception if they are not available.
         // The p2p network does not guarantee sidecar retrieval outside of
         // `MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS` epochs.
         let column_sidecars = self.retrieve_column_sidecars(beacon_block_root)?;
+        ensure!(!column_sidecars.is_empty(), "No column sidecars available");
 
-        if !column_sidecars.is_empty() {
-            // Fulu column sidecars validation
-            for column_sidecar in column_sidecars {
-                if !column_sidecar.verify() {
-                    return Ok(false);
-                }
-                if !verify_data_column_sidecar_kzg_proofs(&column_sidecar)? {
-                    return Ok(false);
-                }
-            }
-            Ok(true)
-        } else {
-            // Fallback to Deneb blobs validation
-            if blob_kzg_commitments.is_empty() {
-                return Ok(true);
+        // Fulu column sidecars validation
+        for column_sidecar in column_sidecars {
+            if !column_sidecar.verify() {
+                return Ok(false);
             }
-
-            self.is_blob_data_available(beacon_block_root, blob_kzg_commitments)
-        }
-    }
-
-    /// Check if blob data is available for a Deneb block.
-    ///
-    /// Spec: https://ethereum.github.io/consensus-specs/specs/deneb/fork-choice/#is_data_available
-    fn is_blob_data_available(
-        &self,
-        beacon_block_root: B256,
-        blob_kzg_commitments: &[KZGCommitment],
-    ) -> anyhow::Result<bool> {
-        let mut blobs = Vec::with_capacity(blob_kzg_commitments.len());
-        let mut proofs = Vec::with_capacity(blob_kzg_commitments.len());
-
-        // Retrieve all blobs and proofs
-        for index in 0..blob_kzg_commitments.len() {
-            let blob_id = BlobIdentifier::new(beacon_block_root, index as u64);
-            if let Some(blob_and_proof) = self.db.blobs_and_proofs_provider().get(blob_id)? {
-                blobs.push(blob_and_proof.blob);
-                proofs.push(blob_and_proof.proof);
-            } else {
-                // Data not available
+            if !verify_data_column_sidecar_kzg_proofs(&column_sidecar)? {
                 return Ok(false);
             }
         }
-
-        // Verify the number of blobs matches
-        if blobs.len() != blob_kzg_commitments.len() {
-            return Ok(false);
-        }
-
-        // Verify blob KZG proofs
-        verify_blob_kzg_proof_batch(&blobs, blob_kzg_commitments, &proofs)
+        Ok(true)
     }
 
     /// Retrieve column sidecars for a block.

crates/storage/src/db/lean.rs

@@ -173,7 +173,7 @@ impl LeanDB {
             return Ok(());
         }
 
-        table_metrics.sort_by(|a, b| b.1.cmp(&a.1));
+        table_metrics.sort_by_key(|b| std::cmp::Reverse(b.1));
         let mut report = String::with_capacity(512);
         let total_mb = total_bytes as f64 / (1024.0 * 1024.0);
         if total_mb >= 1024.0 {

crates/storage/src/db/mod.rs

@@ -15,7 +15,7 @@ use crate::{
         beacon::{
             beacon_block::BeaconBlockTable, beacon_state::BeaconStateTable,
             blobs_and_proofs::BLOB_FOLDER_NAME, block_timeliness::BlockTimelinessTable,
-            checkpoint_states::CheckpointStatesTable,
+            checkpoint_states::CheckpointStatesTable, column_sidecars::COLUMN_FOLDER_NAME,
             equivocating_indices::EQUIVOCATING_INDICES_FIELD,
             finalized_checkpoint::FinalizedCheckpointField, genesis_time::GenesisTimeField,
             justified_checkpoint::JustifiedCheckpointField, latest_messages::LatestMessagesTable,
@@ -89,6 +89,7 @@ impl ReamDB {
         write_txn.commit()?;
 
         fs::create_dir_all(self.data_dir.join(BLOB_FOLDER_NAME))?;
+        fs::create_dir_all(self.data_dir.join(COLUMN_FOLDER_NAME))?;
 
         Ok(BeaconDB {
             db: self.db.clone(),

testing/ef-tests/src/macros/fork_choice.rs

@@ -9,6 +9,7 @@ macro_rules! test_fork_choice {
                 use alloy_primitives::{hex, map::HashMap, B256, hex::FromHex};
                 use ream_bls::BLSSignature;
                 use ream_consensus_beacon::{
+                    data_column_sidecar::{ColumnIdentifier, DataColumnSidecar},
                     attestation::Attestation, attester_slashing::AttesterSlashing, blob_sidecar::BlobIdentifier, electra::{beacon_block::{BeaconBlock, SignedBeaconBlock}, beacon_state::BeaconState},
                 };
                 use ream_consensus_misc::{checkpoint::Checkpoint, polynomial_commitments::kzg_proof::KZGProof};
@@ -67,8 +68,7 @@ macro_rules! test_fork_choice {
                 #[derive(Debug, Deserialize)]
                 pub struct Block {
                     pub block: String,
-                    pub blobs: Option<String>,
-                    pub proofs: Option<Vec<String>>,
+                    pub columns: Option<Vec<String>>,
                     pub valid: Option<bool>,
                 }
 
@@ -152,20 +152,17 @@ macro_rules! test_fork_choice {
                                             panic!("cannot find test asset (block_{blocks:?}.ssz_snappy)")
                                         });
 
-                                    if let (Some(blobs), Some(proof)) = (blocks.blobs, blocks.proofs) {
-                                        let blobs_path = case_dir.join(format!("{}.ssz_snappy", blobs));
-                                        let blobs: VariableList<Blob, U4096> = utils::read_ssz_snappy(&blobs_path).expect("Could not read blob file.");
-                                        let proof: Vec<KZGProof> = proof
-                                            .into_iter()
-                                            .map(|proof| KZGProof::from_hex(proof).expect("could not get KZGProof"))
-                                            .collect();
-                                        let blobs_and_proofs = blobs.into_iter().zip(proof.into_iter()).map(|(blob, proof)| BlobAndProofV1 { blob, proof  } ).collect::<Vec<_>>();
-                                        for (index, blob_and_proof) in blobs_and_proofs.into_iter().enumerate() {
-                                            store.db.blobs_and_proofs_provider().insert(BlobIdentifier::new(block.message.tree_hash_root(), index as u64), blob_and_proof)?;
+                                    let verify_blob_availability = blocks.columns.is_some();
+
+                                    if let Some(columns) = blocks.columns {
+                                        for (index, column) in columns.into_iter().enumerate() {
+                                            let column_path = case_dir.join(format!("{}.ssz_snappy", column));
+                                            let column: DataColumnSidecar = utils::read_ssz_snappy(&column_path).expect("Could not read column file.");
+                                            store.db.column_sidecars_provider().insert(ColumnIdentifier::new(block.message.tree_hash_root(), index as u64), column)?;
                                         }
                                     }
 
-                                    assert_eq!(on_block(&mut store, &block, &mock_engine, true).await.is_ok(), blocks.valid.unwrap_or(true), "Unexpected result on on_block");
+                                    assert_eq!(on_block(&mut store, &block, &mock_engine, verify_blob_availability).await.is_ok(), blocks.valid.unwrap_or(true), "Unexpected result on on_block");
                                 }
                                 ForkChoiceStep::Attestation(attestations) => {
                                     let attestation_path =