From 4073489d0ff17e75b219f115bb2a04dd8497c544 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Arjovsky?= Date: Mon, 9 Mar 2026 18:57:03 +0100 Subject: [PATCH 1/5] dedoded elements in trie layer cache --- crates/blockchain/blockchain.rs | 47 ++-- crates/common/trie/db.rs | 11 +- crates/common/trie/logger.rs | 9 + crates/common/trie/node.rs | 67 ++++-- crates/common/trie/trie.rs | 59 ++++- crates/networking/p2p/sync/healing/storage.rs | 14 +- crates/storage/layering.rs | 222 +++++++++++++----- crates/storage/store.rs | 61 ++--- 8 files changed, 349 insertions(+), 141 deletions(-) diff --git a/crates/blockchain/blockchain.rs b/crates/blockchain/blockchain.rs index d68e6117050..6322e50b2be 100644 --- a/crates/blockchain/blockchain.rs +++ b/crates/blockchain/blockchain.rs @@ -83,7 +83,7 @@ use ethrex_storage::{ AccountUpdatesList, Store, UpdateBatch, error::StoreError, hash_address, hash_key, }; use ethrex_trie::node::{BranchNode, ExtensionNode, LeafNode}; -use ethrex_trie::{Nibbles, Node, NodeRef, Trie, TrieError, TrieNode}; +use ethrex_trie::{Nibbles, Node, NodeRef, Trie, TrieCommitEntry, TrieError}; use ethrex_vm::backends::CachingDatabase; use ethrex_vm::backends::levm::LEVM; use ethrex_vm::backends::levm::db::DatabaseLogger; @@ -262,22 +262,22 @@ enum MerklizationRequest { struct CollectedStateMsg { index: u8, subroot: Box, - state_nodes: Vec, - storage_nodes: Vec<(H256, Vec)>, + state_nodes: Vec, + storage_nodes: Vec<(H256, Vec)>, } struct CollectedStorageMsg { index: u8, prefix: H256, subroot: Box, - nodes: Vec, + nodes: Vec, } #[derive(Default)] struct PreMerkelizedAccountState { info: Option, storage_root: Option>, - nodes: Vec, + nodes: Vec, } /// Work item for BAL state trie shard workers. @@ -688,7 +688,7 @@ impl Blockchain { state.nodes.extend(nodes); } - let mut storage_updates: Vec<(H256, Vec)> = Default::default(); + let mut storage_updates: Vec<(H256, Vec)> = Default::default(); for (hashed_account, state) in account_state { let bucket = hashed_account.as_fixed_bytes()[0] >> 4; @@ -728,7 +728,10 @@ impl Blockchain { let hash = root.commit(Nibbles::default(), &mut state_updates); hash.finalize() } else { - state_updates.push((Nibbles::default(), vec![RLP_NULL])); + state_updates.push(TrieCommitEntry::LeafValue { + path: Nibbles::default(), + value: vec![RLP_NULL], + }); *EMPTY_TRIE_HASH }; @@ -845,7 +848,7 @@ impl Blockchain { // Compute storage roots in parallel let mut storage_roots: Vec> = vec![None; accounts.len()]; - let mut storage_updates: Vec<(H256, Vec)> = Vec::new(); + let mut storage_updates: Vec<(H256, Vec)> = Vec::new(); std::thread::scope(|s| -> Result<(), StoreError> { let accounts_ref = &accounts; @@ -861,8 +864,8 @@ impl Blockchain { .name(format!("bal_storage_worker_{worker_id}")) .spawn_scoped( s, - move || -> Result)>, StoreError> { - let mut results: Vec<(usize, H256, Vec)> = Vec::new(); + move || -> Result)>, StoreError> { + let mut results: Vec<(usize, H256, Vec)> = Vec::new(); // Open one state trie per worker for storage root lookups let state_trie = self.storage.open_state_trie(parent_state_root)?; @@ -879,7 +882,10 @@ impl Blockchain { results.push(( idx, *EMPTY_TRIE_HASH, - vec![(Nibbles::default(), vec![RLP_NULL])], + vec![TrieCommitEntry::LeafValue { + path: Nibbles::default(), + value: vec![RLP_NULL], + }], )); continue; } @@ -967,7 +973,7 @@ impl Blockchain { .name(format!("bal_state_shard_{index}")) .spawn_scoped( s, - move || -> Result<(Box, Vec), StoreError> { + move || -> Result<(Box, Vec), StoreError> { let mut state_trie = self.storage.open_state_trie(parent_state_root)?; @@ -1036,7 +1042,10 @@ impl Blockchain { let hash = root.commit(Nibbles::default(), &mut state_updates); hash.finalize() } else { - state_updates.push((Nibbles::default(), vec![RLP_NULL])); + state_updates.push(TrieCommitEntry::LeafValue { + path: Nibbles::default(), + value: vec![RLP_NULL], + }); *EMPTY_TRIE_HASH }; @@ -1186,7 +1195,10 @@ impl Blockchain { let hash = root.commit(Nibbles::default(), &mut state.nodes); storage_root = Some(hash.finalize()); } else { - state.nodes.push((Nibbles::default(), vec![RLP_NULL])); + state.nodes.push(TrieCommitEntry::LeafValue { + path: Nibbles::default(), + value: vec![RLP_NULL], + }); storage_root = Some(*EMPTY_TRIE_HASH); } } @@ -2799,7 +2811,10 @@ fn branchify(node: Node) -> Box { } } -fn collect_trie(index: u8, mut trie: Trie) -> Result<(Box, Vec), TrieError> { +fn collect_trie( + index: u8, + mut trie: Trie, +) -> Result<(Box, Vec), TrieError> { let root = branchify( trie.root_node()? .map(Arc::unwrap_or_clone) @@ -2807,7 +2822,7 @@ fn collect_trie(index: u8, mut trie: Trie) -> Result<(Box, Vec, Vec>>>; pub trait TrieDB: Send + Sync { fn get(&self, key: Nibbles) -> Result>, TrieError>; fn put_batch(&self, key_values: Vec<(Nibbles, Vec)>) -> Result<(), TrieError>; + + /// Try to get a decoded node directly from a cache. + /// Default returns None — callers fall back to get() + Node::decode(). + fn get_node(&self, _key: Nibbles) -> Result>, TrieError> { + Ok(None) + } // TODO: replace putbatch with this function. fn put_batch_no_alloc(&self, key_values: &[(Nibbles, Node)]) -> Result<(), TrieError> { self.put_batch( @@ -77,7 +83,10 @@ impl InMemoryTrieDB { let hashed_nodes = hashed_nodes .into_iter() - .map(|(k, v)| (k.into_vec(), v)) + .map(|entry| { + let (k, v) = entry.into_rlp_pair(); + (k.into_vec(), v) + }) .collect(); let in_memory_trie = Arc::new(Mutex::new(hashed_nodes)); diff --git a/crates/common/trie/logger.rs b/crates/common/trie/logger.rs index 00eeb9083cb..9a078c598a8 100644 --- a/crates/common/trie/logger.rs +++ b/crates/common/trie/logger.rs @@ -33,6 +33,15 @@ impl TrieLogger { } impl TrieDB for TrieLogger { + fn get_node(&self, key: Nibbles) -> Result>, TrieError> { + if let Some(node) = self.inner_db.get_node(key)? { + let mut lock = self.witness.lock().map_err(|_| TrieError::LockError)?; + lock.insert(node.compute_hash(), (*node).clone()); + return Ok(Some(node)); + } + Ok(None) + } + fn get(&self, key: Nibbles) -> Result>, TrieError> { let result = self.inner_db.get(key)?; if let Some(result) = result.as_ref() diff --git a/crates/common/trie/node.rs b/crates/common/trie/node.rs index 623e0f8a681..9cc88bff1df 100644 --- a/crates/common/trie/node.rs +++ b/crates/common/trie/node.rs @@ -16,7 +16,7 @@ use rkyv::{ with::Skip, }; -use crate::{NodeRLP, TrieDB, error::TrieError, nibbles::Nibbles}; +use crate::{NodeRLP, TrieCommitEntry, TrieDB, error::TrieError, nibbles::Nibbles}; use super::{ValueRLP, node_hash::NodeHash}; @@ -58,11 +58,17 @@ impl NodeRef { NodeRef::Hash(hash @ NodeHash::Inline(_)) => { Ok(Some(Arc::new(Node::decode(hash.as_ref())?))) } - NodeRef::Hash(_) => db - .get(path)? - .filter(|rlp| !rlp.is_empty()) - .map(|rlp| Ok(Arc::new(Node::decode(&rlp)?))) - .transpose(), + NodeRef::Hash(_) => { + // Try decoded cache first + if let Some(node) = db.get_node(path.clone())? { + return Ok(Some(node)); + } + // Fall back to raw bytes + decode + db.get(path)? + .filter(|rlp| !rlp.is_empty()) + .map(|rlp| Ok(Arc::new(Node::decode(&rlp)?))) + .transpose() + } } } @@ -78,14 +84,19 @@ impl NodeRef { NodeRef::Hash(hash @ NodeHash::Inline(_)) => { Ok(Some(Arc::new(Node::decode(hash.as_ref())?))) } - NodeRef::Hash(hash @ NodeHash::Hashed(_)) => db - .get(path)? - .filter(|rlp| !rlp.is_empty()) - .and_then(|rlp| match Node::decode(&rlp) { - Ok(node) => (node.compute_hash() == *hash).then_some(Ok(Arc::new(node))), - Err(err) => Some(Err(TrieError::RLPDecode(err))), - }) - .transpose(), + NodeRef::Hash(hash @ NodeHash::Hashed(_)) => { + // Try decoded cache first (cache contents are trusted) + if let Some(node) = db.get_node(path.clone())? { + return Ok(Some(node)); + } + db.get(path)? + .filter(|rlp| !rlp.is_empty()) + .and_then(|rlp| match Node::decode(&rlp) { + Ok(node) => (node.compute_hash() == *hash).then_some(Ok(Arc::new(node))), + Err(err) => Some(Err(TrieError::RLPDecode(err))), + }) + .transpose() + } } } @@ -108,6 +119,11 @@ impl NodeRef { self.get_node_mut(db, path) } NodeRef::Hash(hash @ NodeHash::Hashed(_)) => { + // Try decoded cache first + if let Some(node) = db.get_node(path.clone())? { + *self = NodeRef::Node(node, OnceLock::from(*hash)); + return self.get_node_mut(db, path); + } let Some(node) = db .get(path.clone())? .filter(|rlp| !rlp.is_empty()) @@ -129,20 +145,20 @@ impl NodeRef { } } - pub fn commit(&mut self, path: Nibbles, acc: &mut Vec<(Nibbles, Vec)>) -> NodeHash { + pub fn commit(&mut self, path: Nibbles, acc: &mut Vec) -> NodeHash { match *self { NodeRef::Node(ref mut node, ref mut hash) => { if let Some(hash) = hash.get() { return *hash; } match Arc::make_mut(node) { - Node::Branch(node) => { - for (choice, node) in &mut node.choices.iter_mut().enumerate() { - node.commit(path.append_new(choice as u8), acc); + Node::Branch(n) => { + for (choice, child) in &mut n.choices.iter_mut().enumerate() { + child.commit(path.append_new(choice as u8), acc); } } - Node::Extension(node) => { - node.child.commit(path.concat(&node.prefix), acc); + Node::Extension(n) => { + n.child.commit(path.concat(&n.prefix), acc); } Node::Leaf(_) => {} } @@ -150,9 +166,16 @@ impl NodeRef { node.encode(&mut buf); let hash = *hash.get_or_init(|| NodeHash::from_encoded(&buf)); if let Node::Leaf(leaf) = node.as_ref() { - acc.push((path.concat(&leaf.partial), leaf.value.clone())); + acc.push(TrieCommitEntry::LeafValue { + path: path.concat(&leaf.partial), + value: leaf.value.clone(), + }); } - acc.push((path, buf)); + acc.push(TrieCommitEntry::Node { + path, + node: node.clone(), + encoded: buf, + }); hash } diff --git a/crates/common/trie/trie.rs b/crates/common/trie/trie.rs index 0b2d3c4a1ff..8caf0c1c7a1 100644 --- a/crates/common/trie/trie.rs +++ b/crates/common/trie/trie.rs @@ -52,6 +52,42 @@ pub type NodeRLP = Vec; /// Represents a node in the Merkle Patricia Trie. pub type TrieNode = (Nibbles, NodeRLP); +/// A structured entry produced by `NodeRef::commit()`. +/// +/// Separates trie nodes (which can be cached as decoded `Arc`) from +/// leaf values (FKV entries) which remain as raw bytes. +#[derive(Debug, Clone)] +pub enum TrieCommitEntry { + /// A trie node (branch/extension/leaf) with both decoded and encoded forms. + Node { + path: Nibbles, + node: Arc, + encoded: Vec, + }, + /// A leaf's application-level value (for FlatKeyValue table), or a deletion marker. + LeafValue { + path: Nibbles, + value: Vec, + }, +} + +impl TrieCommitEntry { + pub fn path(&self) -> &Nibbles { + match self { + TrieCommitEntry::Node { path, .. } => path, + TrieCommitEntry::LeafValue { path, .. } => path, + } + } + + /// Convert to the old (Nibbles, Vec) format for backward compatibility. + pub fn into_rlp_pair(self) -> (Nibbles, Vec) { + match self { + TrieCommitEntry::Node { path, encoded, .. } => (path, encoded), + TrieCommitEntry::LeafValue { path, value } => (path, value), + } + } +} + /// Ethereum-compatible Merkle Patricia Trie pub struct Trie { db: Box, @@ -207,12 +243,12 @@ impl Trie { }) } - /// Returns a list of changes in a TrieNode format since last root hash processed. + /// Returns a list of changes in a TrieCommitEntry format since last root hash processed. /// /// # Returns /// /// A tuple containing the hash and the list of changes. - pub fn collect_changes_since_last_hash(&mut self) -> (H256, Vec) { + pub fn collect_changes_since_last_hash(&mut self) -> (H256, Vec) { let updates = self.commit_without_storing(); let ret_hash = self.hash_no_commit(); (ret_hash, updates) @@ -224,7 +260,8 @@ impl Trie { /// the cached nodes. pub fn commit(&mut self) -> Result<(), TrieError> { let acc = self.commit_without_storing(); - self.db.put_batch(acc)?; + self.db + .put_batch(acc.into_iter().map(|e| e.into_rlp_pair()).collect())?; // Commit the underlying transaction self.db.commit()?; @@ -234,15 +271,25 @@ impl Trie { /// Computes the nodes that would be added if updating the trie. /// Nodes are given with their hash pre-calculated. - pub fn commit_without_storing(&mut self) -> Vec { + pub fn commit_without_storing(&mut self) -> Vec { let mut acc = Vec::new(); if self.root.is_valid() { self.root.commit(Nibbles::default(), &mut acc); } if self.root.compute_hash() == NodeHash::Hashed(*EMPTY_TRIE_HASH) { - acc.push((Nibbles::default(), vec![RLP_NULL])) + acc.push(TrieCommitEntry::LeafValue { + path: Nibbles::default(), + value: vec![RLP_NULL], + }) } - acc.extend(self.pending_removal.drain().map(|nib| (nib, vec![]))); + acc.extend( + self.pending_removal + .drain() + .map(|nib| TrieCommitEntry::LeafValue { + path: nib, + value: vec![], + }), + ); acc } diff --git a/crates/networking/p2p/sync/healing/storage.rs b/crates/networking/p2p/sync/healing/storage.rs index dc9e42c709f..0f6f5159b9d 100644 --- a/crates/networking/p2p/sync/healing/storage.rs +++ b/crates/networking/p2p/sync/healing/storage.rs @@ -21,7 +21,7 @@ use bytes::Bytes; use ethrex_common::{H256, types::AccountState}; use ethrex_rlp::{decode::RLPDecode, encode::RLPEncode, error::RLPDecodeError}; use ethrex_storage::{Store, error::StoreError}; -use ethrex_trie::{EMPTY_TRIE_HASH, Nibbles, Node}; +use ethrex_trie::{EMPTY_TRIE_HASH, Nibbles, Node, TrieCommitEntry}; use rand::random; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use std::{ @@ -222,9 +222,17 @@ pub async fn heal_storage_trie( let mut account_nodes = vec![]; for (path, node) in nodes { for i in 0..path.len() { - account_nodes.push((path.slice(0, i), vec![])); + account_nodes.push(TrieCommitEntry::LeafValue { + path: path.slice(0, i), + value: vec![], + }); } - account_nodes.push((path, node.encode_to_vec())); + let encoded = node.encode_to_vec(); + account_nodes.push(TrieCommitEntry::Node { + path, + node: std::sync::Arc::new(node), + encoded, + }); } encoded_to_write.push((hashed_account, account_nodes)); } diff --git a/crates/storage/layering.rs b/crates/storage/layering.rs index 7d205c15ceb..52b4e61da59 100644 --- a/crates/storage/layering.rs +++ b/crates/storage/layering.rs @@ -1,17 +1,28 @@ use ethrex_common::H256; +use ethrex_trie::{Nibbles, Node, TrieCommitEntry, TrieDB, TrieError}; use fastbloom::AtomicBloomFilter; use rayon::prelude::*; use rustc_hash::{FxBuildHasher, FxHashMap}; use std::{fmt, sync::Arc}; -use ethrex_trie::{Nibbles, TrieDB, TrieError}; - const BLOOM_SIZE: usize = 1_000_000; const FALSE_POSITIVE_RATE: f64 = 0.02; +/// A cached trie node entry holding both the decoded node and its RLP encoding. +#[derive(Clone, Debug)] +pub struct CachedTrieEntry { + pub node: Arc, + pub encoded: Vec, +} + #[derive(Debug, Clone)] struct TrieLayer { - nodes: FxHashMap, Vec>, + /// Account (state) trie nodes, keyed by unprefixed nibble path. + account_nodes: FxHashMap, CachedTrieEntry>, + /// Storage trie nodes, keyed by prefixed nibble path (account_nibbles ++ 0x11 ++ path). + storage_nodes: FxHashMap, CachedTrieEntry>, + /// FKV leaf values (both account and storage), keyed by (possibly prefixed) nibble path. + leaf_values: FxHashMap, Vec>, parent: H256, id: usize, } @@ -24,11 +35,8 @@ struct TrieLayer { /// newest_root -> parent_1 -> parent_2 -> ... -> oldest_root -> (on-disk state) /// ``` /// -/// Each layer stores the trie node diffs produced by one block (regular sync) or one batch -/// of ~1024 blocks (full sync). When the chain reaches `commit_threshold` layers, -/// [`get_commitable`](Self::get_commitable) identifies the layer to flush, and -/// [`commit`](Self::commit) removes it (plus all ancestors) and returns the merged key-values -/// for writing to RocksDB. +/// Each layer stores decoded `Arc` for trie nodes (eliminating `Node::decode()` on +/// cache hits) in separate account/storage maps, plus raw leaf values for the FKV shortcut. /// /// Two commit thresholds are used in practice: /// - **128** — regular block-by-block execution (one layer ≈ one block's trie diff). @@ -94,34 +102,63 @@ impl TrieLayerCache { .expected_items(expected_items.max(BLOOM_SIZE)) } - /// Looks up a trie node `key` starting from the layer identified by `state_root`, + /// Looks up a decoded account trie node starting from the layer identified by `state_root`, /// walking the parent chain toward older layers. - /// - /// Returns `Some(value)` from the first (newest) layer that contains the key, or `None` - /// if no layer has it. A bloom filter is checked first to skip the walk entirely when the - /// key is guaranteed absent from all layers (callers then fall through to the on-disk trie). - pub fn get(&self, state_root: H256, key: &[u8]) -> Option> { - // Fast check to know if any layer may contain the given key. - // We can only be certain it doesn't exist, but if it returns true it may or may not exist (false positive). + pub fn get_account_node(&self, state_root: H256, key: &[u8]) -> Option> { + if !self.bloom.contains(key) { + return None; + } + + let mut current_state_root = state_root; + + while let Some(layer) = self.layers.get(¤t_state_root) { + if let Some(entry) = layer.account_nodes.get(key) { + return Some(entry.node.clone()); + } + current_state_root = layer.parent; + if current_state_root == state_root { + panic!("State cycle found"); + } + } + None + } + + /// Looks up a decoded storage trie node starting from the layer identified by `state_root`, + /// walking the parent chain toward older layers. + /// `key` must be the prefixed nibble path (account_nibbles ++ 0x11 ++ trie_path). + pub fn get_storage_node(&self, state_root: H256, key: &[u8]) -> Option> { + if !self.bloom.contains(key) { + return None; + } + + let mut current_state_root = state_root; + + while let Some(layer) = self.layers.get(¤t_state_root) { + if let Some(entry) = layer.storage_nodes.get(key) { + return Some(entry.node.clone()); + } + current_state_root = layer.parent; + if current_state_root == state_root { + panic!("State cycle found"); + } + } + None + } + + /// Looks up a FKV leaf value starting from the layer identified by `state_root`. + pub fn get_leaf_value(&self, state_root: H256, key: &[u8]) -> Option> { if !self.bloom.contains(key) { - // TrieWrapper goes to db when returning None. return None; } let mut current_state_root = state_root; while let Some(layer) = self.layers.get(¤t_state_root) { - if let Some(value) = layer.nodes.get(key) { + if let Some(value) = layer.leaf_values.get(key) { return Some(value.clone()); } current_state_root = layer.parent; if current_state_root == state_root { - // TODO: check if this is possible in practice - // This can't happen in L1, due to system contracts irreversibly modifying state - // at each block. - // On L2, if no transactions are included in a block, the state root remains the same, - // but we handle that case in put_batch. It may happen, however, if someone modifies - // state with a privileged tx and later reverts it (since it doesn't update nonce). panic!("State cycle found"); } } @@ -142,13 +179,6 @@ impl TrieLayerCache { /// layers. When the count reaches `threshold`, returns the state root of that ancestor layer. /// /// Returns `None` if the chain has fewer than `threshold` layers (nothing to commit yet). - /// - /// This function is used to determine when to trigger a disk commit. We consider a layer "committable" - /// when it has at least `threshold` newer layers on top of it, ensuring that we only commit sufficiently - /// old layers and keep recent ones in memory for fast access. - /// - /// Having a threshold allows both customizing the commit frequency (e.g. full sync vs regular block execution) - /// and avoiding edge cases where there could, theoretically, be a cycle in the layer change. pub(crate) fn get_commitable_with_threshold( &self, mut state_root: H256, @@ -165,26 +195,21 @@ impl TrieLayerCache { None } - /// Inserts a new diff-layer into the cache, keyed by `state_root` and pointing to `parent`. - /// - /// In regular sync each call adds one block's trie diffs. In full sync (batch mode), each - /// call adds diffs for an entire batch of ~1024 blocks. + /// Inserts a new diff-layer into the cache from structured `TrieCommitEntry` entries. /// - /// No-ops if `parent == state_root` (empty block with no state change), or if `state_root` - /// is already present (duplicate insertion guard). + /// Account entries are inserted directly. Storage entries are prefixed with the + /// account hash (nibble-encoded with 0x11 separator) before insertion. pub fn put_batch( &mut self, parent: H256, state_root: H256, - key_values: Vec<(Nibbles, Vec)>, + account_entries: Vec, + storage_entries: Vec<(H256, Vec)>, ) { - if parent == state_root && key_values.is_empty() { + if parent == state_root && account_entries.is_empty() && storage_entries.is_empty() { return; } else if parent == state_root { - // L1 always changes the state root (system contracts run even on empty blocks), so - // this should not happen there. L2 can legitimately keep the same root on empty blocks - // because it has no system contract calls. - tracing::trace!("parent == state_root but key_values not empty"); + tracing::trace!("parent == state_root but entries not empty"); return; } if self.layers.contains_key(&state_root) { @@ -192,19 +217,57 @@ impl TrieLayerCache { return; } - // Add keys to the global bloom filter - for (p, _) in &key_values { - self.bloom.insert(p.as_ref()); + let mut account_nodes = FxHashMap::default(); + let mut storage_nodes = FxHashMap::default(); + let mut leaf_values = FxHashMap::default(); + + for entry in account_entries { + match entry { + TrieCommitEntry::Node { + path, + node, + encoded, + } => { + let key = path.into_vec(); + self.bloom.insert(&key); + account_nodes.insert(key, CachedTrieEntry { node, encoded }); + } + TrieCommitEntry::LeafValue { path, value } => { + let key = path.into_vec(); + self.bloom.insert(&key); + leaf_values.insert(key, value); + } + } } - let nodes: FxHashMap, Vec> = key_values - .into_iter() - .map(|(path, value)| (path.into_vec(), value)) - .collect(); + for (account_hash, entries) in storage_entries { + for entry in entries { + match entry { + TrieCommitEntry::Node { + path, + node, + encoded, + } => { + let prefixed = apply_prefix(Some(account_hash), path); + let key = prefixed.into_vec(); + self.bloom.insert(&key); + storage_nodes.insert(key, CachedTrieEntry { node, encoded }); + } + TrieCommitEntry::LeafValue { path, value } => { + let prefixed = apply_prefix(Some(account_hash), path); + let key = prefixed.into_vec(); + self.bloom.insert(&key); + leaf_values.insert(key, value); + } + } + } + } self.last_id += 1; let entry = TrieLayer { - nodes, + account_nodes, + storage_nodes, + leaf_values, parent, id: self.last_id, }; @@ -216,14 +279,25 @@ impl TrieLayerCache { /// Called after [`commit`](Self::commit) removes layers, since the old filter may contain /// keys from the removed layers (producing unnecessary false positives). pub fn rebuild_bloom(&mut self) { - // Pre-compute total keys for optimal filter sizing - let total_keys: usize = self.layers.values().map(|layer| layer.nodes.len()).sum(); + let total_keys: usize = self + .layers + .values() + .map(|layer| { + layer.account_nodes.len() + layer.storage_nodes.len() + layer.leaf_values.len() + }) + .sum(); let filter = Self::create_filter(total_keys.max(BLOOM_SIZE)); // Parallel insertion - AtomicBloomFilter allows concurrent insert via &self self.layers.par_iter().for_each(|(_, layer)| { - for path in layer.nodes.keys() { + for path in layer.account_nodes.keys() { + filter.insert(path); + } + for path in layer.storage_nodes.keys() { + filter.insert(path); + } + for path in layer.leaf_values.keys() { filter.insert(path); } }); @@ -234,13 +308,8 @@ impl TrieLayerCache { /// Removes the layer at `state_root` and all its ancestors from the cache, returning /// their merged trie node diffs in oldest-first order (suitable for sequential disk write). /// - /// `state_root` must be a key in `self.layers` (as returned by - /// [`get_commitable`](Self::get_commitable) / - /// [`get_commitable_with_threshold`](Self::get_commitable_with_threshold)). - /// If it isn't, the walk exits immediately and returns `None`. - /// - /// After removal, any orphaned layers (older than the committed ones) are pruned, and - /// the bloom filter is rebuilt to remove stale entries. + /// Returns the merged key-value pairs in the same flat `(Vec, Vec)` format + /// as before, suitable for dispatch by key length in `apply_trie_updates`. pub fn commit(&mut self, state_root: H256) -> Option, Vec)>> { let mut layers_to_commit = vec![]; let mut current_state_root = state_root; @@ -256,7 +325,19 @@ impl TrieLayerCache { let nodes_to_commit = layers_to_commit .into_iter() .rev() - .flat_map(|layer| layer.nodes) + .flat_map(|layer| { + layer + .account_nodes + .into_iter() + .map(|(key, entry)| (key, entry.encoded)) + .chain( + layer + .storage_nodes + .into_iter() + .map(|(key, entry)| (key, entry.encoded)), + ) + .chain(layer.leaf_values) + }) .collect(); Some(nodes_to_commit) } @@ -307,6 +388,20 @@ pub fn apply_prefix(prefix: Option, path: Nibbles) -> Nibbles { } impl TrieDB for TrieWrapper { + fn get_node(&self, key: Nibbles) -> Result>, TrieError> { + let cached = if let Some(prefix) = &self.prefix_nibbles { + // Storage trie — look in storage_nodes with prefixed key + let prefixed = prefix.concat(&key); + self.inner + .get_storage_node(self.state_root, prefixed.as_ref()) + } else { + // Account trie — look in account_nodes + self.inner + .get_account_node(self.state_root, key.as_ref()) + }; + Ok(cached) + } + fn flatkeyvalue_computed(&self, key: Nibbles) -> bool { // NOTE: we apply the prefix here, since the underlying TrieDB should // always be for the state trie. @@ -322,7 +417,8 @@ impl TrieDB for TrieWrapper { Some(prefix) => prefix.concat(&key), None => key, }; - if let Some(value) = self.inner.get(self.state_root, key.as_ref()) { + // Check leaf value cache (FKV entries) + if let Some(value) = self.inner.get_leaf_value(self.state_root, key.as_ref()) { return Ok(Some(value)); } self.db.get(key) diff --git a/crates/storage/store.rs b/crates/storage/store.rs index 7c0924354f2..805b176aeb2 100644 --- a/crates/storage/store.rs +++ b/crates/storage/store.rs @@ -36,8 +36,9 @@ use ethrex_rlp::{ decode::{RLPDecode, decode_bytes}, encode::RLPEncode, }; -use ethrex_trie::{EMPTY_TRIE_HASH, Nibbles, Trie, TrieLogger, TrieNode, TrieWitness}; -use ethrex_trie::{Node, NodeRLP}; +use ethrex_trie::{ + EMPTY_TRIE_HASH, Nibbles, Node, NodeRLP, Trie, TrieCommitEntry, TrieLogger, TrieWitness, +}; use lru::LruCache; use rustc_hash::FxBuildHasher; use serde::{Deserialize, Serialize}; @@ -207,7 +208,7 @@ impl Drop for ThreadList { /// /// Each entry contains the hashed account address and the trie nodes /// for that account's storage trie. -pub type StorageTrieNodes = Vec<(H256, Vec<(Nibbles, Vec)>)>; +pub type StorageTrieNodes = Vec<(H256, Vec)>; type StorageTries = HashMap; /// Storage backend type selection. @@ -228,9 +229,9 @@ pub enum EngineType { /// committing them to the database in a single transaction. pub struct UpdateBatch { /// New nodes to add to the state trie. - pub account_updates: Vec, + pub account_updates: Vec, /// Storage trie updates per account (keyed by hashed address). - pub storage_updates: Vec<(H256, Vec)>, + pub storage_updates: Vec<(H256, Vec)>, /// Blocks to store. pub blocks: Vec, /// Receipts to store, grouped by block hash. @@ -244,7 +245,7 @@ pub struct UpdateBatch { } /// Storage trie updates grouped by account address hash. -pub type StorageUpdates = Vec<(H256, Vec<(Nibbles, Vec)>)>; +pub type StorageUpdates = Vec<(H256, Vec)>; /// Collection of account state changes from block execution. /// @@ -253,8 +254,8 @@ pub type StorageUpdates = Vec<(H256, Vec<(Nibbles, Vec)>)>; pub struct AccountUpdatesList { /// Root hash of the state trie after applying these updates. pub state_trie_hash: H256, - /// State trie node updates (path -> RLP-encoded node). - pub state_updates: Vec<(Nibbles, Vec)>, + /// State trie commit entries (nodes + leaf values). + pub state_updates: Vec, /// Storage trie updates per account. pub storage_updates: StorageUpdates, /// New contract bytecode deployments. @@ -1170,8 +1171,9 @@ impl Store { ) -> Result<(), StoreError> { let mut txn = self.backend.begin_write()?; tokio::task::spawn_blocking(move || { - for (address_hash, nodes) in storage_trie_nodes { - for (node_path, node_data) in nodes { + for (address_hash, entries) in storage_trie_nodes { + for entry in entries { + let (node_path, node_data) = entry.into_rlp_pair(); let key = apply_prefix(Some(address_hash), node_path); if node_data.is_empty() { txn.delete(STORAGE_TRIE_NODES, key.as_ref())?; @@ -1886,11 +1888,13 @@ impl Store { let (storage_root, storage_nodes) = storage_trie.collect_changes_since_last_hash(); - storage_trie_nodes.extend( - storage_nodes - .into_iter() - .map(|(path, n)| (apply_prefix(Some(h256_hashed_address), path).into_vec(), n)), - ); + storage_trie_nodes.extend(storage_nodes.into_iter().map(|entry| { + let (path, data) = entry.into_rlp_pair(); + ( + apply_prefix(Some(h256_hashed_address), path).into_vec(), + data, + ) + })); // Add account to trie let account_state = AccountState { @@ -1905,7 +1909,10 @@ impl Store { let (state_root, account_trie_nodes) = genesis_state_trie.collect_changes_since_last_hash(); let account_trie_nodes = account_trie_nodes .into_iter() - .map(|(path, n)| (apply_prefix(None, path).into_vec(), n)) + .map(|entry| { + let (path, data) = entry.into_rlp_pair(); + (apply_prefix(None, path).into_vec(), data) + }) .collect::>(); let mut tx = self.backend.begin_write()?; @@ -2798,14 +2805,12 @@ impl Store { } } -type TrieNodesUpdate = Vec<(Nibbles, Vec)>; - struct TrieUpdate { result_sender: std::sync::mpsc::SyncSender>, parent_state_root: H256, child_state_root: H256, - account_updates: TrieNodesUpdate, - storage_updates: Vec<(H256, TrieNodesUpdate)>, + account_updates: Vec, + storage_updates: Vec<(H256, Vec)>, is_batch: bool, } @@ -2827,22 +2832,18 @@ fn apply_trie_updates( } = trie_update; // Phase 1: update the in-memory diff-layers only, then notify block production. - let new_layer = storage_updates - .into_iter() - .flat_map(|(account_hash, nodes)| { - nodes - .into_iter() - .map(move |(path, node)| (apply_prefix(Some(account_hash), path), node)) - }) - .chain(account_updates) - .collect(); // Read-Copy-Update the trie cache with a new layer. let trie = trie_cache .read() .map_err(|_| StoreError::LockError)? .clone(); let mut trie_mut = (*trie).clone(); - trie_mut.put_batch(parent_state_root, child_state_root, new_layer); + trie_mut.put_batch( + parent_state_root, + child_state_root, + account_updates, + storage_updates, + ); let trie = Arc::new(trie_mut); *trie_cache.write().map_err(|_| StoreError::LockError)? = trie.clone(); // Update finished, signal block processing. From 67d08a8889aa8727b1c7380e0baff2b8d14a9dce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Arjovsky?= Date: Mon, 9 Mar 2026 20:24:45 +0100 Subject: [PATCH 2/5] join stuff again --- crates/storage/layering.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/crates/storage/layering.rs b/crates/storage/layering.rs index 52b4e61da59..7c43df89c1e 100644 --- a/crates/storage/layering.rs +++ b/crates/storage/layering.rs @@ -145,8 +145,11 @@ impl TrieLayerCache { None } - /// Looks up a FKV leaf value starting from the layer identified by `state_root`. - pub fn get_leaf_value(&self, state_root: H256, key: &[u8]) -> Option> { + /// Looks up the RLP-encoded bytes for a key, checking all three maps (leaf values, + /// account nodes, storage nodes). This is the equivalent of the old flat `get()` method + /// and is needed by callers like `has_state_root` that call `TrieDB::get()` expecting + /// to receive encoded node bytes. + pub fn get_encoded(&self, state_root: H256, key: &[u8]) -> Option> { if !self.bloom.contains(key) { return None; } @@ -157,6 +160,12 @@ impl TrieLayerCache { if let Some(value) = layer.leaf_values.get(key) { return Some(value.clone()); } + if let Some(entry) = layer.account_nodes.get(key) { + return Some(entry.encoded.clone()); + } + if let Some(entry) = layer.storage_nodes.get(key) { + return Some(entry.encoded.clone()); + } current_state_root = layer.parent; if current_state_root == state_root { panic!("State cycle found"); @@ -417,8 +426,10 @@ impl TrieDB for TrieWrapper { Some(prefix) => prefix.concat(&key), None => key, }; - // Check leaf value cache (FKV entries) - if let Some(value) = self.inner.get_leaf_value(self.state_root, key.as_ref()) { + // Check all layer cache maps (leaf values, account nodes, storage nodes). + // This is needed because callers like `has_state_root` use `get()` to retrieve + // the root node's encoded bytes, not just FKV leaf values. + if let Some(value) = self.inner.get_encoded(self.state_root, key.as_ref()) { return Ok(Some(value)); } self.db.get(key) From 57df196590c229a14b580b30b0c4faa035d60c90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Arjovsky?= Date: Thu, 12 Mar 2026 12:38:15 +0100 Subject: [PATCH 3/5] format --- crates/common/trie/trie.rs | 5 +---- crates/storage/layering.rs | 3 +-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/common/trie/trie.rs b/crates/common/trie/trie.rs index 8caf0c1c7a1..b078a015f0b 100644 --- a/crates/common/trie/trie.rs +++ b/crates/common/trie/trie.rs @@ -65,10 +65,7 @@ pub enum TrieCommitEntry { encoded: Vec, }, /// A leaf's application-level value (for FlatKeyValue table), or a deletion marker. - LeafValue { - path: Nibbles, - value: Vec, - }, + LeafValue { path: Nibbles, value: Vec }, } impl TrieCommitEntry { diff --git a/crates/storage/layering.rs b/crates/storage/layering.rs index 7c43df89c1e..73c2d66d0fa 100644 --- a/crates/storage/layering.rs +++ b/crates/storage/layering.rs @@ -405,8 +405,7 @@ impl TrieDB for TrieWrapper { .get_storage_node(self.state_root, prefixed.as_ref()) } else { // Account trie — look in account_nodes - self.inner - .get_account_node(self.state_root, key.as_ref()) + self.inner.get_account_node(self.state_root, key.as_ref()) }; Ok(cached) } From fbcb6d9c16c30cd760acc1099c7d85ae89afc223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Arjovsky?= Date: Thu, 12 Mar 2026 13:06:59 +0100 Subject: [PATCH 4/5] trie commit entry type in snap sync --- crates/networking/p2p/sync/snap_sync.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/networking/p2p/sync/snap_sync.rs b/crates/networking/p2p/sync/snap_sync.rs index 38ec77ed7ce..9c1bc77ed65 100644 --- a/crates/networking/p2p/sync/snap_sync.rs +++ b/crates/networking/p2p/sync/snap_sync.rs @@ -806,7 +806,7 @@ pub fn validate_bytecodes(store: Store, state_root: H256) -> bool { // ============================================================================ #[cfg(not(feature = "rocksdb"))] -type StorageRoots = (H256, Vec<(ethrex_trie::Nibbles, Vec)>); +type StorageRoots = (H256, Vec); #[cfg(not(feature = "rocksdb"))] fn compute_storage_roots( From fdb607159a82db5428294b207ed848a7e65bef21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Arjovsky?= Date: Thu, 12 Mar 2026 14:19:20 +0100 Subject: [PATCH 5/5] changelog perf --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a357769155c..326a00964ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Perf +### 2026-03-12 + +- Store decoded `Arc` in TrieLayerCache to eliminate RLP decoding on cache hits, and split cache into separate account/storage/leaf maps + ### 2026-03-05 - Switch hot EVM and mempool HashMaps to FxHashMap for faster hashing [#6303](https://github.com/lambdaclass/ethrex/pull/6303)