From 7c818c1021abd343a71ccabbc461724b8f68ffe3 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Thu, 10 Oct 2024 00:28:35 +0900 Subject: [PATCH 01/82] chore(rpc): use `block_hash` as `BlockId` on `eth_callMany` (#11595) --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 5bc7d73b2221..0aa38a36e9da 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -22,7 +22,7 @@ use reth_primitives::{ }, Header, TransactionSignedEcRecovered, }; -use reth_provider::{ChainSpecProvider, HeaderProvider, StateProvider}; +use reth_provider::{BlockIdReader, ChainSpecProvider, HeaderProvider, StateProvider}; use reth_revm::{database::StateProviderDatabase, db::CacheDB, DatabaseRef}; use reth_rpc_eth_types::{ cache::db::{StateCacheDbRefMutWrapper, StateProviderTraitObjWrapper}, @@ -247,9 +247,19 @@ pub trait EthCall: Call + LoadPendingBlock { state_context.unwrap_or_default(); let transaction_index = transaction_index.unwrap_or_default(); - let target_block = block_number.unwrap_or_default(); + let mut target_block = block_number.unwrap_or_default(); let is_block_target_pending = target_block.is_pending(); + // if it's not pending, we should always use block_hash over block_number to ensure that + // different provider calls query data related to the same block. + if !is_block_target_pending { + target_block = LoadBlock::provider(self) + .block_hash_for_id(target_block) + .map_err(|_| EthApiError::HeaderNotFound(target_block))? + .ok_or_else(|| EthApiError::HeaderNotFound(target_block))? + .into(); + } + let ((cfg, block_env, _), block) = futures::try_join!( self.evm_env_at(target_block), self.block_with_senders(target_block) From adf3a0da6b26fccaaad85c96168338ef3152ae74 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Oct 2024 17:51:25 +0200 Subject: [PATCH 02/82] fix: 7702 gas fields (#11614) --- crates/rpc/rpc-types-compat/src/transaction/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc-types-compat/src/transaction/mod.rs b/crates/rpc/rpc-types-compat/src/transaction/mod.rs index 4a00bf7a4390..9bb2a8b5d92a 100644 --- a/crates/rpc/rpc-types-compat/src/transaction/mod.rs +++ b/crates/rpc/rpc-types-compat/src/transaction/mod.rs @@ -37,11 +37,12 @@ pub trait TransactionCompat: Send + Sync + Unpin + Clone + fmt::Debug { /// Formats gas price and max fee per gas for RPC transaction response w.r.t. network specific /// transaction type. fn gas_price(signed_tx: &TransactionSigned, base_fee: Option) -> GasPrice { + #[allow(unreachable_patterns)] match signed_tx.tx_type() { TxType::Legacy | TxType::Eip2930 => { GasPrice { gas_price: Some(signed_tx.max_fee_per_gas()), max_fee_per_gas: None } } - TxType::Eip1559 | TxType::Eip4844 => { + TxType::Eip1559 | TxType::Eip4844 | TxType::Eip7702 => { // the gas price field for EIP1559 is set to `min(tip, gasFeeCap - baseFee) + // baseFee` let gas_price = base_fee From 40faed6ea05a778642a703627ce912dfb8894cfb Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Wed, 9 Oct 2024 17:59:28 +0200 Subject: [PATCH 03/82] fix(witness): destroyed slots as proof targets (#11596) --- crates/trie/db/tests/witness.rs | 38 +++++++++++++++++++++++++++- crates/trie/trie/src/witness.rs | 45 ++++++++++++++++++++++++--------- 2 files changed, 70 insertions(+), 13 deletions(-) diff --git a/crates/trie/db/tests/witness.rs b/crates/trie/db/tests/witness.rs index cc921f657087..59656383de4c 100644 --- a/crates/trie/db/tests/witness.rs +++ b/crates/trie/db/tests/witness.rs @@ -6,7 +6,7 @@ use alloy_primitives::{ Address, Bytes, B256, U256, }; use alloy_rlp::EMPTY_STRING_CODE; -use reth_primitives::{constants::EMPTY_ROOT_HASH, Account}; +use reth_primitives::{constants::EMPTY_ROOT_HASH, Account, StorageEntry}; use reth_provider::{test_utils::create_test_provider_factory, HashingWriter}; use reth_trie::{proof::Proof, witness::TrieWitness, HashedPostState, HashedStorage, StateRoot}; use reth_trie_db::{DatabaseProof, DatabaseStateRoot, DatabaseTrieWitness}; @@ -55,3 +55,39 @@ fn includes_empty_node_preimage() { // witness includes empty state trie root node assert_eq!(witness.get(&EMPTY_ROOT_HASH), Some(&Bytes::from([EMPTY_STRING_CODE]))); } + +#[test] +fn includes_nodes_for_destroyed_storage_nodes() { + let factory = create_test_provider_factory(); + let provider = factory.provider_rw().unwrap(); + + let address = Address::random(); + let hashed_address = keccak256(address); + let slot = B256::random(); + let hashed_slot = keccak256(slot); + + // Insert account and slot into database + provider.insert_account_for_hashing([(address, Some(Account::default()))]).unwrap(); + provider + .insert_storage_for_hashing([(address, [StorageEntry { key: slot, value: U256::from(1) }])]) + .unwrap(); + + let state_root = StateRoot::from_tx(provider.tx_ref()).root().unwrap(); + let multiproof = Proof::from_tx(provider.tx_ref()) + .multiproof(HashMap::from_iter([(hashed_address, HashSet::from_iter([hashed_slot]))])) + .unwrap(); + + let witness = TrieWitness::from_tx(provider.tx_ref()) + .compute(HashedPostState { + accounts: HashMap::from([(hashed_address, Some(Account::default()))]), + storages: HashMap::from([(hashed_address, HashedStorage::from_iter(true, []))]), // destroyed + }) + .unwrap(); + assert!(witness.contains_key(&state_root)); + for node in multiproof.account_subtree.values() { + assert_eq!(witness.get(&keccak256(node)), Some(node)); + } + for node in multiproof.storages.iter().flat_map(|(_, storage)| storage.subtree.values()) { + assert_eq!(witness.get(&keccak256(node)), Some(node)); + } +} diff --git a/crates/trie/trie/src/witness.rs b/crates/trie/trie/src/witness.rs index d668946e6238..c042a0d8213b 100644 --- a/crates/trie/trie/src/witness.rs +++ b/crates/trie/trie/src/witness.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use crate::{ - hashed_cursor::HashedCursorFactory, + hashed_cursor::{HashedCursor, HashedCursorFactory}, prefix_set::TriePrefixSetsMut, proof::{Proof, StorageProof}, trie_cursor::TrieCursorFactory, @@ -14,7 +14,7 @@ use alloy_primitives::{ }; use alloy_rlp::{BufMut, Decodable, Encodable}; use itertools::{Either, Itertools}; -use reth_execution_errors::TrieWitnessError; +use reth_execution_errors::{StateProofError, TrieWitnessError}; use reth_primitives::constants::EMPTY_ROOT_HASH; use reth_trie_common::{ BranchNode, HashBuilder, Nibbles, StorageMultiProof, TrieAccount, TrieNode, CHILD_INDEX_RANGE, @@ -90,16 +90,7 @@ where return Ok(self.witness) } - let proof_targets = HashMap::from_iter( - state - .accounts - .keys() - .map(|hashed_address| (*hashed_address, HashSet::default())) - .chain(state.storages.iter().map(|(hashed_address, storage)| { - (*hashed_address, storage.storage.keys().copied().collect()) - })), - ); - + let proof_targets = self.get_proof_targets(&state)?; let mut account_multiproof = Proof::new(self.trie_cursor_factory.clone(), self.hashed_cursor_factory.clone()) .with_prefix_sets_mut(self.prefix_sets.clone()) @@ -257,6 +248,36 @@ where Ok(trie_nodes) } + /// Retrieve proof targets for incoming hashed state. + /// This method will aggregate all accounts and slots present in the hash state as well as + /// select all existing slots from the database for the accounts that have been destroyed. + fn get_proof_targets( + &self, + state: &HashedPostState, + ) -> Result>, StateProofError> { + let mut proof_targets = HashMap::default(); + for hashed_address in state.accounts.keys() { + proof_targets.insert(*hashed_address, HashSet::default()); + } + for (hashed_address, storage) in &state.storages { + let mut storage_keys = storage.storage.keys().copied().collect::>(); + if storage.wiped { + // storage for this account was destroyed, gather all slots from the current state + let mut storage_cursor = + self.hashed_cursor_factory.hashed_storage_cursor(*hashed_address)?; + // position cursor at the start + if let Some((hashed_slot, _)) = storage_cursor.seek(B256::ZERO)? { + storage_keys.insert(hashed_slot); + } + while let Some((hashed_slot, _)) = storage_cursor.next()? { + storage_keys.insert(hashed_slot); + } + } + proof_targets.insert(*hashed_address, storage_keys); + } + Ok(proof_targets) + } + fn next_root_from_proofs( trie_nodes: BTreeMap>>, mut trie_node_provider: impl FnMut(Nibbles) -> Result, From 076b1a2c226732591b0743b02dbfbfff070b5e4d Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Oct 2024 18:40:48 +0200 Subject: [PATCH 04/82] chore: bump op-alloy (#11617) --- Cargo.lock | 24 +++++++++---------- Cargo.toml | 8 +++---- crates/e2e-test-utils/src/traits.rs | 4 ++-- crates/optimism/chainspec/src/lib.rs | 10 ++++---- crates/optimism/node/src/engine.rs | 19 +++++++-------- crates/optimism/payload/src/lib.rs | 4 +--- crates/optimism/payload/src/payload.rs | 21 +++++++--------- crates/optimism/rpc/src/eth/receipt.rs | 32 +++++++++++-------------- crates/payload/primitives/src/traits.rs | 4 ++-- crates/primitives/src/alloy_compat.rs | 2 +- 10 files changed, 58 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 863f0f35c18d..9bf1ea2d666d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5174,9 +5174,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "op-alloy-consensus" -version = "0.3.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f7f318f885db6e1455370ca91f74b7faed152c8142f6418f0936d606e582ff" +checksum = "7ea7162170c6f3cad8f67f4dd7108e3f78349fd553da5b8bebff1e7ef8f38896" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5192,9 +5192,9 @@ dependencies = [ [[package]] name = "op-alloy-genesis" -version = "0.3.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8215c87b74d2fbbaff0fd2887868a8341df33a3c495ee01f813e5ddd5be9c46" +checksum = "9f3d31dfbbd8dd898c7512f8ce7d30103980485416f668566100b0ed0994b958" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5206,9 +5206,9 @@ dependencies = [ [[package]] name = "op-alloy-network" -version = "0.3.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd514c4ccd0b3c69fa3e7050cde77db842d4c308ae48f9a3e1ce263e823e45e" +checksum = "d113b325527ba7da271a8793f1c14bdf7f035ce9e0611e668c36fc6812568c7f" dependencies = [ "alloy-consensus", "alloy-network", @@ -5220,9 +5220,9 @@ dependencies = [ [[package]] name = "op-alloy-protocol" -version = "0.3.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa5c397fbe35e07f9c95a571440ca2e90df754e198496d82ff4127de00b89dd9" +checksum = "310873e4fbfc41986716c4fb6000a8b49d025d932d2c261af58271c434b05288" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5237,9 +5237,9 @@ dependencies = [ [[package]] name = "op-alloy-rpc-types" -version = "0.3.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "547d29c5ab957ff32e14edddb93652dad748d2ef6cbe4b0fe8615ce06b0a3ddb" +checksum = "323c65880e2561aa87f74f8af260fd15b9cc930c448c88a60ae95af86c88c634" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5254,9 +5254,9 @@ dependencies = [ [[package]] name = "op-alloy-rpc-types-engine" -version = "0.3.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041122e20b76644cc690bba688671eecdc4626e6384a76eb740535d6ddcef14" +checksum = "349e7b420f45d1a00216ec4c65fcf3f0057a841bc39732c405c85ae782b94121" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", diff --git a/Cargo.toml b/Cargo.toml index 5c772a844429..22ce3e92f6a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -460,10 +460,10 @@ alloy-transport-ipc = { version = "0.4.2", default-features = false } alloy-transport-ws = { version = "0.4.2", default-features = false } # op -op-alloy-rpc-types = "0.3.2" -op-alloy-rpc-types-engine = "0.3.2" -op-alloy-network = "0.3.2" -op-alloy-consensus = "0.3.2" +op-alloy-rpc-types = "0.4" +op-alloy-rpc-types-engine = "0.4" +op-alloy-network = "0.4" +op-alloy-consensus = "0.4" # misc aquamarine = "0.5" diff --git a/crates/e2e-test-utils/src/traits.rs b/crates/e2e-test-utils/src/traits.rs index 7bd5a5b3222f..6786492140b0 100644 --- a/crates/e2e-test-utils/src/traits.rs +++ b/crates/e2e-test-utils/src/traits.rs @@ -1,4 +1,4 @@ -use op_alloy_rpc_types_engine::OptimismExecutionPayloadEnvelopeV3; +use op_alloy_rpc_types_engine::OpExecutionPayloadEnvelopeV3; use reth::rpc::types::engine::{ExecutionPayloadEnvelopeV3, ExecutionPayloadV3}; /// The execution payload envelope type. @@ -7,7 +7,7 @@ pub trait PayloadEnvelopeExt: Send + Sync + std::fmt::Debug { fn execution_payload(&self) -> ExecutionPayloadV3; } -impl PayloadEnvelopeExt for OptimismExecutionPayloadEnvelopeV3 { +impl PayloadEnvelopeExt for OpExecutionPayloadEnvelopeV3 { fn execution_payload(&self) -> ExecutionPayloadV3 { self.execution_payload.clone() } diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index 22316a234d9b..ebd9392b5330 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -224,14 +224,14 @@ impl From for OpChainSpec { #[derive(Default, Debug)] struct OptimismGenesisInfo { - optimism_chain_info: op_alloy_rpc_types::genesis::OptimismChainInfo, + optimism_chain_info: op_alloy_rpc_types::genesis::OpChainInfo, base_fee_params: BaseFeeParamsKind, } impl OptimismGenesisInfo { fn extract_from(genesis: &Genesis) -> Self { let mut info = Self { - optimism_chain_info: op_alloy_rpc_types::genesis::OptimismChainInfo::extract_from( + optimism_chain_info: op_alloy_rpc_types::genesis::OpChainInfo::extract_from( &genesis.config.extra_fields, ) .unwrap_or_default(), @@ -622,7 +622,7 @@ mod tests { #[test] fn parse_genesis_optimism_with_variable_base_fee_params() { - use op_alloy_rpc_types::genesis::OptimismBaseFeeInfo; + use op_alloy_rpc_types::genesis::OpBaseFeeInfo; let geth_genesis = r#" { @@ -673,11 +673,11 @@ mod tests { let optimism_object = genesis.config.extra_fields.get("optimism").unwrap(); let optimism_base_fee_info = - serde_json::from_value::(optimism_object.clone()).unwrap(); + serde_json::from_value::(optimism_object.clone()).unwrap(); assert_eq!( optimism_base_fee_info, - OptimismBaseFeeInfo { + OpBaseFeeInfo { eip1559_elasticity: Some(6), eip1559_denominator: Some(50), eip1559_denominator_canyon: None, diff --git a/crates/optimism/node/src/engine.rs b/crates/optimism/node/src/engine.rs index 11c8f15cfdac..a83f4c696a1c 100644 --- a/crates/optimism/node/src/engine.rs +++ b/crates/optimism/node/src/engine.rs @@ -2,8 +2,7 @@ use std::sync::Arc; use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV2, ExecutionPayloadV1}; use op_alloy_rpc_types_engine::{ - OptimismExecutionPayloadEnvelopeV3, OptimismExecutionPayloadEnvelopeV4, - OptimismPayloadAttributes, + OpExecutionPayloadEnvelopeV3, OpExecutionPayloadEnvelopeV4, OpPayloadAttributes, }; use reth_chainspec::ChainSpec; use reth_node_api::{ @@ -35,13 +34,13 @@ impl EngineTypes for OptimismEngineTypes where T::BuiltPayload: TryInto + TryInto - + TryInto - + TryInto, + + TryInto + + TryInto, { type ExecutionPayloadV1 = ExecutionPayloadV1; type ExecutionPayloadV2 = ExecutionPayloadEnvelopeV2; - type ExecutionPayloadV3 = OptimismExecutionPayloadEnvelopeV3; - type ExecutionPayloadV4 = OptimismExecutionPayloadEnvelopeV4; + type ExecutionPayloadV3 = OpExecutionPayloadEnvelopeV3; + type ExecutionPayloadV4 = OpExecutionPayloadEnvelopeV4; } /// A default payload type for [`OptimismEngineTypes`] @@ -51,7 +50,7 @@ pub struct OptimismPayloadTypes; impl PayloadTypes for OptimismPayloadTypes { type BuiltPayload = OptimismBuiltPayload; - type PayloadAttributes = OptimismPayloadAttributes; + type PayloadAttributes = OpPayloadAttributes; type PayloadBuilderAttributes = OptimismPayloadBuilderAttributes; } @@ -112,12 +111,12 @@ pub fn validate_withdrawals_presence( impl EngineValidator for OptimismEngineValidator where - Types: EngineTypes, + Types: EngineTypes, { fn validate_version_specific_fields( &self, version: EngineApiMessageVersion, - payload_or_attrs: PayloadOrAttributes<'_, OptimismPayloadAttributes>, + payload_or_attrs: PayloadOrAttributes<'_, OpPayloadAttributes>, ) -> Result<(), EngineObjectValidationError> { validate_withdrawals_presence( &self.chain_spec, @@ -138,7 +137,7 @@ where fn ensure_well_formed_attributes( &self, version: EngineApiMessageVersion, - attributes: &OptimismPayloadAttributes, + attributes: &OpPayloadAttributes, ) -> Result<(), EngineObjectValidationError> { validate_version_specific_fields(&self.chain_spec, version, attributes.into())?; diff --git a/crates/optimism/payload/src/lib.rs b/crates/optimism/payload/src/lib.rs index 2bb60594287a..c06b49c53762 100644 --- a/crates/optimism/payload/src/lib.rs +++ b/crates/optimism/payload/src/lib.rs @@ -15,6 +15,4 @@ pub mod builder; pub use builder::OptimismPayloadBuilder; pub mod error; pub mod payload; -pub use payload::{ - OptimismBuiltPayload, OptimismPayloadAttributes, OptimismPayloadBuilderAttributes, -}; +pub use payload::{OpPayloadAttributes, OptimismBuiltPayload, OptimismPayloadBuilderAttributes}; diff --git a/crates/optimism/payload/src/payload.rs b/crates/optimism/payload/src/payload.rs index f1ba24435092..122c2fde5269 100644 --- a/crates/optimism/payload/src/payload.rs +++ b/crates/optimism/payload/src/payload.rs @@ -7,10 +7,8 @@ use alloy_primitives::{Address, B256, U256}; use alloy_rlp::Encodable; use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV2, ExecutionPayloadV1, PayloadId}; /// Re-export for use in downstream arguments. -pub use op_alloy_rpc_types_engine::OptimismPayloadAttributes; -use op_alloy_rpc_types_engine::{ - OptimismExecutionPayloadEnvelopeV3, OptimismExecutionPayloadEnvelopeV4, -}; +pub use op_alloy_rpc_types_engine::OpPayloadAttributes; +use op_alloy_rpc_types_engine::{OpExecutionPayloadEnvelopeV3, OpExecutionPayloadEnvelopeV4}; use reth_chain_state::ExecutedBlock; use reth_chainspec::EthereumHardforks; use reth_optimism_chainspec::OpChainSpec; @@ -40,13 +38,13 @@ pub struct OptimismPayloadBuilderAttributes { } impl PayloadBuilderAttributes for OptimismPayloadBuilderAttributes { - type RpcPayloadAttributes = OptimismPayloadAttributes; + type RpcPayloadAttributes = OpPayloadAttributes; type Error = alloy_rlp::Error; /// Creates a new payload builder for the given parent block and the attributes. /// /// Derives the unique [`PayloadId`] for the given parent and attributes - fn try_new(parent: B256, attributes: OptimismPayloadAttributes) -> Result { + fn try_new(parent: B256, attributes: OpPayloadAttributes) -> Result { let id = payload_id_optimism(&parent, &attributes); let transactions = attributes @@ -213,7 +211,7 @@ impl From for ExecutionPayloadEnvelopeV2 { } } -impl From for OptimismExecutionPayloadEnvelopeV3 { +impl From for OpExecutionPayloadEnvelopeV3 { fn from(value: OptimismBuiltPayload) -> Self { let OptimismBuiltPayload { block, fees, sidecars, chain_spec, attributes, .. } = value; @@ -240,7 +238,7 @@ impl From for OptimismExecutionPayloadEnvelopeV3 { } } } -impl From for OptimismExecutionPayloadEnvelopeV4 { +impl From for OpExecutionPayloadEnvelopeV4 { fn from(value: OptimismBuiltPayload) -> Self { let OptimismBuiltPayload { block, fees, sidecars, chain_spec, attributes, .. } = value; @@ -268,13 +266,10 @@ impl From for OptimismExecutionPayloadEnvelopeV4 { } } -/// Generates the payload id for the configured payload from the [`OptimismPayloadAttributes`]. +/// Generates the payload id for the configured payload from the [`OpPayloadAttributes`]. /// /// Returns an 8-byte identifier by hashing the payload components with sha256 hash. -pub(crate) fn payload_id_optimism( - parent: &B256, - attributes: &OptimismPayloadAttributes, -) -> PayloadId { +pub(crate) fn payload_id_optimism(parent: &B256, attributes: &OpPayloadAttributes) -> PayloadId { use sha2::Digest; let mut hasher = sha2::Sha256::new(); hasher.update(parent.as_slice()); diff --git a/crates/optimism/rpc/src/eth/receipt.rs b/crates/optimism/rpc/src/eth/receipt.rs index 76b03e845a09..fac8d220c761 100644 --- a/crates/optimism/rpc/src/eth/receipt.rs +++ b/crates/optimism/rpc/src/eth/receipt.rs @@ -3,9 +3,7 @@ use alloy_eips::eip2718::Encodable2718; use alloy_rpc_types::{AnyReceiptEnvelope, Log, TransactionReceipt}; use op_alloy_consensus::{OpDepositReceipt, OpDepositReceiptWithBloom, OpReceiptEnvelope}; -use op_alloy_rpc_types::{ - receipt::L1BlockInfo, OpTransactionReceipt, OptimismTransactionReceiptFields, -}; +use op_alloy_rpc_types::{receipt::L1BlockInfo, OpTransactionReceipt, OpTransactionReceiptFields}; use reth_chainspec::ChainSpec; use reth_node_api::{FullNodeComponents, NodeTypes}; use reth_optimism_chainspec::OpChainSpec; @@ -68,7 +66,7 @@ where tx: &TransactionSigned, l1_block_info: revm::L1BlockInfo, receipt: &Receipt, - ) -> Result { + ) -> Result { Ok(OpReceiptFieldsBuilder::default() .l1_block_info(&self.inner.provider().chain_spec(), tx, l1_block_info)? .deposit_nonce(receipt.deposit_nonce) @@ -162,8 +160,8 @@ impl OpReceiptFieldsBuilder { self } - /// Builds the [`OptimismTransactionReceiptFields`] object. - pub const fn build(self) -> OptimismTransactionReceiptFields { + /// Builds the [`OpTransactionReceiptFields`] object. + pub const fn build(self) -> OpTransactionReceiptFields { let Self { l1_block_timestamp: _, // used to compute other fields l1_fee, @@ -177,7 +175,7 @@ impl OpReceiptFieldsBuilder { l1_blob_base_fee_scalar, } = self; - OptimismTransactionReceiptFields { + OpTransactionReceiptFields { l1_block_info: L1BlockInfo { l1_gas_price, l1_gas_used, @@ -201,7 +199,7 @@ pub struct OpReceiptBuilder { /// Transaction type. pub tx_type: TxType, /// Additional OP receipt fields. - pub op_receipt_fields: OptimismTransactionReceiptFields, + pub op_receipt_fields: OpTransactionReceiptFields, } impl OpReceiptBuilder { @@ -234,11 +232,8 @@ impl OpReceiptBuilder { pub fn build(self) -> OpTransactionReceipt { let Self { core_receipt, tx_type, op_receipt_fields } = self; - let OptimismTransactionReceiptFields { - l1_block_info, - deposit_nonce, - deposit_receipt_version, - } = op_receipt_fields; + let OpTransactionReceiptFields { l1_block_info, deposit_nonce, deposit_receipt_version } = + op_receipt_fields; let TransactionReceipt { inner: AnyReceiptEnvelope { inner: receipt_with_bloom, .. }, @@ -261,10 +256,11 @@ impl OpReceiptBuilder { TxType::Legacy => OpReceiptEnvelope::::Legacy(receipt_with_bloom), TxType::Eip2930 => OpReceiptEnvelope::::Eip2930(receipt_with_bloom), TxType::Eip1559 => OpReceiptEnvelope::::Eip1559(receipt_with_bloom), - TxType::Eip4844 => OpReceiptEnvelope::::Eip4844(receipt_with_bloom), - TxType::Eip7702 => { - unimplemented!("not implemented yet for OpReceiptEnvelope") + TxType::Eip4844 => { + // TODO: unreachable + OpReceiptEnvelope::::Eip1559(receipt_with_bloom) } + TxType::Eip7702 => OpReceiptEnvelope::::Eip7702(receipt_with_bloom), TxType::Deposit => { OpReceiptEnvelope::::Deposit(OpDepositReceiptWithBloom:: { receipt: OpDepositReceipt:: { @@ -325,8 +321,8 @@ mod test { /// L1 block info for transaction at index 1 in block 124665056. /// /// - const TX_META_TX_1_OP_MAINNET_BLOCK_124665056: OptimismTransactionReceiptFields = - OptimismTransactionReceiptFields { + const TX_META_TX_1_OP_MAINNET_BLOCK_124665056: OpTransactionReceiptFields = + OpTransactionReceiptFields { l1_block_info: L1BlockInfo { l1_gas_price: Some(1055991687), // since bedrock l1 base fee l1_gas_used: Some(4471), diff --git a/crates/payload/primitives/src/traits.rs b/crates/payload/primitives/src/traits.rs index b0647691f760..6ae6361fdb28 100644 --- a/crates/payload/primitives/src/traits.rs +++ b/crates/payload/primitives/src/traits.rs @@ -4,7 +4,7 @@ use alloy_rpc_types::{ engine::{PayloadAttributes as EthPayloadAttributes, PayloadId}, Withdrawal, }; -use op_alloy_rpc_types_engine::OptimismPayloadAttributes; +use op_alloy_rpc_types_engine::OpPayloadAttributes; use reth_chain_state::ExecutedBlock; use reth_primitives::{SealedBlock, Withdrawals}; use std::{future::Future, pin::Pin}; @@ -145,7 +145,7 @@ impl PayloadAttributes for EthPayloadAttributes { } } -impl PayloadAttributes for OptimismPayloadAttributes { +impl PayloadAttributes for OpPayloadAttributes { fn timestamp(&self) -> u64 { self.payload_attributes.timestamp } diff --git a/crates/primitives/src/alloy_compat.rs b/crates/primitives/src/alloy_compat.rs index f43af019e75b..65099967e6e9 100644 --- a/crates/primitives/src/alloy_compat.rs +++ b/crates/primitives/src/alloy_compat.rs @@ -194,7 +194,7 @@ impl TryFrom> for Transaction { #[cfg(feature = "optimism")] Some(TxType::Deposit) => { let fields = other - .deserialize_into::() + .deserialize_into::() .map_err(|e| ConversionError::Custom(e.to_string()))?; Ok(Self::Deposit(op_alloy_consensus::TxDeposit { source_hash: fields From 33d5eb322d32134c0875219f6e562824fbeca87e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Oct 2024 21:41:50 +0200 Subject: [PATCH 05/82] test: add sanity test for local txs args (#11620) --- crates/node/core/src/args/txpool.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/node/core/src/args/txpool.rs b/crates/node/core/src/args/txpool.rs index 98e125ab9f39..63f6c566ca2b 100644 --- a/crates/node/core/src/args/txpool.rs +++ b/crates/node/core/src/args/txpool.rs @@ -170,4 +170,15 @@ mod tests { let args = CommandParser::::parse_from(["reth"]).args; assert_eq!(args, default_args); } + + #[test] + fn txpool_parse_locals() { + let args = CommandParser::::parse_from([ + "reth", + "--txpool.locals", + "0x0000000000000000000000000000000000000000", + ]) + .args; + assert_eq!(args.locals, vec![Address::ZERO]); + } } From 2b97779880d6aafc8a7787b637485eda0892364e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 10 Oct 2024 02:54:39 +0200 Subject: [PATCH 06/82] fix: persist finalized block (#11623) --- crates/engine/tree/src/persistence.rs | 17 ++++++++++++++++- crates/engine/tree/src/tree/mod.rs | 9 ++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index a748ece8a7c1..231945b3369a 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -4,7 +4,7 @@ use reth_chain_state::ExecutedBlock; use reth_errors::ProviderError; use reth_provider::{ providers::ProviderNodeTypes, writer::UnifiedStorageWriter, BlockHashReader, - DatabaseProviderFactory, ProviderFactory, StaticFileProviderFactory, + DatabaseProviderFactory, FinalizedBlockWriter, ProviderFactory, StaticFileProviderFactory, }; use reth_prune::{PrunerError, PrunerOutput, PrunerWithFactory}; use reth_stages_api::{MetricEvent, MetricEventsSender}; @@ -92,6 +92,10 @@ impl PersistenceService { // we ignore the error because the caller may or may not care about the result let _ = sender.send(res); } + PersistenceAction::SaveFinalizedBlock(finalized_block) => self + .provider + .database_provider_rw()? + .save_finalized_block_number(finalized_block)?, } } Ok(()) @@ -168,6 +172,9 @@ pub enum PersistenceAction { /// Prune associated block data before the given block number, according to already-configured /// prune modes. PruneBefore(u64, oneshot::Sender), + + /// Update the persisted finalized block on disk + SaveFinalizedBlock(u64), } /// A handle to the persistence service @@ -235,6 +242,14 @@ impl PersistenceHandle { self.send_action(PersistenceAction::SaveBlocks(blocks, tx)) } + /// Persists the finalized block number on disk. + pub fn save_finalized_block_number( + &self, + finalized_block: u64, + ) -> Result<(), SendError> { + self.send_action(PersistenceAction::SaveFinalizedBlock(finalized_block)) + } + /// Tells the persistence service to remove blocks above a certain block number. The removed /// blocks are returned by the service. /// diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 0478c73c90d2..5a39bc990176 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -2358,7 +2358,14 @@ where return Err(OnForkChoiceUpdated::invalid_state()) } Ok(Some(finalized)) => { - self.canonical_in_memory_state.set_finalized(finalized); + if Some(finalized.num_hash()) != + self.canonical_in_memory_state.get_finalized_num_hash() + { + // we're also persisting the finalized block on disk so we can reload it on + // restart this is required by optimism which queries the finalized block: + let _ = self.persistence.save_finalized_block_number(finalized.number); + self.canonical_in_memory_state.set_finalized(finalized); + } } Err(err) => { error!(target: "engine::tree", %err, "Failed to fetch finalized block header"); From 701f51bd2d863345c46be20af7a0b62108e7ddda Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Wed, 9 Oct 2024 21:14:39 -0400 Subject: [PATCH 07/82] chore(metrics): remove redundant `starting metrics` log (#11621) --- crates/node/metrics/src/server.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/node/metrics/src/server.rs b/crates/node/metrics/src/server.rs index 06bb490a4365..87521349d4de 100644 --- a/crates/node/metrics/src/server.rs +++ b/crates/node/metrics/src/server.rs @@ -11,7 +11,6 @@ use metrics_process::Collector; use reth_metrics::metrics::Unit; use reth_tasks::TaskExecutor; use std::{convert::Infallible, net::SocketAddr, sync::Arc}; -use tracing::info; /// Configuration for the [`MetricServer`] #[derive(Debug)] @@ -53,8 +52,6 @@ impl MetricServer { let MetricServerConfig { listen_addr, hooks, task_executor, version_info, chain_spec_info } = &self.config; - info!(target: "reth::cli", addr = %listen_addr, "Starting metrics endpoint"); - let hooks = hooks.clone(); self.start_endpoint( *listen_addr, From 672e3111c6749f81571bee5fdbbf4456ee8ade59 Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Thu, 10 Oct 2024 09:50:40 +0200 Subject: [PATCH 08/82] chore(ci): disable hive discord alerts (#11625) --- .github/workflows/hive.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index 3155fd4af2d1..12224a951a42 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -259,11 +259,6 @@ jobs: runs-on: group: Reth steps: - - name: Discord Webhook Action - uses: tsickert/discord-webhook@v6.0.0 - with: - webhook-url: ${{ secrets.RETH_ALERTS_WEBHOOK }} - content: "Failed run: https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}" - name: Slack Webhook Action uses: rtCamp/action-slack-notify@v2 env: From a4b8150201c62c50f2d42f314c03995093b4098b Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 10 Oct 2024 10:15:53 +0200 Subject: [PATCH 09/82] fix: actually commit (#11626) --- crates/engine/tree/src/persistence.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index 231945b3369a..dfddfcfaa89b 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -92,10 +92,11 @@ impl PersistenceService { // we ignore the error because the caller may or may not care about the result let _ = sender.send(res); } - PersistenceAction::SaveFinalizedBlock(finalized_block) => self - .provider - .database_provider_rw()? - .save_finalized_block_number(finalized_block)?, + PersistenceAction::SaveFinalizedBlock(finalized_block) => { + let provider = self.provider.database_provider_rw()?; + provider.save_finalized_block_number(finalized_block)?; + provider.commit()?; + } } } Ok(()) From 58bfa60cea9d80d28149cee6d621bf8b6845ddbf Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Thu, 10 Oct 2024 09:27:01 +0100 Subject: [PATCH 10/82] feat(evm, trie): more metrics (#11613) --- crates/engine/tree/src/tree/metrics.rs | 7 +- crates/engine/tree/src/tree/mod.rs | 9 +- crates/ethereum/evm/src/execute.rs | 2 +- crates/evm/src/either.rs | 6 +- crates/evm/src/execute.rs | 10 +- crates/evm/src/metrics.rs | 101 ++++++++++++++++--- crates/evm/src/noop.rs | 2 +- crates/evm/src/test_utils.rs | 10 +- crates/optimism/evm/src/execute.rs | 2 +- crates/rpc/rpc/src/debug.rs | 2 +- crates/stages/stages/src/stages/execution.rs | 2 +- 11 files changed, 116 insertions(+), 37 deletions(-) diff --git a/crates/engine/tree/src/tree/metrics.rs b/crates/engine/tree/src/tree/metrics.rs index d46c2f05a028..dba80a17914a 100644 --- a/crates/engine/tree/src/tree/metrics.rs +++ b/crates/engine/tree/src/tree/metrics.rs @@ -4,6 +4,7 @@ use reth_metrics::{ metrics::{Counter, Gauge, Histogram}, Metrics, }; +use reth_trie::updates::TrieUpdates; /// Metrics for the `EngineApi`. #[derive(Debug, Default)] @@ -49,6 +50,8 @@ pub(crate) struct EngineMetrics { #[derive(Metrics)] #[metrics(scope = "sync.block_validation")] pub(crate) struct BlockValidationMetrics { + /// Total number of storage tries updated in the state root calculation + pub(crate) state_root_storage_tries_updated_total: Counter, /// Histogram of state root duration pub(crate) state_root_histogram: Histogram, /// Latest state root duration @@ -57,7 +60,9 @@ pub(crate) struct BlockValidationMetrics { impl BlockValidationMetrics { /// Records a new state root time, updating both the histogram and state root gauge - pub(crate) fn record_state_root(&self, elapsed_as_secs: f64) { + pub(crate) fn record_state_root(&self, trie_output: &TrieUpdates, elapsed_as_secs: f64) { + self.state_root_storage_tries_updated_total + .increment(trie_output.storage_tries_ref().len() as u64); self.state_root_duration.set(elapsed_as_secs); self.state_root_histogram.record(elapsed_as_secs); } diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 5a39bc990176..a4491fde2927 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -28,7 +28,7 @@ use reth_chainspec::EthereumHardforks; use reth_consensus::{Consensus, PostExecutionInput}; use reth_engine_primitives::EngineTypes; use reth_errors::{ConsensusError, ProviderResult}; -use reth_evm::execute::{BlockExecutorProvider, Executor}; +use reth_evm::execute::BlockExecutorProvider; use reth_payload_builder::PayloadBuilderHandle; use reth_payload_primitives::{PayloadAttributes, PayloadBuilder, PayloadBuilderAttributes}; use reth_payload_validator::ExecutionPayloadValidator; @@ -2160,10 +2160,7 @@ where let block = block.unseal(); let exec_time = Instant::now(); - let output = self - .metrics - .executor - .metered((&block, U256::MAX).into(), |input| executor.execute(input))?; + let output = self.metrics.executor.execute_metered(executor, (&block, U256::MAX).into())?; trace!(target: "engine::tree", elapsed=?exec_time.elapsed(), ?block_number, "Executed block"); if let Err(err) = self.consensus.validate_block_post_execution( @@ -2227,7 +2224,7 @@ where } let root_elapsed = root_time.elapsed(); - self.metrics.block_validation.record_state_root(root_elapsed.as_secs_f64()); + self.metrics.block_validation.record_state_root(&trie_output, root_elapsed.as_secs_f64()); debug!(target: "engine::tree", ?root_elapsed, ?block_number, "Calculated state root"); let executed = ExecutedBlock { diff --git a/crates/ethereum/evm/src/execute.rs b/crates/ethereum/evm/src/execute.rs index 8c84fafc25fb..6513cf75fad5 100644 --- a/crates/ethereum/evm/src/execute.rs +++ b/crates/ethereum/evm/src/execute.rs @@ -372,7 +372,7 @@ where Ok(BlockExecutionOutput { state: self.state.take_bundle(), receipts, requests, gas_used }) } - fn execute_with_state_witness( + fn execute_with_state_closure( mut self, input: Self::Input<'_>, mut witness: F, diff --git a/crates/evm/src/either.rs b/crates/evm/src/either.rs index e28ec3887f80..8022c68c43dc 100644 --- a/crates/evm/src/either.rs +++ b/crates/evm/src/either.rs @@ -77,7 +77,7 @@ where } } - fn execute_with_state_witness( + fn execute_with_state_closure( self, input: Self::Input<'_>, witness: F, @@ -86,8 +86,8 @@ where F: FnMut(&State), { match self { - Self::Left(a) => a.execute_with_state_witness(input, witness), - Self::Right(b) => b.execute_with_state_witness(input, witness), + Self::Left(a) => a.execute_with_state_closure(input, witness), + Self::Right(b) => b.execute_with_state_closure(input, witness), } } diff --git a/crates/evm/src/execute.rs b/crates/evm/src/execute.rs index 1bd79378127e..145eca29c92b 100644 --- a/crates/evm/src/execute.rs +++ b/crates/evm/src/execute.rs @@ -38,12 +38,12 @@ pub trait Executor { /// The output of the block execution. fn execute(self, input: Self::Input<'_>) -> Result; - /// Executes the EVM with the given input and accepts a witness closure that is invoked with the - /// EVM state after execution. - fn execute_with_state_witness( + /// Executes the EVM with the given input and accepts a state closure that is invoked with + /// the EVM state after execution. + fn execute_with_state_closure( self, input: Self::Input<'_>, - witness: F, + state: F, ) -> Result where F: FnMut(&State); @@ -203,7 +203,7 @@ mod tests { Err(BlockExecutionError::msg("execution unavailable for tests")) } - fn execute_with_state_witness( + fn execute_with_state_closure( self, _: Self::Input<'_>, _: F, diff --git a/crates/evm/src/metrics.rs b/crates/evm/src/metrics.rs index d6ffe0d79c69..fbb2b858b158 100644 --- a/crates/evm/src/metrics.rs +++ b/crates/evm/src/metrics.rs @@ -1,16 +1,18 @@ //! Executor metrics. //! -//! Block processing related to syncing should take care to update the metrics by using e.g. -//! [`ExecutorMetrics::metered`]. +//! Block processing related to syncing should take care to update the metrics by using either +//! [`ExecutorMetrics::execute_metered`] or [`ExecutorMetrics::metered_one`]. use std::time::Instant; use metrics::{Counter, Gauge, Histogram}; -use reth_execution_types::BlockExecutionInput; +use reth_execution_types::{BlockExecutionInput, BlockExecutionOutput}; use reth_metrics::Metrics; use reth_primitives::BlockWithSenders; +use crate::execute::Executor; + /// Executor metrics. -// TODO(onbjerg): add sload/sstore, acc load/acc change, bytecode metrics +// TODO(onbjerg): add sload/sstore #[derive(Metrics, Clone)] #[metrics(scope = "sync.execution")] pub struct ExecutorMetrics { @@ -18,31 +20,106 @@ pub struct ExecutorMetrics { pub gas_processed_total: Counter, /// The instantaneous amount of gas processed per second. pub gas_per_second: Gauge, + /// The Histogram for amount of time taken to execute blocks. pub execution_histogram: Histogram, /// The total amount of time it took to execute the latest block. pub execution_duration: Gauge, + + /// The Histogram for number of accounts loaded when executing the latest block. + pub accounts_loaded_histogram: Histogram, + /// The Histogram for number of storage slots loaded when executing the latest block. + pub storage_slots_loaded_histogram: Histogram, + /// The Histogram for number of bytecodes loaded when executing the latest block. + pub bytecodes_loaded_histogram: Histogram, + + /// The Histogram for number of accounts updated when executing the latest block. + pub accounts_updated_histogram: Histogram, + /// The Histogram for number of storage slots updated when executing the latest block. + pub storage_slots_updated_histogram: Histogram, + /// The Histogram for number of bytecodes updated when executing the latest block. + pub bytecodes_updated_histogram: Histogram, } impl ExecutorMetrics { - /// Execute the given block and update metrics for the execution. - pub fn metered(&self, input: BlockExecutionInput<'_, BlockWithSenders>, f: F) -> R + fn metered(&self, block: &BlockWithSenders, f: F) -> R where - F: FnOnce(BlockExecutionInput<'_, BlockWithSenders>) -> R, + F: FnOnce() -> R, { - let gas_used = input.block.gas_used; - // Execute the block and record the elapsed time. let execute_start = Instant::now(); - let output = f(input); + let output = f(); let execution_duration = execute_start.elapsed().as_secs_f64(); // Update gas metrics. - self.gas_processed_total.increment(gas_used); - self.gas_per_second.set(gas_used as f64 / execution_duration); + self.gas_processed_total.increment(block.gas_used); + self.gas_per_second.set(block.gas_used as f64 / execution_duration); self.execution_histogram.record(execution_duration); self.execution_duration.set(execution_duration); output } + + /// Execute the given block using the provided [`Executor`] and update metrics for the + /// execution. + /// + /// Compared to [`Self::metered_one`], this method additionally updates metrics for the number + /// of accounts, storage slots and bytecodes loaded and updated. + pub fn execute_metered<'a, E, DB, O, Error>( + &self, + executor: E, + input: BlockExecutionInput<'a, BlockWithSenders>, + ) -> Result, Error> + where + E: Executor< + DB, + Input<'a> = BlockExecutionInput<'a, BlockWithSenders>, + Output = BlockExecutionOutput, + Error = Error, + >, + { + let output = self.metered(input.block, || { + executor.execute_with_state_closure(input, |state: &revm::db::State| { + // Update the metrics for the number of accounts, storage slots and bytecodes + // loaded + let accounts = state.cache.accounts.len(); + let storage_slots = state + .cache + .accounts + .values() + .filter_map(|account| { + account.account.as_ref().map(|account| account.storage.len()) + }) + .sum::(); + let bytecodes = state.cache.contracts.len(); + + // Record all state present in the cache state as loaded even though some might have + // been newly created. + // TODO: Consider spitting these into loaded and newly created. + self.accounts_loaded_histogram.record(accounts as f64); + self.storage_slots_loaded_histogram.record(storage_slots as f64); + self.bytecodes_loaded_histogram.record(bytecodes as f64); + }) + })?; + + // Update the metrics for the number of accounts, storage slots and bytecodes updated + let accounts = output.state.state.len(); + let storage_slots = + output.state.state.values().map(|account| account.storage.len()).sum::(); + let bytecodes = output.state.contracts.len(); + + self.accounts_updated_histogram.record(accounts as f64); + self.storage_slots_updated_histogram.record(storage_slots as f64); + self.bytecodes_updated_histogram.record(bytecodes as f64); + + Ok(output) + } + + /// Execute the given block and update metrics for the execution. + pub fn metered_one(&self, input: BlockExecutionInput<'_, BlockWithSenders>, f: F) -> R + where + F: FnOnce(BlockExecutionInput<'_, BlockWithSenders>) -> R, + { + self.metered(input.block, || f(input)) + } } diff --git a/crates/evm/src/noop.rs b/crates/evm/src/noop.rs index 3e01bfc4cc46..4fdc6d367a27 100644 --- a/crates/evm/src/noop.rs +++ b/crates/evm/src/noop.rs @@ -51,7 +51,7 @@ impl Executor for NoopBlockExecutorProvider { Err(BlockExecutionError::msg(UNAVAILABLE_FOR_NOOP)) } - fn execute_with_state_witness( + fn execute_with_state_closure( self, _: Self::Input<'_>, _: F, diff --git a/crates/evm/src/test_utils.rs b/crates/evm/src/test_utils.rs index 45ab2e97734e..fc620bb42c33 100644 --- a/crates/evm/src/test_utils.rs +++ b/crates/evm/src/test_utils.rs @@ -66,26 +66,26 @@ impl Executor for MockExecutorProvider { }) } - fn execute_with_state_witness( + fn execute_with_state_closure( self, - _: Self::Input<'_>, + input: Self::Input<'_>, _: F, ) -> Result where F: FnMut(&State), { - unimplemented!() + >::execute(self, input) } fn execute_with_state_hook( self, - _: Self::Input<'_>, + input: Self::Input<'_>, _: F, ) -> Result where F: OnStateHook, { - unimplemented!() + >::execute(self, input) } } diff --git a/crates/optimism/evm/src/execute.rs b/crates/optimism/evm/src/execute.rs index 4e5918831f76..2491e99b5038 100644 --- a/crates/optimism/evm/src/execute.rs +++ b/crates/optimism/evm/src/execute.rs @@ -381,7 +381,7 @@ where }) } - fn execute_with_state_witness( + fn execute_with_state_closure( mut self, input: Self::Input<'_>, mut witness: F, diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index fb0407039776..b26459c4f040 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -612,7 +612,7 @@ where let mut codes = HashMap::default(); let _ = block_executor - .execute_with_state_witness( + .execute_with_state_closure( (&block.clone().unseal(), block.difficulty).into(), |statedb| { codes = statedb diff --git a/crates/stages/stages/src/stages/execution.rs b/crates/stages/stages/src/stages/execution.rs index a99d8a572e5b..df234e542be6 100644 --- a/crates/stages/stages/src/stages/execution.rs +++ b/crates/stages/stages/src/stages/execution.rs @@ -275,7 +275,7 @@ where // Execute the block let execute_start = Instant::now(); - self.metrics.metered((&block, td).into(), |input| { + self.metrics.metered_one((&block, td).into(), |input| { let sealed = block.header.clone().seal_slow(); let (header, seal) = sealed.into_parts(); From eb9565d22dae8f3c279002695b9bb72fc8abd6a9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 10 Oct 2024 11:39:40 +0200 Subject: [PATCH 11/82] fix: always handle payload building for opstack (#11629) --- crates/chainspec/src/api.rs | 9 +++++++++ crates/engine/service/Cargo.toml | 1 + crates/engine/service/src/service.rs | 7 ++++++- crates/engine/tree/src/engine.rs | 17 +++++++++++++++-- crates/engine/tree/src/tree/mod.rs | 24 +++++++++++++++++++++--- crates/optimism/chainspec/src/lib.rs | 4 ++++ 6 files changed, 56 insertions(+), 6 deletions(-) diff --git a/crates/chainspec/src/api.rs b/crates/chainspec/src/api.rs index fb64e08ae1ef..e481fbede70d 100644 --- a/crates/chainspec/src/api.rs +++ b/crates/chainspec/src/api.rs @@ -46,6 +46,11 @@ pub trait EthChainSpec: Send + Sync + Unpin + Debug { /// The bootnodes for the chain, if any. fn bootnodes(&self) -> Option>; + + /// Returns `true` if this chain contains Optimism configuration. + fn is_optimism(&self) -> bool { + self.chain().is_optimism() + } } impl EthChainSpec for ChainSpec { @@ -92,4 +97,8 @@ impl EthChainSpec for ChainSpec { fn bootnodes(&self) -> Option> { self.bootnodes() } + + fn is_optimism(&self) -> bool { + Self::is_optimism(self) + } } diff --git a/crates/engine/service/Cargo.toml b/crates/engine/service/Cargo.toml index 63d5f1fb9766..c6098bfe6671 100644 --- a/crates/engine/service/Cargo.toml +++ b/crates/engine/service/Cargo.toml @@ -24,6 +24,7 @@ reth-prune.workspace = true reth-stages-api.workspace = true reth-tasks.workspace = true reth-node-types.workspace = true +reth-chainspec.workspace = true # async futures.workspace = true diff --git a/crates/engine/service/src/service.rs b/crates/engine/service/src/service.rs index 880be5c0261b..ed9c1aa1c27e 100644 --- a/crates/engine/service/src/service.rs +++ b/crates/engine/service/src/service.rs @@ -1,11 +1,12 @@ use futures::{Stream, StreamExt}; use pin_project::pin_project; use reth_beacon_consensus::{BeaconConsensusEngineEvent, BeaconEngineMessage, EngineNodeTypes}; +use reth_chainspec::EthChainSpec; use reth_consensus::Consensus; use reth_engine_tree::{ backfill::PipelineSync, download::BasicBlockDownloader, - engine::{EngineApiRequest, EngineApiRequestHandler, EngineHandler}, + engine::{EngineApiKind, EngineApiRequest, EngineApiRequestHandler, EngineHandler}, persistence::PersistenceHandle, tree::{EngineApiTreeHandler, InvalidBlockHook, TreeConfig}, }; @@ -79,6 +80,9 @@ where invalid_block_hook: Box, sync_metrics_tx: MetricEventsSender, ) -> Self { + let engine_kind = + if chain_spec.is_optimism() { EngineApiKind::OpStack } else { EngineApiKind::Ethereum }; + let downloader = BasicBlockDownloader::new(client, consensus.clone()); let persistence_handle = @@ -97,6 +101,7 @@ where canonical_in_memory_state, tree_config, invalid_block_hook, + engine_kind, ); let engine_handler = EngineApiRequestHandler::new(to_tree_tx, from_tree); diff --git a/crates/engine/tree/src/engine.rs b/crates/engine/tree/src/engine.rs index 8172a4469173..c1571ed82178 100644 --- a/crates/engine/tree/src/engine.rs +++ b/crates/engine/tree/src/engine.rs @@ -212,15 +212,28 @@ where } } -/// The type for specifying the kind of engine api -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// The type for specifying the kind of engine api. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum EngineApiKind { /// The chain contains Ethereum configuration. + #[default] Ethereum, /// The chain contains Optimism configuration. OpStack, } +impl EngineApiKind { + /// Returns true if this is the ethereum variant + pub const fn is_ethereum(&self) -> bool { + matches!(self, Self::Ethereum) + } + + /// Returns true if this is the ethereum variant + pub const fn is_opstack(&self) -> bool { + matches!(self, Self::OpStack) + } +} + /// The request variants that the engine API handler can receive. #[derive(Debug)] pub enum EngineApiRequest { diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index a4491fde2927..bc1da6369459 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -66,7 +66,10 @@ pub mod config; mod invalid_block_hook; mod metrics; mod persistence_state; -use crate::{engine::EngineApiRequest, tree::metrics::EngineApiMetrics}; +use crate::{ + engine::{EngineApiKind, EngineApiRequest}, + tree::metrics::EngineApiMetrics, +}; pub use config::TreeConfig; pub use invalid_block_hook::{InvalidBlockHooks, NoopInvalidBlockHook}; pub use persistence_state::PersistenceState; @@ -498,6 +501,8 @@ pub struct EngineApiTreeHandler { metrics: EngineApiMetrics, /// An invalid block hook. invalid_block_hook: Box, + /// The engine API variant of this handler + engine_kind: EngineApiKind, } impl std::fmt::Debug @@ -519,6 +524,7 @@ impl std::fmt::Debug .field("config", &self.config) .field("metrics", &self.metrics) .field("invalid_block_hook", &format!("{:p}", self.invalid_block_hook)) + .field("engine_kind", &self.engine_kind) .finish() } } @@ -545,6 +551,7 @@ where persistence_state: PersistenceState, payload_builder: PayloadBuilderHandle, config: TreeConfig, + engine_kind: EngineApiKind, ) -> Self { let (incoming_tx, incoming) = std::sync::mpsc::channel(); @@ -565,6 +572,7 @@ where metrics: Default::default(), incoming_tx, invalid_block_hook: Box::new(NoopInvalidBlockHook), + engine_kind, } } @@ -589,6 +597,7 @@ where canonical_in_memory_state: CanonicalInMemoryState, config: TreeConfig, invalid_block_hook: Box, + kind: EngineApiKind, ) -> (Sender>>, UnboundedReceiver) { let best_block_number = provider.best_block_number().unwrap_or(0); let header = provider.sealed_header(best_block_number).ok().flatten().unwrap_or_default(); @@ -618,6 +627,7 @@ where persistence_state, payload_builder, config, + kind, ); task.set_invalid_block_hook(invalid_block_hook); let incoming = task.incoming_tx.clone(); @@ -1041,8 +1051,15 @@ where if let Ok(Some(canonical_header)) = self.find_canonical_header(state.head_block_hash) { debug!(target: "engine::tree", head = canonical_header.number, "fcu head block is already canonical"); - // TODO(mattsse): for optimism we'd need to trigger a build job as well because on - // Optimism, the proposers are allowed to reorg their own chain at will. + // For OpStack the proposers are allowed to reorg their own chain at will, so we need to + // always trigger a new payload job if requested. + if self.engine_kind.is_opstack() { + if let Some(attr) = attrs { + debug!(target: "engine::tree", head = canonical_header.number, "handling payload attributes for canonical head"); + let updated = self.process_payload_attributes(attr, &canonical_header, state); + return Ok(TreeOutcome::new(updated)) + } + } // 2. Client software MAY skip an update of the forkchoice state and MUST NOT begin a // payload build process if `forkchoiceState.headBlockHash` references a `VALID` @@ -2680,6 +2697,7 @@ mod tests { PersistenceState::default(), payload_builder, TreeConfig::default(), + EngineApiKind::Ethereum, ); let block_builder = TestBlockBuilder::default().with_chain_spec((*chain_spec).clone()); diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index ebd9392b5330..9b2d1c8c11ba 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -93,6 +93,10 @@ impl EthChainSpec for OpChainSpec { fn bootnodes(&self) -> Option> { self.inner.bootnodes() } + + fn is_optimism(&self) -> bool { + true + } } impl Hardforks for OpChainSpec { From e9a436ae96de77f19af784f77572cc2c0baa2794 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Thu, 10 Oct 2024 12:28:46 +0200 Subject: [PATCH 12/82] chore(trie): make initialization of in-memory trie cursors `pub` (#11628) --- crates/trie/trie/src/trie_cursor/in_memory.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/trie/trie/src/trie_cursor/in_memory.rs b/crates/trie/trie/src/trie_cursor/in_memory.rs index 0f00191378ba..851670f4267a 100644 --- a/crates/trie/trie/src/trie_cursor/in_memory.rs +++ b/crates/trie/trie/src/trie_cursor/in_memory.rs @@ -50,7 +50,7 @@ impl<'a, CF: TrieCursorFactory> TrieCursorFactory for InMemoryTrieCursorFactory< /// It will always give precedence to the data from the trie updates. #[derive(Debug)] pub struct InMemoryAccountTrieCursor<'a, C> { - /// The database cursor. + /// The underlying cursor. cursor: C, /// Forward-only in-memory cursor over storage trie nodes. in_memory_cursor: ForwardInMemoryCursor<'a, Nibbles, BranchNodeCompact>, @@ -61,7 +61,9 @@ pub struct InMemoryAccountTrieCursor<'a, C> { } impl<'a, C: TrieCursor> InMemoryAccountTrieCursor<'a, C> { - const fn new(cursor: C, trie_updates: &'a TrieUpdatesSorted) -> Self { + /// Create new account trie cursor from underlying cursor and reference to + /// [`TrieUpdatesSorted`]. + pub const fn new(cursor: C, trie_updates: &'a TrieUpdatesSorted) -> Self { let in_memory_cursor = ForwardInMemoryCursor::new(&trie_updates.account_nodes); Self { cursor, @@ -160,7 +162,7 @@ impl TrieCursor for InMemoryAccountTrieCursor<'_, C> { pub struct InMemoryStorageTrieCursor<'a, C> { /// The hashed address of the account that trie belongs to. hashed_address: B256, - /// The database cursor. + /// The underlying cursor. cursor: C, /// Forward-only in-memory cursor over storage trie nodes. in_memory_cursor: Option>, @@ -173,7 +175,13 @@ pub struct InMemoryStorageTrieCursor<'a, C> { } impl<'a, C> InMemoryStorageTrieCursor<'a, C> { - fn new(hashed_address: B256, cursor: C, updates: Option<&'a StorageTrieUpdatesSorted>) -> Self { + /// Create new storage trie cursor from underlying cursor and reference to + /// [`StorageTrieUpdatesSorted`]. + pub fn new( + hashed_address: B256, + cursor: C, + updates: Option<&'a StorageTrieUpdatesSorted>, + ) -> Self { let in_memory_cursor = updates.map(|u| ForwardInMemoryCursor::new(&u.storage_nodes)); let removed_nodes = updates.map(|u| &u.removed_nodes); let storage_trie_cleared = updates.map_or(false, |u| u.is_deleted); From 68f38210e02f01fcd455bc5d4e51fef02b22c399 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Thu, 10 Oct 2024 12:28:57 +0200 Subject: [PATCH 13/82] feat(trie): noop hashed cursors (#11627) --- crates/trie/trie/src/hashed_cursor/mod.rs | 3 + crates/trie/trie/src/hashed_cursor/noop.rs | 65 ++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 crates/trie/trie/src/hashed_cursor/noop.rs diff --git a/crates/trie/trie/src/hashed_cursor/mod.rs b/crates/trie/trie/src/hashed_cursor/mod.rs index 9b539cdce7d1..1101f507888f 100644 --- a/crates/trie/trie/src/hashed_cursor/mod.rs +++ b/crates/trie/trie/src/hashed_cursor/mod.rs @@ -6,6 +6,9 @@ use reth_storage_errors::db::DatabaseError; mod post_state; pub use post_state::*; +/// Implementation of noop hashed state cursor. +pub mod noop; + /// The factory trait for creating cursors over the hashed state. pub trait HashedCursorFactory { /// The hashed account cursor type. diff --git a/crates/trie/trie/src/hashed_cursor/noop.rs b/crates/trie/trie/src/hashed_cursor/noop.rs new file mode 100644 index 000000000000..4783d5afd9d4 --- /dev/null +++ b/crates/trie/trie/src/hashed_cursor/noop.rs @@ -0,0 +1,65 @@ +use super::{HashedCursor, HashedCursorFactory, HashedStorageCursor}; +use alloy_primitives::{B256, U256}; +use reth_primitives::Account; +use reth_storage_errors::db::DatabaseError; + +/// Noop hashed cursor factory. +#[derive(Default, Debug)] +#[non_exhaustive] +pub struct NoopHashedCursorFactory; + +impl HashedCursorFactory for NoopHashedCursorFactory { + type AccountCursor = NoopHashedAccountCursor; + type StorageCursor = NoopHashedStorageCursor; + + fn hashed_account_cursor(&self) -> Result { + Ok(NoopHashedAccountCursor::default()) + } + + fn hashed_storage_cursor( + &self, + _hashed_address: B256, + ) -> Result { + Ok(NoopHashedStorageCursor::default()) + } +} + +/// Noop account hashed cursor. +#[derive(Default, Debug)] +#[non_exhaustive] +pub struct NoopHashedAccountCursor; + +impl HashedCursor for NoopHashedAccountCursor { + type Value = Account; + + fn next(&mut self) -> Result, DatabaseError> { + Ok(None) + } + + fn seek(&mut self, _key: B256) -> Result, DatabaseError> { + Ok(None) + } +} + +/// Noop account hashed cursor. +#[derive(Default, Debug)] +#[non_exhaustive] +pub struct NoopHashedStorageCursor; + +impl HashedCursor for NoopHashedStorageCursor { + type Value = U256; + + fn next(&mut self) -> Result, DatabaseError> { + Ok(None) + } + + fn seek(&mut self, _key: B256) -> Result, DatabaseError> { + Ok(None) + } +} + +impl HashedStorageCursor for NoopHashedStorageCursor { + fn is_storage_empty(&mut self) -> Result { + Ok(true) + } +} From b50e710f8452e14adcce8f0a29363a8bdc99c14f Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Thu, 10 Oct 2024 04:31:57 -0600 Subject: [PATCH 14/82] Remove duplicate EngineTypes comment (#11624) --- crates/node/types/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/node/types/src/lib.rs b/crates/node/types/src/lib.rs index 0f6d5a1a370f..2c72e02d3edc 100644 --- a/crates/node/types/src/lib.rs +++ b/crates/node/types/src/lib.rs @@ -40,7 +40,6 @@ pub trait NodeTypes: Send + Sync + Unpin + 'static { pub trait NodeTypesWithEngine: NodeTypes { /// The node's engine types, defining the interaction with the consensus engine. type Engine: EngineTypes; - // type Engine: EngineTypes; } /// A helper trait that is downstream of the [`NodeTypesWithEngine`] trait and adds database to the From 47c3a481de42fc3a16d58c9f927cbf2f5dc90ca3 Mon Sep 17 00:00:00 2001 From: cody-wang-cb Date: Thu, 10 Oct 2024 07:12:27 -0400 Subject: [PATCH 15/82] Broadcast external IP from NAT in enode record (#10274) Co-authored-by: Matthias Seitz --- book/cli/reth/debug/execution.md | 3 +++ book/cli/reth/debug/in-memory-merkle.md | 3 +++ book/cli/reth/debug/merkle.md | 3 +++ book/cli/reth/debug/replay-engine.md | 3 +++ book/cli/reth/node.md | 3 +++ book/cli/reth/p2p.md | 3 +++ book/cli/reth/stage/run.md | 3 +++ book/cli/reth/stage/unwind.md | 3 +++ crates/net/nat/src/lib.rs | 8 ++++++ crates/net/network/src/config.rs | 22 ++++++++++++++++- crates/net/network/src/manager.rs | 2 ++ crates/net/network/src/network.rs | 17 +++++++++---- crates/net/network/tests/it/startup.rs | 33 +++++++++++++++++++++++-- crates/node/core/src/args/network.rs | 11 ++++++++- 14 files changed, 108 insertions(+), 9 deletions(-) diff --git a/book/cli/reth/debug/execution.md b/book/cli/reth/debug/execution.md index 76755a1121e1..202e1452a8ae 100644 --- a/book/cli/reth/debug/execution.md +++ b/book/cli/reth/debug/execution.md @@ -82,6 +82,9 @@ Networking: --enable-discv5-discovery Enable Discv5 discovery + --disable-nat + Disable Nat discovery + --discovery.addr The UDP address to use for devp2p peer discovery version 4 diff --git a/book/cli/reth/debug/in-memory-merkle.md b/book/cli/reth/debug/in-memory-merkle.md index 19061826da5d..534e6d46c69d 100644 --- a/book/cli/reth/debug/in-memory-merkle.md +++ b/book/cli/reth/debug/in-memory-merkle.md @@ -82,6 +82,9 @@ Networking: --enable-discv5-discovery Enable Discv5 discovery + --disable-nat + Disable Nat discovery + --discovery.addr The UDP address to use for devp2p peer discovery version 4 diff --git a/book/cli/reth/debug/merkle.md b/book/cli/reth/debug/merkle.md index bdcbc820f623..19bc38acceb9 100644 --- a/book/cli/reth/debug/merkle.md +++ b/book/cli/reth/debug/merkle.md @@ -82,6 +82,9 @@ Networking: --enable-discv5-discovery Enable Discv5 discovery + --disable-nat + Disable Nat discovery + --discovery.addr The UDP address to use for devp2p peer discovery version 4 diff --git a/book/cli/reth/debug/replay-engine.md b/book/cli/reth/debug/replay-engine.md index 3d3ae85d72a1..7a14b9cf09d4 100644 --- a/book/cli/reth/debug/replay-engine.md +++ b/book/cli/reth/debug/replay-engine.md @@ -82,6 +82,9 @@ Networking: --enable-discv5-discovery Enable Discv5 discovery + --disable-nat + Disable Nat discovery + --discovery.addr The UDP address to use for devp2p peer discovery version 4 diff --git a/book/cli/reth/node.md b/book/cli/reth/node.md index 0e8966764463..231e665cb8d9 100644 --- a/book/cli/reth/node.md +++ b/book/cli/reth/node.md @@ -74,6 +74,9 @@ Networking: --enable-discv5-discovery Enable Discv5 discovery + --disable-nat + Disable Nat discovery + --discovery.addr The UDP address to use for devp2p peer discovery version 4 diff --git a/book/cli/reth/p2p.md b/book/cli/reth/p2p.md index 475e6bdbbb2c..01253705b233 100644 --- a/book/cli/reth/p2p.md +++ b/book/cli/reth/p2p.md @@ -59,6 +59,9 @@ Networking: --enable-discv5-discovery Enable Discv5 discovery + --disable-nat + Disable Nat discovery + --discovery.addr The UDP address to use for devp2p peer discovery version 4 diff --git a/book/cli/reth/stage/run.md b/book/cli/reth/stage/run.md index 5a7a13d37ed4..bfe5ff9d6c63 100644 --- a/book/cli/reth/stage/run.md +++ b/book/cli/reth/stage/run.md @@ -125,6 +125,9 @@ Networking: --enable-discv5-discovery Enable Discv5 discovery + --disable-nat + Disable Nat discovery + --discovery.addr The UDP address to use for devp2p peer discovery version 4 diff --git a/book/cli/reth/stage/unwind.md b/book/cli/reth/stage/unwind.md index 2ca3303648c7..d181b3bcade6 100644 --- a/book/cli/reth/stage/unwind.md +++ b/book/cli/reth/stage/unwind.md @@ -87,6 +87,9 @@ Networking: --enable-discv5-discovery Enable Discv5 discovery + --disable-nat + Disable Nat discovery + --discovery.addr The UDP address to use for devp2p peer discovery version 4 diff --git a/crates/net/nat/src/lib.rs b/crates/net/nat/src/lib.rs index b49677ef87c2..600ba97cd2dd 100644 --- a/crates/net/nat/src/lib.rs +++ b/crates/net/nat/src/lib.rs @@ -61,6 +61,14 @@ impl NatResolver { pub async fn external_addr(self) -> Option { external_addr_with(self).await } + + /// Returns the external ip, if it is [`NatResolver::ExternalIp`] + pub const fn as_external_ip(self) -> Option { + match self { + Self::ExternalIp(ip) => Some(ip), + _ => None, + } + } } impl fmt::Display for NatResolver { diff --git a/crates/net/network/src/config.rs b/crates/net/network/src/config.rs index 9d9183edcf98..72627f5b657d 100644 --- a/crates/net/network/src/config.rs +++ b/crates/net/network/src/config.rs @@ -81,6 +81,8 @@ pub struct NetworkConfig { pub tx_gossip_disabled: bool, /// How to instantiate transactions manager. pub transactions_manager_config: TransactionsManagerConfig, + /// The NAT resolver for external IP + pub nat: Option, } // === impl NetworkConfig === @@ -197,6 +199,8 @@ pub struct NetworkConfigBuilder { block_import: Option>, /// How to instantiate transactions manager. transactions_manager_config: TransactionsManagerConfig, + /// The NAT resolver for external IP + nat: Option, } // === impl NetworkConfigBuilder === @@ -228,6 +232,7 @@ impl NetworkConfigBuilder { tx_gossip_disabled: false, block_import: None, transactions_manager_config: Default::default(), + nat: None, } } @@ -369,6 +374,7 @@ impl NetworkConfigBuilder { self.discovery_v4_builder .get_or_insert_with(Discv4Config::builder) .external_ip_resolver(Some(resolver)); + self.nat = Some(resolver); self } @@ -417,9 +423,15 @@ impl NetworkConfigBuilder { self } + // Disable nat + pub const fn disable_nat(mut self) -> Self { + self.nat = None; + self + } + /// Disables all discovery. pub fn disable_discovery(self) -> Self { - self.disable_discv4_discovery().disable_dns_discovery() + self.disable_discv4_discovery().disable_dns_discovery().disable_nat() } /// Disables all discovery if the given condition is true. @@ -485,6 +497,12 @@ impl NetworkConfigBuilder { self.build(NoopBlockReader::new(chain_spec)) } + /// Sets the NAT resolver for external IP. + pub const fn add_nat(mut self, nat: Option) -> Self { + self.nat = nat; + self + } + /// Consumes the type and creates the actual [`NetworkConfig`] /// for the given client type that can interact with the chain. /// @@ -515,6 +533,7 @@ impl NetworkConfigBuilder { tx_gossip_disabled, block_import, transactions_manager_config, + nat, } = self; discovery_v5_builder = discovery_v5_builder.map(|mut builder| { @@ -581,6 +600,7 @@ impl NetworkConfigBuilder { fork_filter, tx_gossip_disabled, transactions_manager_config, + nat, } } } diff --git a/crates/net/network/src/manager.rs b/crates/net/network/src/manager.rs index e8ffb81c5e1a..3a7f94985fc9 100644 --- a/crates/net/network/src/manager.rs +++ b/crates/net/network/src/manager.rs @@ -188,6 +188,7 @@ impl NetworkManager { extra_protocols, tx_gossip_disabled, transactions_manager_config: _, + nat, } = config; let peers_manager = PeersManager::new(peers_config); @@ -267,6 +268,7 @@ impl NetworkManager { discv4, discv5, event_sender.clone(), + nat, ); Ok(Self { diff --git a/crates/net/network/src/network.rs b/crates/net/network/src/network.rs index de3a0ab7432b..594ad4d155de 100644 --- a/crates/net/network/src/network.rs +++ b/crates/net/network/src/network.rs @@ -9,7 +9,7 @@ use std::{ use alloy_primitives::B256; use enr::Enr; use parking_lot::Mutex; -use reth_discv4::Discv4; +use reth_discv4::{Discv4, NatResolver}; use reth_discv5::Discv5; use reth_eth_wire::{DisconnectReason, NewBlock, NewPooledTransactionHashes, SharedTransactions}; use reth_network_api::{ @@ -65,6 +65,7 @@ impl NetworkHandle { discv4: Option, discv5: Option, event_sender: EventSender, + nat: Option, ) -> Self { let inner = NetworkInner { num_active_peers, @@ -81,6 +82,7 @@ impl NetworkHandle { discv4, discv5, event_sender, + nat, }; Self { inner: Arc::new(inner) } } @@ -216,10 +218,13 @@ impl PeersInfo for NetworkHandle { } else if let Some(record) = self.inner.discv5.as_ref().and_then(|d| d.node_record()) { record } else { - let id = *self.peer_id(); - let mut socket_addr = *self.inner.listener_address.lock(); + let external_ip = self.inner.nat.and_then(|nat| nat.as_external_ip()); - if socket_addr.ip().is_unspecified() { + let mut socket_addr = *self.inner.listener_address.lock(); + if let Some(ip) = external_ip { + // if able to resolve external ip, use it instead and also set the local address + socket_addr.set_ip(ip) + } else if socket_addr.ip().is_unspecified() { // zero address is invalid if socket_addr.ip().is_ipv4() { socket_addr.set_ip(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)); @@ -228,7 +233,7 @@ impl PeersInfo for NetworkHandle { } } - NodeRecord::new(socket_addr, id) + NodeRecord::new(socket_addr, *self.peer_id()) } } @@ -432,6 +437,8 @@ struct NetworkInner { discv5: Option, /// Sender for high level network events. event_sender: EventSender, + /// The NAT resolver + nat: Option, } /// Provides access to modify the network's additional protocol handlers. diff --git a/crates/net/network/tests/it/startup.rs b/crates/net/network/tests/it/startup.rs index 8f7e4ff8c78c..89889a869460 100644 --- a/crates/net/network/tests/it/startup.rs +++ b/crates/net/network/tests/it/startup.rs @@ -1,10 +1,10 @@ use std::{ io, - net::{Ipv4Addr, SocketAddr, SocketAddrV4}, + net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}, }; use reth_chainspec::MAINNET; -use reth_discv4::Discv4Config; +use reth_discv4::{Discv4Config, NatResolver}; use reth_network::{ error::{NetworkError, ServiceKind}, Discovery, NetworkConfigBuilder, NetworkManager, @@ -105,3 +105,32 @@ async fn test_tcp_port_node_record_discovery() { assert_eq!(record.tcp_port, local_addr.port()); assert_ne!(record.udp_port, 0); } + +#[tokio::test(flavor = "multi_thread")] +async fn test_node_record_address_with_nat() { + let secret_key = SecretKey::new(&mut rand::thread_rng()); + let config = NetworkConfigBuilder::new(secret_key) + .add_nat(Some(NatResolver::ExternalIp("10.1.1.1".parse().unwrap()))) + .disable_discv4_discovery() + .disable_dns_discovery() + .build_with_noop_provider(MAINNET.clone()); + + let network = NetworkManager::new(config).await.unwrap(); + let record = network.handle().local_node_record(); + + assert_eq!(record.address, IpAddr::V4(Ipv4Addr::new(10, 1, 1, 1))); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_node_record_address_with_nat_disable_discovery() { + let secret_key = SecretKey::new(&mut rand::thread_rng()); + let config = NetworkConfigBuilder::new(secret_key) + .add_nat(Some(NatResolver::ExternalIp("10.1.1.1".parse().unwrap()))) + .disable_discovery() + .build_with_noop_provider(MAINNET.clone()); + + let network = NetworkManager::new(config).await.unwrap(); + let record = network.handle().local_node_record(); + + assert_eq!(record.address, IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)); +} diff --git a/crates/node/core/src/args/network.rs b/crates/node/core/src/args/network.rs index 04153b93ecdd..d359cfb3f1af 100644 --- a/crates/node/core/src/args/network.rs +++ b/crates/node/core/src/args/network.rs @@ -172,7 +172,7 @@ impl NetworkArgs { DEFAULT_DISCOVERY_ADDR } - } + }; } self.addr @@ -355,6 +355,10 @@ pub struct DiscoveryArgs { #[arg(long, conflicts_with = "disable_discovery")] pub enable_discv5_discovery: bool, + /// Disable Nat discovery. + #[arg(long, conflicts_with = "disable_discovery")] + pub disable_nat: bool, + /// The UDP address to use for devp2p peer discovery version 4. #[arg(id = "discovery.addr", long = "discovery.addr", value_name = "DISCOVERY_ADDR", default_value_t = DEFAULT_DISCOVERY_ADDR)] pub addr: IpAddr, @@ -418,6 +422,10 @@ impl DiscoveryArgs { network_config_builder = network_config_builder.disable_discv4_discovery(); } + if self.disable_discovery || self.disable_nat { + network_config_builder = network_config_builder.disable_nat(); + } + if !self.disable_discovery && self.enable_discv5_discovery { network_config_builder = network_config_builder .discovery_v5(self.discovery_v5_builder(rlpx_tcp_socket, boot_nodes)); @@ -494,6 +502,7 @@ impl Default for DiscoveryArgs { disable_dns_discovery: false, disable_discv4_discovery: false, enable_discv5_discovery: false, + disable_nat: false, addr: DEFAULT_DISCOVERY_ADDR, port: DEFAULT_DISCOVERY_PORT, discv5_addr: None, From 01f382b739b6db371d5a1a762f99ee1c6366c64e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 10 Oct 2024 14:04:18 +0200 Subject: [PATCH 16/82] docs: clarify op-mainnet --debug.tip (#11634) --- book/run/sync-op-mainnet.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/book/run/sync-op-mainnet.md b/book/run/sync-op-mainnet.md index 057860c33494..2a862314a1d5 100644 --- a/book/run/sync-op-mainnet.md +++ b/book/run/sync-op-mainnet.md @@ -9,9 +9,15 @@ To sync OP mainnet, bedrock state needs to be imported as a starting point. Ther **The state snapshot at Bedrock block is required.** It can be exported from [op-geth](https://github.com/testinprod-io/op-erigon/blob/pcw109550/bedrock-db-migration/bedrock-migration.md#export-state) (**.jsonl**) or downloaded directly from [here](https://mega.nz/file/GdZ1xbAT#a9cBv3AqzsTGXYgX7nZc_3fl--tcBmOAIwIA5ND6kwc). +Import the state snapshot + ```sh $ op-reth init-state --without-ovm --chain optimism --datadir op-mainnet world_trie_state.jsonl +``` +Sync the node to a recent finalized block (e.g. 125200000) to catch up close to the tip, before pairing with op-node. + +```sh $ op-reth node --chain optimism --datadir op-mainnet --debug.tip 0x098f87b75c8b861c775984f9d5dbe7b70cbbbc30fc15adb03a5044de0144f2d0 # block #125200000 ``` From 8a11830272303d6050bd027abd9ca99ca178a91d Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Thu, 10 Oct 2024 14:24:24 +0200 Subject: [PATCH 17/82] feat(ci): add Kurtosis workflow (#11615) --- .github/assets/kurtosis_network_params.yaml | 15 ++++ .github/workflows/kurtosis.yml | 95 +++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 .github/assets/kurtosis_network_params.yaml create mode 100644 .github/workflows/kurtosis.yml diff --git a/.github/assets/kurtosis_network_params.yaml b/.github/assets/kurtosis_network_params.yaml new file mode 100644 index 000000000000..9c104de49500 --- /dev/null +++ b/.github/assets/kurtosis_network_params.yaml @@ -0,0 +1,15 @@ +participants: + - el_type: geth + cl_type: lighthouse + - el_type: reth + el_extra_params: + - --engine.experimental + el_image: "ghcr.io/paradigmxyz/reth:kurtosis-ci" + cl_type: teku +additional_services: + - assertoor +assertoor_params: + run_block_proposal_check: true + run_transaction_test: true + run_blob_transaction_test: true + run_opcodes_transaction_test: true diff --git a/.github/workflows/kurtosis.yml b/.github/workflows/kurtosis.yml new file mode 100644 index 000000000000..74d26dbd3eea --- /dev/null +++ b/.github/workflows/kurtosis.yml @@ -0,0 +1,95 @@ +# Runs `assertoor` tests on a `kurtosis` testnet. + +name: kurtosis + +on: + workflow_dispatch: + schedule: + # every day + - cron: "0 1 * * *" + +env: + CARGO_TERM_COLOR: always + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + prepare-reth: + if: github.repository == 'paradigmxyz/reth' + timeout-minutes: 45 + runs-on: + group: Reth + steps: + - uses: actions/checkout@v4 + - run: mkdir artifacts + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - name: Build reth + run: | + cargo build --features asm-keccak --profile hivetests --bin reth --locked + mkdir dist && cp ./target/hivetests/reth ./dist/reth + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and export reth image + uses: docker/build-push-action@v6 + with: + context: . + file: .github/assets/hive/Dockerfile + tags: ghcr.io/paradigmxyz/reth:kurtosis-ci + outputs: type=docker,dest=./artifacts/reth_image.tar + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Upload reth image + uses: actions/upload-artifact@v4 + with: + name: artifacts + path: ./artifacts + + test: + timeout-minutes: 60 + strategy: + fail-fast: false + name: run kurtosis + runs-on: + group: Reth + needs: + - prepare-reth + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download reth image + uses: actions/download-artifact@v4 + with: + name: artifacts + path: /tmp + + - name: Load Docker image + run: | + docker load -i /tmp/reth_image.tar & + wait + docker image ls -a + + - name: Run kurtosis + uses: ethpandaops/kurtosis-assertoor-github-action@v1 + with: + ethereum_package_args: '.github/assets/kurtosis_network_params.yaml' + + notify-on-error: + needs: test + if: failure() + runs-on: + group: Reth + steps: + - name: Slack Webhook Action + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Failed run: https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}" + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} From 90cb3629a5fdde03c1b6f937cc02bf45e1ef9e59 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Thu, 10 Oct 2024 14:13:14 +0100 Subject: [PATCH 18/82] feat(exex): commit only notifications with unfinalized blocks to WAL (#11638) --- crates/exex/exex/src/manager.rs | 124 ++++++++++++++----- crates/node/builder/src/launch/exex.rs | 6 +- crates/stages/stages/src/stages/execution.rs | 15 ++- 3 files changed, 108 insertions(+), 37 deletions(-) diff --git a/crates/exex/exex/src/manager.rs b/crates/exex/exex/src/manager.rs index 31dc822222b6..8c1518f3090f 100644 --- a/crates/exex/exex/src/manager.rs +++ b/crates/exex/exex/src/manager.rs @@ -34,6 +34,18 @@ use tokio_util::sync::{PollSendError, PollSender, ReusableBoxFuture}; /// or 17 minutes of 1-second blocks. pub const DEFAULT_EXEX_MANAGER_CAPACITY: usize = 1024; +/// The source of the notification. +/// +/// This distinguishment is needed to not commit any pipeline notificatations to [WAL](`Wal`), +/// because they are already finalized. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExExNotificationSource { + /// The notification was sent from the pipeline. + Pipeline, + /// The notification was sent from the blockchain tree. + BlockchainTree, +} + /// Metrics for an `ExEx`. #[derive(Metrics)] #[metrics(scope = "exex")] @@ -197,7 +209,7 @@ pub struct ExExManager

{ exex_handles: Vec, /// [`ExExNotification`] channel from the [`ExExManagerHandle`]s. - handle_rx: UnboundedReceiver, + handle_rx: UnboundedReceiver<(ExExNotificationSource, ExExNotification)>, /// The minimum notification ID currently present in the buffer. min_id: usize, @@ -429,14 +441,23 @@ where // Drain handle notifications while this.buffer.len() < this.max_capacity { - if let Poll::Ready(Some(notification)) = this.handle_rx.poll_recv(cx) { - debug!( - target: "exex::manager", - committed_tip = ?notification.committed_chain().map(|chain| chain.tip().number), - reverted_tip = ?notification.reverted_chain().map(|chain| chain.tip().number), - "Received new notification" - ); - this.wal.commit(¬ification)?; + if let Poll::Ready(Some((source, notification))) = this.handle_rx.poll_recv(cx) { + let committed_tip = notification.committed_chain().map(|chain| chain.tip().number); + let reverted_tip = notification.reverted_chain().map(|chain| chain.tip().number); + debug!(target: "exex::manager", ?committed_tip, ?reverted_tip, "Received new notification"); + + // Commit to WAL only notifications from blockchain tree. Pipeline notifications + // always contain only finalized blocks. + match source { + ExExNotificationSource::BlockchainTree => { + debug!(target: "exex::manager", ?committed_tip, ?reverted_tip, "Committing notification to WAL"); + this.wal.commit(¬ification)?; + } + ExExNotificationSource::Pipeline => { + debug!(target: "exex::manager", ?committed_tip, ?reverted_tip, "Notification was sent from pipeline, skipping WAL commit"); + } + } + this.push_notification(notification); continue } @@ -491,7 +512,7 @@ where #[derive(Debug)] pub struct ExExManagerHandle { /// Channel to send notifications to the `ExEx` manager. - exex_tx: UnboundedSender, + exex_tx: UnboundedSender<(ExExNotificationSource, ExExNotification)>, /// The number of `ExEx`'s running on the node. num_exexs: usize, /// A watch channel denoting whether the manager is ready for new notifications or not. @@ -533,8 +554,12 @@ impl ExExManagerHandle { /// Synchronously send a notification over the channel to all execution extensions. /// /// Senders should call [`Self::has_capacity`] first. - pub fn send(&self, notification: ExExNotification) -> Result<(), SendError> { - self.exex_tx.send(notification) + pub fn send( + &self, + source: ExExNotificationSource, + notification: ExExNotification, + ) -> Result<(), SendError<(ExExNotificationSource, ExExNotification)>> { + self.exex_tx.send((source, notification)) } /// Asynchronously send a notification over the channel to all execution extensions. @@ -543,10 +568,11 @@ impl ExExManagerHandle { /// capacity in the channel, the future will wait. pub async fn send_async( &mut self, + source: ExExNotificationSource, notification: ExExNotification, - ) -> Result<(), SendError> { + ) -> Result<(), SendError<(ExExNotificationSource, ExExNotification)>> { self.ready().await; - self.exex_tx.send(notification) + self.exex_tx.send((source, notification)) } /// Get the current capacity of the `ExEx` manager's internal notification buffer. @@ -610,16 +636,16 @@ impl Clone for ExExManagerHandle { mod tests { use super::*; use alloy_primitives::B256; - use futures::StreamExt; + use futures::{StreamExt, TryStreamExt}; use rand::Rng; use reth_db_common::init::init_genesis; use reth_evm_ethereum::execute::EthExecutorProvider; use reth_primitives::SealedBlockWithSenders; use reth_provider::{ providers::BlockchainProvider2, test_utils::create_test_provider_factory, BlockReader, - Chain, TransactionVariant, + BlockWriter, Chain, DatabaseProviderFactory, TransactionVariant, }; - use reth_testing_utils::generators; + use reth_testing_utils::generators::{self, random_block, BlockParams}; fn empty_finalized_header_stream() -> ForkChoiceStream { let (tx, rx) = watch::channel(None); @@ -959,9 +985,21 @@ mod tests { }; // Send notifications to go over the max capacity - exex_manager.handle.exex_tx.send(notification.clone()).unwrap(); - exex_manager.handle.exex_tx.send(notification.clone()).unwrap(); - exex_manager.handle.exex_tx.send(notification).unwrap(); + exex_manager + .handle + .exex_tx + .send((ExExNotificationSource::BlockchainTree, notification.clone())) + .unwrap(); + exex_manager + .handle + .exex_tx + .send((ExExNotificationSource::BlockchainTree, notification.clone())) + .unwrap(); + exex_manager + .handle + .exex_tx + .send((ExExNotificationSource::BlockchainTree, notification)) + .unwrap(); // Pin the ExExManager to call the poll method let mut pinned_manager = std::pin::pin!(exex_manager); @@ -1177,6 +1215,18 @@ mod tests { .sealed_block_with_senders(genesis_hash.into(), TransactionVariant::NoHash) .unwrap() .ok_or_else(|| eyre::eyre!("genesis block not found"))?; + + let block = random_block( + &mut rng, + genesis_block.number + 1, + BlockParams { parent: Some(genesis_hash), ..Default::default() }, + ) + .seal_with_senders() + .unwrap(); + let provider_rw = provider_factory.database_provider_rw().unwrap(); + provider_rw.insert_block(block.clone()).unwrap(); + provider_rw.commit().unwrap(); + let provider = BlockchainProvider2::new(provider_factory).unwrap(); let temp_dir = tempfile::tempdir().unwrap(); @@ -1190,33 +1240,49 @@ mod tests { wal.handle(), ); - let notification = ExExNotification::ChainCommitted { + let genesis_notification = ExExNotification::ChainCommitted { new: Arc::new(Chain::new(vec![genesis_block.clone()], Default::default(), None)), }; + let notification = ExExNotification::ChainCommitted { + new: Arc::new(Chain::new(vec![block.clone()], Default::default(), None)), + }; let (finalized_headers_tx, rx) = watch::channel(None); + finalized_headers_tx.send(Some(genesis_block.header.clone()))?; let finalized_header_stream = ForkChoiceStream::new(rx); let mut exex_manager = std::pin::pin!(ExExManager::new( provider, vec![exex_handle], - 1, + 2, wal, finalized_header_stream )); let mut cx = Context::from_waker(futures::task::noop_waker_ref()); - exex_manager.handle().send(notification.clone())?; + exex_manager + .handle() + .send(ExExNotificationSource::Pipeline, genesis_notification.clone())?; + exex_manager.handle().send(ExExNotificationSource::BlockchainTree, notification.clone())?; assert!(exex_manager.as_mut().poll(&mut cx)?.is_pending()); - assert_eq!(notifications.next().await.unwrap().unwrap(), notification.clone()); + assert_eq!( + notifications.try_poll_next_unpin(&mut cx)?, + Poll::Ready(Some(genesis_notification)) + ); + assert!(exex_manager.as_mut().poll(&mut cx)?.is_pending()); + assert_eq!( + notifications.try_poll_next_unpin(&mut cx)?, + Poll::Ready(Some(notification.clone())) + ); + // WAL shouldn't contain the genesis notification, because it's finalized assert_eq!( exex_manager.wal.iter_notifications()?.collect::>>()?, [notification.clone()] ); - finalized_headers_tx.send(Some(genesis_block.header.clone()))?; + finalized_headers_tx.send(Some(block.header.clone()))?; assert!(exex_manager.as_mut().poll(&mut cx).is_pending()); // WAL isn't finalized because the ExEx didn't emit the `FinishedHeight` event assert_eq!( @@ -1229,7 +1295,7 @@ mod tests { .send(ExExEvent::FinishedHeight((rng.gen::(), rng.gen::()).into())) .unwrap(); - finalized_headers_tx.send(Some(genesis_block.header.clone()))?; + finalized_headers_tx.send(Some(block.header.clone()))?; assert!(exex_manager.as_mut().poll(&mut cx).is_pending()); // WAL isn't finalized because the ExEx emitted a `FinishedHeight` event with a // non-canonical block @@ -1239,12 +1305,12 @@ mod tests { ); // Send a `FinishedHeight` event with a canonical block - events_tx.send(ExExEvent::FinishedHeight(genesis_block.num_hash())).unwrap(); + events_tx.send(ExExEvent::FinishedHeight(block.num_hash())).unwrap(); - finalized_headers_tx.send(Some(genesis_block.header.clone()))?; + finalized_headers_tx.send(Some(block.header.clone()))?; assert!(exex_manager.as_mut().poll(&mut cx).is_pending()); // WAL is finalized - assert!(exex_manager.wal.iter_notifications()?.next().is_none()); + assert_eq!(exex_manager.wal.iter_notifications()?.next().transpose()?, None); Ok(()) } diff --git a/crates/node/builder/src/launch/exex.rs b/crates/node/builder/src/launch/exex.rs index 5441f393d3c9..a3640690c1dc 100644 --- a/crates/node/builder/src/launch/exex.rs +++ b/crates/node/builder/src/launch/exex.rs @@ -6,7 +6,8 @@ use futures::future; use reth_chain_state::ForkChoiceSubscriptions; use reth_chainspec::EthChainSpec; use reth_exex::{ - ExExContext, ExExHandle, ExExManager, ExExManagerHandle, Wal, DEFAULT_EXEX_MANAGER_CAPACITY, + ExExContext, ExExHandle, ExExManager, ExExManagerHandle, ExExNotificationSource, Wal, + DEFAULT_EXEX_MANAGER_CAPACITY, }; use reth_node_api::{FullNodeComponents, NodeTypes}; use reth_primitives::Head; @@ -47,6 +48,7 @@ impl ExExLauncher { return Ok(None) } + info!(target: "reth::cli", "Loading ExEx Write-Ahead Log..."); let exex_wal = Wal::new( config_container .config @@ -127,7 +129,7 @@ impl ExExLauncher { async move { while let Ok(notification) = canon_state_notifications.recv().await { handle - .send_async(notification.into()) + .send_async(ExExNotificationSource::BlockchainTree, notification.into()) .await .expect("blockchain tree notification could not be sent to exex manager"); } diff --git a/crates/stages/stages/src/stages/execution.rs b/crates/stages/stages/src/stages/execution.rs index df234e542be6..7bb6ebc59e09 100644 --- a/crates/stages/stages/src/stages/execution.rs +++ b/crates/stages/stages/src/stages/execution.rs @@ -9,7 +9,7 @@ use reth_evm::{ metrics::ExecutorMetrics, }; use reth_execution_types::{Chain, ExecutionOutcome}; -use reth_exex::{ExExManagerHandle, ExExNotification}; +use reth_exex::{ExExManagerHandle, ExExNotification, ExExNotificationSource}; use reth_primitives::{Header, SealedHeader, StaticFileSegment}; use reth_primitives_traits::format_gas_throughput; use reth_provider::{ @@ -389,9 +389,10 @@ where // NOTE: We can ignore the error here, since an error means that the channel is closed, // which means the manager has died, which then in turn means the node is shutting down. - let _ = self - .exex_manager_handle - .send(ExExNotification::ChainCommitted { new: Arc::new(chain) }); + let _ = self.exex_manager_handle.send( + ExExNotificationSource::Pipeline, + ExExNotification::ChainCommitted { new: Arc::new(chain) }, + ); Ok(()) } @@ -477,8 +478,10 @@ where // NOTE: We can ignore the error here, since an error means that the channel is closed, // which means the manager has died, which then in turn means the node is shutting down. - let _ = - self.exex_manager_handle.send(ExExNotification::ChainReverted { old: Arc::new(chain) }); + let _ = self.exex_manager_handle.send( + ExExNotificationSource::Pipeline, + ExExNotification::ChainReverted { old: Arc::new(chain) }, + ); Ok(()) } From 456d507797cec7f7e1dc64c34f7310df6dd899d4 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Thu, 10 Oct 2024 14:48:30 +0100 Subject: [PATCH 19/82] feat(ci): move book clippy and tests to matrix (#11618) --- .github/workflows/lint.yml | 25 +++++++++++++++---------- .github/workflows/unit.yml | 37 +++++++++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e8161557fde9..fd71a8c636e4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,16 +11,21 @@ env: jobs: clippy-binaries: - name: clippy / ${{ matrix.network }} + name: clippy binaries / ${{ matrix.type }} runs-on: ubuntu-latest timeout-minutes: 30 strategy: matrix: include: - - binary: reth - network: ethereum - - binary: op-reth - network: optimism + - type: ethereum + args: --bin reth --workspace + features: "ethereum asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs" + - type: optimism + args: --bin op-reth --workspace + features: "optimism asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs" + - type: book + args: --manifest-path book/sources/Cargo.toml --workspace --bins + features: "" steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@clippy @@ -30,12 +35,12 @@ jobs: - uses: Swatinem/rust-cache@v2 with: cache-on-failure: true + - if: "${{ matrix.type == 'book' }}" + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Run clippy on binaries - run: cargo clippy --bin "${{ matrix.binary }}" --workspace --features "${{ matrix.network }} asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs" - env: - RUSTFLAGS: -D warnings - - name: Run clippy on book binary sources - run: cargo clippy --manifest-path book/sources/Cargo.toml --workspace --bins + run: cargo clippy ${{ matrix.args }} --features "${{ matrix.features }}" env: RUSTFLAGS: -D warnings diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 1056a6fb58c8..defd9a6f535d 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -18,15 +18,34 @@ concurrency: jobs: test: - name: test / ${{ matrix.network }} (${{ matrix.partition }}/2) + name: test / ${{ matrix.type }} (${{ matrix.partition }}/${{ matrix.total_partitions }}) runs-on: group: Reth env: RUST_BACKTRACE: 1 strategy: matrix: - partition: [1, 2] - network: ["ethereum", "optimism"] + include: + - type: ethereum + args: --features "asm-keccak ethereum" --locked + partition: 1 + total_partitions: 2 + - type: ethereum + args: --features "asm-keccak ethereum" --locked + partition: 2 + total_partitions: 2 + - type: optimism + args: --features "asm-keccak optimism" --locked + partition: 1 + total_partitions: 2 + - type: optimism + args: --features "asm-keccak optimism" --locked + partition: 2 + total_partitions: 2 + - type: book + args: --manifest-path book/sources/Cargo.toml + partition: 1 + total_partitions: 1 timeout-minutes: 30 steps: - uses: actions/checkout@v4 @@ -35,18 +54,16 @@ jobs: with: cache-on-failure: true - uses: taiki-e/install-action@nextest + - if: "${{ matrix.type == 'book' }}" + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Run tests run: | cargo nextest run \ - --locked --features "asm-keccak ${{ matrix.network }}" \ - --workspace --exclude ef-tests \ + ${{ matrix.args }} --workspace --exclude ef-tests \ --partition hash:${{ matrix.partition }}/2 \ -E "!kind(test)" - - name: Run tests on book sources - run: | - cargo nextest run \ - --manifest-path book/sources/Cargo.toml --workspace \ - -E "!kind(test)" state: name: Ethereum state tests From 58bac0ee301612c22d123cce61d9505734a696b9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 10 Oct 2024 16:59:04 +0200 Subject: [PATCH 20/82] chore: release 1.1.0 (#11640) --- Cargo.lock | 236 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 119 insertions(+), 119 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9bf1ea2d666d..37da7b57510b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2564,7 +2564,7 @@ dependencies = [ [[package]] name = "ef-tests" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -5268,7 +5268,7 @@ dependencies = [ [[package]] name = "op-reth" -version = "1.0.8" +version = "1.1.0" dependencies = [ "clap", "reth-cli-util", @@ -6230,7 +6230,7 @@ dependencies = [ [[package]] name = "reth" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6303,7 +6303,7 @@ dependencies = [ [[package]] name = "reth-auto-seal-consensus" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -6333,7 +6333,7 @@ dependencies = [ [[package]] name = "reth-basic-payload-builder" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -6356,7 +6356,7 @@ dependencies = [ [[package]] name = "reth-beacon-consensus" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -6407,7 +6407,7 @@ dependencies = [ [[package]] name = "reth-bench" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-json-rpc", @@ -6442,7 +6442,7 @@ dependencies = [ [[package]] name = "reth-blockchain-tree" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6480,7 +6480,7 @@ dependencies = [ [[package]] name = "reth-blockchain-tree-api" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "reth-consensus", @@ -6492,7 +6492,7 @@ dependencies = [ [[package]] name = "reth-chain-state" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6520,7 +6520,7 @@ dependencies = [ [[package]] name = "reth-chainspec" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-chains", "alloy-eips", @@ -6541,7 +6541,7 @@ dependencies = [ [[package]] name = "reth-cli" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-genesis", "clap", @@ -6553,7 +6553,7 @@ dependencies = [ [[package]] name = "reth-cli-commands" -version = "1.0.8" +version = "1.1.0" dependencies = [ "ahash", "alloy-eips", @@ -6614,7 +6614,7 @@ dependencies = [ [[package]] name = "reth-cli-runner" -version = "1.0.8" +version = "1.1.0" dependencies = [ "reth-tasks", "tokio", @@ -6623,7 +6623,7 @@ dependencies = [ [[package]] name = "reth-cli-util" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -6640,7 +6640,7 @@ dependencies = [ [[package]] name = "reth-codecs" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6663,7 +6663,7 @@ dependencies = [ [[package]] name = "reth-codecs-derive" -version = "1.0.8" +version = "1.1.0" dependencies = [ "convert_case", "proc-macro2", @@ -6674,7 +6674,7 @@ dependencies = [ [[package]] name = "reth-config" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "eyre", @@ -6690,7 +6690,7 @@ dependencies = [ [[package]] name = "reth-consensus" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "auto_impl", @@ -6700,7 +6700,7 @@ dependencies = [ [[package]] name = "reth-consensus-common" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -6715,7 +6715,7 @@ dependencies = [ [[package]] name = "reth-consensus-debug-client" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6738,7 +6738,7 @@ dependencies = [ [[package]] name = "reth-db" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "arbitrary", @@ -6779,7 +6779,7 @@ dependencies = [ [[package]] name = "reth-db-api" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -6806,7 +6806,7 @@ dependencies = [ [[package]] name = "reth-db-common" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -6834,7 +6834,7 @@ dependencies = [ [[package]] name = "reth-db-models" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "arbitrary", @@ -6850,7 +6850,7 @@ dependencies = [ [[package]] name = "reth-discv4" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -6876,7 +6876,7 @@ dependencies = [ [[package]] name = "reth-discv5" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -6900,7 +6900,7 @@ dependencies = [ [[package]] name = "reth-dns-discovery" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-chains", "alloy-primitives", @@ -6928,7 +6928,7 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -6965,7 +6965,7 @@ dependencies = [ [[package]] name = "reth-e2e-test-utils" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7003,7 +7003,7 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "1.0.8" +version = "1.1.0" dependencies = [ "aes", "alloy-primitives", @@ -7033,7 +7033,7 @@ dependencies = [ [[package]] name = "reth-engine-local" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -7063,7 +7063,7 @@ dependencies = [ [[package]] name = "reth-engine-primitives" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "reth-execution-types", @@ -7075,7 +7075,7 @@ dependencies = [ [[package]] name = "reth-engine-service" -version = "1.0.8" +version = "1.1.0" dependencies = [ "futures", "pin-project", @@ -7103,7 +7103,7 @@ dependencies = [ [[package]] name = "reth-engine-tree" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7150,7 +7150,7 @@ dependencies = [ [[package]] name = "reth-engine-util" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -7180,7 +7180,7 @@ dependencies = [ [[package]] name = "reth-errors" -version = "1.0.8" +version = "1.1.0" dependencies = [ "reth-blockchain-tree-api", "reth-consensus", @@ -7192,7 +7192,7 @@ dependencies = [ [[package]] name = "reth-eth-wire" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7227,7 +7227,7 @@ dependencies = [ [[package]] name = "reth-eth-wire-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7250,7 +7250,7 @@ dependencies = [ [[package]] name = "reth-ethereum-cli" -version = "1.0.8" +version = "1.1.0" dependencies = [ "clap", "eyre", @@ -7261,7 +7261,7 @@ dependencies = [ [[package]] name = "reth-ethereum-consensus" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "reth-chainspec", @@ -7273,7 +7273,7 @@ dependencies = [ [[package]] name = "reth-ethereum-engine-primitives" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7292,7 +7292,7 @@ dependencies = [ [[package]] name = "reth-ethereum-forks" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-chains", "alloy-primitives", @@ -7311,7 +7311,7 @@ dependencies = [ [[package]] name = "reth-ethereum-payload-builder" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "reth-basic-payload-builder", @@ -7335,7 +7335,7 @@ dependencies = [ [[package]] name = "reth-etl" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "rayon", @@ -7345,7 +7345,7 @@ dependencies = [ [[package]] name = "reth-evm" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7367,7 +7367,7 @@ dependencies = [ [[package]] name = "reth-evm-ethereum" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7390,7 +7390,7 @@ dependencies = [ [[package]] name = "reth-execution-errors" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7405,7 +7405,7 @@ dependencies = [ [[package]] name = "reth-execution-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7422,7 +7422,7 @@ dependencies = [ [[package]] name = "reth-exex" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7467,7 +7467,7 @@ dependencies = [ [[package]] name = "reth-exex-test-utils" -version = "1.0.8" +version = "1.1.0" dependencies = [ "eyre", "futures-util", @@ -7499,7 +7499,7 @@ dependencies = [ [[package]] name = "reth-exex-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7515,7 +7515,7 @@ dependencies = [ [[package]] name = "reth-fs-util" -version = "1.0.8" +version = "1.1.0" dependencies = [ "serde", "serde_json", @@ -7524,7 +7524,7 @@ dependencies = [ [[package]] name = "reth-invalid-block-hooks" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -7548,7 +7548,7 @@ dependencies = [ [[package]] name = "reth-ipc" -version = "1.0.8" +version = "1.1.0" dependencies = [ "async-trait", "bytes", @@ -7570,7 +7570,7 @@ dependencies = [ [[package]] name = "reth-libmdbx" -version = "1.0.8" +version = "1.1.0" dependencies = [ "bitflags 2.6.0", "byteorder", @@ -7591,7 +7591,7 @@ dependencies = [ [[package]] name = "reth-mdbx-sys" -version = "1.0.8" +version = "1.1.0" dependencies = [ "bindgen 0.70.1", "cc", @@ -7599,7 +7599,7 @@ dependencies = [ [[package]] name = "reth-metrics" -version = "1.0.8" +version = "1.1.0" dependencies = [ "futures", "metrics", @@ -7610,14 +7610,14 @@ dependencies = [ [[package]] name = "reth-net-banlist" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", ] [[package]] name = "reth-net-nat" -version = "1.0.8" +version = "1.1.0" dependencies = [ "futures-util", "if-addrs", @@ -7631,7 +7631,7 @@ dependencies = [ [[package]] name = "reth-network" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7691,7 +7691,7 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types-admin", @@ -7713,7 +7713,7 @@ dependencies = [ [[package]] name = "reth-network-p2p" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7733,7 +7733,7 @@ dependencies = [ [[package]] name = "reth-network-peers" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -7749,7 +7749,7 @@ dependencies = [ [[package]] name = "reth-network-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "humantime-serde", "reth-ethereum-forks", @@ -7762,7 +7762,7 @@ dependencies = [ [[package]] name = "reth-nippy-jar" -version = "1.0.8" +version = "1.1.0" dependencies = [ "anyhow", "bincode", @@ -7780,7 +7780,7 @@ dependencies = [ [[package]] name = "reth-node-api" -version = "1.0.8" +version = "1.1.0" dependencies = [ "reth-engine-primitives", "reth-evm", @@ -7797,7 +7797,7 @@ dependencies = [ [[package]] name = "reth-node-builder" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types", @@ -7861,7 +7861,7 @@ dependencies = [ [[package]] name = "reth-node-core" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -7911,7 +7911,7 @@ dependencies = [ [[package]] name = "reth-node-ethereum" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -7947,7 +7947,7 @@ dependencies = [ [[package]] name = "reth-node-events" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -7969,7 +7969,7 @@ dependencies = [ [[package]] name = "reth-node-metrics" -version = "1.0.8" +version = "1.1.0" dependencies = [ "eyre", "http", @@ -7995,7 +7995,7 @@ dependencies = [ [[package]] name = "reth-node-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "reth-chainspec", "reth-db-api", @@ -8004,7 +8004,7 @@ dependencies = [ [[package]] name = "reth-optimism-chainspec" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-chains", "alloy-genesis", @@ -8022,7 +8022,7 @@ dependencies = [ [[package]] name = "reth-optimism-cli" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8066,7 +8066,7 @@ dependencies = [ [[package]] name = "reth-optimism-consensus" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "reth-chainspec", @@ -8081,7 +8081,7 @@ dependencies = [ [[package]] name = "reth-optimism-evm" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8107,7 +8107,7 @@ dependencies = [ [[package]] name = "reth-optimism-forks" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-chains", "alloy-primitives", @@ -8118,7 +8118,7 @@ dependencies = [ [[package]] name = "reth-optimism-node" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-genesis", @@ -8171,7 +8171,7 @@ dependencies = [ [[package]] name = "reth-optimism-payload-builder" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8204,7 +8204,7 @@ dependencies = [ [[package]] name = "reth-optimism-primitives" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "reth-primitives", @@ -8213,7 +8213,7 @@ dependencies = [ [[package]] name = "reth-optimism-rpc" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8252,7 +8252,7 @@ dependencies = [ [[package]] name = "reth-optimism-storage" -version = "1.0.8" +version = "1.1.0" dependencies = [ "reth-codecs", "reth-db-api", @@ -8263,7 +8263,7 @@ dependencies = [ [[package]] name = "reth-payload-builder" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types", @@ -8284,7 +8284,7 @@ dependencies = [ [[package]] name = "reth-payload-primitives" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types", @@ -8305,7 +8305,7 @@ dependencies = [ [[package]] name = "reth-payload-validator" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-rpc-types", "reth-chainspec", @@ -8315,7 +8315,7 @@ dependencies = [ [[package]] name = "reth-primitives" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8360,7 +8360,7 @@ dependencies = [ [[package]] name = "reth-primitives-traits" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8388,7 +8388,7 @@ dependencies = [ [[package]] name = "reth-provider" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8438,7 +8438,7 @@ dependencies = [ [[package]] name = "reth-prune" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "assert_matches", @@ -8467,7 +8467,7 @@ dependencies = [ [[package]] name = "reth-prune-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "arbitrary", @@ -8487,7 +8487,7 @@ dependencies = [ [[package]] name = "reth-revm" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "reth-chainspec", @@ -8504,7 +8504,7 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -8572,7 +8572,7 @@ dependencies = [ [[package]] name = "reth-rpc-api" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-json-rpc", @@ -8598,7 +8598,7 @@ dependencies = [ [[package]] name = "reth-rpc-api-testing-util" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types", @@ -8617,7 +8617,7 @@ dependencies = [ [[package]] name = "reth-rpc-builder" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-network", "alloy-primitives", @@ -8669,7 +8669,7 @@ dependencies = [ [[package]] name = "reth-rpc-engine-api" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8705,7 +8705,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-api" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-dyn-abi", "alloy-eips", @@ -8745,7 +8745,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8788,7 +8788,7 @@ dependencies = [ [[package]] name = "reth-rpc-layer" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-rpc-types-engine", "http", @@ -8803,7 +8803,7 @@ dependencies = [ [[package]] name = "reth-rpc-server-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -8818,7 +8818,7 @@ dependencies = [ [[package]] name = "reth-rpc-types-compat" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8834,7 +8834,7 @@ dependencies = [ [[package]] name = "reth-stages" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8884,7 +8884,7 @@ dependencies = [ [[package]] name = "reth-stages-api" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "aquamarine", @@ -8912,7 +8912,7 @@ dependencies = [ [[package]] name = "reth-stages-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "arbitrary", @@ -8929,7 +8929,7 @@ dependencies = [ [[package]] name = "reth-static-file" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "assert_matches", @@ -8954,7 +8954,7 @@ dependencies = [ [[package]] name = "reth-static-file-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "clap", @@ -8965,7 +8965,7 @@ dependencies = [ [[package]] name = "reth-storage-api" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8983,7 +8983,7 @@ dependencies = [ [[package]] name = "reth-storage-errors" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8995,7 +8995,7 @@ dependencies = [ [[package]] name = "reth-tasks" -version = "1.0.8" +version = "1.1.0" dependencies = [ "auto_impl", "dyn-clone", @@ -9012,7 +9012,7 @@ dependencies = [ [[package]] name = "reth-testing-utils" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9025,7 +9025,7 @@ dependencies = [ [[package]] name = "reth-tokio-util" -version = "1.0.8" +version = "1.1.0" dependencies = [ "tokio", "tokio-stream", @@ -9034,7 +9034,7 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "1.0.8" +version = "1.1.0" dependencies = [ "clap", "eyre", @@ -9048,7 +9048,7 @@ dependencies = [ [[package]] name = "reth-transaction-pool" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9093,7 +9093,7 @@ dependencies = [ [[package]] name = "reth-trie" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -9124,7 +9124,7 @@ dependencies = [ [[package]] name = "reth-trie-common" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-genesis", @@ -9148,7 +9148,7 @@ dependencies = [ [[package]] name = "reth-trie-db" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -9183,7 +9183,7 @@ dependencies = [ [[package]] name = "reth-trie-parallel" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", diff --git a/Cargo.toml b/Cargo.toml index 22ce3e92f6a6..db115c89cccd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "1.0.8" +version = "1.1.0" edition = "2021" rust-version = "1.81" license = "MIT OR Apache-2.0" From d027b7b9389a9e8ab88f9f7171490696ad423a36 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Thu, 10 Oct 2024 16:04:31 +0100 Subject: [PATCH 21/82] feat(bin): make experimental engine default (#11612) --- bin/reth/src/main.rs | 25 ++++++++++++++++++++----- book/cli/reth/node.md | 7 ++++++- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/bin/reth/src/main.rs b/bin/reth/src/main.rs index 7153cdcc6c88..f424163a24fe 100644 --- a/bin/reth/src/main.rs +++ b/bin/reth/src/main.rs @@ -14,15 +14,24 @@ use reth_node_builder::{ }; use reth_node_ethereum::{node::EthereumAddOns, EthereumNode}; use reth_provider::providers::BlockchainProvider2; +use reth_tracing::tracing::warn; +use tracing::info; /// Parameters for configuring the engine #[derive(Debug, Clone, Args, PartialEq, Eq)] #[command(next_help_heading = "Engine")] pub struct EngineArgs { - /// Enable the engine2 experimental features on reth binary + /// Enable the experimental engine features on reth binary + /// + /// DEPRECATED: experimental engine is default now, use --engine.legacy to enable the legacy + /// functionality #[arg(long = "engine.experimental", default_value = "false")] pub experimental: bool, + /// Enable the legacy engine on reth binary + #[arg(long = "engine.legacy", default_value = "false")] + pub legacy: bool, + /// Configure persistence threshold for engine experimental. #[arg(long = "engine.persistence-threshold", requires = "experimental", default_value_t = DEFAULT_PERSISTENCE_THRESHOLD)] pub persistence_threshold: u64, @@ -36,6 +45,7 @@ impl Default for EngineArgs { fn default() -> Self { Self { experimental: false, + legacy: false, persistence_threshold: DEFAULT_PERSISTENCE_THRESHOLD, memory_block_buffer_target: DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, } @@ -52,9 +62,13 @@ fn main() { if let Err(err) = Cli::::parse().run(|builder, engine_args| async move { - let enable_engine2 = engine_args.experimental; - match enable_engine2 { - true => { + if engine_args.experimental { + warn!(target: "reth::cli", "Experimental engine is default now, and the --engine.experimental flag is deprecated. To enable the legacy functionality, use --engine.legacy."); + } + + let use_legacy_engine = engine_args.legacy; + match use_legacy_engine { + false => { let engine_tree_config = TreeConfig::default() .with_persistence_threshold(engine_args.persistence_threshold) .with_memory_block_buffer_target(engine_args.memory_block_buffer_target); @@ -73,7 +87,8 @@ fn main() { .await?; handle.node_exit_future.await } - false => { + true => { + info!(target: "reth::cli", "Running with legacy engine"); let handle = builder.launch_node(EthereumNode::default()).await?; handle.node_exit_future.await } diff --git a/book/cli/reth/node.md b/book/cli/reth/node.md index 231e665cb8d9..73a8063a852b 100644 --- a/book/cli/reth/node.md +++ b/book/cli/reth/node.md @@ -666,7 +666,12 @@ Pruning: Engine: --engine.experimental - Enable the engine2 experimental features on reth binary + Enable the experimental engine features on reth binary + + DEPRECATED: experimental engine is default now, use --engine.legacy to enable the legacy functionality + + --engine.legacy + Enable the legacy engine on reth binary --engine.persistence-threshold Configure persistence threshold for engine experimental From 1d2c6d5ada38fb86f3edc169a10ee6b78003472b Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Fri, 11 Oct 2024 00:43:58 +0900 Subject: [PATCH 22/82] chore(rpc): don't recover sender if we have it (#11645) --- crates/rpc/rpc-eth-api/src/helpers/transaction.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index c25b18c2f67b..94dd04414874 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -370,12 +370,9 @@ pub trait EthTransactions: LoadTransaction { let gas_limit = estimated_gas; request.set_gas_limit(gas_limit.to()); - let signed_tx = self.sign_request(&from, request).await?; + let transaction = self.sign_request(&from, request).await?.with_signer(from); - let recovered = - signed_tx.into_ecrecovered().ok_or(EthApiError::InvalidTransactionSignature)?; - - let pool_transaction = <::Pool as TransactionPool>::Transaction::try_from_consensus(recovered.into()).map_err(|_| EthApiError::TransactionConversionError)?; + let pool_transaction = <::Pool as TransactionPool>::Transaction::try_from_consensus(transaction.into()).map_err(|_| EthApiError::TransactionConversionError)?; // submit the transaction to the pool with a `Local` origin let hash = LoadTransaction::pool(self) From fca1cd8181c38ddd3b63744ed9c9cc5dec1fa31e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 10 Oct 2024 18:54:06 +0200 Subject: [PATCH 23/82] fix: don't unwrap missing requests (#11646) --- crates/storage/codecs/src/alloy/header.rs | 10 ++++++++++ .../provider/src/providers/database/provider.rs | 12 ++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/crates/storage/codecs/src/alloy/header.rs b/crates/storage/codecs/src/alloy/header.rs index 526bc69b1369..3a17ed1fdcd2 100644 --- a/crates/storage/codecs/src/alloy/header.rs +++ b/crates/storage/codecs/src/alloy/header.rs @@ -181,4 +181,14 @@ mod tests { let len = header.to_compact(&mut encoded_header); assert_eq!(header, Header::from_compact(&encoded_header, len).0); } + + #[test] + fn test_extra_fields_missing() { + let mut header = HOLESKY_BLOCK; + header.extra_fields = None; + + let mut encoded_header = vec![]; + let len = header.to_compact(&mut encoded_header); + assert_eq!(header, Header::from_compact(&encoded_header, len).0); + } } diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 1afd4da3fa8c..767f92db98f2 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -2151,10 +2151,8 @@ impl RequestsProvider ) -> ProviderResult> { if self.chain_spec.is_prague_active_at_timestamp(timestamp) { if let Some(number) = self.convert_hash_or_number(id)? { - // If we are past Prague, then all blocks should have a requests list, even if - // empty - let requests = self.tx.get::(number)?.unwrap_or_default(); - return Ok(Some(requests)) + let requests = self.tx.get::(number)?; + return Ok(requests) } } Ok(None) @@ -3483,10 +3481,8 @@ impl(block_number, requests)?; - durations_recorder.record_relative(metrics::Action::InsertBlockRequests); - } + self.tx.put::(block_number, requests)?; + durations_recorder.record_relative(metrics::Action::InsertBlockRequests); } let block_indices = StoredBlockBodyIndices { first_tx_num, tx_count }; From 250785f833246ba7c9a1e48b47cd0c2a4606ed09 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 10 Oct 2024 19:15:06 +0200 Subject: [PATCH 24/82] chore: preempt single block downloading (#11647) --- crates/net/p2p/src/full_block.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/net/p2p/src/full_block.rs b/crates/net/p2p/src/full_block.rs index 91b786e410ca..0116f1348810 100644 --- a/crates/net/p2p/src/full_block.rs +++ b/crates/net/p2p/src/full_block.rs @@ -177,6 +177,9 @@ where fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = self.get_mut(); + // preemptive yield point + let mut budget = 4; + loop { match ready!(this.request.poll(cx)) { ResponseResult::Header(res) => { @@ -232,6 +235,14 @@ where if let Some(res) = this.take_block() { return Poll::Ready(res) } + + // ensure we still have enough budget for another iteration + budget -= 1; + if budget == 0 { + // make sure we're woken up again + cx.waker().wake_by_ref(); + return Poll::Pending + } } } } From 1ba631ba9581973e7c6cadeea92cfe1802aceb4a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 10 Oct 2024 19:32:15 +0200 Subject: [PATCH 25/82] feat: store safe block num as well (#11648) --- crates/blockchain-tree/src/externals.rs | 2 +- crates/chain-state/src/chain_info.rs | 8 +++- crates/chain-state/src/in_memory.rs | 17 ++++++--- crates/cli/commands/src/stage/unwind.rs | 2 +- .../consensus/beacon/src/engine/test_utils.rs | 9 ++++- crates/engine/tree/src/persistence.rs | 18 ++++++++- crates/engine/tree/src/tree/mod.rs | 13 +++++-- crates/stages/api/src/pipeline/mod.rs | 4 +- crates/storage/db/src/tables/mod.rs | 11 ++++-- .../src/providers/blockchain_provider.rs | 20 ++++++++-- .../src/providers/database/provider.rs | 37 ++++++++++++++----- crates/storage/provider/src/providers/mod.rs | 16 ++++++-- .../provider/src/traits/finalized_block.rs | 15 ++++++-- crates/storage/provider/src/traits/mod.rs | 2 +- 14 files changed, 130 insertions(+), 44 deletions(-) diff --git a/crates/blockchain-tree/src/externals.rs b/crates/blockchain-tree/src/externals.rs index a4f72f6d33d7..719852c12ac0 100644 --- a/crates/blockchain-tree/src/externals.rs +++ b/crates/blockchain-tree/src/externals.rs @@ -7,7 +7,7 @@ use reth_db_api::{cursor::DbCursorRO, transaction::DbTx}; use reth_node_types::NodeTypesWithDB; use reth_primitives::StaticFileSegment; use reth_provider::{ - providers::ProviderNodeTypes, FinalizedBlockReader, FinalizedBlockWriter, ProviderFactory, + providers::ProviderNodeTypes, ChainStateBlockReader, ChainStateBlockWriter, ProviderFactory, StaticFileProviderFactory, StatsReader, }; use reth_storage_errors::provider::ProviderResult; diff --git a/crates/chain-state/src/chain_info.rs b/crates/chain-state/src/chain_info.rs index d9e2c03e2738..d36d9f47e438 100644 --- a/crates/chain-state/src/chain_info.rs +++ b/crates/chain-state/src/chain_info.rs @@ -21,9 +21,13 @@ pub struct ChainInfoTracker { impl ChainInfoTracker { /// Create a new chain info container for the given canonical head and finalized header if it /// exists. - pub fn new(head: SealedHeader, finalized: Option) -> Self { + pub fn new( + head: SealedHeader, + finalized: Option, + safe: Option, + ) -> Self { let (finalized_block, _) = watch::channel(finalized); - let (safe_block, _) = watch::channel(None); + let (safe_block, _) = watch::channel(safe); Self { inner: Arc::new(ChainInfoInner { diff --git a/crates/chain-state/src/in_memory.rs b/crates/chain-state/src/in_memory.rs index 07120cf8ee31..fb67608ebda7 100644 --- a/crates/chain-state/src/in_memory.rs +++ b/crates/chain-state/src/in_memory.rs @@ -173,12 +173,13 @@ impl CanonicalInMemoryState { numbers: BTreeMap, pending: Option, finalized: Option, + safe: Option, ) -> Self { let in_memory_state = InMemoryState::new(blocks, numbers, pending); let header = in_memory_state .head_state() .map_or_else(SealedHeader::default, |state| state.block_ref().block().header.clone()); - let chain_info_tracker = ChainInfoTracker::new(header, finalized); + let chain_info_tracker = ChainInfoTracker::new(header, finalized, safe); let (canon_state_notification_sender, _) = broadcast::channel(CANON_STATE_NOTIFICATION_CHANNEL_SIZE); @@ -193,13 +194,17 @@ impl CanonicalInMemoryState { /// Create an empty state. pub fn empty() -> Self { - Self::new(HashMap::default(), BTreeMap::new(), None, None) + Self::new(HashMap::default(), BTreeMap::new(), None, None, None) } /// Create a new in memory state with the given local head and finalized header /// if it exists. - pub fn with_head(head: SealedHeader, finalized: Option) -> Self { - let chain_info_tracker = ChainInfoTracker::new(head, finalized); + pub fn with_head( + head: SealedHeader, + finalized: Option, + safe: Option, + ) -> Self { + let chain_info_tracker = ChainInfoTracker::new(head, finalized, safe); let in_memory_state = InMemoryState::default(); let (canon_state_notification_sender, _) = broadcast::channel(CANON_STATE_NOTIFICATION_CHANNEL_SIZE); @@ -1255,7 +1260,7 @@ mod tests { numbers.insert(2, block2.block().hash()); numbers.insert(3, block3.block().hash()); - let canonical_state = CanonicalInMemoryState::new(blocks, numbers, None, None); + let canonical_state = CanonicalInMemoryState::new(blocks, numbers, None, None, None); let historical: StateProviderBox = Box::new(MockStateProvider); @@ -1297,7 +1302,7 @@ mod tests { let mut numbers = BTreeMap::new(); numbers.insert(1, hash); - let state = CanonicalInMemoryState::new(blocks, numbers, None, None); + let state = CanonicalInMemoryState::new(blocks, numbers, None, None, None); let chain: Vec<_> = state.canonical_chain().collect(); assert_eq!(chain.len(), 1); diff --git a/crates/cli/commands/src/stage/unwind.rs b/crates/cli/commands/src/stage/unwind.rs index 19305554eaa3..a5c9956c95b2 100644 --- a/crates/cli/commands/src/stage/unwind.rs +++ b/crates/cli/commands/src/stage/unwind.rs @@ -17,7 +17,7 @@ use reth_node_builder::{NodeTypesWithDB, NodeTypesWithEngine}; use reth_node_core::args::NetworkArgs; use reth_provider::{ providers::ProviderNodeTypes, BlockExecutionWriter, BlockNumReader, ChainSpecProvider, - FinalizedBlockReader, FinalizedBlockWriter, ProviderFactory, StaticFileProviderFactory, + ChainStateBlockReader, ChainStateBlockWriter, ProviderFactory, StaticFileProviderFactory, }; use reth_prune::PruneModes; use reth_stages::{ diff --git a/crates/consensus/beacon/src/engine/test_utils.rs b/crates/consensus/beacon/src/engine/test_utils.rs index 4dfd9c87d321..633ae03d8ad9 100644 --- a/crates/consensus/beacon/src/engine/test_utils.rs +++ b/crates/consensus/beacon/src/engine/test_utils.rs @@ -398,8 +398,13 @@ where let (header, seal) = sealed.into_parts(); let genesis_block = SealedHeader::new(header, seal); - let blockchain_provider = - BlockchainProvider::with_blocks(provider_factory.clone(), tree, genesis_block, None); + let blockchain_provider = BlockchainProvider::with_blocks( + provider_factory.clone(), + tree, + genesis_block, + None, + None, + ); let pruner = Pruner::new_with_factory( provider_factory.clone(), diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index dfddfcfaa89b..25c1f0ed7030 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -4,7 +4,7 @@ use reth_chain_state::ExecutedBlock; use reth_errors::ProviderError; use reth_provider::{ providers::ProviderNodeTypes, writer::UnifiedStorageWriter, BlockHashReader, - DatabaseProviderFactory, FinalizedBlockWriter, ProviderFactory, StaticFileProviderFactory, + ChainStateBlockWriter, DatabaseProviderFactory, ProviderFactory, StaticFileProviderFactory, }; use reth_prune::{PrunerError, PrunerOutput, PrunerWithFactory}; use reth_stages_api::{MetricEvent, MetricEventsSender}; @@ -97,6 +97,11 @@ impl PersistenceService { provider.save_finalized_block_number(finalized_block)?; provider.commit()?; } + PersistenceAction::SaveSafeBlock(safe_block) => { + let provider = self.provider.database_provider_rw()?; + provider.save_safe_block_number(safe_block)?; + provider.commit()?; + } } } Ok(()) @@ -176,6 +181,9 @@ pub enum PersistenceAction { /// Update the persisted finalized block on disk SaveFinalizedBlock(u64), + + /// Update the persisted safe block on disk + SaveSafeBlock(u64), } /// A handle to the persistence service @@ -251,6 +259,14 @@ impl PersistenceHandle { self.send_action(PersistenceAction::SaveFinalizedBlock(finalized_block)) } + /// Persists the finalized block number on disk. + pub fn save_safe_block_number( + &self, + safe_block: u64, + ) -> Result<(), SendError> { + self.send_action(PersistenceAction::SaveSafeBlock(safe_block)) + } + /// Tells the persistence service to remove blocks above a certain block number. The removed /// blocks are returned by the service. /// diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index bc1da6369459..3eadbbd522db 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -2401,8 +2401,13 @@ where // if the safe block is not known, we can't update the safe block return Err(OnForkChoiceUpdated::invalid_state()) } - Ok(Some(finalized)) => { - self.canonical_in_memory_state.set_safe(finalized); + Ok(Some(safe)) => { + if Some(safe.num_hash()) != self.canonical_in_memory_state.get_safe_num_hash() { + // we're also persisting the safe block on disk so we can reload it on + // restart this is required by optimism which queries the safe block: + let _ = self.persistence.save_safe_block_number(safe.number); + self.canonical_in_memory_state.set_safe(safe); + } } Err(err) => { error!(target: "engine::tree", %err, "Failed to fetch safe block header"); @@ -2680,7 +2685,7 @@ mod tests { let (header, seal) = sealed.into_parts(); let header = SealedHeader::new(header, seal); let engine_api_tree_state = EngineApiTreeState::new(10, 10, header.num_hash()); - let canonical_in_memory_state = CanonicalInMemoryState::with_head(header, None); + let canonical_in_memory_state = CanonicalInMemoryState::with_head(header, None, None); let (to_payload_service, _payload_command_rx) = unbounded_channel(); let payload_builder = PayloadBuilderHandle::new(to_payload_service); @@ -2744,7 +2749,7 @@ mod tests { let last_executed_block = blocks.last().unwrap().clone(); let pending = Some(BlockState::new(last_executed_block)); self.tree.canonical_in_memory_state = - CanonicalInMemoryState::new(state_by_hash, hash_by_number, pending, None); + CanonicalInMemoryState::new(state_by_hash, hash_by_number, pending, None, None); self.blocks = blocks.clone(); self.persist_blocks( diff --git a/crates/stages/api/src/pipeline/mod.rs b/crates/stages/api/src/pipeline/mod.rs index 19b68b384853..1f6d9341ad6d 100644 --- a/crates/stages/api/src/pipeline/mod.rs +++ b/crates/stages/api/src/pipeline/mod.rs @@ -7,8 +7,8 @@ pub use event::*; use futures_util::Future; use reth_primitives_traits::constants::BEACON_CONSENSUS_REORG_UNWIND_DEPTH; use reth_provider::{ - providers::ProviderNodeTypes, writer::UnifiedStorageWriter, DatabaseProviderFactory, - FinalizedBlockReader, FinalizedBlockWriter, ProviderFactory, StageCheckpointReader, + providers::ProviderNodeTypes, writer::UnifiedStorageWriter, ChainStateBlockReader, + ChainStateBlockWriter, DatabaseProviderFactory, ProviderFactory, StageCheckpointReader, StageCheckpointWriter, StaticFileProviderFactory, }; use reth_prune::PrunerBuilder; diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index 384139618163..83a063903e08 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -416,6 +416,8 @@ tables! { pub enum ChainStateKey { /// Last finalized block key LastFinalizedBlock, + /// Last finalized block key + LastSafeBlockBlock, } impl Encode for ChainStateKey { @@ -424,16 +426,17 @@ impl Encode for ChainStateKey { fn encode(self) -> Self::Encoded { match self { Self::LastFinalizedBlock => [0], + Self::LastSafeBlockBlock => [1], } } } impl Decode for ChainStateKey { fn decode(value: &[u8]) -> Result { - if value == [0] { - Ok(Self::LastFinalizedBlock) - } else { - Err(reth_db_api::DatabaseError::Decode) + match value { + [0] => Ok(Self::LastFinalizedBlock), + [1] => Ok(Self::LastSafeBlockBlock), + _ => Err(reth_db_api::DatabaseError::Decode), } } } diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 54f28b77b912..a0811db2aee2 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1,8 +1,8 @@ use crate::{ providers::StaticFileProvider, AccountReader, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, CanonChainTracker, CanonStateNotifications, - CanonStateSubscriptions, ChainSpecProvider, ChangeSetReader, DatabaseProviderFactory, - DatabaseProviderRO, EvmEnvProvider, FinalizedBlockReader, HeaderProvider, ProviderError, + CanonStateSubscriptions, ChainSpecProvider, ChainStateBlockReader, ChangeSetReader, + DatabaseProviderFactory, DatabaseProviderRO, EvmEnvProvider, HeaderProvider, ProviderError, ProviderFactory, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, RequestsProvider, StageCheckpointReader, StateProviderBox, StateProviderFactory, StateReader, StaticFileProviderFactory, TransactionVariant, TransactionsProvider, WithdrawalsProvider, @@ -93,9 +93,23 @@ impl BlockchainProvider2 { .map(|num| provider.sealed_header(num)) .transpose()? .flatten(); + let safe_header = provider + .last_safe_block_number()? + .or_else(|| { + // for the purpose of this we can also use the finalized block if we don't have the + // safe block + provider.last_finalized_block_number().ok().flatten() + }) + .map(|num| provider.sealed_header(num)) + .transpose()? + .flatten(); Ok(Self { database, - canonical_in_memory_state: CanonicalInMemoryState::with_head(latest, finalized_header), + canonical_in_memory_state: CanonicalInMemoryState::with_head( + latest, + finalized_header, + safe_header, + ), }) } diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 767f92db98f2..c9b0af3d33ca 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -7,13 +7,14 @@ use crate::{ }, writer::UnifiedStorageWriter, AccountReader, BlockExecutionReader, BlockExecutionWriter, BlockHashReader, BlockNumReader, - BlockReader, BlockWriter, BundleStateInit, DBProvider, EvmEnvProvider, FinalizedBlockReader, - FinalizedBlockWriter, HashingWriter, HeaderProvider, HeaderSyncGap, HeaderSyncGapProvider, - HistoricalStateProvider, HistoryWriter, LatestStateProvider, OriginalValuesKnown, - ProviderError, PruneCheckpointReader, PruneCheckpointWriter, RequestsProvider, RevertsInit, - StageCheckpointReader, StateChangeWriter, StateProviderBox, StateReader, StateWriter, - StaticFileProviderFactory, StatsReader, StorageReader, StorageTrieWriter, TransactionVariant, - TransactionsProvider, TransactionsProviderExt, TrieWriter, WithdrawalsProvider, + BlockReader, BlockWriter, BundleStateInit, ChainStateBlockReader, ChainStateBlockWriter, + DBProvider, EvmEnvProvider, HashingWriter, HeaderProvider, HeaderSyncGap, + HeaderSyncGapProvider, HistoricalStateProvider, HistoryWriter, LatestStateProvider, + OriginalValuesKnown, ProviderError, PruneCheckpointReader, PruneCheckpointWriter, + RequestsProvider, RevertsInit, StageCheckpointReader, StateChangeWriter, StateProviderBox, + StateReader, StateWriter, StaticFileProviderFactory, StatsReader, StorageReader, + StorageTrieWriter, TransactionVariant, TransactionsProvider, TransactionsProviderExt, + TrieWriter, WithdrawalsProvider, }; use alloy_eips::BlockHashOrNumber; use alloy_primitives::{keccak256, Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, U256}; @@ -3596,7 +3597,7 @@ impl StatsReader for DatabaseProvider { } } -impl FinalizedBlockReader for DatabaseProvider { +impl ChainStateBlockReader for DatabaseProvider { fn last_finalized_block_number(&self) -> ProviderResult> { let mut finalized_blocks = self .tx @@ -3608,14 +3609,32 @@ impl FinalizedBlockReader for DatabaseProvider ProviderResult> { + let mut finalized_blocks = self + .tx + .cursor_read::()? + .walk(Some(tables::ChainStateKey::LastSafeBlockBlock))? + .take(1) + .collect::, _>>()?; + + let last_finalized_block_number = finalized_blocks.pop_first().map(|pair| pair.1); + Ok(last_finalized_block_number) + } } -impl FinalizedBlockWriter for DatabaseProvider { +impl ChainStateBlockWriter for DatabaseProvider { fn save_finalized_block_number(&self, block_number: BlockNumber) -> ProviderResult<()> { Ok(self .tx .put::(tables::ChainStateKey::LastFinalizedBlock, block_number)?) } + + fn save_safe_block_number(&self, block_number: BlockNumber) -> ProviderResult<()> { + Ok(self + .tx + .put::(tables::ChainStateKey::LastSafeBlockBlock, block_number)?) + } } impl DBProvider for DatabaseProvider { diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index 5fed81c155d1..561e1d974362 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -1,9 +1,9 @@ use crate::{ AccountReader, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, BlockchainTreePendingStateProvider, CanonChainTracker, CanonStateNotifications, - CanonStateSubscriptions, ChainSpecProvider, ChangeSetReader, DatabaseProviderFactory, - EvmEnvProvider, FinalizedBlockReader, FullExecutionDataProvider, HeaderProvider, ProviderError, - PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, RequestsProvider, + CanonStateSubscriptions, ChainSpecProvider, ChainStateBlockReader, ChangeSetReader, + DatabaseProviderFactory, EvmEnvProvider, FullExecutionDataProvider, HeaderProvider, + ProviderError, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, RequestsProvider, StageCheckpointReader, StateProviderBox, StateProviderFactory, StaticFileProviderFactory, TransactionVariant, TransactionsProvider, TreeViewer, WithdrawalsProvider, }; @@ -109,8 +109,9 @@ impl BlockchainProvider { tree: Arc, latest: SealedHeader, finalized: Option, + safe: Option, ) -> Self { - Self { database, tree, chain_info: ChainInfoTracker::new(latest, finalized) } + Self { database, tree, chain_info: ChainInfoTracker::new(latest, finalized, safe) } } /// Create a new provider using only the database and the tree, fetching the latest header from @@ -128,11 +129,18 @@ impl BlockchainProvider { .transpose()? .flatten(); + let safe_header = provider + .last_safe_block_number()? + .map(|num| provider.sealed_header(num)) + .transpose()? + .flatten(); + Ok(Self::with_blocks( database, tree, SealedHeader::new(latest_header, best.best_hash), finalized_header, + safe_header, )) } diff --git a/crates/storage/provider/src/traits/finalized_block.rs b/crates/storage/provider/src/traits/finalized_block.rs index 5509db0aa939..98a6d9d0e343 100644 --- a/crates/storage/provider/src/traits/finalized_block.rs +++ b/crates/storage/provider/src/traits/finalized_block.rs @@ -1,16 +1,23 @@ use alloy_primitives::BlockNumber; use reth_errors::ProviderResult; -/// Functionality to read the last known finalized block from the database. -pub trait FinalizedBlockReader: Send + Sync { +/// Functionality to read the last known chain blocks from the database. +pub trait ChainStateBlockReader: Send + Sync { /// Returns the last finalized block number. /// /// If no finalized block has been written yet, this returns `None`. fn last_finalized_block_number(&self) -> ProviderResult>; + /// Returns the last safe block number. + /// + /// If no safe block has been written yet, this returns `None`. + fn last_safe_block_number(&self) -> ProviderResult>; } -/// Functionality to write the last known finalized block to the database. -pub trait FinalizedBlockWriter: Send + Sync { +/// Functionality to write the last known chain blocks to the database. +pub trait ChainStateBlockWriter: Send + Sync { /// Saves the given finalized block number in the DB. fn save_finalized_block_number(&self, block_number: BlockNumber) -> ProviderResult<()>; + + /// Saves the given safe block number in the DB. + fn save_safe_block_number(&self, block_number: BlockNumber) -> ProviderResult<()>; } diff --git a/crates/storage/provider/src/traits/mod.rs b/crates/storage/provider/src/traits/mod.rs index 8bae4f67ae85..c31c7c1e2f21 100644 --- a/crates/storage/provider/src/traits/mod.rs +++ b/crates/storage/provider/src/traits/mod.rs @@ -42,4 +42,4 @@ mod tree_viewer; pub use tree_viewer::TreeViewer; mod finalized_block; -pub use finalized_block::{FinalizedBlockReader, FinalizedBlockWriter}; +pub use finalized_block::{ChainStateBlockReader, ChainStateBlockWriter}; From cb123084bf6d22f5b6b8a185697f8333a20525ae Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 11 Oct 2024 04:53:19 +0200 Subject: [PATCH 26/82] docs: `LoadFee::eip1559_fees` returns base fee, not max fee per gas (#11652) --- crates/rpc/rpc-eth-api/src/helpers/fee.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/fee.rs b/crates/rpc/rpc-eth-api/src/helpers/fee.rs index b6dcef4708ef..52ccabebf620 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/fee.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/fee.rs @@ -284,15 +284,15 @@ pub trait LoadFee: LoadBlock { /// Returns the EIP-1559 fees if they are set, otherwise fetches a suggested gas price for /// EIP-1559 transactions. /// - /// Returns (`max_fee`, `priority_fee`) + /// Returns (`base_fee`, `priority_fee`) fn eip1559_fees( &self, - max_fee_per_gas: Option, + base_fee: Option, max_priority_fee_per_gas: Option, ) -> impl Future> + Send { async move { - let max_fee_per_gas = match max_fee_per_gas { - Some(max_fee_per_gas) => max_fee_per_gas, + let base_fee = match base_fee { + Some(base_fee) => base_fee, None => { // fetch pending base fee let base_fee = self @@ -311,7 +311,7 @@ pub trait LoadFee: LoadBlock { Some(max_priority_fee_per_gas) => max_priority_fee_per_gas, None => self.suggested_priority_fee().await?, }; - Ok((max_fee_per_gas, max_priority_fee_per_gas)) + Ok((base_fee, max_priority_fee_per_gas)) } } From ad485277deef9cf6359409b0a7d208ba9ab67e71 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 11 Oct 2024 08:28:07 +0200 Subject: [PATCH 27/82] feat: propagate helper (#11654) Co-authored-by: Oliver --- crates/net/network/src/transactions/mod.rs | 25 ++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index 0c488ff919dc..bc77de9f463b 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -153,6 +153,14 @@ impl TransactionsHandle { self.send(TransactionsCommand::PropagateTransactionsTo(transactions, peer)) } + /// Manually propagate the given transactions to all peers. + /// + /// It's up to the [`TransactionsManager`] whether the transactions are sent as hashes or in + /// full. + pub fn propagate_transactions(&self, transactions: Vec) { + self.send(TransactionsCommand::PropagateTransactions(transactions)) + } + /// Request the transaction hashes known by specific peers. pub async fn get_transaction_hashes( &self, @@ -398,8 +406,14 @@ where trace!(target: "net::tx", num_hashes=?hashes.len(), "Start propagating transactions"); - // This fetches all transaction from the pool, including the 4844 blob transactions but - // __without__ their sidecar, because 4844 transactions are only ever announced as hashes. + self.propagate_all(hashes); + } + + /// Propagates the given transactions to the peers + /// + /// This fetches all transaction from the pool, including the 4844 blob transactions but + /// __without__ their sidecar, because 4844 transactions are only ever announced as hashes. + fn propagate_all(&mut self, hashes: Vec) { let propagated = self.propagate_transactions( self.pool.get_all(hashes).into_iter().map(PropagateTransaction::new).collect(), ); @@ -872,11 +886,12 @@ where let peers = self.peers.keys().copied().collect::>(); tx.send(peers).ok(); } - TransactionsCommand::PropagateTransactionsTo(txs, _peer) => { - if let Some(propagated) = self.propagate_full_transactions_to_peer(txs, _peer) { + TransactionsCommand::PropagateTransactionsTo(txs, peer) => { + if let Some(propagated) = self.propagate_full_transactions_to_peer(txs, peer) { self.pool.on_propagated(propagated); } } + TransactionsCommand::PropagateTransactions(txs) => self.propagate_all(txs), TransactionsCommand::GetTransactionHashes { peers, tx } => { let mut res = HashMap::with_capacity(peers.len()); for peer_id in peers { @@ -1653,6 +1668,8 @@ enum TransactionsCommand { GetActivePeers(oneshot::Sender>), /// Propagate a collection of full transactions to a specific peer. PropagateTransactionsTo(Vec, PeerId), + /// Propagate a collection of full transactions to all peers. + PropagateTransactions(Vec), /// Request transaction hashes known by specific peers from the [`TransactionsManager`]. GetTransactionHashes { peers: Vec, From 8fc703cf82dd97c023935374caac83d4cbb7e02d Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Fri, 11 Oct 2024 10:21:59 +0200 Subject: [PATCH 28/82] chore(ci): remove assertoor workflow (#11656) --- .github/workflows/assertoor.yml | 230 -------------------------------- 1 file changed, 230 deletions(-) delete mode 100644 .github/workflows/assertoor.yml diff --git a/.github/workflows/assertoor.yml b/.github/workflows/assertoor.yml deleted file mode 100644 index a5028f7ff31a..000000000000 --- a/.github/workflows/assertoor.yml +++ /dev/null @@ -1,230 +0,0 @@ -name: Assertoor Tests - -on: - workflow_dispatch: - schedule: - - cron: '0 0 * * *' - -jobs: - get_tests: - name: "Run assertoor tests on reth pairs" - runs-on: ubuntu-latest - outputs: - test_result: ${{ steps.test_result.outputs.test_result }} - test_status: ${{ steps.test_result.outputs.test_status }} - failed_test_status: ${{ steps.test_result.outputs.failed_test_status }} - if: github.repository == 'paradigmxyz/reth' - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - name: Setup Kurtosis - shell: bash - run: | - echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list - sudo apt update - sudo apt install kurtosis-cli - kurtosis analytics disable - - - name: Run Kurtosis - shell: bash - id: services - run: | - export github_sha=${{ github.sha }} - export github_repository=${{ github.repository }} - - cat etc/assertoor/assertoor-template.yaml | envsubst > etc/assertoor/assertoor.yaml - - kurtosis run github.com/ethpandaops/ethereum-package --enclave assertoor-${{ github.run_id }} --args-file etc/assertoor/assertoor.yaml - - enclave_dump=$(kurtosis enclave inspect assertoor-${{ github.run_id }}) - - assertoor_url=$(echo "$enclave_dump" | grep assertoor | grep http | sed 's/.*\(http:\/\/[0-9.:]\+\).*/\1/') - echo "assertoor_url: ${assertoor_url}" - echo "assertoor_url=${assertoor_url}" >> $GITHUB_OUTPUT - - - name: Await test completion - shell: bash - id: test_result - run: | - assertoor_url="${{ steps.services.outputs.assertoor_url }}" - - YELLOW='\033[1;33m' - GRAY='\033[0;37m' - GREEN='\033[0;32m' - RED='\033[0;31m' - NC='\033[0m' - - # print assertor logs - assertoor_container=$(docker container list | grep assertoor | sed 's/^\([^ ]\+\) .*$/\1/') - docker logs -f $assertoor_container & - - # helper to fetch task status for specific test id - get_tasks_status() { - tasks=$(curl -s ${assertoor_url}/api/v1/test_run/$1 | jq -c ".data.tasks[] | {index, parent_index, name, title, status, result}") - declare -A task_graph_map - task_graph_map[0]="" - - while read task; do - task_id=$(echo "$task" | jq -r ".index") - task_parent=$(echo "$task" | jq -r ".parent_index") - task_name=$(echo "$task" | jq -r ".name") - task_title=$(echo "$task" | jq -r ".title") - task_status=$(echo "$task" | jq -r ".status") - task_result=$(echo "$task" | jq -r ".result") - - task_graph="${task_graph_map[$task_parent]}" - task_graph_map[$task_id]="$task_graph |" - if [ ! -z "$task_graph" ]; then - task_graph="${task_graph}- " - fi - - if [ "$task_status" == "pending" ]; then - task_status="${GRAY}pending ${NC}" - elif [ "$task_status" == "running" ]; then - task_status="${YELLOW}running ${NC}" - elif [ "$task_status" == "complete" ]; then - task_status="${GREEN}complete${NC}" - fi - - if [ "$task_result" == "none" ]; then - task_result="${GRAY}none ${NC}" - elif [ "$task_result" == "success" ]; then - task_result="${GREEN}success${NC}" - elif [ "$task_result" == "failure" ]; then - task_result="${RED}failure${NC}" - fi - - echo -e " $(printf '%-4s' "$task_id")\t$task_status\t$task_result\t$(printf '%-50s' "$task_graph$task_name") \t$task_title" - done <<< $(echo "$tasks") - } - - # poll & check test status - final_test_result="" - failed_test_id="" - while true - do - pending_tests=0 - failed_tests=0 - total_tests=0 - running_test="" - - status_lines=() - task_lines="" - status_lines+=("$(date +'%Y-%m-%d %H:%M:%S') Test Status:") - - tests=$(curl -s ${assertoor_url}/api/v1/test_runs | jq -c ".data[] | {run_id, test_id, name, status}") - while read test; do - if [ -z "$test" ]; then - continue - fi - run_id=$(echo "$test" | jq -r ".run_id") - test_id=$(echo "$test" | jq -r ".test_id") - test_name=$(echo "$test" | jq -r ".name") - test_status=$(echo "$test" | jq -r ".status") - - if [ "$test_status" == "pending" ]; then - pending_tests=$(expr $pending_tests + 1) - status_name="${GRAY}pending${NC}" - elif [ "$test_status" == "running" ]; then - pending_tests=$(expr $pending_tests + 1) - running_test="$run_id" - status_name="${YELLOW}running${NC}" - - elif [ "$test_status" == "success" ]; then - status_name="${GREEN}success${NC}" - elif [ "$test_status" == "failure" ]; then - failed_tests=$(expr $failed_tests + 1) - failed_test_id="$run_id" - status_name="${RED}failure${NC}" - else - status_name="$test_status" - fi - status_lines+=(" $(printf '%-3s' "$test_id") $status_name \t$test_name") - total_tests=$(expr $total_tests + 1) - done <<< $(echo "$tests") - - for status_line in "${status_lines[@]}" - do - echo -e "$status_line" - done - - if ! [ -z "$running_test" ]; then - task_lines=$(get_tasks_status "$running_test") - echo "Active Test Task Status:" - echo "$task_lines" - fi - - if [ $failed_tests -gt 0 ]; then - final_test_result="failure" - break - fi - if [ $total_tests -gt 0 ] && [ $pending_tests -le 0 ]; then - final_test_result="success" - break - fi - - sleep 60 - done - - # save test results & status to github output - echo "test_result=$(echo "$final_test_result")" >> $GITHUB_OUTPUT - echo "test_status<> $GITHUB_OUTPUT - for status_line in "${status_lines[@]}" - do - echo -e "$status_line" >> $GITHUB_OUTPUT - done - echo "EOF" >> $GITHUB_OUTPUT - - if ! [ -z "$failed_test_id" ]; then - echo "failed_test_status<> $GITHUB_OUTPUT - get_tasks_status "$failed_test_id" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - else - echo "failed_test_status=" >> $GITHUB_OUTPUT - fi - - - name: Generate dump and remove kurtosis enclave - shell: bash - run: | - mkdir -p ./temp/dump - cd ./temp/dump - cp ../../etc/assertoor/assertoor.yaml ./kurtosis-params.yaml - - kurtosis enclave dump assertoor-${{ github.run_id }} - kurtosis enclave rm -f assertoor-${{ github.run_id }} - - - name: Upload dump artifact - uses: actions/upload-artifact@v4 - with: - name: "kurtosis-enclave-dump-${{ github.run_id }}" - path: ./temp/dump - - - name: Return test result - shell: bash - run: | - test_result="${{ steps.test_result.outputs.test_result }}" - test_status=$( - cat <<"EOF" - ${{ steps.test_result.outputs.test_status }} - EOF - ) - failed_test_status=$( - cat <<"EOF" - ${{ steps.test_result.outputs.failed_test_status }} - EOF - ) - - echo "Test Result: $test_result" - echo "$test_status" - - if ! [ "$test_result" == "success" ]; then - echo "" - echo "Failed Test Task Status:" - echo "$failed_test_status" - - echo "" - echo "See 'Await test completion' task for detailed logs about this failure!" - echo "" - - exit 1 # fail action - fi From c4411991e1e5a10f2cfe99056b650557f4da62fb Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 11 Oct 2024 11:45:44 +0200 Subject: [PATCH 29/82] feat: add override for additional_validation_tasks (#11655) --- crates/node/builder/src/components/pool.rs | 3 +++ crates/optimism/node/src/node.rs | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/node/builder/src/components/pool.rs b/crates/node/builder/src/components/pool.rs index 234455913c6f..436a80c52e00 100644 --- a/crates/node/builder/src/components/pool.rs +++ b/crates/node/builder/src/components/pool.rs @@ -52,6 +52,8 @@ pub struct PoolBuilderConfigOverrides { pub minimal_protocol_basefee: Option, /// Addresses that will be considered as local. Above exemptions apply. pub local_addresses: HashSet

, + /// Additional tasks to validate new transactions. + pub additional_validation_tasks: Option, } impl PoolBuilderConfigOverrides { @@ -65,6 +67,7 @@ impl PoolBuilderConfigOverrides { max_account_slots, minimal_protocol_basefee, local_addresses, + additional_validation_tasks: _, } = self; if let Some(pending_limit) = pending_limit { diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index caeab3741a2e..b6e64f7e0e43 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -189,7 +189,11 @@ where )) .with_head_timestamp(ctx.head().timestamp) .kzg_settings(ctx.kzg_settings()?) - .with_additional_tasks(ctx.config().txpool.additional_validation_tasks) + .with_additional_tasks( + pool_config_overrides + .additional_validation_tasks + .unwrap_or_else(|| ctx.config().txpool.additional_validation_tasks), + ) .build_with_tasks(ctx.provider().clone(), ctx.task_executor().clone(), blob_store.clone()) .map(|validator| { OpTransactionValidator::new(validator) From 6d3aa5a0d5df5a42b6fcf38a9937e3255052bd47 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 11 Oct 2024 14:17:07 +0200 Subject: [PATCH 30/82] Revert "fix(net): batch `P2PStream` sends" (#11658) --- crates/net/eth-wire/src/p2pstream.rs | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/crates/net/eth-wire/src/p2pstream.rs b/crates/net/eth-wire/src/p2pstream.rs index 76075838bc76..9bb8fe7c399c 100644 --- a/crates/net/eth-wire/src/p2pstream.rs +++ b/crates/net/eth-wire/src/p2pstream.rs @@ -614,24 +614,19 @@ where /// Returns `Poll::Ready(Ok(()))` when no buffered items remain. fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let mut this = self.project(); - let poll_res = loop { - match this.inner.as_mut().poll_ready(cx) { - Poll::Pending => break Poll::Pending, - Poll::Ready(Err(err)) => break Poll::Ready(Err(err.into())), - Poll::Ready(Ok(())) => { + loop { + match ready!(this.inner.as_mut().poll_flush(cx)) { + Err(err) => return Poll::Ready(Err(err.into())), + Ok(()) => { let Some(message) = this.outgoing_messages.pop_front() else { - break Poll::Ready(Ok(())) + return Poll::Ready(Ok(())) }; if let Err(err) = this.inner.as_mut().start_send(message) { - break Poll::Ready(Err(err.into())) + return Poll::Ready(Err(err.into())) } } } - }; - - ready!(this.inner.as_mut().poll_flush(cx))?; - - poll_res + } } fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { From c29c1f5fa8949f3a6eb111b9c699af08873ab5a9 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Fri, 11 Oct 2024 15:34:52 +0200 Subject: [PATCH 31/82] chore(net): log p2p stream flush error (#11659) --- crates/net/eth-wire/src/p2pstream.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/net/eth-wire/src/p2pstream.rs b/crates/net/eth-wire/src/p2pstream.rs index 9bb8fe7c399c..9882e39787e2 100644 --- a/crates/net/eth-wire/src/p2pstream.rs +++ b/crates/net/eth-wire/src/p2pstream.rs @@ -616,7 +616,13 @@ where let mut this = self.project(); loop { match ready!(this.inner.as_mut().poll_flush(cx)) { - Err(err) => return Poll::Ready(Err(err.into())), + Err(err) => { + trace!(target: "net::p2p", + %err, + "error flushing p2p stream" + ); + return Poll::Ready(Err(err.into())) + } Ok(()) => { let Some(message) = this.outgoing_messages.pop_front() else { return Poll::Ready(Ok(())) From 6a56ae75b0342fc187d9546db5c4f83ec407607b Mon Sep 17 00:00:00 2001 From: Ayene <2958807+ayenesimo1i@users.noreply.github.com> Date: Fri, 11 Oct 2024 18:31:11 +0300 Subject: [PATCH 32/82] fix(docs): remove ci link (#11665) Co-authored-by: Federico Gimenez --- docs/repo/ci.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/repo/ci.md b/docs/repo/ci.md index da84e001f376..5ed2cec0091e 100644 --- a/docs/repo/ci.md +++ b/docs/repo/ci.md @@ -26,7 +26,7 @@ The CI runs a couple of workflows: ### Integration Testing -- **[assertoor]**: Runs Assertoor tests on Reth pairs. +- **[kurtosis]**: Spins up a Kurtosis testnet and runs Assertoor tests on Reth pairs. - **[hive]**: Runs `ethereum/hive` tests. ### Linting and Checks @@ -48,7 +48,7 @@ The CI runs a couple of workflows: [dependencies]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/dependencies.yml [stale]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/stale.yml [docker]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/docker.yml -[assertoor]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/assertoor.yml +[kurtosis]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/kurtosis.yml [hive]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/hive.yml [lint]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/lint.yml [lint-actions]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/lint-actions.yml From ad2a2f2101d7407efec67293f5f9509faa0fb5f6 Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Sat, 12 Oct 2024 00:27:24 +0800 Subject: [PATCH 33/82] chore(test): use collect void realloc (#11669) --- crates/transaction-pool/src/test_utils/mock.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/transaction-pool/src/test_utils/mock.rs b/crates/transaction-pool/src/test_utils/mock.rs index 6b470bb6fb1f..e2b5f373e446 100644 --- a/crates/transaction-pool/src/test_utils/mock.rs +++ b/crates/transaction-pool/src/test_utils/mock.rs @@ -1338,10 +1338,8 @@ impl MockTransactionDistribution { nonce_range: Range, rng: &mut impl rand::Rng, ) -> MockTransactionSet { - let mut txs = Vec::new(); - for nonce in nonce_range { - txs.push(self.tx(nonce, rng).with_sender(sender)); - } + let txs = + nonce_range.map(|nonce| self.tx(nonce, rng).with_sender(sender)).collect::>(); MockTransactionSet::new(txs) } From 0affb976a02881ab4aaa3b56b7d6f3e0a9ebbc9a Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Sat, 12 Oct 2024 01:49:29 +0800 Subject: [PATCH 34/82] transaction-pool:rm redundance clone (#11667) --- crates/transaction-pool/src/pool/best.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/transaction-pool/src/pool/best.rs b/crates/transaction-pool/src/pool/best.rs index 5880a73f5101..52f25a9db8dc 100644 --- a/crates/transaction-pool/src/pool/best.rs +++ b/crates/transaction-pool/src/pool/best.rs @@ -132,9 +132,8 @@ impl BestTransactions { /// created and inserts them fn add_new_transactions(&mut self) { while let Some(pending_tx) = self.try_recv() { - let tx = pending_tx.transaction.clone(); // same logic as PendingPool::add_transaction/PendingPool::best_with_unlocked - let tx_id = *tx.id(); + let tx_id = *pending_tx.transaction.id(); if self.ancestor(&tx_id).is_none() { self.independent.insert(pending_tx.clone()); } From d8b7f6014f90c86afc0415b0200929752a415a63 Mon Sep 17 00:00:00 2001 From: Francis Li Date: Fri, 11 Oct 2024 10:55:22 -0700 Subject: [PATCH 35/82] fix(rpc): add missing codes for witness (#11673) --- crates/rpc/rpc/src/debug.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index b26459c4f040..738b91ad0eca 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -34,7 +34,7 @@ use reth_rpc_server_types::{result::internal_rpc_err, ToRpcResult}; use reth_tasks::pool::BlockingTaskGuard; use reth_trie::{HashedPostState, HashedStorage}; use revm::{ - db::CacheDB, + db::{CacheDB, State}, primitives::{db::DatabaseCommit, BlockEnv, CfgEnvWithHandlerCfg, Env, EnvWithHandlerCfg}, }; use revm_inspectors::tracing::{ @@ -614,12 +614,23 @@ where let _ = block_executor .execute_with_state_closure( (&block.clone().unseal(), block.difficulty).into(), - |statedb| { + |statedb: &State<_>| { codes = statedb .cache .contracts .iter() .map(|(hash, code)| (*hash, code.original_bytes())) + .chain( + // cache state does not have all the contracts, especially when + // a contract is created within the block + // the contract only exists in bundle state, therefore we need + // to include them as well + statedb + .bundle_state + .contracts + .iter() + .map(|(hash, code)| (*hash, code.original_bytes())), + ) .collect(); for (address, account) in &statedb.cache.accounts { From bca11aa2dd2ad2a0fdb43853ca1c5047fcb43e8f Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Fri, 11 Oct 2024 19:56:33 +0200 Subject: [PATCH 36/82] clippy: add `from_iter_instead_of_collect` warn (#11666) --- Cargo.toml | 1 + crates/evm/execution-types/src/chain.rs | 2 +- crates/rpc/rpc-builder/src/metrics.rs | 9 ++-- crates/rpc/rpc/src/debug.rs | 6 +-- .../src/providers/database/provider.rs | 8 ++-- .../storage/provider/src/test_utils/blocks.rs | 47 +++++++++---------- crates/transaction-pool/benches/reorder.rs | 10 ++-- crates/transaction-pool/src/maintain.rs | 2 +- crates/trie/db/src/state.rs | 32 +++++++------ crates/trie/db/tests/post_state.rs | 44 +++++++++-------- crates/trie/trie/src/proof.rs | 2 +- crates/trie/trie/src/updates.rs | 2 +- 12 files changed, 85 insertions(+), 80 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index db115c89cccd..efae22f8ed11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -183,6 +183,7 @@ equatable_if_let = "warn" explicit_into_iter_loop = "warn" explicit_iter_loop = "warn" flat_map_option = "warn" +from_iter_instead_of_collect = "warn" if_not_else = "warn" implicit_clone = "warn" imprecise_flops = "warn" diff --git a/crates/evm/execution-types/src/chain.rs b/crates/evm/execution-types/src/chain.rs index 25bc39ea3257..30f1f4cd2fc3 100644 --- a/crates/evm/execution-types/src/chain.rs +++ b/crates/evm/execution-types/src/chain.rs @@ -52,7 +52,7 @@ impl Chain { execution_outcome: ExecutionOutcome, trie_updates: Option, ) -> Self { - let blocks = BTreeMap::from_iter(blocks.into_iter().map(|b| (b.number, b))); + let blocks = blocks.into_iter().map(|b| (b.number, b)).collect::>(); debug_assert!(!blocks.is_empty(), "Chain should have at least one block"); Self { blocks, execution_outcome, trie_updates } diff --git a/crates/rpc/rpc-builder/src/metrics.rs b/crates/rpc/rpc-builder/src/metrics.rs index 08fd38898558..57283ded3796 100644 --- a/crates/rpc/rpc-builder/src/metrics.rs +++ b/crates/rpc/rpc-builder/src/metrics.rs @@ -30,9 +30,12 @@ impl RpcRequestMetrics { Self { inner: Arc::new(RpcServerMetricsInner { connection_metrics: transport.connection_metrics(), - call_metrics: HashMap::from_iter(module.method_names().map(|method| { - (method, RpcServerCallMetrics::new_with_labels(&[("method", method)])) - })), + call_metrics: module + .method_names() + .map(|method| { + (method, RpcServerCallMetrics::new_with_labels(&[("method", method)])) + }) + .collect(), }), } } diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 738b91ad0eca..fbef6f7a7e0b 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -663,11 +663,7 @@ where let state = state_provider.witness(Default::default(), hashed_state).map_err(Into::into)?; - Ok(ExecutionWitness { - state: HashMap::from_iter(state.into_iter()), - codes, - keys: Some(keys), - }) + Ok(ExecutionWitness { state: state.into_iter().collect(), codes, keys: Some(keys) }) }) .await } diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index c9b0af3d33ca..33fed1e80ce1 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -2914,10 +2914,10 @@ impl HashingWriter for DatabaseProvider()?; // Hash the address and key and apply them to HashedStorage (if Storage is None diff --git a/crates/storage/provider/src/test_utils/blocks.rs b/crates/storage/provider/src/test_utils/blocks.rs index daed906646d3..f7928319b48c 100644 --- a/crates/storage/provider/src/test_utils/blocks.rs +++ b/crates/storage/provider/src/test_utils/blocks.rs @@ -324,11 +324,10 @@ fn block3( ) .state_storage( address, - HashMap::from_iter( - slot_range - .clone() - .map(|slot| (U256::from(slot), (U256::ZERO, U256::from(slot)))), - ), + slot_range + .clone() + .map(|slot| (U256::from(slot), (U256::ZERO, U256::from(slot)))) + .collect(), ) .revert_account_info(number, address, Some(None)) .revert_storage(number, address, Vec::new()); @@ -393,20 +392,18 @@ fn block4( ) .state_storage( address, - HashMap::from_iter( - slot_range.clone().map(|slot| { - (U256::from(slot), (U256::from(slot), U256::from(slot * 2))) - }), - ), + slot_range + .clone() + .map(|slot| (U256::from(slot), (U256::from(slot), U256::from(slot * 2)))) + .collect(), ) } else { bundle_state_builder.state_address(address).state_storage( address, - HashMap::from_iter( - slot_range - .clone() - .map(|slot| (U256::from(slot), (U256::from(slot), U256::ZERO))), - ), + slot_range + .clone() + .map(|slot| (U256::from(slot), (U256::from(slot), U256::ZERO))) + .collect(), ) }; // record previous account info @@ -423,7 +420,7 @@ fn block4( .revert_storage( number, address, - Vec::from_iter(slot_range.clone().map(|slot| (U256::from(slot), U256::from(slot)))), + slot_range.clone().map(|slot| (U256::from(slot), U256::from(slot))).collect(), ); } let execution_outcome = ExecutionOutcome::new( @@ -485,12 +482,11 @@ fn block5( ) .state_storage( address, - HashMap::from_iter( - slot_range - .clone() - .take(50) - .map(|slot| (U256::from(slot), (U256::from(slot), U256::from(slot * 4)))), - ), + slot_range + .clone() + .take(50) + .map(|slot| (U256::from(slot), (U256::from(slot), U256::from(slot * 4)))) + .collect(), ); bundle_state_builder = if idx % 2 == 0 { bundle_state_builder @@ -506,9 +502,10 @@ fn block5( .revert_storage( number, address, - Vec::from_iter( - slot_range.clone().map(|slot| (U256::from(slot), U256::from(slot * 2))), - ), + slot_range + .clone() + .map(|slot| (U256::from(slot), U256::from(slot * 2))) + .collect(), ) } else { bundle_state_builder.revert_address(number, address) diff --git a/crates/transaction-pool/benches/reorder.rs b/crates/transaction-pool/benches/reorder.rs index 1836ce40c085..9fc21629753f 100644 --- a/crates/transaction-pool/benches/reorder.rs +++ b/crates/transaction-pool/benches/reorder.rs @@ -206,10 +206,12 @@ mod implementations { self.base_fee = Some(base_fee); let drained = self.inner.drain(); - self.inner = BinaryHeap::from_iter(drained.map(|mock| { - let priority = mock.tx.effective_tip_per_gas(base_fee).expect("set"); - MockTransactionWithPriority { tx: mock.tx, priority } - })); + self.inner = drained + .map(|mock| { + let priority = mock.tx.effective_tip_per_gas(base_fee).expect("set"); + MockTransactionWithPriority { tx: mock.tx, priority } + }) + .collect(); } } } diff --git a/crates/transaction-pool/src/maintain.rs b/crates/transaction-pool/src/maintain.rs index 66b98614737e..36c7067d4a83 100644 --- a/crates/transaction-pool/src/maintain.rs +++ b/crates/transaction-pool/src/maintain.rs @@ -490,7 +490,7 @@ impl MaintainedPoolState { } } -/// A unique `ChangedAccount` identified by its address that can be used for deduplication +/// A unique [`ChangedAccount`] identified by its address that can be used for deduplication #[derive(Eq)] struct ChangedAccountEntry(ChangedAccount); diff --git a/crates/trie/db/src/state.rs b/crates/trie/db/src/state.rs index 5acb9e0d1b49..4d46183dfda4 100644 --- a/crates/trie/db/src/state.rs +++ b/crates/trie/db/src/state.rs @@ -244,22 +244,24 @@ impl DatabaseHashedPostState for HashedPostState { } } - let hashed_accounts = HashMap::from_iter( - accounts.into_iter().map(|(address, info)| (keccak256(address), info)), - ); + let hashed_accounts = + accounts.into_iter().map(|(address, info)| (keccak256(address), info)).collect(); - let hashed_storages = HashMap::from_iter(storages.into_iter().map(|(address, storage)| { - ( - keccak256(address), - HashedStorage::from_iter( - // The `wiped` flag indicates only whether previous storage entries - // should be looked up in db or not. For reverts it's a noop since all - // wiped changes had been written as storage reverts. - false, - storage.into_iter().map(|(slot, value)| (keccak256(slot), value)), - ), - ) - })); + let hashed_storages = storages + .into_iter() + .map(|(address, storage)| { + ( + keccak256(address), + HashedStorage::from_iter( + // The `wiped` flag indicates only whether previous storage entries + // should be looked up in db or not. For reverts it's a noop since all + // wiped changes had been written as storage reverts. + false, + storage.into_iter().map(|(slot, value)| (keccak256(slot), value)), + ), + ) + }) + .collect(); Ok(Self { accounts: hashed_accounts, storages: hashed_storages }) } diff --git a/crates/trie/db/tests/post_state.rs b/crates/trie/db/tests/post_state.rs index ceadf7cde590..ce6f10d76aed 100644 --- a/crates/trie/db/tests/post_state.rs +++ b/crates/trie/db/tests/post_state.rs @@ -55,7 +55,7 @@ fn assert_storage_cursor_order( #[test] fn post_state_only_accounts() { let accounts = - Vec::from_iter((1..11).map(|key| (B256::with_last_byte(key), Account::default()))); + (1..11).map(|key| (B256::with_last_byte(key), Account::default())).collect::>(); let mut hashed_post_state = HashedPostState::default(); for (hashed_address, account) in &accounts { @@ -73,7 +73,7 @@ fn post_state_only_accounts() { #[test] fn db_only_accounts() { let accounts = - Vec::from_iter((1..11).map(|key| (B256::with_last_byte(key), Account::default()))); + (1..11).map(|key| (B256::with_last_byte(key), Account::default())).collect::>(); let db = create_test_rw_db(); db.update(|tx| { @@ -96,7 +96,7 @@ fn db_only_accounts() { fn account_cursor_correct_order() { // odd keys are in post state, even keys are in db let accounts = - Vec::from_iter((1..111).map(|key| (B256::with_last_byte(key), Account::default()))); + (1..111).map(|key| (B256::with_last_byte(key), Account::default())).collect::>(); let db = create_test_rw_db(); db.update(|tx| { @@ -121,9 +121,9 @@ fn account_cursor_correct_order() { fn removed_accounts_are_discarded() { // odd keys are in post state, even keys are in db let accounts = - Vec::from_iter((1..111).map(|key| (B256::with_last_byte(key), Account::default()))); + (1..111).map(|key| (B256::with_last_byte(key), Account::default())).collect::>(); // accounts 5, 9, 11 should be considered removed from post state - let removed_keys = Vec::from_iter([5, 9, 11].into_iter().map(B256::with_last_byte)); + let removed_keys = [5, 9, 11].into_iter().map(B256::with_last_byte).collect::>(); let db = create_test_rw_db(); db.update(|tx| { @@ -150,9 +150,9 @@ fn removed_accounts_are_discarded() { #[test] fn post_state_accounts_take_precedence() { - let accounts = Vec::from_iter((1..10).map(|key| { - (B256::with_last_byte(key), Account { nonce: key as u64, ..Default::default() }) - })); + let accounts = (1..10) + .map(|key| (B256::with_last_byte(key), Account { nonce: key as u64, ..Default::default() })) + .collect::>(); let db = create_test_rw_db(); db.update(|tx| { @@ -224,7 +224,7 @@ fn storage_is_empty() { } let db_storage = - BTreeMap::from_iter((0..10).map(|key| (B256::with_last_byte(key), U256::from(key)))); + (0..10).map(|key| (B256::with_last_byte(key), U256::from(key))).collect::>(); db.update(|tx| { for (slot, value) in &db_storage { // insert zero value accounts to the database @@ -299,9 +299,10 @@ fn storage_is_empty() { fn storage_cursor_correct_order() { let address = B256::random(); let db_storage = - BTreeMap::from_iter((1..11).map(|key| (B256::with_last_byte(key), U256::from(key)))); - let post_state_storage = - BTreeMap::from_iter((11..21).map(|key| (B256::with_last_byte(key), U256::from(key)))); + (1..11).map(|key| (B256::with_last_byte(key), U256::from(key))).collect::>(); + let post_state_storage = (11..21) + .map(|key| (B256::with_last_byte(key), U256::from(key))) + .collect::>(); let db = create_test_rw_db(); db.update(|tx| { @@ -334,10 +335,12 @@ fn storage_cursor_correct_order() { fn zero_value_storage_entries_are_discarded() { let address = B256::random(); let db_storage = - BTreeMap::from_iter((0..10).map(|key| (B256::with_last_byte(key), U256::from(key)))); // every even number is changed to zero value - let post_state_storage = BTreeMap::from_iter((0..10).map(|key| { - (B256::with_last_byte(key), if key % 2 == 0 { U256::ZERO } else { U256::from(key) }) - })); + (0..10).map(|key| (B256::with_last_byte(key), U256::from(key))).collect::>(); // every even number is changed to zero value + let post_state_storage = (0..10) + .map(|key| { + (B256::with_last_byte(key), if key % 2 == 0 { U256::ZERO } else { U256::from(key) }) + }) + .collect::>(); let db = create_test_rw_db(); db.update(|tx| { @@ -371,9 +374,10 @@ fn zero_value_storage_entries_are_discarded() { fn wiped_storage_is_discarded() { let address = B256::random(); let db_storage = - BTreeMap::from_iter((1..11).map(|key| (B256::with_last_byte(key), U256::from(key)))); - let post_state_storage = - BTreeMap::from_iter((11..21).map(|key| (B256::with_last_byte(key), U256::from(key)))); + (1..11).map(|key| (B256::with_last_byte(key), U256::from(key))).collect::>(); + let post_state_storage = (11..21) + .map(|key| (B256::with_last_byte(key), U256::from(key))) + .collect::>(); let db = create_test_rw_db(); db.update(|tx| { @@ -404,7 +408,7 @@ fn wiped_storage_is_discarded() { fn post_state_storages_take_precedence() { let address = B256::random(); let storage = - BTreeMap::from_iter((1..10).map(|key| (B256::with_last_byte(key), U256::from(key)))); + (1..10).map(|key| (B256::with_last_byte(key), U256::from(key))).collect::>(); let db = create_test_rw_db(); db.update(|tx| { diff --git a/crates/trie/trie/src/proof.rs b/crates/trie/trie/src/proof.rs index d31d63fd9a8b..e99d686aca7f 100644 --- a/crates/trie/trie/src/proof.rs +++ b/crates/trie/trie/src/proof.rs @@ -100,7 +100,7 @@ where let walker = TrieWalker::new(trie_cursor, prefix_set.freeze()); // Create a hash builder to rebuild the root node since it is not available in the database. - let retainer = ProofRetainer::from_iter(targets.keys().map(Nibbles::unpack)); + let retainer = targets.keys().map(Nibbles::unpack).collect(); let mut hash_builder = HashBuilder::default().with_proof_retainer(retainer); let mut storages = HashMap::default(); diff --git a/crates/trie/trie/src/updates.rs b/crates/trie/trie/src/updates.rs index 03e80cf52e53..6d1bcab63d8f 100644 --- a/crates/trie/trie/src/updates.rs +++ b/crates/trie/trie/src/updates.rs @@ -236,7 +236,7 @@ mod serde_nibbles_set { S: Serializer, { let mut storage_nodes = - Vec::from_iter(map.iter().map(|elem| alloy_primitives::hex::encode(elem.pack()))); + map.iter().map(|elem| alloy_primitives::hex::encode(elem.pack())).collect::>(); storage_nodes.sort_unstable(); storage_nodes.serialize(serializer) } From 160e4b2ce7a7ab4ff70a4cae7644a3dd01cc25ee Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Fri, 11 Oct 2024 20:11:58 +0200 Subject: [PATCH 37/82] tx-pool: simplify `FinalizedBlockTracker` update logic (#11664) --- crates/transaction-pool/src/maintain.rs | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/crates/transaction-pool/src/maintain.rs b/crates/transaction-pool/src/maintain.rs index 36c7067d4a83..aaf2d6d12d85 100644 --- a/crates/transaction-pool/src/maintain.rs +++ b/crates/transaction-pool/src/maintain.rs @@ -454,20 +454,11 @@ impl FinalizedBlockTracker { /// Updates the tracked finalized block and returns the new finalized block if it changed fn update(&mut self, finalized_block: Option) -> Option { - match (self.last_finalized_block, finalized_block) { - (Some(last), Some(finalized)) => { - self.last_finalized_block = Some(finalized); - if last < finalized { - Some(finalized) - } else { - None - } - } - (None, Some(finalized)) => { - self.last_finalized_block = Some(finalized); - Some(finalized) - } - _ => None, + let finalized = finalized_block?; + if self.last_finalized_block.replace(finalized).map_or(true, |last| last < finalized) { + Some(finalized) + } else { + None } } } From b51299b86bc8447e37d3a3c9a4a8329091d58936 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Fri, 11 Oct 2024 20:22:20 +0200 Subject: [PATCH 38/82] refac: small refactor in `BlockchainProvider2` (#11660) --- .../provider/src/providers/blockchain_provider.rs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index a0811db2aee2..dc4264210b27 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -681,7 +681,7 @@ impl HeaderProvider for BlockchainProvider2 { { last_finalized_num_hash.number } else { - self.database.last_block_number()? + self.last_block_number()? } } else { // Otherwise, return what we have on disk for the input block @@ -841,12 +841,7 @@ impl BlockReader for BlockchainProvider2 { id, |db_provider| db_provider.ommers(id), |block_state| { - if self - .database - .chain_spec() - .final_paris_total_difficulty(block_state.number()) - .is_some() - { + if self.chain_spec().final_paris_total_difficulty(block_state.number()).is_some() { return Ok(Some(Vec::new())) } @@ -1174,7 +1169,7 @@ impl WithdrawalsProvider for BlockchainProvider2 { id: BlockHashOrNumber, timestamp: u64, ) -> ProviderResult> { - if !self.database.chain_spec().is_shanghai_active_at_timestamp(timestamp) { + if !self.chain_spec().is_shanghai_active_at_timestamp(timestamp) { return Ok(None) } @@ -1210,7 +1205,7 @@ impl RequestsProvider for BlockchainProvider2 { id: BlockHashOrNumber, timestamp: u64, ) -> ProviderResult> { - if !self.database.chain_spec().is_prague_active_at_timestamp(timestamp) { + if !self.chain_spec().is_prague_active_at_timestamp(timestamp) { return Ok(None) } From f2440c763520e16315b3f60241ff9c9b52a4c50c Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Fri, 11 Oct 2024 20:52:36 +0200 Subject: [PATCH 39/82] test: add unit tests for `ChainInfoTracker` (#11642) Co-authored-by: Alexey Shekhirin --- Cargo.lock | 1 + crates/chain-state/Cargo.toml | 1 + crates/chain-state/src/chain_info.rs | 203 ++++++++++++++++++++++++++- 3 files changed, 201 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 37da7b57510b..22e1360a22e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6511,6 +6511,7 @@ dependencies = [ "reth-metrics", "reth-primitives", "reth-storage-api", + "reth-testing-utils", "reth-trie", "revm", "tokio", diff --git a/crates/chain-state/Cargo.toml b/crates/chain-state/Cargo.toml index 63016918c5cb..c9691bec411c 100644 --- a/crates/chain-state/Cargo.toml +++ b/crates/chain-state/Cargo.toml @@ -47,6 +47,7 @@ rand = { workspace = true, optional = true } revm = { workspace = true, optional = true } [dev-dependencies] +reth-testing-utils.workspace = true alloy-signer.workspace = true alloy-signer-local.workspace = true alloy-consensus.workspace = true diff --git a/crates/chain-state/src/chain_info.rs b/crates/chain-state/src/chain_info.rs index d36d9f47e438..3c75544ac460 100644 --- a/crates/chain-state/src/chain_info.rs +++ b/crates/chain-state/src/chain_info.rs @@ -95,14 +95,12 @@ impl ChainInfoTracker { /// Returns the safe header of the chain. pub fn get_safe_num_hash(&self) -> Option { - let h = self.inner.safe_block.borrow(); - h.as_ref().map(|h| h.num_hash()) + self.inner.safe_block.borrow().as_ref().map(SealedHeader::num_hash) } /// Returns the finalized header of the chain. pub fn get_finalized_num_hash(&self) -> Option { - let h = self.inner.finalized_block.borrow(); - h.as_ref().map(|h| h.num_hash()) + self.inner.finalized_block.borrow().as_ref().map(SealedHeader::num_hash) } /// Sets the canonical head of the chain. @@ -169,3 +167,200 @@ struct ChainInfoInner { /// The block that the beacon node considers finalized. finalized_block: watch::Sender>, } + +#[cfg(test)] +mod tests { + use super::*; + use reth_testing_utils::{generators, generators::random_header}; + + #[test] + fn test_chain_info() { + // Create a random header + let mut rng = generators::rng(); + let header = random_header(&mut rng, 10, None); + + // Create a new chain info tracker with the header + let tracker = ChainInfoTracker::new(header.clone(), None, None); + + // Fetch the chain information from the tracker + let chain_info = tracker.chain_info(); + + // Verify that the chain information matches the header + assert_eq!(chain_info.best_number, header.number); + assert_eq!(chain_info.best_hash, header.hash()); + } + + #[test] + fn test_on_forkchoice_update_received() { + // Create a random block header + let mut rng = generators::rng(); + let header = random_header(&mut rng, 10, None); + + // Create a new chain info tracker with the header + let tracker = ChainInfoTracker::new(header, None, None); + + // Assert that there has been no forkchoice update yet (the timestamp is None) + assert!(tracker.last_forkchoice_update_received_at().is_none()); + + // Call the method to record the receipt of a forkchoice update + tracker.on_forkchoice_update_received(); + + // Assert that there is now a timestamp indicating when the forkchoice update was received + assert!(tracker.last_forkchoice_update_received_at().is_some()); + } + + #[test] + fn test_on_transition_configuration_exchanged() { + // Create a random header + let mut rng = generators::rng(); + let header = random_header(&mut rng, 10, None); + + // Create a new chain info tracker with the header + let tracker = ChainInfoTracker::new(header, None, None); + + // Assert that there has been no transition configuration exchange yet (the timestamp is + // None) + assert!(tracker.last_transition_configuration_exchanged_at().is_none()); + + // Call the method to record the transition configuration exchange + tracker.on_transition_configuration_exchanged(); + + // Assert that there is now a timestamp indicating when the transition configuration + // exchange occurred + assert!(tracker.last_transition_configuration_exchanged_at().is_some()); + } + + #[test] + fn test_set_canonical_head() { + // Create a random number generator + let mut rng = generators::rng(); + // Generate two random headers for testing + let header1 = random_header(&mut rng, 10, None); + let header2 = random_header(&mut rng, 20, None); + + // Create a new chain info tracker with the first header + let tracker = ChainInfoTracker::new(header1, None, None); + + // Set the second header as the canonical head of the tracker + tracker.set_canonical_head(header2.clone()); + + // Assert that the tracker now uses the second header as its canonical head + let canonical_head = tracker.get_canonical_head(); + assert_eq!(canonical_head, header2); + } + + #[test] + fn test_set_safe() { + // Create a random number generator + let mut rng = generators::rng(); + + // Case 1: basic test + // Generate two random headers for the test + let header1 = random_header(&mut rng, 10, None); + let header2 = random_header(&mut rng, 20, None); + + // Create a new chain info tracker with the first header (header1) + let tracker = ChainInfoTracker::new(header1, None, None); + + // Call the set_safe method with the second header (header2) + tracker.set_safe(header2.clone()); + + // Verify that the tracker now has header2 as the safe block + let safe_header = tracker.get_safe_header(); + assert!(safe_header.is_some()); // Ensure a safe header is present + let safe_header = safe_header.unwrap(); + assert_eq!(safe_header, header2); + + // Case 2: call with the same header as the current safe block + // Call set_safe again with the same header (header2) + tracker.set_safe(header2.clone()); + + // Verify that nothing changes and the safe header remains the same + let same_safe_header = tracker.get_safe_header(); + assert!(same_safe_header.is_some()); + let same_safe_header = same_safe_header.unwrap(); + assert_eq!(same_safe_header, header2); + + // Case 3: call with a different (new) header + // Generate a third header with a higher block number + let header3 = random_header(&mut rng, 30, None); + + // Call set_safe with this new header (header3) + tracker.set_safe(header3.clone()); + + // Verify that the safe header is updated with the new header + let updated_safe_header = tracker.get_safe_header(); + assert!(updated_safe_header.is_some()); + let updated_safe_header = updated_safe_header.unwrap(); + assert_eq!(updated_safe_header, header3); + } + + #[test] + fn test_set_finalized() { + // Create a random number generator + let mut rng = generators::rng(); + + // Generate random headers for testing + let header1 = random_header(&mut rng, 10, None); + let header2 = random_header(&mut rng, 20, None); + let header3 = random_header(&mut rng, 30, None); + + // Create a new chain info tracker with the first header + let tracker = ChainInfoTracker::new(header1, None, None); + + // Initial state: finalize header should be None + assert!(tracker.get_finalized_header().is_none()); + + // Set the second header as the finalized header + tracker.set_finalized(header2.clone()); + + // Assert that the tracker now uses the second header as its finalized block + let finalized_header = tracker.get_finalized_header(); + assert!(finalized_header.is_some()); + let finalized_header = finalized_header.unwrap(); + assert_eq!(finalized_header, header2); + + // Case 2: attempt to set the same finalized header again + tracker.set_finalized(header2.clone()); + + // The finalized header should remain unchanged + let unchanged_finalized_header = tracker.get_finalized_header(); + assert_eq!(unchanged_finalized_header.unwrap(), header2); // Should still be header2 + + // Case 3: set a higher block number as finalized + tracker.set_finalized(header3.clone()); + + // The finalized header should now be updated to header3 + let updated_finalized_header = tracker.get_finalized_header(); + assert!(updated_finalized_header.is_some()); + assert_eq!(updated_finalized_header.unwrap(), header3); + } + + #[test] + fn test_get_finalized_num_hash() { + // Create a random header + let mut rng = generators::rng(); + let finalized_header = random_header(&mut rng, 10, None); + + // Create a new chain info tracker with the finalized header + let tracker = + ChainInfoTracker::new(finalized_header.clone(), Some(finalized_header.clone()), None); + + // Assert that the BlockNumHash returned matches the finalized header + assert_eq!(tracker.get_finalized_num_hash(), Some(finalized_header.num_hash())); + } + + #[test] + fn test_get_safe_num_hash() { + // Create a random header + let mut rng = generators::rng(); + let safe_header = random_header(&mut rng, 10, None); + + // Create a new chain info tracker with the safe header + let tracker = ChainInfoTracker::new(safe_header.clone(), None, None); + tracker.set_safe(safe_header.clone()); + + // Assert that the BlockNumHash returned matches the safe header + assert_eq!(tracker.get_safe_num_hash(), Some(safe_header.num_hash())); + } +} From 5e1bd04941604156c9c212f19174044592410d37 Mon Sep 17 00:00:00 2001 From: 0xDmtri <0xDmtri@protonmail.com> Date: Fri, 11 Oct 2024 20:53:34 +0200 Subject: [PATCH 40/82] chore(examples): fix db-access example with RO provider (#11670) Co-authored-by: Matthias Seitz --- examples/db-access/src/main.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/db-access/src/main.rs b/examples/db-access/src/main.rs index ab018a0b07a6..5772461bd7a7 100644 --- a/examples/db-access/src/main.rs +++ b/examples/db-access/src/main.rs @@ -1,6 +1,7 @@ use alloy_primitives::{Address, Sealable, B256}; use alloy_rpc_types::{Filter, FilteredParams}; use reth_chainspec::ChainSpecBuilder; +use reth_db::{open_db_read_only, DatabaseEnv}; use reth_node_ethereum::EthereumNode; use reth_node_types::NodeTypesWithDBAdapter; use reth_primitives::SealedHeader; @@ -8,7 +9,7 @@ use reth_provider::{ providers::StaticFileProvider, AccountReader, BlockReader, BlockSource, HeaderProvider, ProviderFactory, ReceiptProvider, StateProvider, TransactionsProvider, }; -use std::path::Path; +use std::{path::Path, sync::Arc}; // Providers are zero cost abstractions on top of an opened MDBX Transaction // exposing a familiar API to query the chain's information without requiring knowledge @@ -20,17 +21,16 @@ fn main() -> eyre::Result<()> { // Opens a RO handle to the database file. let db_path = std::env::var("RETH_DB_PATH")?; let db_path = Path::new(&db_path); + let db = open_db_read_only(db_path.join("db").as_path(), Default::default())?; // Instantiate a provider factory for Ethereum mainnet using the provided DB. // TODO: Should the DB version include the spec so that you do not need to specify it here? let spec = ChainSpecBuilder::mainnet().build(); - let factory = - ProviderFactory::>::new_with_database_path( - db_path, - spec.into(), - Default::default(), - StaticFileProvider::read_only(db_path.join("static_files"), false)?, - )?; + let factory = ProviderFactory::>>::new( + db.into(), + spec.into(), + StaticFileProvider::read_only(db_path.join("static_files"), true)?, + ); // This call opens a RO transaction on the database. To write to the DB you'd need to call // the `provider_rw` function and look for the `Writer` variants of the traits. From 75dda1c398153b674c760402056ec52a8c2cceaf Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Sat, 12 Oct 2024 08:24:06 +0400 Subject: [PATCH 41/82] fix: always poll new pool imports (#11675) --- crates/net/network/src/transactions/mod.rs | 46 +++++++++++----------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index bc77de9f463b..439f92bada9a 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -1255,29 +1255,6 @@ where // yield back control to tokio. See `NetworkManager` for more context on the design // pattern. - // Advance pool imports (flush txns to pool). - // - // Note, this is done in batches. A batch is filled from one `Transactions` - // broadcast messages or one `PooledTransactions` response at a time. The - // minimum batch size is 1 transaction (and might often be the case with blob - // transactions). - // - // The smallest decodable transaction is an empty legacy transaction, 10 bytes - // (2 MiB / 10 bytes > 200k transactions). - // - // Since transactions aren't validated until they are inserted into the pool, - // this can potentially validate >200k transactions. More if the message size - // is bigger than the soft limit on a `PooledTransactions` response which is - // 2 MiB (`Transactions` broadcast messages is smaller, 128 KiB). - let maybe_more_pool_imports = metered_poll_nested_stream_with_budget!( - poll_durations.acc_pending_imports, - "net::tx", - "Batched pool imports stream", - DEFAULT_BUDGET_TRY_DRAIN_PENDING_POOL_IMPORTS, - this.pool_imports.poll_next_unpin(cx), - |batch_results| this.on_batch_import_result(batch_results) - ); - // Advance network/peer related events (update peers map). let maybe_more_network_events = metered_poll_nested_stream_with_budget!( poll_durations.acc_network_events, @@ -1351,6 +1328,29 @@ where |event| this.on_network_tx_event(event), ); + // Advance pool imports (flush txns to pool). + // + // Note, this is done in batches. A batch is filled from one `Transactions` + // broadcast messages or one `PooledTransactions` response at a time. The + // minimum batch size is 1 transaction (and might often be the case with blob + // transactions). + // + // The smallest decodable transaction is an empty legacy transaction, 10 bytes + // (2 MiB / 10 bytes > 200k transactions). + // + // Since transactions aren't validated until they are inserted into the pool, + // this can potentially validate >200k transactions. More if the message size + // is bigger than the soft limit on a `PooledTransactions` response which is + // 2 MiB (`Transactions` broadcast messages is smaller, 128 KiB). + let maybe_more_pool_imports = metered_poll_nested_stream_with_budget!( + poll_durations.acc_pending_imports, + "net::tx", + "Batched pool imports stream", + DEFAULT_BUDGET_TRY_DRAIN_PENDING_POOL_IMPORTS, + this.pool_imports.poll_next_unpin(cx), + |batch_results| this.on_batch_import_result(batch_results) + ); + // Tries to drain hashes pending fetch cache if the tx manager currently has // capacity for this (fetch txns). // From 43fe46f0d340fdf27e6b1b51916d0ecf9f6c314b Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Sat, 12 Oct 2024 10:06:19 +0200 Subject: [PATCH 42/82] fix(net): decrease budget for header reqs to process before yielding thread (#11636) --- crates/net/network/src/budget.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/net/network/src/budget.rs b/crates/net/network/src/budget.rs index 0b5e3d3a90bb..2f878cdb2e88 100644 --- a/crates/net/network/src/budget.rs +++ b/crates/net/network/src/budget.rs @@ -5,8 +5,8 @@ pub const DEFAULT_BUDGET_TRY_DRAIN_STREAM: u32 = 10; /// Default budget to try and drain headers and bodies download streams. /// -/// Default is 4 iterations. -pub const DEFAULT_BUDGET_TRY_DRAIN_DOWNLOADERS: u32 = 4; +/// Default is 1 iteration. +pub const DEFAULT_BUDGET_TRY_DRAIN_DOWNLOADERS: u32 = 1; /// Default budget to try and drain [`Swarm`](crate::swarm::Swarm). /// From 5c84daba11d5ce38b1649504f0bae947e2b7c362 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 12 Oct 2024 13:19:48 +0200 Subject: [PATCH 43/82] fix: dont remove txs manually (#11683) --- crates/consensus/auto-seal/src/task.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/crates/consensus/auto-seal/src/task.rs b/crates/consensus/auto-seal/src/task.rs index e4873615f1d0..cb0586d44407 100644 --- a/crates/consensus/auto-seal/src/task.rs +++ b/crates/consensus/auto-seal/src/task.rs @@ -113,7 +113,6 @@ where let to_engine = this.to_engine.clone(); let client = this.client.clone(); let chain_spec = Arc::clone(&this.chain_spec); - let pool = this.pool.clone(); let events = this.pipe_line_events.take(); let executor = this.block_executor.clone(); @@ -139,11 +138,6 @@ where &executor, ) { Ok((new_header, _bundle_state)) => { - // clear all transactions from pool - pool.remove_transactions( - transactions.iter().map(|tx| tx.hash()).collect(), - ); - let state = ForkchoiceState { head_block_hash: new_header.hash(), finalized_block_hash: new_header.hash(), From 8ebf10b4ac3f5e95752fa28d2d8a9b14585730b9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 12 Oct 2024 14:42:46 +0200 Subject: [PATCH 44/82] fix: only +1 on the pool nonce (#11680) --- crates/rpc/rpc-eth-api/src/helpers/state.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/state.rs b/crates/rpc/rpc-eth-api/src/helpers/state.rs index d601e43d90a8..7b11ce6afe63 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/state.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/state.rs @@ -284,8 +284,8 @@ pub trait LoadState: EthApiTypes { Self: SpawnBlocking, { self.spawn_blocking_io(move |this| { - // first fetch the on chain nonce - let nonce = this + // first fetch the on chain nonce of the account + let on_chain_account_nonce = this .state_at_block_id_or_latest(block_id)? .account_nonce(address) .map_err(Self::Error::from_eth_err)? @@ -297,20 +297,24 @@ pub trait LoadState: EthApiTypes { this.pool().get_highest_transaction_by_sender(address) { { - // and the corresponding txcount is nonce + 1 - let next_nonce = - nonce.max(highest_pool_tx.nonce()).checked_add(1).ok_or_else(|| { + // and the corresponding txcount is nonce + 1 of the highest tx in the pool + // (on chain nonce is increased after tx) + let next_tx_nonce = + highest_pool_tx.nonce().checked_add(1).ok_or_else(|| { Self::Error::from(EthApiError::InvalidTransaction( RpcInvalidTransactionError::NonceMaxValue, )) })?; - let tx_count = nonce.max(next_nonce); + // guard against drifts in the pool + let next_tx_nonce = on_chain_account_nonce.max(next_tx_nonce); + + let tx_count = on_chain_account_nonce.max(next_tx_nonce); return Ok(U256::from(tx_count)); } } } - Ok(U256::from(nonce)) + Ok(U256::from(on_chain_account_nonce)) }) } From db1d64b1c8126abf7573ccb4a0a1a757c1a100b2 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Sat, 12 Oct 2024 14:46:03 +0200 Subject: [PATCH 45/82] test: more unit tests for `HashedPostState` (#11663) --- crates/trie/trie/src/state.rs | 148 ++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) diff --git a/crates/trie/trie/src/state.rs b/crates/trie/trie/src/state.rs index 3b0af5cd879b..2af48dfff798 100644 --- a/crates/trie/trie/src/state.rs +++ b/crates/trie/trie/src/state.rs @@ -347,6 +347,15 @@ impl HashedStorageSorted { #[cfg(test)] mod tests { + use alloy_primitives::Bytes; + use revm::{ + db::{ + states::{plain_account::PlainStorage, StorageSlot}, + PlainAccount, StorageWithOriginalValues, + }, + primitives::{AccountInfo, Bytecode}, + }; + use super::*; #[test] @@ -422,4 +431,143 @@ mod tests { ); assert_eq!(account_storage.map(|st| st.wiped), Some(true)); } + + #[test] + fn test_hashed_post_state_from_bundle_state() { + // Prepare a random Ethereum address as a key for the account. + let address = Address::random(); + + // Create a mock account info object. + let account_info = AccountInfo { + balance: U256::from(123), + nonce: 42, + code_hash: B256::random(), + code: Some(Bytecode::LegacyRaw(Bytes::from(vec![1, 2]))), + }; + + let mut storage = StorageWithOriginalValues::default(); + storage.insert( + U256::from(1), + StorageSlot { present_value: U256::from(4), ..Default::default() }, + ); + + // Create a `BundleAccount` struct to represent the account and its storage. + let account = BundleAccount { + status: AccountStatus::Changed, + info: Some(account_info.clone()), + storage, + original_info: None, + }; + + // Create a vector of tuples representing the bundle state. + let state = vec![(&address, &account)]; + + // Convert the bundle state into a hashed post state. + let hashed_state = HashedPostState::from_bundle_state(state); + + // Validate the hashed post state. + assert_eq!(hashed_state.accounts.len(), 1); + assert_eq!(hashed_state.storages.len(), 1); + + // Validate the account info. + assert_eq!( + *hashed_state.accounts.get(&keccak256(address)).unwrap(), + Some(account_info.into()) + ); + } + + #[test] + fn test_hashed_post_state_from_cache_state() { + // Prepare a random Ethereum address. + let address = Address::random(); + + // Create mock account info. + let account_info = AccountInfo { + balance: U256::from(500), + nonce: 5, + code_hash: B256::random(), + code: None, + }; + + let mut storage = PlainStorage::default(); + storage.insert(U256::from(1), U256::from(35636)); + + // Create a `CacheAccount` with the mock account info. + let account = CacheAccount { + account: Some(PlainAccount { info: account_info.clone(), storage }), + status: AccountStatus::Changed, + }; + + // Create a vector of tuples representing the cache state. + let state = vec![(&address, &account)]; + + // Convert the cache state into a hashed post state. + let hashed_state = HashedPostState::from_cache_state(state); + + // Validate the hashed post state. + assert_eq!(hashed_state.accounts.len(), 1); + assert_eq!(hashed_state.storages.len(), 1); + + // Validate the account info. + assert_eq!( + *hashed_state.accounts.get(&keccak256(address)).unwrap(), + Some(account_info.into()) + ); + } + + #[test] + fn test_hashed_post_state_with_accounts() { + // Prepare random addresses and mock account info. + let address_1 = Address::random(); + let address_2 = Address::random(); + + let account_info_1 = AccountInfo { + balance: U256::from(1000), + nonce: 1, + code_hash: B256::random(), + code: None, + }; + + // Create hashed accounts with addresses. + let account_1 = (keccak256(address_1), Some(account_info_1.into())); + let account_2 = (keccak256(address_2), None); + + // Add accounts to the hashed post state. + let hashed_state = HashedPostState::default().with_accounts(vec![account_1, account_2]); + + // Validate the hashed post state. + assert_eq!(hashed_state.accounts.len(), 2); + assert!(hashed_state.accounts.contains_key(&keccak256(address_1))); + assert!(hashed_state.accounts.contains_key(&keccak256(address_2))); + } + + #[test] + fn test_hashed_post_state_with_storages() { + // Prepare random addresses and mock storage entries. + let address_1 = Address::random(); + let address_2 = Address::random(); + + let storage_1 = (keccak256(address_1), HashedStorage::new(false)); + let storage_2 = (keccak256(address_2), HashedStorage::new(true)); + + // Add storages to the hashed post state. + let hashed_state = HashedPostState::default().with_storages(vec![storage_1, storage_2]); + + // Validate the hashed post state. + assert_eq!(hashed_state.storages.len(), 2); + assert!(hashed_state.storages.contains_key(&keccak256(address_1))); + assert!(hashed_state.storages.contains_key(&keccak256(address_2))); + } + + #[test] + fn test_hashed_post_state_is_empty() { + // Create an empty hashed post state and validate it's empty. + let empty_state = HashedPostState::default(); + assert!(empty_state.is_empty()); + + // Add an account and validate the state is no longer empty. + let non_empty_state = HashedPostState::default() + .with_accounts(vec![(keccak256(Address::random()), Some(Account::default()))]); + assert!(!non_empty_state.is_empty()); + } } From b365bd52f260ba3459f305926c3d6c3264913ed1 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 12 Oct 2024 15:27:52 +0200 Subject: [PATCH 46/82] docs: complete sentence (#11685) --- crates/net/network/src/transactions/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/net/network/src/transactions/config.rs b/crates/net/network/src/transactions/config.rs index 81ec293ea1ff..b838f7cfe71b 100644 --- a/crates/net/network/src/transactions/config.rs +++ b/crates/net/network/src/transactions/config.rs @@ -47,7 +47,7 @@ pub enum TransactionPropagationMode { } impl TransactionPropagationMode { - /// Returns the number of peers that should + /// Returns the number of peers full transactions should be propagated to. pub(crate) fn full_peer_count(&self, peer_count: usize) -> usize { match self { Self::Sqrt => (peer_count as f64).sqrt().round() as usize, From 9ec4c000246f84594ec7169e88e31a5ae496cdfe Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 12 Oct 2024 15:28:08 +0200 Subject: [PATCH 47/82] chore: we dont need sat here (#11678) --- crates/net/network/src/budget.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/net/network/src/budget.rs b/crates/net/network/src/budget.rs index 2f878cdb2e88..6eba076f6b9f 100644 --- a/crates/net/network/src/budget.rs +++ b/crates/net/network/src/budget.rs @@ -55,7 +55,7 @@ macro_rules! poll_nested_stream_with_budget { let mut f = $on_ready_some; f(item); - budget = budget.saturating_sub(1); + budget -= 1; if budget == 0 { break true } From de736a53cc7216141d900f4e145f06e892d6cdd7 Mon Sep 17 00:00:00 2001 From: Delweng Date: Sat, 12 Oct 2024 21:29:20 +0800 Subject: [PATCH 48/82] chore(stages): reduce the progress logging (#11653) Signed-off-by: jsvisa --- .../stages/src/stages/hashing_account.rs | 20 +++++---- .../stages/src/stages/hashing_storage.rs | 20 +++++---- crates/stages/stages/src/stages/headers.rs | 24 +++++----- crates/stages/stages/src/stages/tx_lookup.rs | 8 +++- crates/stages/stages/src/stages/utils.rs | 44 ++++++++++++++----- 5 files changed, 74 insertions(+), 42 deletions(-) diff --git a/crates/stages/stages/src/stages/hashing_account.rs b/crates/stages/stages/src/stages/hashing_account.rs index 14afb37d81db..2fdccd7837fb 100644 --- a/crates/stages/stages/src/stages/hashing_account.rs +++ b/crates/stages/stages/src/stages/hashing_account.rs @@ -1,3 +1,4 @@ +use crate::log_progress; use alloy_primitives::{keccak256, B256}; use itertools::Itertools; use reth_config::config::{EtlConfig, HashingConfig}; @@ -18,6 +19,7 @@ use std::{ fmt::Debug, ops::{Range, RangeInclusive}, sync::mpsc::{self, Receiver}, + time::Instant, }; use tracing::*; @@ -186,16 +188,16 @@ where let mut hashed_account_cursor = tx.cursor_write::>()?; - let total_hashes = collector.len(); - let interval = (total_hashes / 10).max(1); + let total = collector.len(); + let mut last_log = Instant::now(); for (index, item) in collector.iter()?.enumerate() { - if index > 0 && index % interval == 0 { - info!( - target: "sync::stages::hashing_account", - progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), - "Inserting hashes" - ); - } + log_progress!( + "sync::stages::hashing_account", + index, + total, + last_log, + "Inserting hashes" + ); let (key, value) = item?; hashed_account_cursor diff --git a/crates/stages/stages/src/stages/hashing_storage.rs b/crates/stages/stages/src/stages/hashing_storage.rs index ef070d30c6d6..0646032935f1 100644 --- a/crates/stages/stages/src/stages/hashing_storage.rs +++ b/crates/stages/stages/src/stages/hashing_storage.rs @@ -1,3 +1,4 @@ +use crate::log_progress; use alloy_primitives::{bytes::BufMut, keccak256, B256}; use itertools::Itertools; use reth_config::config::{EtlConfig, HashingConfig}; @@ -19,6 +20,7 @@ use reth_storage_errors::provider::ProviderResult; use std::{ fmt::Debug, sync::mpsc::{self, Receiver}, + time::Instant, }; use tracing::*; @@ -117,17 +119,17 @@ where collect(&mut channels, &mut collector)?; - let total_hashes = collector.len(); - let interval = (total_hashes / 10).max(1); + let total = collector.len(); + let mut last_log = Instant::now(); let mut cursor = tx.cursor_dup_write::()?; for (index, item) in collector.iter()?.enumerate() { - if index > 0 && index % interval == 0 { - info!( - target: "sync::stages::hashing_storage", - progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), - "Inserting hashes" - ); - } + log_progress!( + "sync::stages::hashing_storage", + index, + total, + last_log, + "Inserting hashes" + ); let (addr_key, value) = item?; cursor.append_dup( diff --git a/crates/stages/stages/src/stages/headers.rs b/crates/stages/stages/src/stages/headers.rs index 199e015c2dce..eaa2ea01f0d8 100644 --- a/crates/stages/stages/src/stages/headers.rs +++ b/crates/stages/stages/src/stages/headers.rs @@ -1,3 +1,4 @@ +use crate::log_progress; use alloy_primitives::{BlockHash, BlockNumber, Bytes, B256}; use futures_util::StreamExt; use reth_config::config::EtlConfig; @@ -25,6 +26,7 @@ use reth_storage_errors::provider::ProviderError; use std::{ sync::Arc, task::{ready, Context, Poll}, + time::Instant, }; use tokio::sync::watch; use tracing::*; @@ -95,9 +97,9 @@ where provider: &impl DBProvider, static_file_provider: StaticFileProvider, ) -> Result { - let total_headers = self.header_collector.len(); + let total = self.header_collector.len(); - info!(target: "sync::stages::headers", total = total_headers, "Writing headers"); + info!(target: "sync::stages::headers", total, "Writing headers"); // Consistency check of expected headers in static files vs DB is done on provider::sync_gap // when poll_execute_ready is polled. @@ -113,13 +115,11 @@ where // Although headers were downloaded in reverse order, the collector iterates it in ascending // order let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?; - let interval = (total_headers / 10).max(1); + let mut last_log = Instant::now(); for (index, header) in self.header_collector.iter()?.enumerate() { let (_, header_buf) = header?; - if index > 0 && index % interval == 0 && total_headers > 100 { - info!(target: "sync::stages::headers", progress = %format!("{:.2}%", (index as f64 / total_headers as f64) * 100.0), "Writing headers"); - } + log_progress!("sync::stages::headers", index, total, last_log, "Writing headers"); let sealed_header: SealedHeader = bincode::deserialize::>(&header_buf) @@ -147,7 +147,7 @@ where writer.append_header(&header, td, &header_hash)?; } - info!(target: "sync::stages::headers", total = total_headers, "Writing headers hash index"); + info!(target: "sync::stages::headers", total, "Writing headers hash index"); let mut cursor_header_numbers = provider.tx_ref().cursor_write::>()?; @@ -168,9 +168,13 @@ where for (index, hash_to_number) in self.hash_collector.iter()?.enumerate() { let (hash, number) = hash_to_number?; - if index > 0 && index % interval == 0 && total_headers > 100 { - info!(target: "sync::stages::headers", progress = %format!("{:.2}%", (index as f64 / total_headers as f64) * 100.0), "Writing headers hash index"); - } + log_progress!( + "sync::stages::headers", + index, + total, + last_log, + "Writing headers hash index" + ); if first_sync { cursor_header_numbers.append( diff --git a/crates/stages/stages/src/stages/tx_lookup.rs b/crates/stages/stages/src/stages/tx_lookup.rs index 60c958abf862..c5ccff54c525 100644 --- a/crates/stages/stages/src/stages/tx_lookup.rs +++ b/crates/stages/stages/src/stages/tx_lookup.rs @@ -1,3 +1,4 @@ +use super::utils::LOG_INTERVAL; use alloy_primitives::{TxHash, TxNumber}; use num_traits::Zero; use reth_config::config::{EtlConfig, TransactionLookupConfig}; @@ -17,6 +18,7 @@ use reth_stages_api::{ UnwindInput, UnwindOutput, }; use reth_storage_errors::provider::ProviderError; +use std::time::Instant; use tracing::*; /// The transaction lookup stage. @@ -147,16 +149,18 @@ where .cursor_write::>()?; let total_hashes = hash_collector.len(); - let interval = (total_hashes / 10).max(1); + let mut last_log = Instant::now(); for (index, hash_to_number) in hash_collector.iter()?.enumerate() { let (hash, number) = hash_to_number?; - if index > 0 && index % interval == 0 { + let now = Instant::now(); + if now.duration_since(last_log) >= LOG_INTERVAL { info!( target: "sync::stages::transaction_lookup", ?append_only, progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), "Inserting hashes" ); + last_log = now; } let key = RawKey::::from_vec(hash); diff --git a/crates/stages/stages/src/stages/utils.rs b/crates/stages/stages/src/stages/utils.rs index caf039faca10..af6594cd7861 100644 --- a/crates/stages/stages/src/stages/utils.rs +++ b/crates/stages/stages/src/stages/utils.rs @@ -12,12 +12,36 @@ use reth_db_api::{ use reth_etl::Collector; use reth_provider::DBProvider; use reth_stages_api::StageError; -use std::{collections::HashMap, hash::Hash, ops::RangeBounds}; +use std::{ + collections::HashMap, + hash::Hash, + ops::RangeBounds, + time::{Duration, Instant}, +}; use tracing::info; /// Number of blocks before pushing indices from cache to [`Collector`] const DEFAULT_CACHE_THRESHOLD: u64 = 100_000; +/// Log interval for progress. +pub(crate) const LOG_INTERVAL: Duration = Duration::from_secs(5); + +/// Log progress at a regular interval. +#[macro_export] +macro_rules! log_progress { + ($target:expr, $index:expr, $total:expr, $last_log:expr, $message:expr) => { + let now = std::time::Instant::now(); + if now.duration_since($last_log) >= $crate::stages::utils::LOG_INTERVAL { + info!( + target: $target, + progress = %format!("{:.2}%", ($index as f64 / $total as f64) * 100.0), + $message + ); + $last_log = now; + } + } +} + /// Collects all history (`H`) indices for a range of changesets (`CS`) and stores them in a /// [`Collector`]. /// @@ -65,18 +89,16 @@ where }; // observability - let total_changesets = provider.tx_ref().entries::()?; - let interval = (total_changesets / 1000).max(1); + let total = provider.tx_ref().entries::()?; + let mut last_log = Instant::now(); let mut flush_counter = 0; let mut current_block_number = u64::MAX; - for (idx, entry) in changeset_cursor.walk_range(range)?.enumerate() { + for (index, entry) in changeset_cursor.walk_range(range)?.enumerate() { let (block_number, key) = partial_key_factory(entry?); cache.entry(key).or_default().push(block_number); - if idx > 0 && idx % interval == 0 && total_changesets > 1000 { - info!(target: "sync::stages::index_history", progress = %format!("{:.4}%", (idx as f64 / total_changesets as f64) * 100.0), "Collecting indices"); - } + log_progress!("sync::stages::index_history", index, total, last_log, "Collecting indices"); // Make sure we only flush the cache every DEFAULT_CACHE_THRESHOLD blocks. if current_block_number != block_number { @@ -120,17 +142,15 @@ where let mut current_list = Vec::::new(); // observability - let total_entries = collector.len(); - let interval = (total_entries / 100).max(1); + let total = collector.len(); + let mut last_log = Instant::now(); for (index, element) in collector.iter()?.enumerate() { let (k, v) = element?; let sharded_key = decode_key(k)?; let new_list = BlockNumberList::decompress_owned(v)?; - if index > 0 && index % interval == 0 && total_entries > 100 { - info!(target: "sync::stages::index_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices"); - } + log_progress!("sync::stages::index_history", index, total, last_log, "Writing indices"); // AccountsHistory: `Address`. // StorageHistory: `Address.StorageKey`. From 86c6ba5d8da703cdd948f03f18a8ca2de645322a Mon Sep 17 00:00:00 2001 From: Delweng Date: Sat, 12 Oct 2024 23:18:43 +0800 Subject: [PATCH 49/82] chore(forkid): simplify and add comment of the set_head_priv (#11686) Signed-off-by: jsvisa --- crates/ethereum-forks/src/forkid.rs | 34 +++++++++++------------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/crates/ethereum-forks/src/forkid.rs b/crates/ethereum-forks/src/forkid.rs index 8a344e2dd2cc..6876d0eb926d 100644 --- a/crates/ethereum-forks/src/forkid.rs +++ b/crates/ethereum-forks/src/forkid.rs @@ -258,32 +258,24 @@ impl ForkFilter { } fn set_head_priv(&mut self, head: Head) -> Option { - let recompute_cache = { - let head_in_past = match self.cache.epoch_start { - ForkFilterKey::Block(epoch_start_block) => head.number < epoch_start_block, - ForkFilterKey::Time(epoch_start_time) => head.timestamp < epoch_start_time, - }; - let head_in_future = match self.cache.epoch_end { - Some(ForkFilterKey::Block(epoch_end_block)) => head.number >= epoch_end_block, - Some(ForkFilterKey::Time(epoch_end_time)) => head.timestamp >= epoch_end_time, - None => false, - }; - - head_in_past || head_in_future + let head_in_past = match self.cache.epoch_start { + ForkFilterKey::Block(epoch_start_block) => head.number < epoch_start_block, + ForkFilterKey::Time(epoch_start_time) => head.timestamp < epoch_start_time, }; - - // recompute the cache - let transition = if recompute_cache { - let past = self.current(); - self.cache = Cache::compute_cache(&self.forks, head); - Some(ForkTransition { current: self.current(), past }) - } else { - None + let head_in_future = match self.cache.epoch_end { + Some(ForkFilterKey::Block(epoch_end_block)) => head.number >= epoch_end_block, + Some(ForkFilterKey::Time(epoch_end_time)) => head.timestamp >= epoch_end_time, + None => false, }; self.head = head; - transition + // Recompute the cache if the head is in the past or future epoch. + (head_in_past || head_in_future).then(|| { + let past = self.current(); + self.cache = Cache::compute_cache(&self.forks, head); + ForkTransition { current: self.current(), past } + }) } /// Set the current head. From c47621754fc4fe1b97639d5c479be65a66dc11d7 Mon Sep 17 00:00:00 2001 From: Kien Trinh <51135161+kien6034@users.noreply.github.com> Date: Sat, 12 Oct 2024 22:45:47 +0700 Subject: [PATCH 50/82] feat: add helper function to modify node builder (#11682) --- crates/node/builder/src/builder/mod.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crates/node/builder/src/builder/mod.rs b/crates/node/builder/src/builder/mod.rs index 8b57781ea97c..15bdc730a85d 100644 --- a/crates/node/builder/src/builder/mod.rs +++ b/crates/node/builder/src/builder/mod.rs @@ -162,6 +162,14 @@ impl NodeBuilder<(), ChainSpec> { pub const fn new(config: NodeConfig) -> Self { Self { config, database: () } } + + /// Apply a function to the builder + pub fn apply(self, f: F) -> Self + where + F: FnOnce(Self) -> Self, + { + f(self) + } } impl NodeBuilder { @@ -400,6 +408,14 @@ where &self.builder.config } + /// Apply a function to the builder + pub fn apply(self, f: F) -> Self + where + F: FnOnce(Self) -> Self, + { + f(self) + } + /// Sets the hook that is run once the node's components are initialized. pub fn on_component_initialized(self, hook: F) -> Self where From 53bd6872dbbac4b938140b1320902610523114bb Mon Sep 17 00:00:00 2001 From: greged93 <82421016+greged93@users.noreply.github.com> Date: Sat, 12 Oct 2024 17:57:25 +0200 Subject: [PATCH 51/82] feat(op): opchainspec builder (#11630) Co-authored-by: Matthias Seitz --- crates/chainspec/src/spec.rs | 67 ++-------- crates/optimism/chainspec/src/lib.rs | 165 ++++++++++++++++++++++-- crates/optimism/evm/src/config.rs | 15 ++- crates/optimism/evm/src/execute.rs | 19 +-- crates/optimism/node/tests/e2e/utils.rs | 14 +- 5 files changed, 179 insertions(+), 101 deletions(-) diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index d6ca92aa24eb..f80a20924394 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -755,13 +755,19 @@ impl ChainSpecBuilder { } /// Add the given fork with the given activation condition to the spec. - pub fn with_fork(mut self, fork: EthereumHardfork, condition: ForkCondition) -> Self { + pub fn with_fork(mut self, fork: H, condition: ForkCondition) -> Self { self.hardforks.insert(fork, condition); self } + /// Add the given chain hardforks to the spec. + pub fn with_forks(mut self, forks: ChainHardforks) -> Self { + self.hardforks = forks; + self + } + /// Remove the given fork from the spec. - pub fn without_fork(mut self, fork: EthereumHardfork) -> Self { + pub fn without_fork(mut self, fork: H) -> Self { self.hardforks.remove(fork); self } @@ -876,63 +882,6 @@ impl ChainSpecBuilder { self } - /// Enable Bedrock at genesis - #[cfg(feature = "optimism")] - pub fn bedrock_activated(mut self) -> Self { - self = self.paris_activated(); - self.hardforks - .insert(reth_optimism_forks::OptimismHardfork::Bedrock, ForkCondition::Block(0)); - self - } - - /// Enable Regolith at genesis - #[cfg(feature = "optimism")] - pub fn regolith_activated(mut self) -> Self { - self = self.bedrock_activated(); - self.hardforks - .insert(reth_optimism_forks::OptimismHardfork::Regolith, ForkCondition::Timestamp(0)); - self - } - - /// Enable Canyon at genesis - #[cfg(feature = "optimism")] - pub fn canyon_activated(mut self) -> Self { - self = self.regolith_activated(); - // Canyon also activates changes from L1's Shanghai hardfork - self.hardforks.insert(EthereumHardfork::Shanghai, ForkCondition::Timestamp(0)); - self.hardforks - .insert(reth_optimism_forks::OptimismHardfork::Canyon, ForkCondition::Timestamp(0)); - self - } - - /// Enable Ecotone at genesis - #[cfg(feature = "optimism")] - pub fn ecotone_activated(mut self) -> Self { - self = self.canyon_activated(); - self.hardforks.insert(EthereumHardfork::Cancun, ForkCondition::Timestamp(0)); - self.hardforks - .insert(reth_optimism_forks::OptimismHardfork::Ecotone, ForkCondition::Timestamp(0)); - self - } - - /// Enable Fjord at genesis - #[cfg(feature = "optimism")] - pub fn fjord_activated(mut self) -> Self { - self = self.ecotone_activated(); - self.hardforks - .insert(reth_optimism_forks::OptimismHardfork::Fjord, ForkCondition::Timestamp(0)); - self - } - - /// Enable Granite at genesis - #[cfg(feature = "optimism")] - pub fn granite_activated(mut self) -> Self { - self = self.fjord_activated(); - self.hardforks - .insert(reth_optimism_forks::OptimismHardfork::Granite, ForkCondition::Timestamp(0)); - self - } - /// Build the resulting [`ChainSpec`]. /// /// # Panics diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index 9b2d1c8c11ba..5ebc18f67688 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -16,25 +16,154 @@ mod dev; mod op; mod op_sepolia; -use std::fmt::Display; - +use alloy_chains::Chain; use alloy_genesis::Genesis; use alloy_primitives::{Parity, Signature, B256, U256}; pub use base::BASE_MAINNET; pub use base_sepolia::BASE_SEPOLIA; +use derive_more::{Constructor, Deref, From, Into}; pub use dev::OP_DEV; +use once_cell::sync::OnceCell; pub use op::OP_MAINNET; pub use op_sepolia::OP_SEPOLIA; - -use derive_more::{Constructor, Deref, Into}; -use once_cell::sync::OnceCell; use reth_chainspec::{ - BaseFeeParams, BaseFeeParamsKind, ChainSpec, DepositContract, EthChainSpec, EthereumHardforks, - ForkFilter, ForkId, Hardforks, Head, + BaseFeeParams, BaseFeeParamsKind, ChainSpec, ChainSpecBuilder, DepositContract, EthChainSpec, + EthereumHardforks, ForkFilter, ForkId, Hardforks, Head, }; -use reth_ethereum_forks::{ChainHardforks, EthereumHardfork, ForkCondition}; +use reth_ethereum_forks::{ChainHardforks, EthereumHardfork, ForkCondition, Hardfork}; use reth_network_peers::NodeRecord; use reth_primitives_traits::Header; +use std::fmt::Display; + +/// Chain spec builder for a OP stack chain. +#[derive(Debug, Default, From)] +pub struct OpChainSpecBuilder { + /// [`ChainSpecBuilder`] + inner: ChainSpecBuilder, +} + +impl OpChainSpecBuilder { + /// Construct a new builder from the base mainnet chain spec. + pub fn base_mainnet() -> Self { + let mut inner = ChainSpecBuilder::default() + .chain(BASE_MAINNET.chain) + .genesis(BASE_MAINNET.genesis.clone()); + let forks = BASE_MAINNET.hardforks.clone(); + inner = inner.with_forks(forks); + + Self { inner } + } + + /// Construct a new builder from the optimism mainnet chain spec. + pub fn optimism_mainnet() -> Self { + let mut inner = + ChainSpecBuilder::default().chain(OP_MAINNET.chain).genesis(OP_MAINNET.genesis.clone()); + let forks = OP_MAINNET.hardforks.clone(); + inner = inner.with_forks(forks); + + Self { inner } + } +} + +impl OpChainSpecBuilder { + /// Set the chain ID + pub fn chain(mut self, chain: Chain) -> Self { + self.inner = self.inner.chain(chain); + self + } + + /// Set the genesis block. + pub fn genesis(mut self, genesis: Genesis) -> Self { + self.inner = self.inner.genesis(genesis); + self + } + + /// Add the given fork with the given activation condition to the spec. + pub fn with_fork(mut self, fork: H, condition: ForkCondition) -> Self { + self.inner = self.inner.with_fork(fork, condition); + self + } + + /// Add the given forks with the given activation condition to the spec. + pub fn with_forks(mut self, forks: ChainHardforks) -> Self { + self.inner = self.inner.with_forks(forks); + self + } + + /// Remove the given fork from the spec. + pub fn without_fork(mut self, fork: reth_optimism_forks::OptimismHardfork) -> Self { + self.inner = self.inner.without_fork(fork); + self + } + + /// Enable Bedrock at genesis + pub fn bedrock_activated(mut self) -> Self { + self.inner = self.inner.paris_activated(); + self.inner = self + .inner + .with_fork(reth_optimism_forks::OptimismHardfork::Bedrock, ForkCondition::Block(0)); + self + } + + /// Enable Regolith at genesis + pub fn regolith_activated(mut self) -> Self { + self = self.bedrock_activated(); + self.inner = self.inner.with_fork( + reth_optimism_forks::OptimismHardfork::Regolith, + ForkCondition::Timestamp(0), + ); + self + } + + /// Enable Canyon at genesis + pub fn canyon_activated(mut self) -> Self { + self = self.regolith_activated(); + // Canyon also activates changes from L1's Shanghai hardfork + self.inner = self.inner.with_fork(EthereumHardfork::Shanghai, ForkCondition::Timestamp(0)); + self.inner = self + .inner + .with_fork(reth_optimism_forks::OptimismHardfork::Canyon, ForkCondition::Timestamp(0)); + self + } + + /// Enable Ecotone at genesis + pub fn ecotone_activated(mut self) -> Self { + self = self.canyon_activated(); + self.inner = self.inner.with_fork(EthereumHardfork::Cancun, ForkCondition::Timestamp(0)); + self.inner = self + .inner + .with_fork(reth_optimism_forks::OptimismHardfork::Ecotone, ForkCondition::Timestamp(0)); + self + } + + /// Enable Fjord at genesis + pub fn fjord_activated(mut self) -> Self { + self = self.ecotone_activated(); + self.inner = self + .inner + .with_fork(reth_optimism_forks::OptimismHardfork::Fjord, ForkCondition::Timestamp(0)); + self + } + + /// Enable Granite at genesis + pub fn granite_activated(mut self) -> Self { + self = self.fjord_activated(); + self.inner = self + .inner + .with_fork(reth_optimism_forks::OptimismHardfork::Granite, ForkCondition::Timestamp(0)); + self + } + + /// Build the resulting [`OpChainSpec`]. + /// + /// # Panics + /// + /// This function panics if the chain ID and genesis is not set ([`Self::chain`] and + /// [`Self::genesis`]) + pub fn build(self) -> OpChainSpec { + OpChainSpec { inner: self.inner.build() } + } +} /// OP stack chain spec type. #[derive(Debug, Clone, Deref, Into, Constructor, PartialEq, Eq)] @@ -286,6 +415,8 @@ mod tests { #[test] fn base_mainnet_forkids() { + let base_mainnet = OpChainSpecBuilder::base_mainnet().build(); + let _ = base_mainnet.genesis_hash.set(BASE_MAINNET.genesis_hash.get().copied().unwrap()); test_fork_ids( &BASE_MAINNET, &[ @@ -372,8 +503,12 @@ mod tests { #[test] fn op_mainnet_forkids() { + let op_mainnet = OpChainSpecBuilder::optimism_mainnet().build(); + // for OP mainnet we have to do this because the genesis header can't be properly computed + // from the genesis.json file + let _ = op_mainnet.genesis_hash.set(OP_MAINNET.genesis_hash()); test_fork_ids( - &OP_MAINNET, + &op_mainnet, &[ ( Head { number: 0, ..Default::default() }, @@ -483,9 +618,19 @@ mod tests { ) } + #[test] + fn latest_base_mainnet_fork_id_with_builder() { + let base_mainnet = OpChainSpecBuilder::base_mainnet().build(); + assert_eq!( + ForkId { hash: ForkHash([0xbc, 0x38, 0xf9, 0xca]), next: 0 }, + base_mainnet.latest_fork_id() + ) + } + #[test] fn is_bedrock_active() { - assert!(!OP_MAINNET.is_bedrock_active_at_block(1)) + let op_mainnet = OpChainSpecBuilder::optimism_mainnet().build(); + assert!(!op_mainnet.is_bedrock_active_at_block(1)) } #[test] diff --git a/crates/optimism/evm/src/config.rs b/crates/optimism/evm/src/config.rs index 95d2c9a66fa9..668fcba4ddc4 100644 --- a/crates/optimism/evm/src/config.rs +++ b/crates/optimism/evm/src/config.rs @@ -1,5 +1,5 @@ -use reth_chainspec::ChainSpec; use reth_ethereum_forks::{EthereumHardfork, Head}; +use reth_optimism_chainspec::OpChainSpec; use reth_optimism_forks::OptimismHardfork; /// Returns the revm [`SpecId`](revm_primitives::SpecId) at the given timestamp. @@ -9,7 +9,7 @@ use reth_optimism_forks::OptimismHardfork; /// This is only intended to be used after the Bedrock, when hardforks are activated by /// timestamp. pub fn revm_spec_by_timestamp_after_bedrock( - chain_spec: &ChainSpec, + chain_spec: &OpChainSpec, timestamp: u64, ) -> revm_primitives::SpecId { if chain_spec.fork(OptimismHardfork::Granite).active_at_timestamp(timestamp) { @@ -28,7 +28,7 @@ pub fn revm_spec_by_timestamp_after_bedrock( } /// Map the latest active hardfork at the given block to a revm [`SpecId`](revm_primitives::SpecId). -pub fn revm_spec(chain_spec: &ChainSpec, block: &Head) -> revm_primitives::SpecId { +pub fn revm_spec(chain_spec: &OpChainSpec, block: &Head) -> revm_primitives::SpecId { if chain_spec.fork(OptimismHardfork::Granite).active_at_head(block) { revm_primitives::GRANITE } else if chain_spec.fork(OptimismHardfork::Fjord).active_at_head(block) { @@ -79,12 +79,13 @@ pub fn revm_spec(chain_spec: &ChainSpec, block: &Head) -> revm_primitives::SpecI mod tests { use super::*; use reth_chainspec::ChainSpecBuilder; + use reth_optimism_chainspec::{OpChainSpec, OpChainSpecBuilder}; #[test] fn test_revm_spec_by_timestamp_after_merge() { #[inline(always)] - fn op_cs(f: impl FnOnce(ChainSpecBuilder) -> ChainSpecBuilder) -> ChainSpec { - let cs = ChainSpecBuilder::mainnet().chain(reth_chainspec::Chain::from_id(10)); + fn op_cs(f: impl FnOnce(OpChainSpecBuilder) -> OpChainSpecBuilder) -> OpChainSpec { + let cs = ChainSpecBuilder::mainnet().chain(reth_chainspec::Chain::from_id(10)).into(); f(cs).build() } assert_eq!( @@ -116,8 +117,8 @@ mod tests { #[test] fn test_to_revm_spec() { #[inline(always)] - fn op_cs(f: impl FnOnce(ChainSpecBuilder) -> ChainSpecBuilder) -> ChainSpec { - let cs = ChainSpecBuilder::mainnet().chain(reth_chainspec::Chain::from_id(10)); + fn op_cs(f: impl FnOnce(OpChainSpecBuilder) -> OpChainSpecBuilder) -> OpChainSpec { + let cs = ChainSpecBuilder::mainnet().chain(reth_chainspec::Chain::from_id(10)).into(); f(cs).build() } assert_eq!( diff --git a/crates/optimism/evm/src/execute.rs b/crates/optimism/evm/src/execute.rs index 2491e99b5038..f4abb8c887e2 100644 --- a/crates/optimism/evm/src/execute.rs +++ b/crates/optimism/evm/src/execute.rs @@ -513,8 +513,8 @@ mod tests { use crate::OpChainSpec; use alloy_consensus::TxEip1559; use alloy_primitives::{b256, Address, StorageKey, StorageValue}; - use reth_chainspec::{ChainSpecBuilder, MIN_TRANSACTION_GAS}; - use reth_optimism_chainspec::{optimism_deposit_tx_signature, BASE_MAINNET}; + use reth_chainspec::MIN_TRANSACTION_GAS; + use reth_optimism_chainspec::{optimism_deposit_tx_signature, OpChainSpecBuilder}; use reth_primitives::{Account, Block, BlockBody, Signature, Transaction, TransactionSigned}; use reth_revm::{ database::StateProviderDatabase, test_utils::StateProviderTest, L1_BLOCK_CONTRACT, @@ -548,8 +548,7 @@ mod tests { db } - fn executor_provider(chain_spec: Arc) -> OpExecutorProvider { - let chain_spec = Arc::new(OpChainSpec::new(Arc::unwrap_or_clone(chain_spec))); + fn executor_provider(chain_spec: Arc) -> OpExecutorProvider { OpExecutorProvider { evm_config: OptimismEvmConfig::new(chain_spec.clone()), chain_spec } } @@ -572,11 +571,7 @@ mod tests { let account = Account { balance: U256::MAX, ..Account::default() }; db.insert_account(addr, account, None, HashMap::default()); - let chain_spec = Arc::new( - ChainSpecBuilder::from(&Arc::new(BASE_MAINNET.inner.clone())) - .regolith_activated() - .build(), - ); + let chain_spec = Arc::new(OpChainSpecBuilder::base_mainnet().regolith_activated().build()); let tx = TransactionSigned::from_transaction_and_signature( Transaction::Eip1559(TxEip1559 { @@ -656,11 +651,7 @@ mod tests { db.insert_account(addr, account, None, HashMap::default()); - let chain_spec = Arc::new( - ChainSpecBuilder::from(&Arc::new(BASE_MAINNET.inner.clone())) - .canyon_activated() - .build(), - ); + let chain_spec = Arc::new(OpChainSpecBuilder::base_mainnet().canyon_activated().build()); let tx = TransactionSigned::from_transaction_and_signature( Transaction::Eip1559(TxEip1559 { diff --git a/crates/optimism/node/tests/e2e/utils.rs b/crates/optimism/node/tests/e2e/utils.rs index 1e9ffa652f1c..863bf254e494 100644 --- a/crates/optimism/node/tests/e2e/utils.rs +++ b/crates/optimism/node/tests/e2e/utils.rs @@ -1,15 +1,13 @@ -use std::sync::Arc; - use alloy_genesis::Genesis; use alloy_primitives::{Address, B256}; use reth::{rpc::types::engine::PayloadAttributes, tasks::TaskManager}; -use reth_chainspec::ChainSpecBuilder; use reth_e2e_test_utils::{transaction::TransactionTestContext, wallet::Wallet, NodeHelperType}; -use reth_optimism_chainspec::{OpChainSpec, BASE_MAINNET}; +use reth_optimism_chainspec::OpChainSpecBuilder; use reth_optimism_node::{ node::OptimismAddOns, OptimismBuiltPayload, OptimismNode, OptimismPayloadBuilderAttributes, }; use reth_payload_builder::EthPayloadBuilderAttributes; +use std::sync::Arc; use tokio::sync::Mutex; /// Optimism Node Helper type @@ -19,13 +17,7 @@ pub(crate) async fn setup(num_nodes: usize) -> eyre::Result<(Vec, TaskMa let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); reth_e2e_test_utils::setup( num_nodes, - Arc::new(OpChainSpec::new( - ChainSpecBuilder::default() - .chain(BASE_MAINNET.chain) - .genesis(genesis) - .ecotone_activated() - .build(), - )), + Arc::new(OpChainSpecBuilder::base_mainnet().genesis(genesis).ecotone_activated().build()), false, ) .await From 67c57250770ed3ce5c75b948b8ec4921f9b0be6d Mon Sep 17 00:00:00 2001 From: Delweng Date: Sun, 13 Oct 2024 02:53:33 +0800 Subject: [PATCH 52/82] chore(clippy): fix the very complex type used (#11689) Signed-off-by: jsvisa Co-authored-by: Matthias Seitz --- crates/rpc/rpc-eth-api/src/helpers/block.rs | 9 +++++++-- crates/rpc/rpc-eth-api/src/helpers/call.rs | 6 ++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index d7e081d80f80..898037f4a586 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -13,6 +13,11 @@ use crate::{FromEthApiError, FullEthApiTypes, RpcBlock, RpcReceipt}; use super::{LoadPendingBlock, LoadReceipt, SpawnBlocking}; +/// Result type of the fetched block receipts. +pub type BlockReceiptsResult = Result>>, E>; +/// Result type of the fetched block and its receipts. +pub type BlockAndReceiptsResult = Result>)>, E>; + /// Block related functions for the [`EthApiServer`](crate::EthApiServer) trait in the /// `eth_` namespace. pub trait EthBlocks: LoadBlock { @@ -108,7 +113,7 @@ pub trait EthBlocks: LoadBlock { fn block_receipts( &self, block_id: BlockId, - ) -> impl Future>>, Self::Error>> + Send + ) -> impl Future> + Send where Self: LoadReceipt; @@ -116,7 +121,7 @@ pub trait EthBlocks: LoadBlock { fn load_block_and_receipts( &self, block_id: BlockId, - ) -> impl Future>)>, Self::Error>> + Send + ) -> impl Future> + Send where Self: LoadReceipt, { diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 0aa38a36e9da..5f6f3d77f555 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -41,6 +41,9 @@ use tracing::trace; use super::{LoadBlock, LoadPendingBlock, LoadState, LoadTransaction, SpawnBlocking, Trace}; +/// Result type for `eth_simulateV1` RPC method. +pub type SimulatedBlocksResult = Result>>, E>; + /// Execution related functions for the [`EthApiServer`](crate::EthApiServer) trait in /// the `eth_` namespace. pub trait EthCall: Call + LoadPendingBlock { @@ -62,8 +65,7 @@ pub trait EthCall: Call + LoadPendingBlock { &self, payload: SimulatePayload, block: Option, - ) -> impl Future>>, Self::Error>> - + Send + ) -> impl Future> + Send where Self: LoadBlock + FullEthApiTypes, { From c03399d1eb2f6a54c751661dadf8fa8fa4197ad7 Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Sun, 13 Oct 2024 11:25:56 +0200 Subject: [PATCH 53/82] chore(ci): unpin clippy (#11697) --- .github/workflows/lint.yml | 2 -- crates/storage/provider/src/providers/blockchain_provider.rs | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index fd71a8c636e4..efa38857e06b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,7 +30,6 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@clippy with: - toolchain: nightly-2024-09-25 components: clippy - uses: Swatinem/rust-cache@v2 with: @@ -52,7 +51,6 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@nightly with: - toolchain: nightly-2024-09-25 components: clippy - uses: Swatinem/rust-cache@v2 with: diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index dc4264210b27..9b88cab136f5 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -197,8 +197,9 @@ impl BlockchainProvider2 { for (_, block_body) in block_bodies { let mut block_receipts = Vec::with_capacity(block_body.tx_count as usize); for tx_num in block_body.tx_num_range() { - let receipt = - receipt_iter.next().ok_or(ProviderError::ReceiptNotFound(tx_num.into()))?; + let receipt = receipt_iter + .next() + .ok_or_else(|| ProviderError::ReceiptNotFound(tx_num.into()))?; block_receipts.push(Some(receipt)); } receipts.push(block_receipts); From ffb78b3dc756c15067f25329855b15aca000aee3 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Sun, 13 Oct 2024 12:58:55 +0200 Subject: [PATCH 54/82] refactor(tree): small refac for `BlockBuffer` (#11691) --- crates/blockchain-tree/src/block_buffer.rs | 30 ++++++++++++---------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/crates/blockchain-tree/src/block_buffer.rs b/crates/blockchain-tree/src/block_buffer.rs index e116463e4af6..5d4ca2705cb1 100644 --- a/crates/blockchain-tree/src/block_buffer.rs +++ b/crates/blockchain-tree/src/block_buffer.rs @@ -2,7 +2,7 @@ use crate::metrics::BlockBufferMetrics; use alloy_primitives::{BlockHash, BlockNumber}; use reth_network::cache::LruCache; use reth_primitives::SealedBlockWithSenders; -use std::collections::{btree_map, hash_map, BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; /// Contains the tree of pending blocks that cannot be executed due to missing parent. /// It allows to store unconnected blocks for potential future inclusion. @@ -83,6 +83,7 @@ impl BlockBuffer { } self.metrics.blocks.set(self.blocks.len() as f64); } + /// Removes the given block from the buffer and also all the children of the block. /// /// This is used to get all the blocks that are dependent on the block that is included. @@ -93,10 +94,11 @@ impl BlockBuffer { &mut self, parent_hash: &BlockHash, ) -> Vec { - // remove parent block if present - let mut removed = self.remove_block(parent_hash).into_iter().collect::>(); - - removed.extend(self.remove_children(vec![*parent_hash])); + let removed = self + .remove_block(parent_hash) + .into_iter() + .chain(self.remove_children(vec![*parent_hash])) + .collect(); self.metrics.blocks.set(self.blocks.len() as f64); removed } @@ -126,10 +128,10 @@ impl BlockBuffer { /// Remove block entry fn remove_from_earliest_blocks(&mut self, number: BlockNumber, hash: &BlockHash) { - if let btree_map::Entry::Occupied(mut entry) = self.earliest_blocks.entry(number) { - entry.get_mut().remove(hash); - if entry.get().is_empty() { - entry.remove(); + if let Some(entry) = self.earliest_blocks.get_mut(&number) { + entry.remove(hash); + if entry.is_empty() { + self.earliest_blocks.remove(&number); } } } @@ -137,13 +139,13 @@ impl BlockBuffer { /// Remove from parent child connection. This method does not remove children. fn remove_from_parent(&mut self, parent_hash: BlockHash, hash: &BlockHash) { // remove from parent to child connection, but only for this block parent. - if let hash_map::Entry::Occupied(mut entry) = self.parent_to_child.entry(parent_hash) { - entry.get_mut().remove(hash); + if let Some(entry) = self.parent_to_child.get_mut(&parent_hash) { + entry.remove(hash); // if set is empty remove block entry. - if entry.get().is_empty() { - entry.remove(); + if entry.is_empty() { + self.parent_to_child.remove(&parent_hash); } - }; + } } /// Removes block from inner collections. From 6d8d327129a75f758e207ff15ffa5b6937d87129 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sun, 13 Oct 2024 16:32:52 +0200 Subject: [PATCH 55/82] chore: set request budget to 2 (#11699) Co-authored-by: Oliver --- crates/net/network/src/budget.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/net/network/src/budget.rs b/crates/net/network/src/budget.rs index 6eba076f6b9f..5f5d88861815 100644 --- a/crates/net/network/src/budget.rs +++ b/crates/net/network/src/budget.rs @@ -5,8 +5,8 @@ pub const DEFAULT_BUDGET_TRY_DRAIN_STREAM: u32 = 10; /// Default budget to try and drain headers and bodies download streams. /// -/// Default is 1 iteration. -pub const DEFAULT_BUDGET_TRY_DRAIN_DOWNLOADERS: u32 = 1; +/// Default is 2 iterations. +pub const DEFAULT_BUDGET_TRY_DRAIN_DOWNLOADERS: u32 = 2; /// Default budget to try and drain [`Swarm`](crate::swarm::Swarm). /// From 661b260f6172c047e8530e2331b2c84141e03c2b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 13 Oct 2024 14:39:44 +0000 Subject: [PATCH 56/82] chore(deps): weekly `cargo update` (#11696) Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com> --- Cargo.lock | 352 ++++++++++++++++++++++++++--------------------------- 1 file changed, 170 insertions(+), 182 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22e1360a22e7..538816c7c404 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] @@ -97,10 +97,11 @@ checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "alloy-chains" -version = "0.1.34" +version = "0.1.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8158b4878c67837e5413721cc44298e6a2d88d39203175ea025e51892a16ba4c" +checksum = "156bfc5dcd52ef9a5f33381701fa03310317e14c65093a9430d3e3557b08dcd3" dependencies = [ + "alloy-primitives", "alloy-rlp", "arbitrary", "num_enum", @@ -129,9 +130,9 @@ dependencies = [ [[package]] name = "alloy-dyn-abi" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b499852e1d0e9b8c6db0f24c48998e647c0d5762a01090f955106a7700e4611" +checksum = "f95d76a38cae906fd394a5afb0736aaceee5432efe76addfd71048e623e208af" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -205,9 +206,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a438d4486b5d525df3b3004188f9d5cd1d65cd30ecc41e5a3ccef6f6342e8af9" +checksum = "03c66eec1acdd96b39b995b8f5ee5239bc0c871d62c527ae1ac9fd1d7fecd455" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -297,7 +298,7 @@ dependencies = [ "getrandom 0.2.15", "hashbrown 0.15.0", "hex-literal", - "indexmap 2.5.0", + "indexmap 2.6.0", "itoa", "k256", "keccak-asm", @@ -599,9 +600,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68e7f6e8fe5b443f82b3f1e15abfa191128f71569148428e49449d01f6f49e8b" +checksum = "661c516eb1fa3294cc7f2fb8955b3b609d639c282ac81a4eedb14d3046db503a" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", @@ -613,14 +614,14 @@ dependencies = [ [[package]] name = "alloy-sol-macro-expander" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b96ce28d2fde09abb6135f410c41fad670a3a770b6776869bd852f1df102e6f" +checksum = "ecbabb8fc3d75a0c2cea5215be22e7a267e3efde835b0f2a8922f5e3f5d47683" dependencies = [ "alloy-sol-macro-input", "const-hex", "heck", - "indexmap 2.5.0", + "indexmap 2.6.0", "proc-macro-error2", "proc-macro2", "quote", @@ -631,9 +632,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-input" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "906746396a8296537745711630d9185746c0b50c033d5e9d18b0a6eba3d53f90" +checksum = "16517f2af03064485150d89746b8ffdcdbc9b6eeb3d536fb66efd7c2846fbc75" dependencies = [ "const-hex", "dunce", @@ -646,9 +647,9 @@ dependencies = [ [[package]] name = "alloy-sol-type-parser" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc85178909a49c8827ffccfc9103a7ce1767ae66a801b69bdc326913870bf8e6" +checksum = "c07ebb0c1674ff8cbb08378d7c2e0e27919d2a2dae07ad3bca26174deda8d389" dependencies = [ "serde", "winnow", @@ -656,9 +657,9 @@ dependencies = [ [[package]] name = "alloy-sol-types" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d86a533ce22525969661b25dfe296c112d35eb6861f188fd284f8bd4bb3842ae" +checksum = "8e448d879903624863f608c552d10efb0e0905ddbee98b0049412799911eb062" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -1017,9 +1018,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.12" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" +checksum = "998282f8f49ccd6116b0ed8a4de0fbd3151697920e7c7533416d6e25e76434a7" dependencies = [ "brotli", "flate2", @@ -1204,26 +1205,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bindgen" -version = "0.69.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" -dependencies = [ - "bitflags 2.6.0", - "cexpr", - "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", - "proc-macro2", - "quote", - "regex", - "rustc-hash 1.1.0", - "shlex", - "syn 2.0.79", -] - [[package]] name = "bindgen" version = "0.70.1" @@ -1334,7 +1315,7 @@ dependencies = [ "bitflags 2.6.0", "boa_interner", "boa_macros", - "indexmap 2.5.0", + "indexmap 2.6.0", "num-bigint", "rustc-hash 2.0.0", ] @@ -1360,7 +1341,7 @@ dependencies = [ "fast-float", "hashbrown 0.14.5", "icu_normalizer", - "indexmap 2.5.0", + "indexmap 2.6.0", "intrusive-collections", "itertools 0.13.0", "num-bigint", @@ -1406,7 +1387,7 @@ dependencies = [ "boa_gc", "boa_macros", "hashbrown 0.14.5", - "indexmap 2.5.0", + "indexmap 2.6.0", "once_cell", "phf", "rustc-hash 2.0.0", @@ -1474,9 +1455,9 @@ dependencies = [ [[package]] name = "brotli" -version = "6.0.0" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1526,9 +1507,9 @@ dependencies = [ [[package]] name = "bytemuck_derive" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc8b54b395f2fcfbb3d90c47b01c7f444d94d05bdeb775811dec868ac3bbc26" +checksum = "bcfcc3cd946cb52f0bbfdbbcfa2f4e24f75ebb6c0e1002f7c25904fada18b9ec" dependencies = [ "proc-macro2", "quote", @@ -1620,9 +1601,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.24" +version = "1.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938" +checksum = "b16803a61b81d9eabb7eae2588776c4c1e584b738ede45fdbb4c972cec1e9945" dependencies = [ "jobserver", "libc", @@ -1715,9 +1696,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.19" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7be5744db7978a28d9df86a214130d106a89ce49644cbc4e3f0c22c3fba30615" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", "clap_derive", @@ -1725,9 +1706,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.19" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5fbc17d3ef8278f55b282b2a2e75ae6f6c7d4bb70ed3d0382375104bfafdb4b" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ "anstream", "anstyle", @@ -3140,9 +3121,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -3155,9 +3136,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -3165,15 +3146,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -3182,9 +3163,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" @@ -3203,9 +3184,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -3214,15 +3195,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-timer" @@ -3236,9 +3217,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -3318,9 +3299,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" @@ -3397,7 +3378,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.5.0", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -3448,6 +3429,8 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" dependencies = [ + "allocator-api2", + "equivalent", "foldhash", "serde", ] @@ -3979,13 +3962,13 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "arbitrary", "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.0", "serde", ] @@ -4002,7 +3985,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ "ahash", - "indexmap 2.5.0", + "indexmap 2.6.0", "is-terminal", "itoa", "log", @@ -4100,15 +4083,15 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "iri-string" -version = "0.7.6" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44bd7eced44cfe2cebc674adb2a7124a754a4b5269288d22e9f39f8fada3562d" +checksum = "dc0f0a572e8ffe56e2ff4f769f32ffe919282c3916799f8b68688b6030063bea" dependencies = [ "memchr", "serde", @@ -4140,15 +4123,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -4195,18 +4169,18 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ "wasm-bindgen", ] [[package]] name = "jsonrpsee" -version = "0.24.5" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "126b48a5acc3c52fbd5381a77898cb60e145123179588a29e7ac48f9c06e401b" +checksum = "02f01f48e04e0d7da72280ab787c9943695699c9b32b99158ece105e8ad0afea" dependencies = [ "jsonrpsee-client-transport", "jsonrpsee-core", @@ -4222,9 +4196,9 @@ dependencies = [ [[package]] name = "jsonrpsee-client-transport" -version = "0.24.5" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf679a8e0e083c77997f7c4bb4ca826577105906027ae462aac70ff348d02c6a" +checksum = "d80eccbd47a7b9f1e67663fd846928e941cb49c65236e297dd11c9ea3c5e3387" dependencies = [ "base64 0.22.1", "futures-channel", @@ -4247,9 +4221,9 @@ dependencies = [ [[package]] name = "jsonrpsee-core" -version = "0.24.5" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0e503369a76e195b65af35058add0e6900b794a4e9a9316900ddd3a87a80477" +checksum = "3c2709a32915d816a6e8f625bf72cf74523ebe5d8829f895d6b041b1d3137818" dependencies = [ "async-trait", "bytes", @@ -4274,9 +4248,9 @@ dependencies = [ [[package]] name = "jsonrpsee-http-client" -version = "0.24.5" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c0caba4a6a8efbafeec9baa986aa22a75a96c29d3e4b0091b0098d6470efb5" +checksum = "cc54db939002b030e794fbfc9d5a925aa2854889c5a2f0352b0bffa54681707e" dependencies = [ "async-trait", "base64 0.22.1", @@ -4299,9 +4273,9 @@ dependencies = [ [[package]] name = "jsonrpsee-proc-macros" -version = "0.24.5" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc660a9389e2748e794a40673a4155d501f32db667757cdb80edeff0306b489b" +checksum = "3a9a4b2eaba8cc928f49c4ccf4fcfa65b690a73997682da99ed08f3393b51f07" dependencies = [ "heck", "proc-macro-crate", @@ -4312,9 +4286,9 @@ dependencies = [ [[package]] name = "jsonrpsee-server" -version = "0.24.5" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af6e6c9b6d975edcb443565d648b605f3e85a04ec63aa6941811a8894cc9cded" +checksum = "e30110d0f2d7866c8cc6c86483bdab2eb9f4d2f0e20db55518b2bca84651ba8e" dependencies = [ "futures-util", "http", @@ -4339,9 +4313,9 @@ dependencies = [ [[package]] name = "jsonrpsee-types" -version = "0.24.5" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8fb16314327cbc94fdf7965ef7e4422509cd5597f76d137bd104eb34aeede67" +checksum = "1ca331cd7b3fe95b33432825c2d4c9f5a43963e207fdc01ae67f9fd80ab0930f" dependencies = [ "http", "serde", @@ -4351,9 +4325,9 @@ dependencies = [ [[package]] name = "jsonrpsee-wasm-client" -version = "0.24.5" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0da62b43702bd5640ea305d35df95da30abc878e79a7b4b01feda3beaf35d3c" +checksum = "5c603d97578071dc44d79d3cfaf0775437638fd5adc33c6b622dfe4fa2ec812d" dependencies = [ "jsonrpsee-client-transport", "jsonrpsee-core", @@ -4362,9 +4336,9 @@ dependencies = [ [[package]] name = "jsonrpsee-ws-client" -version = "0.24.5" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39aabf5d6c6f22da8d5b808eea1fab0736059f11fb42f71f141b14f404e5046a" +checksum = "755ca3da1c67671f1fae01cd1a47f41dfb2233a8f19a643e587ab0a663942044" dependencies = [ "http", "jsonrpsee-client-transport", @@ -4465,12 +4439,6 @@ dependencies = [ "spin", ] -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "libc" version = "0.2.159" @@ -4514,11 +4482,11 @@ dependencies = [ [[package]] name = "libproc" -version = "0.14.8" +version = "0.14.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae9ea4b75e1a81675429dafe43441df1caea70081e82246a8cccf514884a88bb" +checksum = "e78a09b56be5adbcad5aa1197371688dc6bb249a26da3bca2011ee2fb987ebfb" dependencies = [ - "bindgen 0.69.4", + "bindgen", "errno", "libc", ] @@ -4640,11 +4608,11 @@ dependencies = [ [[package]] name = "lru" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.0", ] [[package]] @@ -4739,7 +4707,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4f0c8427b39666bf970460908b213ec09b3b350f20c0c2eabcbba51704a08e6" dependencies = [ "base64 0.22.1", - "indexmap 2.5.0", + "indexmap 2.6.0", "metrics", "metrics-util", "quanta", @@ -4748,17 +4716,18 @@ dependencies = [ [[package]] name = "metrics-process" -version = "2.1.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb524e5438255eaa8aa74214d5a62713b77b2c3c6e3c0bbeee65cfd9a58948ba" +checksum = "e69e6ced169644e186e060ddc15f3923fdf06862c811a867bb1e5e7c7824f4d0" dependencies = [ + "libc", "libproc", "mach2", "metrics", "once_cell", - "procfs", + "procfs 0.17.0", "rlimit", - "windows 0.57.0", + "windows 0.58.0", ] [[package]] @@ -5150,21 +5119,18 @@ dependencies = [ [[package]] name = "object" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.20.1" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" -dependencies = [ - "portable-atomic", -] +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "oorandom" @@ -5510,18 +5476,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" dependencies = [ "proc-macro2", "quote", @@ -5777,9 +5743,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" dependencies = [ "unicode-ident", ] @@ -5795,7 +5761,19 @@ dependencies = [ "flate2", "hex", "lazy_static", - "procfs-core", + "procfs-core 0.16.0", + "rustix", +] + +[[package]] +name = "procfs" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" +dependencies = [ + "bitflags 2.6.0", + "hex", + "procfs-core 0.17.0", "rustix", ] @@ -5810,6 +5788,16 @@ dependencies = [ "hex", ] +[[package]] +name = "procfs-core" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" +dependencies = [ + "bitflags 2.6.0", + "hex", +] + [[package]] name = "proptest" version = "1.5.0" @@ -6057,9 +6045,9 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.1.0" +version = "11.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d" +checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0" dependencies = [ "bitflags 2.6.0", ] @@ -7578,7 +7566,7 @@ dependencies = [ "criterion", "dashmap 6.1.0", "derive_more 1.0.0", - "indexmap 2.5.0", + "indexmap 2.6.0", "parking_lot 0.12.3", "pprof", "rand 0.8.5", @@ -7594,7 +7582,7 @@ dependencies = [ name = "reth-mdbx-sys" version = "1.1.0" dependencies = [ - "bindgen 0.70.1", + "bindgen", "cc", ] @@ -7979,7 +7967,7 @@ dependencies = [ "metrics-exporter-prometheus", "metrics-process", "metrics-util", - "procfs", + "procfs 0.16.0", "reqwest", "reth-chainspec", "reth-db-api", @@ -9499,9 +9487,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.13" +version = "0.23.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" +checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" dependencies = [ "log", "once_cell", @@ -9632,18 +9620,18 @@ dependencies = [ [[package]] name = "scc" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836f1e0f4963ef5288b539b643b35e043e76a32d0f4e47e67febf69576527f50" +checksum = "553f8299af7450cda9a52d3a370199904e7a46b5ffd1bef187c4a6af3bb6db69" dependencies = [ "sdd", ] [[package]] name = "schannel" -version = "0.1.24" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" dependencies = [ "windows-sys 0.59.0", ] @@ -9673,9 +9661,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sdd" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a7b59a5d9b0099720b417b6325d91a52cbf5b3dcb5041d864be53eefa58abc" +checksum = "49c1eeaf4b6a87c7479688c6d52b9f1153cedd3c489300564f932b065c6eab95" [[package]] name = "sec1" @@ -9800,7 +9788,7 @@ version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "itoa", "memchr", "ryu", @@ -9852,15 +9840,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.10.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9720086b3357bcb44fce40117d769a4d068c70ecfa190850a980a71755f66fcc" +checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_derive", "serde_json", @@ -9870,9 +9858,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.10.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f1abbfe725f27678f4663bcacb75a83e829fd464c25d78dd038a3a29e307cec" +checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" dependencies = [ "darling", "proc-macro2", @@ -10260,9 +10248,9 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab661c8148c2261222a4d641ad5477fd4bea79406a99056096a0b41b35617a5" +checksum = "20e7b52ad118b2153644eea95c6fc740b6c1555b2344fdab763fc9de4075f665" dependencies = [ "paste", "proc-macro2", @@ -10670,7 +10658,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", @@ -11019,9 +11007,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" @@ -11214,9 +11202,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", "once_cell", @@ -11225,9 +11213,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", @@ -11240,9 +11228,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.43" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" +checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" dependencies = [ "cfg-if", "js-sys", @@ -11252,9 +11240,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -11262,9 +11250,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", @@ -11275,9 +11263,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "wasm-streams" @@ -11294,9 +11282,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.70" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" dependencies = [ "js-sys", "wasm-bindgen", From 176496189dbd5300fc4c778aca671a55345bd386 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 14 Oct 2024 10:29:58 +0200 Subject: [PATCH 57/82] Revert "chore(stages): reduce the progress logging " (#11698) --- .../stages/src/stages/hashing_account.rs | 20 ++++----- .../stages/src/stages/hashing_storage.rs | 20 ++++----- crates/stages/stages/src/stages/headers.rs | 24 +++++----- crates/stages/stages/src/stages/tx_lookup.rs | 8 +--- crates/stages/stages/src/stages/utils.rs | 44 +++++-------------- 5 files changed, 42 insertions(+), 74 deletions(-) diff --git a/crates/stages/stages/src/stages/hashing_account.rs b/crates/stages/stages/src/stages/hashing_account.rs index 2fdccd7837fb..14afb37d81db 100644 --- a/crates/stages/stages/src/stages/hashing_account.rs +++ b/crates/stages/stages/src/stages/hashing_account.rs @@ -1,4 +1,3 @@ -use crate::log_progress; use alloy_primitives::{keccak256, B256}; use itertools::Itertools; use reth_config::config::{EtlConfig, HashingConfig}; @@ -19,7 +18,6 @@ use std::{ fmt::Debug, ops::{Range, RangeInclusive}, sync::mpsc::{self, Receiver}, - time::Instant, }; use tracing::*; @@ -188,16 +186,16 @@ where let mut hashed_account_cursor = tx.cursor_write::>()?; - let total = collector.len(); - let mut last_log = Instant::now(); + let total_hashes = collector.len(); + let interval = (total_hashes / 10).max(1); for (index, item) in collector.iter()?.enumerate() { - log_progress!( - "sync::stages::hashing_account", - index, - total, - last_log, - "Inserting hashes" - ); + if index > 0 && index % interval == 0 { + info!( + target: "sync::stages::hashing_account", + progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), + "Inserting hashes" + ); + } let (key, value) = item?; hashed_account_cursor diff --git a/crates/stages/stages/src/stages/hashing_storage.rs b/crates/stages/stages/src/stages/hashing_storage.rs index 0646032935f1..ef070d30c6d6 100644 --- a/crates/stages/stages/src/stages/hashing_storage.rs +++ b/crates/stages/stages/src/stages/hashing_storage.rs @@ -1,4 +1,3 @@ -use crate::log_progress; use alloy_primitives::{bytes::BufMut, keccak256, B256}; use itertools::Itertools; use reth_config::config::{EtlConfig, HashingConfig}; @@ -20,7 +19,6 @@ use reth_storage_errors::provider::ProviderResult; use std::{ fmt::Debug, sync::mpsc::{self, Receiver}, - time::Instant, }; use tracing::*; @@ -119,17 +117,17 @@ where collect(&mut channels, &mut collector)?; - let total = collector.len(); - let mut last_log = Instant::now(); + let total_hashes = collector.len(); + let interval = (total_hashes / 10).max(1); let mut cursor = tx.cursor_dup_write::()?; for (index, item) in collector.iter()?.enumerate() { - log_progress!( - "sync::stages::hashing_storage", - index, - total, - last_log, - "Inserting hashes" - ); + if index > 0 && index % interval == 0 { + info!( + target: "sync::stages::hashing_storage", + progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), + "Inserting hashes" + ); + } let (addr_key, value) = item?; cursor.append_dup( diff --git a/crates/stages/stages/src/stages/headers.rs b/crates/stages/stages/src/stages/headers.rs index eaa2ea01f0d8..199e015c2dce 100644 --- a/crates/stages/stages/src/stages/headers.rs +++ b/crates/stages/stages/src/stages/headers.rs @@ -1,4 +1,3 @@ -use crate::log_progress; use alloy_primitives::{BlockHash, BlockNumber, Bytes, B256}; use futures_util::StreamExt; use reth_config::config::EtlConfig; @@ -26,7 +25,6 @@ use reth_storage_errors::provider::ProviderError; use std::{ sync::Arc, task::{ready, Context, Poll}, - time::Instant, }; use tokio::sync::watch; use tracing::*; @@ -97,9 +95,9 @@ where provider: &impl DBProvider, static_file_provider: StaticFileProvider, ) -> Result { - let total = self.header_collector.len(); + let total_headers = self.header_collector.len(); - info!(target: "sync::stages::headers", total, "Writing headers"); + info!(target: "sync::stages::headers", total = total_headers, "Writing headers"); // Consistency check of expected headers in static files vs DB is done on provider::sync_gap // when poll_execute_ready is polled. @@ -115,11 +113,13 @@ where // Although headers were downloaded in reverse order, the collector iterates it in ascending // order let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?; - let mut last_log = Instant::now(); + let interval = (total_headers / 10).max(1); for (index, header) in self.header_collector.iter()?.enumerate() { let (_, header_buf) = header?; - log_progress!("sync::stages::headers", index, total, last_log, "Writing headers"); + if index > 0 && index % interval == 0 && total_headers > 100 { + info!(target: "sync::stages::headers", progress = %format!("{:.2}%", (index as f64 / total_headers as f64) * 100.0), "Writing headers"); + } let sealed_header: SealedHeader = bincode::deserialize::>(&header_buf) @@ -147,7 +147,7 @@ where writer.append_header(&header, td, &header_hash)?; } - info!(target: "sync::stages::headers", total, "Writing headers hash index"); + info!(target: "sync::stages::headers", total = total_headers, "Writing headers hash index"); let mut cursor_header_numbers = provider.tx_ref().cursor_write::>()?; @@ -168,13 +168,9 @@ where for (index, hash_to_number) in self.hash_collector.iter()?.enumerate() { let (hash, number) = hash_to_number?; - log_progress!( - "sync::stages::headers", - index, - total, - last_log, - "Writing headers hash index" - ); + if index > 0 && index % interval == 0 && total_headers > 100 { + info!(target: "sync::stages::headers", progress = %format!("{:.2}%", (index as f64 / total_headers as f64) * 100.0), "Writing headers hash index"); + } if first_sync { cursor_header_numbers.append( diff --git a/crates/stages/stages/src/stages/tx_lookup.rs b/crates/stages/stages/src/stages/tx_lookup.rs index c5ccff54c525..60c958abf862 100644 --- a/crates/stages/stages/src/stages/tx_lookup.rs +++ b/crates/stages/stages/src/stages/tx_lookup.rs @@ -1,4 +1,3 @@ -use super::utils::LOG_INTERVAL; use alloy_primitives::{TxHash, TxNumber}; use num_traits::Zero; use reth_config::config::{EtlConfig, TransactionLookupConfig}; @@ -18,7 +17,6 @@ use reth_stages_api::{ UnwindInput, UnwindOutput, }; use reth_storage_errors::provider::ProviderError; -use std::time::Instant; use tracing::*; /// The transaction lookup stage. @@ -149,18 +147,16 @@ where .cursor_write::>()?; let total_hashes = hash_collector.len(); - let mut last_log = Instant::now(); + let interval = (total_hashes / 10).max(1); for (index, hash_to_number) in hash_collector.iter()?.enumerate() { let (hash, number) = hash_to_number?; - let now = Instant::now(); - if now.duration_since(last_log) >= LOG_INTERVAL { + if index > 0 && index % interval == 0 { info!( target: "sync::stages::transaction_lookup", ?append_only, progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), "Inserting hashes" ); - last_log = now; } let key = RawKey::::from_vec(hash); diff --git a/crates/stages/stages/src/stages/utils.rs b/crates/stages/stages/src/stages/utils.rs index af6594cd7861..caf039faca10 100644 --- a/crates/stages/stages/src/stages/utils.rs +++ b/crates/stages/stages/src/stages/utils.rs @@ -12,36 +12,12 @@ use reth_db_api::{ use reth_etl::Collector; use reth_provider::DBProvider; use reth_stages_api::StageError; -use std::{ - collections::HashMap, - hash::Hash, - ops::RangeBounds, - time::{Duration, Instant}, -}; +use std::{collections::HashMap, hash::Hash, ops::RangeBounds}; use tracing::info; /// Number of blocks before pushing indices from cache to [`Collector`] const DEFAULT_CACHE_THRESHOLD: u64 = 100_000; -/// Log interval for progress. -pub(crate) const LOG_INTERVAL: Duration = Duration::from_secs(5); - -/// Log progress at a regular interval. -#[macro_export] -macro_rules! log_progress { - ($target:expr, $index:expr, $total:expr, $last_log:expr, $message:expr) => { - let now = std::time::Instant::now(); - if now.duration_since($last_log) >= $crate::stages::utils::LOG_INTERVAL { - info!( - target: $target, - progress = %format!("{:.2}%", ($index as f64 / $total as f64) * 100.0), - $message - ); - $last_log = now; - } - } -} - /// Collects all history (`H`) indices for a range of changesets (`CS`) and stores them in a /// [`Collector`]. /// @@ -89,16 +65,18 @@ where }; // observability - let total = provider.tx_ref().entries::()?; - let mut last_log = Instant::now(); + let total_changesets = provider.tx_ref().entries::()?; + let interval = (total_changesets / 1000).max(1); let mut flush_counter = 0; let mut current_block_number = u64::MAX; - for (index, entry) in changeset_cursor.walk_range(range)?.enumerate() { + for (idx, entry) in changeset_cursor.walk_range(range)?.enumerate() { let (block_number, key) = partial_key_factory(entry?); cache.entry(key).or_default().push(block_number); - log_progress!("sync::stages::index_history", index, total, last_log, "Collecting indices"); + if idx > 0 && idx % interval == 0 && total_changesets > 1000 { + info!(target: "sync::stages::index_history", progress = %format!("{:.4}%", (idx as f64 / total_changesets as f64) * 100.0), "Collecting indices"); + } // Make sure we only flush the cache every DEFAULT_CACHE_THRESHOLD blocks. if current_block_number != block_number { @@ -142,15 +120,17 @@ where let mut current_list = Vec::::new(); // observability - let total = collector.len(); - let mut last_log = Instant::now(); + let total_entries = collector.len(); + let interval = (total_entries / 100).max(1); for (index, element) in collector.iter()?.enumerate() { let (k, v) = element?; let sharded_key = decode_key(k)?; let new_list = BlockNumberList::decompress_owned(v)?; - log_progress!("sync::stages::index_history", index, total, last_log, "Writing indices"); + if index > 0 && index % interval == 0 && total_entries > 100 { + info!(target: "sync::stages::index_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices"); + } // AccountsHistory: `Address`. // StorageHistory: `Address.StorageKey`. From a049dff0b774f2766ce1bef97020f6b390dadc60 Mon Sep 17 00:00:00 2001 From: Kien Trinh <51135161+kien6034@users.noreply.github.com> Date: Mon, 14 Oct 2024 16:11:24 +0700 Subject: [PATCH 58/82] tests(node-builder): basic exex test update for apply function (#11695) Co-authored-by: Matthias Seitz --- crates/node/builder/src/builder/mod.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/node/builder/src/builder/mod.rs b/crates/node/builder/src/builder/mod.rs index 15bdc730a85d..9f7254bad737 100644 --- a/crates/node/builder/src/builder/mod.rs +++ b/crates/node/builder/src/builder/mod.rs @@ -177,6 +177,11 @@ impl NodeBuilder { pub const fn config(&self) -> &NodeConfig { &self.config } + + /// Returns a mutable reference to the node builder's config. + pub fn config_mut(&mut self) -> &mut NodeConfig { + &mut self.config + } } impl NodeBuilder { From 482468579f6cf3c4ef07fe3cccc22cb88998f035 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Mon, 14 Oct 2024 11:12:29 +0200 Subject: [PATCH 59/82] chore(sdk): define trait `Receipt` (#11643) --- crates/primitives-traits/Cargo.toml | 2 +- crates/primitives-traits/src/lib.rs | 3 +++ crates/primitives-traits/src/receipt.rs | 29 +++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 crates/primitives-traits/src/receipt.rs diff --git a/crates/primitives-traits/Cargo.toml b/crates/primitives-traits/Cargo.toml index 2fec75666568..b34987327ee8 100644 --- a/crates/primitives-traits/Cargo.toml +++ b/crates/primitives-traits/Cargo.toml @@ -64,4 +64,4 @@ arbitrary = [ "dep:proptest", "dep:proptest-arbitrary-interop", ] -serde-bincode-compat = ["serde_with", "alloy-consensus/serde-bincode-compat"] +serde-bincode-compat = ["serde_with", "alloy-consensus/serde-bincode-compat"] \ No newline at end of file diff --git a/crates/primitives-traits/src/lib.rs b/crates/primitives-traits/src/lib.rs index ccc3ea13baf6..9c244226b2c7 100644 --- a/crates/primitives-traits/src/lib.rs +++ b/crates/primitives-traits/src/lib.rs @@ -20,6 +20,9 @@ pub use constants::gas_units::{format_gas, format_gas_throughput}; pub mod account; pub use account::{Account, Bytecode}; +pub mod receipt; +pub use receipt::Receipt; + mod integer_list; pub use integer_list::{IntegerList, IntegerListError}; diff --git a/crates/primitives-traits/src/receipt.rs b/crates/primitives-traits/src/receipt.rs new file mode 100644 index 000000000000..e2d19e4d4ff5 --- /dev/null +++ b/crates/primitives-traits/src/receipt.rs @@ -0,0 +1,29 @@ +//! Receipt abstraction + +use alloc::fmt; + +use alloy_consensus::TxReceipt; +use reth_codecs::Compact; +use serde::{Deserialize, Serialize}; + +/// Helper trait that unifies all behaviour required by receipt to support full node operations. +pub trait FullReceipt: Receipt + Compact {} + +impl FullReceipt for T where T: Receipt + Compact {} + +/// Abstraction of a receipt. +pub trait Receipt: + TxReceipt + + Clone + + fmt::Debug + + PartialEq + + Eq + + Default + + alloy_rlp::Encodable + + alloy_rlp::Decodable + + Serialize + + for<'de> Deserialize<'de> +{ + /// Returns transaction type. + fn tx_type(&self) -> u8; +} From a129f62aaa4ccfa9926498804fa5b70bf6f72784 Mon Sep 17 00:00:00 2001 From: Varun Doshi <61531351+varun-doshi@users.noreply.github.com> Date: Mon, 14 Oct 2024 14:49:04 +0530 Subject: [PATCH 60/82] feat: reset pruned numbers on stage drop (#11491) Co-authored-by: Alexey Shekhirin --- crates/cli/commands/src/stage/drop.rs | 114 ++++++++++++-------------- 1 file changed, 51 insertions(+), 63 deletions(-) diff --git a/crates/cli/commands/src/stage/drop.rs b/crates/cli/commands/src/stage/drop.rs index e324c9851502..9e0396404b37 100644 --- a/crates/cli/commands/src/stage/drop.rs +++ b/crates/cli/commands/src/stage/drop.rs @@ -4,7 +4,7 @@ use clap::Parser; use itertools::Itertools; use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_cli::chainspec::ChainSpecParser; -use reth_db::{static_file::iter_static_files, tables}; +use reth_db::{mdbx::tx::Tx, static_file::iter_static_files, tables, DatabaseError}; use reth_db_api::transaction::{DbTx, DbTxMut}; use reth_db_common::{ init::{insert_genesis_header, insert_genesis_history, insert_genesis_state}, @@ -69,42 +69,28 @@ impl> Command tx.clear::()?; tx.clear::()?; tx.clear::()?; - tx.put::( - StageId::Headers.to_string(), - Default::default(), - )?; + reset_stage_checkpoint(tx, StageId::Headers)?; + insert_genesis_header(&provider_rw.0, &static_file_provider, &self.env.chain)?; } StageEnum::Bodies => { tx.clear::()?; tx.clear::()?; + reset_prune_checkpoint(tx, PruneSegment::Transactions)?; + tx.clear::()?; tx.clear::()?; tx.clear::()?; tx.clear::()?; - tx.put::( - StageId::Bodies.to_string(), - Default::default(), - )?; + reset_stage_checkpoint(tx, StageId::Bodies)?; + insert_genesis_header(&provider_rw.0, &static_file_provider, &self.env.chain)?; } StageEnum::Senders => { tx.clear::()?; // Reset pruned numbers to not count them in the next rerun's stage progress - if let Some(mut prune_checkpoint) = - tx.get::(PruneSegment::SenderRecovery)? - { - prune_checkpoint.block_number = None; - prune_checkpoint.tx_number = None; - tx.put::( - PruneSegment::SenderRecovery, - prune_checkpoint, - )?; - } - tx.put::( - StageId::SenderRecovery.to_string(), - Default::default(), - )?; + reset_prune_checkpoint(tx, PruneSegment::SenderRecovery)?; + reset_stage_checkpoint(tx, StageId::SenderRecovery)?; } StageEnum::Execution => { tx.clear::()?; @@ -113,53 +99,38 @@ impl> Command tx.clear::()?; tx.clear::()?; tx.clear::()?; - tx.put::( - StageId::Execution.to_string(), - Default::default(), - )?; + + reset_prune_checkpoint(tx, PruneSegment::Receipts)?; + reset_prune_checkpoint(tx, PruneSegment::ContractLogs)?; + reset_stage_checkpoint(tx, StageId::Execution)?; + let alloc = &self.env.chain.genesis().alloc; insert_genesis_state(&provider_rw.0, alloc.iter())?; } StageEnum::AccountHashing => { tx.clear::()?; - tx.put::( - StageId::AccountHashing.to_string(), - Default::default(), - )?; + reset_stage_checkpoint(tx, StageId::AccountHashing)?; } StageEnum::StorageHashing => { tx.clear::()?; - tx.put::( - StageId::StorageHashing.to_string(), - Default::default(), - )?; + reset_stage_checkpoint(tx, StageId::StorageHashing)?; } StageEnum::Hashing => { // Clear hashed accounts tx.clear::()?; - tx.put::( - StageId::AccountHashing.to_string(), - Default::default(), - )?; + reset_stage_checkpoint(tx, StageId::AccountHashing)?; // Clear hashed storages tx.clear::()?; - tx.put::( - StageId::StorageHashing.to_string(), - Default::default(), - )?; + reset_stage_checkpoint(tx, StageId::StorageHashing)?; } StageEnum::Merkle => { tx.clear::()?; tx.clear::()?; - tx.put::( - StageId::MerkleExecute.to_string(), - Default::default(), - )?; - tx.put::( - StageId::MerkleUnwind.to_string(), - Default::default(), - )?; + + reset_stage_checkpoint(tx, StageId::MerkleExecute)?; + reset_stage_checkpoint(tx, StageId::MerkleUnwind)?; + tx.delete::( StageId::MerkleExecute.to_string(), None, @@ -168,22 +139,17 @@ impl> Command StageEnum::AccountHistory | StageEnum::StorageHistory => { tx.clear::()?; tx.clear::()?; - tx.put::( - StageId::IndexAccountHistory.to_string(), - Default::default(), - )?; - tx.put::( - StageId::IndexStorageHistory.to_string(), - Default::default(), - )?; + + reset_stage_checkpoint(tx, StageId::IndexAccountHistory)?; + reset_stage_checkpoint(tx, StageId::IndexStorageHistory)?; + insert_genesis_history(&provider_rw.0, self.env.chain.genesis().alloc.iter())?; } StageEnum::TxLookup => { tx.clear::()?; - tx.put::( - StageId::TransactionLookup.to_string(), - Default::default(), - )?; + reset_prune_checkpoint(tx, PruneSegment::TransactionLookup)?; + + reset_stage_checkpoint(tx, StageId::TransactionLookup)?; insert_genesis_header(&provider_rw.0, &static_file_provider, &self.env.chain)?; } } @@ -195,3 +161,25 @@ impl> Command Ok(()) } } + +fn reset_prune_checkpoint( + tx: &Tx, + prune_segment: PruneSegment, +) -> Result<(), DatabaseError> { + if let Some(mut prune_checkpoint) = tx.get::(prune_segment)? { + prune_checkpoint.block_number = None; + prune_checkpoint.tx_number = None; + tx.put::(prune_segment, prune_checkpoint)?; + } + + Ok(()) +} + +fn reset_stage_checkpoint( + tx: &Tx, + stage_id: StageId, +) -> Result<(), DatabaseError> { + tx.put::(stage_id.to_string(), Default::default())?; + + Ok(()) +} From d2233fcc0d2d745e131ace1c4689116792dd8f8c Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Mon, 14 Oct 2024 11:22:27 +0200 Subject: [PATCH 61/82] feat: new `reth-trie-sparse` crate (#11707) --- Cargo.lock | 4 ++++ Cargo.toml | 1 + crates/trie/sparse/Cargo.toml | 12 ++++++++++++ crates/trie/sparse/src/lib.rs | 1 + 4 files changed, 18 insertions(+) create mode 100644 crates/trie/sparse/Cargo.toml create mode 100644 crates/trie/sparse/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 538816c7c404..3e3fd18a8442 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9197,6 +9197,10 @@ dependencies = [ "tracing", ] +[[package]] +name = "reth-trie-sparse" +version = "1.1.0" + [[package]] name = "revm" version = "14.0.3" diff --git a/Cargo.toml b/Cargo.toml index efae22f8ed11..628d7d47b225 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -123,6 +123,7 @@ members = [ "crates/trie/common", "crates/trie/db", "crates/trie/parallel/", + "crates/trie/sparse", "crates/trie/trie", "examples/beacon-api-sidecar-fetcher/", "examples/beacon-api-sse/", diff --git a/crates/trie/sparse/Cargo.toml b/crates/trie/sparse/Cargo.toml new file mode 100644 index 000000000000..4ebb56145e1f --- /dev/null +++ b/crates/trie/sparse/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "reth-trie-sparse" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = "Sparse MPT implementation" + +[lints] +workspace = true diff --git a/crates/trie/sparse/src/lib.rs b/crates/trie/sparse/src/lib.rs new file mode 100644 index 000000000000..5d3d4a5b6f8d --- /dev/null +++ b/crates/trie/sparse/src/lib.rs @@ -0,0 +1 @@ +//! The implementation of sparse MPT. From 7c2c3a6a5c54432d2ea820a77616064338322021 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 14 Oct 2024 11:46:00 +0200 Subject: [PATCH 62/82] fix: respect --debug.terminate --debug.max-block (#11710) --- crates/node/builder/src/launch/common.rs | 9 +++++++++ crates/node/builder/src/launch/engine.rs | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 99e9b29368a0..3e8f92e707c0 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -806,6 +806,15 @@ where Ok(initial_target) } + /// Returns true if the node should terminate after the initial backfill run. + /// + /// This is the case if any of these configs are set: + /// `--debug.max-block` + /// `--debug.terminate` + pub const fn terminate_after_initial_backfill(&self) -> bool { + self.node_config().debug.terminate || self.node_config().debug.max_block.is_some() + } + /// Check if the pipeline is consistent (all stages have the checkpoint block numbers no less /// than the checkpoint of the first stage). /// diff --git a/crates/node/builder/src/launch/engine.rs b/crates/node/builder/src/launch/engine.rs index 46ffacbf717a..782cc7bbb1b0 100644 --- a/crates/node/builder/src/launch/engine.rs +++ b/crates/node/builder/src/launch/engine.rs @@ -334,6 +334,8 @@ where .fuse(); let chainspec = ctx.chain_spec(); let (exit, rx) = oneshot::channel(); + let terminate_after_backfill = ctx.terminate_after_initial_backfill(); + info!(target: "reth::cli", "Starting consensus engine"); ctx.task_executor().spawn_critical("consensus engine", async move { if let Some(initial_target) = initial_target { @@ -357,6 +359,11 @@ where debug!(target: "reth::cli", "Event: {event}"); match event { ChainEvent::BackfillSyncFinished => { + if terminate_after_backfill { + debug!(target: "reth::cli", "Terminating after initial backfill"); + break + } + network_handle.update_sync_state(SyncState::Idle); } ChainEvent::BackfillSyncStarted => { From b6371018944e8cba6f5b4c56e76ff211b9e820b1 Mon Sep 17 00:00:00 2001 From: caglarkaya Date: Mon, 14 Oct 2024 12:50:07 +0300 Subject: [PATCH 63/82] docs: move ExEx book examples (#11616) Co-authored-by: Alexey Shekhirin --- book/developers/exex/remote.md | 349 +----------------- book/developers/exex/tracking-state.md | 138 +------ book/sources/Cargo.toml | 2 + book/sources/exex/remote/Cargo.toml | 52 +++ book/sources/exex/remote/build.rs | 4 + book/sources/exex/remote/proto/exex.proto | 13 + book/sources/exex/remote/src/consumer.rs | 32 ++ book/sources/exex/remote/src/exex.rs | 87 +++++ book/sources/exex/remote/src/exex_1.rs | 40 ++ book/sources/exex/remote/src/exex_2.rs | 49 +++ book/sources/exex/remote/src/exex_3.rs | 65 ++++ book/sources/exex/remote/src/exex_4.rs | 84 +++++ book/sources/exex/remote/src/lib.rs | 3 + book/sources/exex/tracking-state/Cargo.toml | 14 + book/sources/exex/tracking-state/src/bin/1.rs | 57 +++ book/sources/exex/tracking-state/src/bin/2.rs | 73 ++++ 16 files changed, 587 insertions(+), 475 deletions(-) create mode 100644 book/sources/exex/remote/Cargo.toml create mode 100644 book/sources/exex/remote/build.rs create mode 100644 book/sources/exex/remote/proto/exex.proto create mode 100644 book/sources/exex/remote/src/consumer.rs create mode 100644 book/sources/exex/remote/src/exex.rs create mode 100644 book/sources/exex/remote/src/exex_1.rs create mode 100644 book/sources/exex/remote/src/exex_2.rs create mode 100644 book/sources/exex/remote/src/exex_3.rs create mode 100644 book/sources/exex/remote/src/exex_4.rs create mode 100644 book/sources/exex/remote/src/lib.rs create mode 100644 book/sources/exex/tracking-state/Cargo.toml create mode 100644 book/sources/exex/tracking-state/src/bin/1.rs create mode 100644 book/sources/exex/tracking-state/src/bin/2.rs diff --git a/book/developers/exex/remote.md b/book/developers/exex/remote.md index 4344e28b34fc..0ec704308ff4 100644 --- a/book/developers/exex/remote.md +++ b/book/developers/exex/remote.md @@ -25,41 +25,7 @@ We will also need a bunch of dependencies. Some of them you know from the [Hello but some of specific to what we need now. ```toml -[package] -name = "remote-exex" -version = "0.1.0" -edition = "2021" - -[dependencies] -# reth -reth = { git = "https://github.com/paradigmxyz/reth.git" } -reth-exex = { git = "https://github.com/paradigmxyz/reth.git", features = ["serde"] } -reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth.git"} -reth-tracing = { git = "https://github.com/paradigmxyz/reth.git" } - -# async -tokio = { version = "1", features = ["full"] } -tokio-stream = "0.1" -futures-util = "0.3" - -# grpc -tonic = "0.11" -prost = "0.12" -bincode = "1" - -# misc -eyre = "0.6" - -[build-dependencies] -tonic-build = "0.11" - -[[bin]] -name = "exex" -path = "src/exex.rs" - -[[bin]] -name = "consumer" -path = "src/consumer.rs" +{{#include ../../sources/exex/remote/Cargo.toml}} ``` We also added a build dependency for Tonic. We will use it to generate the Rust code for our @@ -87,26 +53,12 @@ For an example of a full schema, see the [Remote ExEx](https://github.com/paradi ```protobuf -syntax = "proto3"; - -package exex; - -service RemoteExEx { - rpc Subscribe(SubscribeRequest) returns (stream ExExNotification) {} -} - -message SubscribeRequest {} - -message ExExNotification { - bytes data = 1; -} +{{#include ../../sources/exex/remote/proto/exex.proto}} ``` To instruct Tonic to generate the Rust code using this `.proto`, add the following lines to your `lib.rs` file: ```rust,norun,noplayground,ignore -pub mod proto { - tonic::include_proto!("exex"); -} +{{#include ../../sources/exex/remote/src/lib.rs}} ``` ## ExEx and gRPC server @@ -119,52 +71,7 @@ Let's create a minimal gRPC server that listens on the port `:10000`, and spawn the [NodeBuilder](https://reth.rs/docs/reth/builder/struct.NodeBuilder.html)'s [task executor](https://reth.rs/docs/reth/tasks/struct.TaskExecutor.html). ```rust,norun,noplayground,ignore -use remote_exex::proto::{ - self, - remote_ex_ex_server::{RemoteExEx, RemoteExExServer}, -}; -use reth_exex::ExExNotification; -use reth_node_ethereum::EthereumNode; -use reth_tracing::tracing::info; -use std::sync::Arc; -use tokio::sync::{broadcast, mpsc}; -use tokio_stream::wrappers::ReceiverStream; -use tonic::{transport::Server, Request, Response, Status}; - -struct ExExService {} - -#[tonic::async_trait] -impl RemoteExEx for ExExService { - type SubscribeStream = ReceiverStream>; - - async fn subscribe( - &self, - _request: Request, - ) -> Result, Status> { - let (_tx, rx) = mpsc::channel(1); - - Ok(Response::new(ReceiverStream::new(rx))) - } -} - -fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _| async move { - let server = Server::builder() - .add_service(RemoteExExServer::new(ExExService {})) - .serve("[::1]:10000".parse().unwrap()); - - let handle = builder.node(EthereumNode::default()).launch().await?; - - handle - .node - .task_executor - .spawn_critical("gRPC server", async move { - server.await.expect("failed to start gRPC server") - }); - - handle.wait_for_node_exit().await - }) -} +{{#include ../../sources/exex/remote/src/exex_1.rs}} ``` Currently, it does not send anything on the stream. @@ -175,40 +82,7 @@ Let's create this channel in the `main` function where we will have both gRPC se and save the sender part (that way we will be able to create new receivers) of this channel in our gRPC server. ```rust,norun,noplayground,ignore -// ... -use reth_exex::{ExExNotification}; - -struct ExExService { - notifications: Arc>, -} - -... - -fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _| async move { - let notifications = Arc::new(broadcast::channel(1).0); - - let server = Server::builder() - .add_service(RemoteExExServer::new(ExExService { - notifications: notifications.clone(), - })) - .serve("[::1]:10000".parse().unwrap()); - - let handle = builder - .node(EthereumNode::default()) - .launch() - .await?; - - handle - .node - .task_executor - .spawn_critical("gRPC server", async move { - server.await.expect("failed to start gRPC server") - }); - - handle.wait_for_node_exit().await - }) -} +{{#include ../../sources/exex/remote/src/exex_2.rs}} ``` And with that, we're ready to handle incoming notifications, serialize them with [bincode](https://docs.rs/bincode/) @@ -218,37 +92,7 @@ For each incoming request, we spawn a separate tokio task that will run in the b and then return the stream receiver to the client. ```rust,norun,noplayground,ignore -// ... - -#[tonic::async_trait] -impl RemoteExEx for ExExService { - type SubscribeStream = ReceiverStream>; - - async fn subscribe( - &self, - _request: Request, - ) -> Result, Status> { - let (tx, rx) = mpsc::channel(1); - - let mut notifications = self.notifications.subscribe(); - tokio::spawn(async move { - while let Ok(notification) = notifications.recv().await { - let proto_notification = proto::ExExNotification { - data: bincode::serialize(¬ification).expect("failed to serialize"), - }; - tx.send(Ok(proto_notification)) - .await - .expect("failed to send notification to client"); - - info!("Notification sent to the gRPC client"); - } - }); - - Ok(Response::new(ReceiverStream::new(rx))) - } -} - -// ... +{{#rustdoc_include ../../sources/exex/remote/src/exex_3.rs:snippet}} ``` That's it for the gRPC server part! It doesn't receive anything on the `notifications` channel yet, @@ -267,65 +111,14 @@ Don't forget to emit `ExExEvent::FinishedHeight` ```rust,norun,noplayground,ignore -// ... - -use futures_util::StreamExt; -use reth_exex::{ExExContext, ExExEvent}; - -async fn remote_exex( - mut ctx: ExExContext, - notifications: Arc>, -) -> eyre::Result<()> { - while let Some(notification) = ctx.notifications.next().await { - if let Some(committed_chain) = notification.committed_chain() { - ctx.events - .send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; - } - - info!("Notification sent to the gRPC server"); - let _ = notifications.send(notification); - } - - Ok(()) -} - -// ... +{{#rustdoc_include ../../sources/exex/remote/src/exex_4.rs:snippet}} ``` All that's left is to connect all pieces together: install our ExEx in the node and pass the sender part of communication channel to it. ```rust,norun,noplayground,ignore -// ... - -fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _| async move { - let notifications = Arc::new(broadcast::channel(1).0); - - let server = Server::builder() - .add_service(RemoteExExServer::new(ExExService { - notifications: notifications.clone(), - })) - .serve("[::1]:10000".parse().unwrap()); - - let handle = builder - .node(EthereumNode::default()) - .install_exex("remote-exex", |ctx| async move { - Ok(remote_exex(ctx, notifications)) - }) - .launch() - .await?; - - handle - .node - .task_executor - .spawn_critical("gRPC server", async move { - server.await.expect("failed to start gRPC server") - }); - - handle.wait_for_node_exit().await - }) -} +{{#rustdoc_include ../../sources/exex/remote/src/exex.rs:snippet}} ``` ### Full `exex.rs` code @@ -334,98 +127,7 @@ fn main() -> eyre::Result<()> { Click to expand ```rust,norun,noplayground,ignore -use std::sync::Arc; - -use futures_util::StreamExt; -use remote_exex::proto::{ - self, - remote_ex_ex_server::{RemoteExEx, RemoteExExServer}, -}; -use reth::api::FullNodeComponents; -use reth_exex::{ExExContext, ExExEvent, ExExNotification}; -use reth_node_ethereum::EthereumNode; -use reth_tracing::tracing::info; -use tokio::sync::{broadcast, mpsc}; -use tokio_stream::wrappers::ReceiverStream; -use tonic::{transport::Server, Request, Response, Status}; - -struct ExExService { - notifications: Arc>, -} - -#[tonic::async_trait] -impl RemoteExEx for ExExService { - type SubscribeStream = ReceiverStream>; - - async fn subscribe( - &self, - _request: Request, - ) -> Result, Status> { - let (tx, rx) = mpsc::channel(1); - - let mut notifications = self.notifications.subscribe(); - tokio::spawn(async move { - while let Ok(notification) = notifications.recv().await { - let proto_notification = proto::ExExNotification { - data: bincode::serialize(¬ification).expect("failed to serialize"), - }; - tx.send(Ok(proto_notification)) - .await - .expect("failed to send notification to client"); - - info!(?notification, "Notification sent to the gRPC client"); - } - }); - - Ok(Response::new(ReceiverStream::new(rx))) - } -} - -async fn remote_exex( - mut ctx: ExExContext, - notifications: Arc>, -) -> eyre::Result<()> { - while let Some(notification) = ctx.notifications.next().await { - if let Some(committed_chain) = notification.committed_chain() { - ctx.events - .send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; - } - - info!(?notification, "Notification sent to the gRPC server"); - let _ = notifications.send(notification); - } - - Ok(()) -} - -fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _| async move { - let notifications = Arc::new(broadcast::channel(1).0); - - let server = Server::builder() - .add_service(RemoteExExServer::new(ExExService { - notifications: notifications.clone(), - })) - .serve("[::1]:10000".parse().unwrap()); - - let handle = builder - .node(EthereumNode::default()) - .install_exex("remote-exex", |ctx| async move { - Ok(remote_exex(ctx, notifications)) - }) - .launch() - .await?; - - handle - .node - .task_executor - .spawn_critical("gRPC server", async move { - server.await.expect("failed to start gRPC server") - }); - - handle.wait_for_node_exit().await - }) -} +{{#include ../../sources/exex/remote/src/exex.rs}} ``` @@ -442,38 +144,7 @@ because notifications can get very heavy ```rust,norun,noplayground,ignore -use remote_exex::proto::{remote_ex_ex_client::RemoteExExClient, SubscribeRequest}; -use reth_exex::ExExNotification; -use reth_tracing::{tracing::info, RethTracer, Tracer}; - -#[tokio::main] -async fn main() -> eyre::Result<()> { - let _ = RethTracer::new().init()?; - - let mut client = RemoteExExClient::connect("http://[::1]:10000") - .await? - .max_encoding_message_size(usize::MAX) - .max_decoding_message_size(usize::MAX); - - let mut stream = client.subscribe(SubscribeRequest {}).await?.into_inner(); - while let Some(notification) = stream.message().await? { - let notification: ExExNotification = bincode::deserialize(¬ification.data)?; - - match notification { - ExExNotification::ChainCommitted { new } => { - info!(committed_chain = ?new.range(), "Received commit"); - } - ExExNotification::ChainReorged { old, new } => { - info!(from_chain = ?old.range(), to_chain = ?new.range(), "Received reorg"); - } - ExExNotification::ChainReverted { old } => { - info!(reverted_chain = ?old.range(), "Received revert"); - } - }; - } - - Ok(()) -} +{{#include ../../sources/exex/remote/src/consumer.rs}} ``` ## Running diff --git a/book/developers/exex/tracking-state.md b/book/developers/exex/tracking-state.md index 52c73e618029..d2a9fe6ca3e3 100644 --- a/book/developers/exex/tracking-state.md +++ b/book/developers/exex/tracking-state.md @@ -19,63 +19,7 @@ because you can't access variables inside the function to assert the state of yo ```rust,norun,noplayground,ignore -use std::{ - future::Future, - pin::Pin, - task::{ready, Context, Poll}, -}; - -use futures_util::StreamExt; -use reth::api::FullNodeComponents; -use reth_exex::{ExExContext, ExExEvent, ExExNotification}; -use reth_node_ethereum::EthereumNode; -use reth_tracing::tracing::info; - -struct MyExEx { - ctx: ExExContext, -} - -impl Future for MyExEx { - type Output = eyre::Result<()>; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let this = self.get_mut(); - - while let Some(notification) = ready!(this.ctx.notifications.poll_next_unpin(cx)) { - match ¬ification { - ExExNotification::ChainCommitted { new } => { - info!(committed_chain = ?new.range(), "Received commit"); - } - ExExNotification::ChainReorged { old, new } => { - info!(from_chain = ?old.range(), to_chain = ?new.range(), "Received reorg"); - } - ExExNotification::ChainReverted { old } => { - info!(reverted_chain = ?old.range(), "Received revert"); - } - }; - - if let Some(committed_chain) = notification.committed_chain() { - this.ctx - .events - .send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; - } - } - - Poll::Ready(Ok(())) - } -} - -fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _| async move { - let handle = builder - .node(EthereumNode::default()) - .install_exex("my-exex", |ctx| async move { Ok(MyExEx { ctx }) }) - .launch() - .await?; - - handle.wait_for_node_exit().await - }) -} +{{#include ../../sources/exex/tracking-state/src/bin/1.rs}} ``` For those who are not familiar with how async Rust works on a lower level, that may seem scary, @@ -96,85 +40,7 @@ With all that done, we're now free to add more fields to our `MyExEx` struct, an Our ExEx will count the number of transactions in each block and log it to the console. ```rust,norun,noplayground,ignore -use std::{ - future::Future, - pin::Pin, - task::{ready, Context, Poll}, -}; - -use futures_util::StreamExt; -use reth::{api::FullNodeComponents, primitives::BlockNumber}; -use reth_exex::{ExExContext, ExExEvent}; -use reth_node_ethereum::EthereumNode; -use reth_tracing::tracing::info; - -struct MyExEx { - ctx: ExExContext, - /// First block that was committed since the start of the ExEx. - first_block: Option, - /// Total number of transactions committed. - transactions: u64, -} - -impl MyExEx { - fn new(ctx: ExExContext) -> Self { - Self { - ctx, - first_block: None, - transactions: 0, - } - } -} - -impl Future for MyExEx { - type Output = eyre::Result<()>; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let this = self.get_mut(); - - while let Some(notification) = ready!(this.ctx.notifications.poll_next_unpin(cx)) { - if let Some(reverted_chain) = notification.reverted_chain() { - this.transactions = this.transactions.saturating_sub( - reverted_chain - .blocks_iter() - .map(|b| b.body.len() as u64) - .sum(), - ); - } - - if let Some(committed_chain) = notification.committed_chain() { - this.first_block.get_or_insert(committed_chain.first().number); - - this.transactions += committed_chain - .blocks_iter() - .map(|b| b.body.len() as u64) - .sum::(); - - this.ctx - .events - .send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; - } - - if let Some(first_block) = this.first_block { - info!(%first_block, transactions = %this.transactions, "Total number of transactions"); - } - } - - Poll::Ready(Ok(())) - } -} - -fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _| async move { - let handle = builder - .node(EthereumNode::default()) - .install_exex("my-exex", |ctx| async move { Ok(MyExEx::new(ctx)) }) - .launch() - .await?; - - handle.wait_for_node_exit().await - }) -} +{{#include ../../sources/exex/tracking-state/src/bin/2.rs}} ``` As you can see, we added two fields to our ExEx struct: diff --git a/book/sources/Cargo.toml b/book/sources/Cargo.toml index c04c8567f94d..1529af952b92 100644 --- a/book/sources/Cargo.toml +++ b/book/sources/Cargo.toml @@ -1,6 +1,8 @@ [workspace] members = [ "exex/hello-world", + "exex/remote", + "exex/tracking-state", ] # Explicitly set the resolver to version 2, which is the default for packages with edition >= 2021 diff --git a/book/sources/exex/remote/Cargo.toml b/book/sources/exex/remote/Cargo.toml new file mode 100644 index 000000000000..6eeb848cacfe --- /dev/null +++ b/book/sources/exex/remote/Cargo.toml @@ -0,0 +1,52 @@ +[package] +name = "remote-exex" +version = "0.1.0" +edition = "2021" + +[dependencies] +# reth +reth = { git = "https://github.com/paradigmxyz/reth.git" } +reth-exex = { git = "https://github.com/paradigmxyz/reth.git", features = ["serde"] } +reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth.git"} +reth-node-api = { git = "https://github.com/paradigmxyz/reth.git"} +reth-tracing = { git = "https://github.com/paradigmxyz/reth.git" } + +# async +tokio = { version = "1", features = ["full"] } +tokio-stream = "0.1" +futures-util = "0.3" + +# grpc +tonic = "0.11" +prost = "0.12" +bincode = "1" + +# misc +eyre = "0.6" + +[build-dependencies] +tonic-build = "0.11" + +[[bin]] +name = "exex_1" +path = "src/exex_1.rs" + +[[bin]] +name = "exex_2" +path = "src/exex_2.rs" + +[[bin]] +name = "exex_3" +path = "src/exex_3.rs" + +[[bin]] +name = "exex_4" +path = "src/exex_4.rs" + +[[bin]] +name = "exex" +path = "src/exex.rs" + +[[bin]] +name = "consumer" +path = "src/consumer.rs" \ No newline at end of file diff --git a/book/sources/exex/remote/build.rs b/book/sources/exex/remote/build.rs new file mode 100644 index 000000000000..8e66f2a30a6b --- /dev/null +++ b/book/sources/exex/remote/build.rs @@ -0,0 +1,4 @@ +fn main() -> Result<(), Box> { + tonic_build::compile_protos("proto/exex.proto")?; + Ok(()) +} diff --git a/book/sources/exex/remote/proto/exex.proto b/book/sources/exex/remote/proto/exex.proto new file mode 100644 index 000000000000..9bb180b5f215 --- /dev/null +++ b/book/sources/exex/remote/proto/exex.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package exex; + +service RemoteExEx { + rpc Subscribe(SubscribeRequest) returns (stream ExExNotification) {} +} + +message SubscribeRequest {} + +message ExExNotification { + bytes data = 1; +} \ No newline at end of file diff --git a/book/sources/exex/remote/src/consumer.rs b/book/sources/exex/remote/src/consumer.rs new file mode 100644 index 000000000000..a0400f4bbfde --- /dev/null +++ b/book/sources/exex/remote/src/consumer.rs @@ -0,0 +1,32 @@ +use remote_exex::proto::{remote_ex_ex_client::RemoteExExClient, SubscribeRequest}; +use reth_exex::ExExNotification; +use reth_tracing::{tracing::info, RethTracer, Tracer}; + +#[tokio::main] +async fn main() -> eyre::Result<()> { + let _ = RethTracer::new().init()?; + + let mut client = RemoteExExClient::connect("http://[::1]:10000") + .await? + .max_encoding_message_size(usize::MAX) + .max_decoding_message_size(usize::MAX); + + let mut stream = client.subscribe(SubscribeRequest {}).await?.into_inner(); + while let Some(notification) = stream.message().await? { + let notification: ExExNotification = bincode::deserialize(¬ification.data)?; + + match notification { + ExExNotification::ChainCommitted { new } => { + info!(committed_chain = ?new.range(), "Received commit"); + } + ExExNotification::ChainReorged { old, new } => { + info!(from_chain = ?old.range(), to_chain = ?new.range(), "Received reorg"); + } + ExExNotification::ChainReverted { old } => { + info!(reverted_chain = ?old.range(), "Received revert"); + } + }; + } + + Ok(()) +} diff --git a/book/sources/exex/remote/src/exex.rs b/book/sources/exex/remote/src/exex.rs new file mode 100644 index 000000000000..1ae4785db8b8 --- /dev/null +++ b/book/sources/exex/remote/src/exex.rs @@ -0,0 +1,87 @@ +use futures_util::TryStreamExt; +use remote_exex::proto::{ + self, + remote_ex_ex_server::{RemoteExEx, RemoteExExServer}, +}; +use reth_exex::{ExExContext, ExExEvent, ExExNotification}; +use reth_node_api::FullNodeComponents; +use reth_node_ethereum::EthereumNode; +use reth_tracing::tracing::info; +use std::sync::Arc; +use tokio::sync::{broadcast, mpsc}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{transport::Server, Request, Response, Status}; + +struct ExExService { + notifications: Arc>, +} + +#[tonic::async_trait] +impl RemoteExEx for ExExService { + type SubscribeStream = ReceiverStream>; + + async fn subscribe( + &self, + _request: Request, + ) -> Result, Status> { + let (tx, rx) = mpsc::channel(1); + + let mut notifications = self.notifications.subscribe(); + tokio::spawn(async move { + while let Ok(notification) = notifications.recv().await { + let proto_notification = proto::ExExNotification { + data: bincode::serialize(¬ification).expect("failed to serialize"), + }; + tx.send(Ok(proto_notification)) + .await + .expect("failed to send notification to client"); + + info!("Notification sent to the gRPC client"); + } + }); + + Ok(Response::new(ReceiverStream::new(rx))) + } +} + +async fn remote_exex( + mut ctx: ExExContext, + notifications: Arc>, +) -> eyre::Result<()> { + while let Some(notification) = ctx.notifications.try_next().await? { + if let Some(committed_chain) = notification.committed_chain() { + ctx.events.send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; + } + + info!("Notification sent to the gRPC server"); + let _ = notifications.send(notification); + } + + Ok(()) +} + +// ANCHOR: snippet +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let notifications = Arc::new(broadcast::channel(1).0); + + let server = Server::builder() + .add_service(RemoteExExServer::new(ExExService { + notifications: notifications.clone(), + })) + .serve("[::1]:10000".parse().unwrap()); + + let handle = builder + .node(EthereumNode::default()) + .install_exex("remote-exex", |ctx| async move { Ok(remote_exex(ctx, notifications)) }) + .launch() + .await?; + + handle.node.task_executor.spawn_critical("gRPC server", async move { + server.await.expect("failed to start gRPC server") + }); + + handle.wait_for_node_exit().await + }) +} +// ANCHOR_END: snippet diff --git a/book/sources/exex/remote/src/exex_1.rs b/book/sources/exex/remote/src/exex_1.rs new file mode 100644 index 000000000000..09a4bcc064d5 --- /dev/null +++ b/book/sources/exex/remote/src/exex_1.rs @@ -0,0 +1,40 @@ +use remote_exex::proto::{ + self, + remote_ex_ex_server::{RemoteExEx, RemoteExExServer}, +}; +use reth_node_ethereum::EthereumNode; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{transport::Server, Request, Response, Status}; + +struct ExExService {} + +#[tonic::async_trait] +impl RemoteExEx for ExExService { + type SubscribeStream = ReceiverStream>; + + async fn subscribe( + &self, + _request: Request, + ) -> Result, Status> { + let (_tx, rx) = mpsc::channel(1); + + Ok(Response::new(ReceiverStream::new(rx))) + } +} + +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let server = Server::builder() + .add_service(RemoteExExServer::new(ExExService {})) + .serve("[::1]:10000".parse().unwrap()); + + let handle = builder.node(EthereumNode::default()).launch().await?; + + handle.node.task_executor.spawn_critical("gRPC server", async move { + server.await.expect("failed to start gRPC server") + }); + + handle.wait_for_node_exit().await + }) +} diff --git a/book/sources/exex/remote/src/exex_2.rs b/book/sources/exex/remote/src/exex_2.rs new file mode 100644 index 000000000000..c4f51ddaea33 --- /dev/null +++ b/book/sources/exex/remote/src/exex_2.rs @@ -0,0 +1,49 @@ +use remote_exex::proto::{ + self, + remote_ex_ex_server::{RemoteExEx, RemoteExExServer}, +}; +use reth_exex::ExExNotification; +use reth_node_ethereum::EthereumNode; +use std::sync::Arc; +use tokio::sync::{broadcast, mpsc}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{transport::Server, Request, Response, Status}; + +#[allow(dead_code)] +struct ExExService { + notifications: Arc>, +} + +#[tonic::async_trait] +impl RemoteExEx for ExExService { + type SubscribeStream = ReceiverStream>; + + async fn subscribe( + &self, + _request: Request, + ) -> Result, Status> { + let (_tx, rx) = mpsc::channel(1); + + Ok(Response::new(ReceiverStream::new(rx))) + } +} + +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let notifications = Arc::new(broadcast::channel(1).0); + + let server = Server::builder() + .add_service(RemoteExExServer::new(ExExService { + notifications: notifications.clone(), + })) + .serve("[::1]:10000".parse().unwrap()); + + let handle = builder.node(EthereumNode::default()).launch().await?; + + handle.node.task_executor.spawn_critical("gRPC server", async move { + server.await.expect("failed to start gRPC server") + }); + + handle.wait_for_node_exit().await + }) +} diff --git a/book/sources/exex/remote/src/exex_3.rs b/book/sources/exex/remote/src/exex_3.rs new file mode 100644 index 000000000000..9f264cf34531 --- /dev/null +++ b/book/sources/exex/remote/src/exex_3.rs @@ -0,0 +1,65 @@ +use remote_exex::proto::{ + self, + remote_ex_ex_server::{RemoteExEx, RemoteExExServer}, +}; +use reth_exex::ExExNotification; +use reth_node_ethereum::EthereumNode; +use reth_tracing::tracing::info; +use std::sync::Arc; +use tokio::sync::{broadcast, mpsc}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{transport::Server, Request, Response, Status}; + +struct ExExService { + notifications: Arc>, +} + +// ANCHOR: snippet +#[tonic::async_trait] +impl RemoteExEx for ExExService { + type SubscribeStream = ReceiverStream>; + + async fn subscribe( + &self, + _request: Request, + ) -> Result, Status> { + let (tx, rx) = mpsc::channel(1); + + let mut notifications = self.notifications.subscribe(); + tokio::spawn(async move { + while let Ok(notification) = notifications.recv().await { + let proto_notification = proto::ExExNotification { + data: bincode::serialize(¬ification).expect("failed to serialize"), + }; + tx.send(Ok(proto_notification)) + .await + .expect("failed to send notification to client"); + + info!("Notification sent to the gRPC client"); + } + }); + + Ok(Response::new(ReceiverStream::new(rx))) + } +} +// ANCHOR_END: snippet + +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let notifications = Arc::new(broadcast::channel(1).0); + + let server = Server::builder() + .add_service(RemoteExExServer::new(ExExService { + notifications: notifications.clone(), + })) + .serve("[::1]:10000".parse().unwrap()); + + let handle = builder.node(EthereumNode::default()).launch().await?; + + handle.node.task_executor.spawn_critical("gRPC server", async move { + server.await.expect("failed to start gRPC server") + }); + + handle.wait_for_node_exit().await + }) +} diff --git a/book/sources/exex/remote/src/exex_4.rs b/book/sources/exex/remote/src/exex_4.rs new file mode 100644 index 000000000000..24c7bf2c2f11 --- /dev/null +++ b/book/sources/exex/remote/src/exex_4.rs @@ -0,0 +1,84 @@ +use futures_util::TryStreamExt; +use remote_exex::proto::{ + self, + remote_ex_ex_server::{RemoteExEx, RemoteExExServer}, +}; +use reth_exex::{ExExContext, ExExEvent, ExExNotification}; +use reth_node_api::FullNodeComponents; +use reth_node_ethereum::EthereumNode; +use reth_tracing::tracing::info; +use std::sync::Arc; +use tokio::sync::{broadcast, mpsc}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{transport::Server, Request, Response, Status}; + +struct ExExService { + notifications: Arc>, +} + +#[tonic::async_trait] +impl RemoteExEx for ExExService { + type SubscribeStream = ReceiverStream>; + + async fn subscribe( + &self, + _request: Request, + ) -> Result, Status> { + let (tx, rx) = mpsc::channel(1); + + let mut notifications = self.notifications.subscribe(); + tokio::spawn(async move { + while let Ok(notification) = notifications.recv().await { + let proto_notification = proto::ExExNotification { + data: bincode::serialize(¬ification).expect("failed to serialize"), + }; + tx.send(Ok(proto_notification)) + .await + .expect("failed to send notification to client"); + + info!("Notification sent to the gRPC client"); + } + }); + + Ok(Response::new(ReceiverStream::new(rx))) + } +} + +// ANCHOR: snippet +#[allow(dead_code)] +async fn remote_exex( + mut ctx: ExExContext, + notifications: Arc>, +) -> eyre::Result<()> { + while let Some(notification) = ctx.notifications.try_next().await? { + if let Some(committed_chain) = notification.committed_chain() { + ctx.events.send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; + } + + info!("Notification sent to the gRPC server"); + let _ = notifications.send(notification); + } + + Ok(()) +} +// ANCHOR_END: snippet + +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let notifications = Arc::new(broadcast::channel(1).0); + + let server = Server::builder() + .add_service(RemoteExExServer::new(ExExService { + notifications: notifications.clone(), + })) + .serve("[::1]:10000".parse().unwrap()); + + let handle = builder.node(EthereumNode::default()).launch().await?; + + handle.node.task_executor.spawn_critical("gRPC server", async move { + server.await.expect("failed to start gRPC server") + }); + + handle.wait_for_node_exit().await + }) +} diff --git a/book/sources/exex/remote/src/lib.rs b/book/sources/exex/remote/src/lib.rs new file mode 100644 index 000000000000..9abb458bd3c0 --- /dev/null +++ b/book/sources/exex/remote/src/lib.rs @@ -0,0 +1,3 @@ +pub mod proto { + tonic::include_proto!("exex"); +} diff --git a/book/sources/exex/tracking-state/Cargo.toml b/book/sources/exex/tracking-state/Cargo.toml new file mode 100644 index 000000000000..3ce21b0c3403 --- /dev/null +++ b/book/sources/exex/tracking-state/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "tracking-state" +version = "0.1.0" +edition = "2021" + +[dependencies] +reth = { git = "https://github.com/paradigmxyz/reth.git" } +reth-exex = { git = "https://github.com/paradigmxyz/reth.git", features = ["serde"] } +reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth.git"} +reth-tracing = { git = "https://github.com/paradigmxyz/reth.git" } + +eyre = "0.6" # Easy error handling +futures-util = "0.3" # Stream utilities for consuming notifications +alloy-primitives = "0.8.7" diff --git a/book/sources/exex/tracking-state/src/bin/1.rs b/book/sources/exex/tracking-state/src/bin/1.rs new file mode 100644 index 000000000000..0d42e0791a17 --- /dev/null +++ b/book/sources/exex/tracking-state/src/bin/1.rs @@ -0,0 +1,57 @@ +use std::{ + future::Future, + pin::Pin, + task::{ready, Context, Poll}, +}; + +use futures_util::{FutureExt, TryStreamExt}; +use reth::api::FullNodeComponents; +use reth_exex::{ExExContext, ExExEvent, ExExNotification}; +use reth_node_ethereum::EthereumNode; +use reth_tracing::tracing::info; + +struct MyExEx { + ctx: ExExContext, +} + +impl Future for MyExEx { + type Output = eyre::Result<()>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + while let Some(notification) = ready!(this.ctx.notifications.try_next().poll_unpin(cx))? { + match ¬ification { + ExExNotification::ChainCommitted { new } => { + info!(committed_chain = ?new.range(), "Received commit"); + } + ExExNotification::ChainReorged { old, new } => { + info!(from_chain = ?old.range(), to_chain = ?new.range(), "Received reorg"); + } + ExExNotification::ChainReverted { old } => { + info!(reverted_chain = ?old.range(), "Received revert"); + } + }; + + if let Some(committed_chain) = notification.committed_chain() { + this.ctx + .events + .send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; + } + } + + Poll::Ready(Ok(())) + } +} + +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let handle = builder + .node(EthereumNode::default()) + .install_exex("my-exex", |ctx| async move { Ok(MyExEx { ctx }) }) + .launch() + .await?; + + handle.wait_for_node_exit().await + }) +} diff --git a/book/sources/exex/tracking-state/src/bin/2.rs b/book/sources/exex/tracking-state/src/bin/2.rs new file mode 100644 index 000000000000..9416810668f5 --- /dev/null +++ b/book/sources/exex/tracking-state/src/bin/2.rs @@ -0,0 +1,73 @@ +use std::{ + future::Future, + pin::Pin, + task::{ready, Context, Poll}, +}; + +use alloy_primitives::BlockNumber; +use futures_util::{FutureExt, TryStreamExt}; +use reth::api::FullNodeComponents; +use reth_exex::{ExExContext, ExExEvent}; +use reth_node_ethereum::EthereumNode; +use reth_tracing::tracing::info; + +struct MyExEx { + ctx: ExExContext, + /// First block that was committed since the start of the ExEx. + first_block: Option, + /// Total number of transactions committed. + transactions: u64, +} + +impl MyExEx { + fn new(ctx: ExExContext) -> Self { + Self { ctx, first_block: None, transactions: 0 } + } +} + +impl Future for MyExEx { + type Output = eyre::Result<()>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + while let Some(notification) = ready!(this.ctx.notifications.try_next().poll_unpin(cx))? { + if let Some(reverted_chain) = notification.reverted_chain() { + this.transactions = this.transactions.saturating_sub( + reverted_chain.blocks_iter().map(|b| b.body.transactions.len() as u64).sum(), + ); + } + + if let Some(committed_chain) = notification.committed_chain() { + this.first_block.get_or_insert(committed_chain.first().number); + + this.transactions += committed_chain + .blocks_iter() + .map(|b| b.body.transactions.len() as u64) + .sum::(); + + this.ctx + .events + .send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; + } + + if let Some(first_block) = this.first_block { + info!(%first_block, transactions = %this.transactions, "Total number of transactions"); + } + } + + Poll::Ready(Ok(())) + } +} + +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let handle = builder + .node(EthereumNode::default()) + .install_exex("my-exex", |ctx| async move { Ok(MyExEx::new(ctx)) }) + .launch() + .await?; + + handle.wait_for_node_exit().await + }) +} From a087b52dc21a0ec59b8e63dd2f8c3222739e9962 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Mon, 14 Oct 2024 11:58:15 +0100 Subject: [PATCH 64/82] perf(rpc): do not clone filter matcher on every block tracing (#11714) --- crates/rpc/rpc/src/trace.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index 687762b74b5e..f7b2f7ab7eac 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -251,7 +251,8 @@ where &self, filter: TraceFilter, ) -> Result, Eth::Error> { - let matcher = filter.matcher(); + // We'll reuse the matcher across multiple blocks that are traced in parallel + let matcher = Arc::new(filter.matcher()); let TraceFilter { from_block, to_block, after, count, .. } = filter; let start = from_block.unwrap_or(0); let end = if let Some(to_block) = to_block { From 9c8360e5327794fdc9a4917ccd254d0adf441f72 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Mon, 14 Oct 2024 13:41:50 +0200 Subject: [PATCH 65/82] fix(net): remove outdated debug assert `TransactionFetcher` (#11713) --- crates/net/network/src/transactions/fetcher.rs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/crates/net/network/src/transactions/fetcher.rs b/crates/net/network/src/transactions/fetcher.rs index 9276219d593b..3e856951552a 100644 --- a/crates/net/network/src/transactions/fetcher.rs +++ b/crates/net/network/src/transactions/fetcher.rs @@ -912,17 +912,6 @@ impl TransactionFetcher { // fallback peers let GetPooledTxResponse { peer_id, mut requested_hashes, result } = response; - debug_assert!( - self.active_peers.get(&peer_id).is_some(), - "`{}` has been removed from `@active_peers` before inflight request(s) resolved, broken invariant `@active_peers` and `@inflight_requests`, `%peer_id`: {}, `@hashes_fetch_inflight_and_pending_fetch` for `%requested_hashes`: {:?}", - peer_id, - peer_id, - requested_hashes.iter().map(|hash| { - let metadata = self.hashes_fetch_inflight_and_pending_fetch.get(hash); - (*hash, metadata.map(|m| (m.retries, m.tx_encoded_length))) - }).collect::)>)>>() - ); - self.decrement_inflight_request_count_for(&peer_id); match result { From c05a90054263f6da4ff581a5a7d933261767f0c5 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Mon, 14 Oct 2024 21:44:28 +0900 Subject: [PATCH 66/82] perf(rpc): use `Arc` on cache and rpc (#11635) --- Cargo.lock | 1 + crates/ethereum/payload/src/lib.rs | 2 +- crates/evm/src/lib.rs | 10 +- crates/optimism/evm/src/l1.rs | 13 +- crates/optimism/node/src/txpool.rs | 2 +- crates/optimism/payload/src/builder.rs | 4 +- crates/optimism/rpc/src/eth/block.rs | 2 +- crates/optimism/rpc/src/eth/receipt.rs | 5 +- crates/primitives/src/transaction/mod.rs | 5 + crates/rpc/rpc-eth-api/src/helpers/block.rs | 25 +-- crates/rpc/rpc-eth-api/src/helpers/call.rs | 27 +-- crates/rpc/rpc-eth-api/src/helpers/fee.rs | 14 +- .../rpc-eth-api/src/helpers/pending_block.rs | 2 +- crates/rpc/rpc-eth-api/src/helpers/trace.rs | 10 +- .../rpc-eth-api/src/helpers/transaction.rs | 30 +-- crates/rpc/rpc-eth-types/Cargo.toml | 1 + crates/rpc/rpc-eth-types/src/cache/mod.rs | 180 +++++------------- crates/rpc/rpc-eth-types/src/fee_history.rs | 17 +- crates/rpc/rpc-eth-types/src/gas_oracle.rs | 14 +- crates/rpc/rpc-eth-types/src/logs_utils.rs | 7 +- crates/rpc/rpc/src/debug.rs | 17 +- crates/rpc/rpc/src/eth/filter.rs | 5 +- crates/rpc/rpc/src/trace.rs | 10 +- 23 files changed, 168 insertions(+), 235 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3e3fd18a8442..3c5639eadab0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8745,6 +8745,7 @@ dependencies = [ "alloy-sol-types", "derive_more 1.0.0", "futures", + "itertools 0.13.0", "jsonrpsee-core", "jsonrpsee-types", "metrics", diff --git a/crates/ethereum/payload/src/lib.rs b/crates/ethereum/payload/src/lib.rs index 5d2fcde2f7e5..97237efa8b60 100644 --- a/crates/ethereum/payload/src/lib.rs +++ b/crates/ethereum/payload/src/lib.rs @@ -228,7 +228,7 @@ where let env = EnvWithHandlerCfg::new_with_cfg_env( initialized_cfg.clone(), initialized_block_env.clone(), - evm_config.tx_env(&tx), + evm_config.tx_env(tx.as_signed(), tx.signer()), ); // Configure the environment for the block. diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs index 6fcb3d9f8c3d..e0d45f04cd31 100644 --- a/crates/evm/src/lib.rs +++ b/crates/evm/src/lib.rs @@ -11,11 +11,9 @@ extern crate alloc; -use core::ops::Deref; - use crate::builder::RethEvmBuilder; use alloy_primitives::{Address, Bytes, B256, U256}; -use reth_primitives::{TransactionSigned, TransactionSignedEcRecovered}; +use reth_primitives::TransactionSigned; use reth_primitives_traits::BlockHeader; use revm::{Database, Evm, GetInspector}; use revm_primitives::{BlockEnv, CfgEnvWithHandlerCfg, Env, EnvWithHandlerCfg, SpecId, TxEnv}; @@ -111,10 +109,10 @@ pub trait ConfigureEvmEnv: Send + Sync + Unpin + Clone + 'static { /// The header type used by the EVM. type Header: BlockHeader; - /// Returns a [`TxEnv`] from a [`TransactionSignedEcRecovered`]. - fn tx_env(&self, transaction: &TransactionSignedEcRecovered) -> TxEnv { + /// Returns a [`TxEnv`] from a [`TransactionSigned`] and [`Address`]. + fn tx_env(&self, transaction: &TransactionSigned, signer: Address) -> TxEnv { let mut tx_env = TxEnv::default(); - self.fill_tx_env(&mut tx_env, transaction.deref(), transaction.signer()); + self.fill_tx_env(&mut tx_env, transaction, signer); tx_env } diff --git a/crates/optimism/evm/src/l1.rs b/crates/optimism/evm/src/l1.rs index 18ccbed9518a..3412501eb99b 100644 --- a/crates/optimism/evm/src/l1.rs +++ b/crates/optimism/evm/src/l1.rs @@ -6,7 +6,7 @@ use reth_chainspec::ChainSpec; use reth_execution_errors::BlockExecutionError; use reth_optimism_chainspec::OpChainSpec; use reth_optimism_forks::OptimismHardfork; -use reth_primitives::Block; +use reth_primitives::BlockBody; use revm::{ primitives::{Bytecode, HashMap, SpecId}, DatabaseCommit, L1BlockInfo, @@ -31,9 +31,8 @@ const L1_BLOCK_ECOTONE_SELECTOR: [u8; 4] = hex!("440a5e20"); /// transaction in the L2 block. /// /// Returns an error if the L1 info transaction is not found, if the block is empty. -pub fn extract_l1_info(block: &Block) -> Result { - let l1_info_tx_data = block - .body +pub fn extract_l1_info(body: &BlockBody) -> Result { + let l1_info_tx_data = body .transactions .first() .ok_or_else(|| OptimismBlockExecutionError::L1BlockInfoError { @@ -302,7 +301,7 @@ mod tests { use alloy_eips::eip2718::Decodable2718; use reth_optimism_chainspec::OP_MAINNET; use reth_optimism_forks::OptimismHardforks; - use reth_primitives::{BlockBody, TransactionSigned}; + use reth_primitives::{Block, BlockBody, TransactionSigned}; use super::*; @@ -318,7 +317,7 @@ mod tests { body: BlockBody { transactions: vec![l1_info_tx], ..Default::default() }, }; - let l1_info: L1BlockInfo = extract_l1_info(&mock_block).unwrap(); + let l1_info: L1BlockInfo = extract_l1_info(&mock_block.body).unwrap(); assert_eq!(l1_info.l1_base_fee, U256::from(652_114)); assert_eq!(l1_info.l1_fee_overhead, Some(U256::from(2100))); assert_eq!(l1_info.l1_base_fee_scalar, U256::from(1_000_000)); @@ -358,7 +357,7 @@ mod tests { // test - let l1_block_info: L1BlockInfo = extract_l1_info(&block).unwrap(); + let l1_block_info: L1BlockInfo = extract_l1_info(&block.body).unwrap(); assert_eq!(l1_block_info.l1_base_fee, expected_l1_base_fee); assert_eq!(l1_block_info.l1_base_fee_scalar, expected_l1_base_fee_scalar); diff --git a/crates/optimism/node/src/txpool.rs b/crates/optimism/node/src/txpool.rs index 7ed2a161d0e4..09aa76fefb8b 100644 --- a/crates/optimism/node/src/txpool.rs +++ b/crates/optimism/node/src/txpool.rs @@ -99,7 +99,7 @@ where /// Update the L1 block info. fn update_l1_block_info(&self, block: &Block) { self.block_info.timestamp.store(block.timestamp, Ordering::Relaxed); - if let Ok(cost_addition) = reth_optimism_evm::extract_l1_info(block) { + if let Ok(cost_addition) = reth_optimism_evm::extract_l1_info(&block.body) { *self.block_info.l1_block_info.write() = cost_addition; } } diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index 5e9c1b5d18be..2ff3d3342037 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -275,7 +275,7 @@ where let env = EnvWithHandlerCfg::new_with_cfg_env( initialized_cfg.clone(), initialized_block_env.clone(), - evm_config.tx_env(&sequencer_tx), + evm_config.tx_env(sequencer_tx.as_signed(), sequencer_tx.signer()), ); let mut evm = evm_config.evm_with_env(&mut db, env); @@ -356,7 +356,7 @@ where let env = EnvWithHandlerCfg::new_with_cfg_env( initialized_cfg.clone(), initialized_block_env.clone(), - evm_config.tx_env(&tx), + evm_config.tx_env(tx.as_signed(), tx.signer()), ); // Configure the environment for the block. diff --git a/crates/optimism/rpc/src/eth/block.rs b/crates/optimism/rpc/src/eth/block.rs index d5066be0c620..dfdd09608563 100644 --- a/crates/optimism/rpc/src/eth/block.rs +++ b/crates/optimism/rpc/src/eth/block.rs @@ -45,7 +45,7 @@ where let block = block.unseal(); let l1_block_info = - reth_optimism_evm::extract_l1_info(&block).map_err(OpEthApiError::from)?; + reth_optimism_evm::extract_l1_info(&block.body).map_err(OpEthApiError::from)?; return block .body diff --git a/crates/optimism/rpc/src/eth/receipt.rs b/crates/optimism/rpc/src/eth/receipt.rs index fac8d220c761..cc685104133c 100644 --- a/crates/optimism/rpc/src/eth/receipt.rs +++ b/crates/optimism/rpc/src/eth/receipt.rs @@ -40,9 +40,8 @@ where meta.block_hash.into(), )))?; - let block = block.unseal(); let l1_block_info = - reth_optimism_evm::extract_l1_info(&block).map_err(OpEthApiError::from)?; + reth_optimism_evm::extract_l1_info(&block.body).map_err(OpEthApiError::from)?; Ok(OpReceiptBuilder::new( &self.inner.provider().chain_spec(), @@ -353,7 +352,7 @@ mod test { }; let l1_block_info = - reth_optimism_evm::extract_l1_info(&block).expect("should extract l1 info"); + reth_optimism_evm::extract_l1_info(&block.body).expect("should extract l1 info"); // test assert!(OP_MAINNET.is_fjord_active_at_timestamp(BLOCK_124665056_TIMESTAMP)); diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 7ef2c0c1fb76..3622b953107b 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -1472,6 +1472,11 @@ impl TransactionSignedEcRecovered { self.signer } + /// Returns a reference to [`TransactionSigned`] + pub const fn as_signed(&self) -> &TransactionSigned { + &self.signed_transaction + } + /// Transform back to [`TransactionSigned`] pub fn into_signed(self) -> TransactionSigned { self.signed_transaction diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index 898037f4a586..5d8496abdae2 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -64,7 +64,7 @@ pub trait EthBlocks: LoadBlock { } let block = from_block::( - block.unseal(), + (*block).clone().unseal(), total_difficulty.unwrap_or_default(), full.into(), Some(block_hash), @@ -100,10 +100,10 @@ pub trait EthBlocks: LoadBlock { Ok(self .cache() - .get_block_transactions(block_hash) + .get_sealed_block_with_senders(block_hash) .await .map_err(Self::Error::from_eth_err)? - .map(|txs| txs.len())) + .map(|b| b.body.transactions.len())) } } @@ -150,6 +150,7 @@ pub trait EthBlocks: LoadBlock { .get_block_and_receipts(block_hash) .await .map_err(Self::Error::from_eth_err) + .map(|b| b.map(|(b, r)| (b.block.clone(), r))) } Ok(None) @@ -208,23 +209,11 @@ pub trait LoadBlock: LoadPendingBlock + SpawnBlocking { /// Data access in default (L1) trait method implementations. fn cache(&self) -> &EthStateCache; - /// Returns the block object for the given block id. - fn block( - &self, - block_id: BlockId, - ) -> impl Future, Self::Error>> + Send { - async move { - self.block_with_senders(block_id) - .await - .map(|maybe_block| maybe_block.map(|block| block.block)) - } - } - /// Returns the block object for the given block id. fn block_with_senders( &self, block_id: BlockId, - ) -> impl Future, Self::Error>> + Send { + ) -> impl Future>, Self::Error>> + Send { async move { if block_id.is_pending() { // Pending block can be fetched directly without need for caching @@ -232,11 +221,11 @@ pub trait LoadBlock: LoadPendingBlock + SpawnBlocking { .pending_block_with_senders() .map_err(Self::Error::from_eth_err)?; return if maybe_pending.is_some() { - Ok(maybe_pending) + Ok(maybe_pending.map(Arc::new)) } else { // If no pending block from provider, try to get local pending block return match self.local_pending_block().await? { - Some((block, _)) => Ok(Some(block)), + Some((block, _)) => Ok(Some(Arc::new(block))), None => Ok(None), }; }; diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 5f6f3d77f555..ff6e930069f8 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -5,7 +5,7 @@ use crate::{ AsEthApiError, FromEthApiError, FromEvmError, FullEthApiTypes, IntoEthApiError, RpcBlock, }; use alloy_eips::{eip1559::calc_next_block_base_fee, eip2930::AccessListResult}; -use alloy_primitives::{Bytes, TxKind, B256, U256}; +use alloy_primitives::{Address, Bytes, TxKind, B256, U256}; use alloy_rpc_types::{ simulate::{SimBlock, SimulatePayload, SimulatedBlock}, state::{EvmOverrides, StateOverride}, @@ -20,7 +20,7 @@ use reth_primitives::{ BlockEnv, CfgEnvWithHandlerCfg, EnvWithHandlerCfg, ExecutionResult, HaltReason, ResultAndState, TransactTo, TxEnv, }, - Header, TransactionSignedEcRecovered, + Header, TransactionSigned, }; use reth_provider::{BlockIdReader, ChainSpecProvider, HeaderProvider, StateProvider}; use reth_revm::{database::StateProviderDatabase, db::CacheDB, DatabaseRef}; @@ -92,7 +92,8 @@ pub trait EthCall: Call + LoadPendingBlock { // Gas cap for entire operation let total_gas_limit = self.call_gas_limit(); - let base_block = self.block(block).await?.ok_or(EthApiError::HeaderNotFound(block))?; + let base_block = + self.block_with_senders(block).await?.ok_or(EthApiError::HeaderNotFound(block))?; let mut parent_hash = base_block.header.hash(); let total_difficulty = LoadPendingBlock::provider(self) .header_td_by_number(block_env.number.to()) @@ -292,12 +293,12 @@ pub trait EthCall: Call + LoadPendingBlock { if replay_block_txs { // only need to replay the transactions in the block if not all transactions are // to be replayed - let transactions = block.into_transactions_ecrecovered().take(num_txs); - for tx in transactions { + let transactions = block.transactions_with_sender().take(num_txs); + for (signer, tx) in transactions { let env = EnvWithHandlerCfg::new_with_cfg_env( cfg.clone(), block_env.clone(), - Call::evm_config(&this).tx_env(&tx), + Call::evm_config(&this).tx_env(tx, *signer), ); let (res, _) = this.transact(&mut db, env)?; db.commit(res.state); @@ -605,11 +606,11 @@ pub trait Call: LoadState + SpawnBlocking { // we need to get the state of the parent block because we're essentially replaying the // block the transaction is included in let parent_block = block.parent_hash; - let block_txs = block.into_transactions_ecrecovered(); let this = self.clone(); self.spawn_with_state_at_block(parent_block.into(), move |state| { let mut db = CacheDB::new(StateProviderDatabase::new(state)); + let block_txs = block.transactions_with_sender(); // replay all transactions prior to the targeted transaction this.replay_transactions_until( @@ -623,7 +624,7 @@ pub trait Call: LoadState + SpawnBlocking { let env = EnvWithHandlerCfg::new_with_cfg_env( cfg, block_env, - Call::evm_config(&this).tx_env(&tx), + Call::evm_config(&this).tx_env(tx.as_signed(), tx.signer()), ); let (res, _) = this.transact(&mut db, env)?; @@ -641,30 +642,30 @@ pub trait Call: LoadState + SpawnBlocking { /// /// Note: This assumes the target transaction is in the given iterator. /// Returns the index of the target transaction in the given iterator. - fn replay_transactions_until( + fn replay_transactions_until<'a, DB, I>( &self, db: &mut CacheDB, cfg: CfgEnvWithHandlerCfg, block_env: BlockEnv, - transactions: impl IntoIterator, + transactions: I, target_tx_hash: B256, ) -> Result where DB: DatabaseRef, EthApiError: From, + I: IntoIterator, { let env = EnvWithHandlerCfg::new_with_cfg_env(cfg, block_env, Default::default()); let mut evm = self.evm_config().evm_with_env(db, env); let mut index = 0; - for tx in transactions { + for (sender, tx) in transactions { if tx.hash() == target_tx_hash { // reached the target transaction break } - let sender = tx.signer(); - self.evm_config().fill_tx_env(evm.tx_mut(), &tx.into_signed(), sender); + self.evm_config().fill_tx_env(evm.tx_mut(), tx, *sender); evm.transact_commit().map_err(Self::Error::from_evm_err)?; index += 1; } diff --git a/crates/rpc/rpc-eth-api/src/helpers/fee.rs b/crates/rpc/rpc-eth-api/src/helpers/fee.rs index 52ccabebf620..34ba6dc7e4ec 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/fee.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/fee.rs @@ -172,8 +172,8 @@ pub trait EthFees: LoadFee { // Percentiles were specified, so we need to collect reward percentile ino if let Some(percentiles) = &reward_percentiles { - let (transactions, receipts) = LoadFee::cache(self) - .get_transactions_and_receipts(header.hash()) + let (block, receipts) = LoadFee::cache(self) + .get_block_and_receipts(header.hash()) .await .map_err(Self::Error::from_eth_err)? .ok_or(EthApiError::InvalidBlockRange)?; @@ -182,7 +182,7 @@ pub trait EthFees: LoadFee { percentiles, header.gas_used, header.base_fee_per_gas.unwrap_or_default(), - &transactions, + &block.body.transactions, &receipts, ) .unwrap_or_default(), @@ -296,7 +296,7 @@ pub trait LoadFee: LoadBlock { None => { // fetch pending base fee let base_fee = self - .block(BlockNumberOrTag::Pending.into()) + .block_with_senders(BlockNumberOrTag::Pending.into()) .await? .ok_or(EthApiError::HeaderNotFound(BlockNumberOrTag::Pending.into()))? .base_fee_per_gas @@ -332,7 +332,7 @@ pub trait LoadFee: LoadBlock { /// /// See also: fn gas_price(&self) -> impl Future> + Send { - let header = self.block(BlockNumberOrTag::Latest.into()); + let header = self.block_with_senders(BlockNumberOrTag::Latest.into()); let suggested_tip = self.suggested_priority_fee(); async move { let (header, suggested_tip) = futures::try_join!(header, suggested_tip)?; @@ -344,9 +344,9 @@ pub trait LoadFee: LoadBlock { /// Returns a suggestion for a base fee for blob transactions. fn blob_base_fee(&self) -> impl Future> + Send { async move { - self.block(BlockNumberOrTag::Latest.into()) + self.block_with_senders(BlockNumberOrTag::Latest.into()) .await? - .and_then(|h: reth_primitives::SealedBlock| h.next_block_blob_fee()) + .and_then(|h| h.next_block_blob_fee()) .ok_or(EthApiError::ExcessBlobGasNotSet.into()) .map(U256::from) } diff --git a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs index 8014851e3f93..03597289f151 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs @@ -323,7 +323,7 @@ pub trait LoadPendingBlock: EthApiTypes { let env = Env::boxed( cfg.cfg_env.clone(), block_env.clone(), - Self::evm_config(self).tx_env(&tx), + Self::evm_config(self).tx_env(tx.as_signed(), tx.signer()), ); let mut evm = revm::Evm::builder().with_env(env).with_db(&mut db).build(); diff --git a/crates/rpc/rpc-eth-api/src/helpers/trace.rs b/crates/rpc/rpc-eth-api/src/helpers/trace.rs index 7b5d13b2a7dd..2dc1c5b4746d 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/trace.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/trace.rs @@ -191,11 +191,11 @@ pub trait Trace: LoadState { // block the transaction is included in let parent_block = block.parent_hash; let parent_beacon_block_root = block.parent_beacon_block_root; - let block_txs = block.into_transactions_ecrecovered(); let this = self.clone(); self.spawn_with_state_at_block(parent_block.into(), move |state| { let mut db = CacheDB::new(StateProviderDatabase::new(state)); + let block_txs = block.transactions_with_sender(); // apply relevant system calls let mut system_caller = SystemCaller::new( @@ -227,7 +227,7 @@ pub trait Trace: LoadState { let env = EnvWithHandlerCfg::new_with_cfg_env( cfg, block_env, - Call::evm_config(&this).tx_env(&tx), + Call::evm_config(&this).tx_env(tx.as_signed(), tx.signer()), ); let (res, _) = this.inspect(StateCacheDbRefMutWrapper(&mut db), env, &mut inspector)?; @@ -356,10 +356,10 @@ pub trait Trace: LoadState { let mut results = Vec::with_capacity(max_transactions); let mut transactions = block - .into_transactions_ecrecovered() + .transactions_with_sender() .take(max_transactions) .enumerate() - .map(|(idx, tx)| { + .map(|(idx, (signer, tx))| { let tx_info = TransactionInfo { hash: Some(tx.hash()), index: Some(idx as u64), @@ -367,7 +367,7 @@ pub trait Trace: LoadState { block_number: Some(block_number), base_fee: Some(base_fee), }; - let tx_env = Trace::evm_config(&this).tx_env(&tx); + let tx_env = Trace::evm_config(&this).tx_env(tx, *signer); (tx_info, tx_env) }) .peekable(); diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index 94dd04414874..4c2493717a04 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -18,6 +18,7 @@ use reth_rpc_eth_types::{ }; use reth_rpc_types_compat::transaction::{from_recovered, from_recovered_with_block_context}; use reth_transaction_pool::{PoolTransaction, TransactionOrigin, TransactionPool}; +use std::sync::Arc; use crate::{FromEthApiError, FullEthApiTypes, IntoEthApiError, RpcReceipt, RpcTransaction}; @@ -79,7 +80,11 @@ pub trait EthTransactions: LoadTransaction { block: B256, ) -> impl Future>, Self::Error>> + Send { async move { - self.cache().get_block_transactions(block).await.map_err(Self::Error::from_eth_err) + self.cache() + .get_sealed_block_with_senders(block) + .await + .map(|b| b.map(|b| b.body.transactions.clone())) + .map_err(Self::Error::from_eth_err) } } @@ -194,7 +199,7 @@ pub trait EthTransactions: LoadTransaction { let block_hash = block.hash(); let block_number = block.number; let base_fee_per_gas = block.base_fee_per_gas; - if let Some(tx) = block.into_transactions_ecrecovered().nth(index) { + if let Some((signer, tx)) = block.transactions_with_sender().nth(index) { let tx_info = TransactionInfo { hash: Some(tx.hash()), block_hash: Some(block_hash), @@ -204,7 +209,8 @@ pub trait EthTransactions: LoadTransaction { }; return Ok(Some(from_recovered_with_block_context::( - tx, tx_info, + tx.clone().with_signer(*signer), + tx_info, ))) } } @@ -270,10 +276,10 @@ pub trait EthTransactions: LoadTransaction { let base_fee_per_gas = block.base_fee_per_gas; block - .into_transactions_ecrecovered() + .transactions_with_sender() .enumerate() - .find(|(_, tx)| tx.signer() == sender && tx.nonce() == nonce) - .map(|(index, tx)| { + .find(|(_, (signer, tx))| **signer == sender && tx.nonce() == nonce) + .map(|(index, (signer, tx))| { let tx_info = TransactionInfo { hash: Some(tx.hash()), block_hash: Some(block_hash), @@ -282,7 +288,8 @@ pub trait EthTransactions: LoadTransaction { index: Some(index as u64), }; from_recovered_with_block_context::( - tx, tx_info, + tx.clone().with_signer(*signer), + tx_info, ) }) }) @@ -544,8 +551,9 @@ pub trait LoadTransaction: SpawnBlocking + FullEthApiTypes { fn transaction_and_block( &self, hash: B256, - ) -> impl Future, Self::Error>> - + Send { + ) -> impl Future< + Output = Result)>, Self::Error>, + > + Send { async move { let (transaction, at) = match self.transaction_by_hash_at(hash).await? { None => return Ok(None), @@ -559,10 +567,10 @@ pub trait LoadTransaction: SpawnBlocking + FullEthApiTypes { }; let block = self .cache() - .get_block_with_senders(block_hash) + .get_sealed_block_with_senders(block_hash) .await .map_err(Self::Error::from_eth_err)?; - Ok(block.map(|block| (transaction, (*block).clone().seal(block_hash)))) + Ok(block.map(|block| (transaction, block))) } } } diff --git a/crates/rpc/rpc-eth-types/Cargo.toml b/crates/rpc/rpc-eth-types/Cargo.toml index 2c6f51ff462b..46a0d7b5c323 100644 --- a/crates/rpc/rpc-eth-types/Cargo.toml +++ b/crates/rpc/rpc-eth-types/Cargo.toml @@ -58,6 +58,7 @@ derive_more.workspace = true schnellru.workspace = true rand.workspace = true tracing.workspace = true +itertools.workspace = true [dev-dependencies] serde_json.workspace = true diff --git a/crates/rpc/rpc-eth-types/src/cache/mod.rs b/crates/rpc/rpc-eth-types/src/cache/mod.rs index e8353ecc4e16..cbf05f2764e4 100644 --- a/crates/rpc/rpc-eth-types/src/cache/mod.rs +++ b/crates/rpc/rpc-eth-types/src/cache/mod.rs @@ -7,8 +7,7 @@ use reth_errors::{ProviderError, ProviderResult}; use reth_evm::{provider::EvmEnvProvider, ConfigureEvm}; use reth_execution_types::Chain; use reth_primitives::{ - Block, BlockHashOrNumber, BlockWithSenders, Header, Receipt, SealedBlock, - SealedBlockWithSenders, TransactionSigned, TransactionSignedEcRecovered, + BlockHashOrNumber, Header, Receipt, SealedBlockWithSenders, TransactionSigned, }; use reth_storage_api::{BlockReader, StateProviderFactory, TransactionVariant}; use reth_tasks::{TaskSpawner, TokioTaskExecutor}; @@ -33,13 +32,13 @@ pub mod db; pub mod metrics; pub mod multi_consumer; -/// The type that can send the response to a requested [`Block`] +/// The type that can send the response to a requested [`SealedBlockWithSenders`] type BlockTransactionsResponseSender = oneshot::Sender>>>; -/// The type that can send the response to a requested [`BlockWithSenders`] +/// The type that can send the response to a requested [`SealedBlockWithSenders`] type BlockWithSendersResponseSender = - oneshot::Sender>>>; + oneshot::Sender>>>; /// The type that can send the response to the requested receipts of a block. type ReceiptsResponseSender = oneshot::Sender>>>>; @@ -49,7 +48,7 @@ type EnvResponseSender = oneshot::Sender = MultiConsumerLruCache< B256, - Arc, + Arc, L, Either, >; @@ -142,92 +141,18 @@ impl EthStateCache { this } - /// Requests the [`Block`] for the block hash - /// - /// Returns `None` if the block does not exist. - pub async fn get_block(&self, block_hash: B256) -> ProviderResult> { - let (response_tx, rx) = oneshot::channel(); - let _ = self.to_service.send(CacheAction::GetBlockWithSenders { block_hash, response_tx }); - let block_with_senders_res = - rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)?; - - if let Ok(Some(block_with_senders)) = block_with_senders_res { - Ok(Some(block_with_senders.block.clone())) - } else { - Ok(None) - } - } - - /// Requests the [`Block`] for the block hash, sealed with the given block hash. - /// - /// Returns `None` if the block does not exist. - pub async fn get_sealed_block(&self, block_hash: B256) -> ProviderResult> { - Ok(self.get_block(block_hash).await?.map(|block| block.seal(block_hash))) - } - - /// Requests the transactions of the [`Block`] - /// - /// Returns `None` if the block does not exist. - pub async fn get_block_transactions( - &self, - block_hash: B256, - ) -> ProviderResult>> { - let (response_tx, rx) = oneshot::channel(); - let _ = self.to_service.send(CacheAction::GetBlockTransactions { block_hash, response_tx }); - rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)? - } - - /// Requests the ecrecovered transactions of the [`Block`] - /// - /// Returns `None` if the block does not exist. - pub async fn get_block_transactions_ecrecovered( - &self, - block_hash: B256, - ) -> ProviderResult>> { - Ok(self - .get_block_with_senders(block_hash) - .await? - .map(|block| (*block).clone().into_transactions_ecrecovered().collect())) - } - - /// Fetches both transactions and receipts for the given block hash. - pub async fn get_transactions_and_receipts( - &self, - block_hash: B256, - ) -> ProviderResult, Arc>)>> { - let transactions = self.get_block_transactions(block_hash); - let receipts = self.get_receipts(block_hash); - - let (transactions, receipts) = futures::try_join!(transactions, receipts)?; - - Ok(transactions.zip(receipts)) - } - - /// Requests the [`BlockWithSenders`] for the block hash + /// Requests the [`SealedBlockWithSenders`] for the block hash /// /// Returns `None` if the block does not exist. - pub async fn get_block_with_senders( + pub async fn get_sealed_block_with_senders( &self, block_hash: B256, - ) -> ProviderResult>> { + ) -> ProviderResult>> { let (response_tx, rx) = oneshot::channel(); let _ = self.to_service.send(CacheAction::GetBlockWithSenders { block_hash, response_tx }); rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)? } - /// Requests the [`SealedBlockWithSenders`] for the block hash - /// - /// Returns `None` if the block does not exist. - pub async fn get_sealed_block_with_senders( - &self, - block_hash: B256, - ) -> ProviderResult> { - Ok(self - .get_block_with_senders(block_hash) - .await? - .map(|block| (*block).clone().seal(block_hash))) - } - /// Requests the [Receipt] for the block hash /// /// Returns `None` if the block was not found. @@ -244,8 +169,8 @@ impl EthStateCache { pub async fn get_block_and_receipts( &self, block_hash: B256, - ) -> ProviderResult>)>> { - let block = self.get_sealed_block(block_hash); + ) -> ProviderResult, Arc>)>> { + let block = self.get_sealed_block_with_senders(block_hash); let receipts = self.get_receipts(block_hash); let (block, receipts) = futures::try_join!(block, receipts)?; @@ -292,7 +217,7 @@ pub(crate) struct EthStateCacheService< LimitReceipts = ByLength, LimitEnvs = ByLength, > where - LimitBlocks: Limiter>, + LimitBlocks: Limiter>, LimitReceipts: Limiter>>, LimitEnvs: Limiter, { @@ -325,7 +250,7 @@ where fn on_new_block( &mut self, block_hash: B256, - res: ProviderResult>>, + res: ProviderResult>>, ) { if let Some(queued) = self.full_block_cache.remove(&block_hash) { // send the response to queued senders @@ -367,7 +292,11 @@ where } } - fn on_reorg_block(&mut self, block_hash: B256, res: ProviderResult>) { + fn on_reorg_block( + &mut self, + block_hash: B256, + res: ProviderResult>, + ) { let res = res.map(|b| b.map(Arc::new)); if let Some(queued) = self.full_block_cache.remove(&block_hash) { // send the response to queued senders @@ -441,7 +370,7 @@ where // Only look in the database to prevent situations where we // looking up the tree is blocking let block_sender = provider - .block_with_senders( + .sealed_block_with_senders( BlockHashOrNumber::Hash(block_hash), TransactionVariant::WithHash, ) @@ -453,36 +382,6 @@ where })); } } - CacheAction::GetBlockTransactions { block_hash, response_tx } => { - // check if block is cached - if let Some(block) = this.full_block_cache.get(&block_hash) { - let _ = response_tx.send(Ok(Some(block.body.transactions.clone()))); - continue - } - - // block is not in the cache, request it if this is the first consumer - if this.full_block_cache.queue(block_hash, Either::Right(response_tx)) { - let provider = this.provider.clone(); - let action_tx = this.action_tx.clone(); - let rate_limiter = this.rate_limiter.clone(); - this.action_task_spawner.spawn_blocking(Box::pin(async move { - // Acquire permit - let _permit = rate_limiter.acquire().await; - // Only look in the database to prevent situations where we - // looking up the tree is blocking - let res = provider - .block_with_senders( - BlockHashOrNumber::Hash(block_hash), - TransactionVariant::WithHash, - ) - .map(|b| b.map(Arc::new)); - let _ = action_tx.send(CacheAction::BlockWithSendersResult { - block_hash, - res, - }); - })); - } - } CacheAction::GetReceipts { block_hash, response_tx } => { // check if block is cached if let Some(receipts) = this.receipts_cache.get(&block_hash).cloned() { @@ -574,7 +473,7 @@ where } CacheAction::CacheNewCanonicalChain { chain_change } => { for block in chain_change.blocks { - this.on_new_block(block.hash(), Ok(Some(Arc::new(block.unseal())))); + this.on_new_block(block.hash(), Ok(Some(Arc::new(block)))); } for block_receipts in chain_change.receipts { @@ -588,7 +487,7 @@ where } CacheAction::RemoveReorgedChain { chain_change } => { for block in chain_change.blocks { - this.on_reorg_block(block.hash(), Ok(Some(block.unseal()))); + this.on_reorg_block(block.hash(), Ok(Some(block))); } for block_receipts in chain_change.receipts { @@ -610,15 +509,36 @@ where /// All message variants sent through the channel enum CacheAction { - GetBlockWithSenders { block_hash: B256, response_tx: BlockWithSendersResponseSender }, - GetBlockTransactions { block_hash: B256, response_tx: BlockTransactionsResponseSender }, - GetEnv { block_hash: B256, response_tx: EnvResponseSender }, - GetReceipts { block_hash: B256, response_tx: ReceiptsResponseSender }, - BlockWithSendersResult { block_hash: B256, res: ProviderResult>> }, - ReceiptsResult { block_hash: B256, res: ProviderResult>>> }, - EnvResult { block_hash: B256, res: Box> }, - CacheNewCanonicalChain { chain_change: ChainChange }, - RemoveReorgedChain { chain_change: ChainChange }, + GetBlockWithSenders { + block_hash: B256, + response_tx: BlockWithSendersResponseSender, + }, + GetEnv { + block_hash: B256, + response_tx: EnvResponseSender, + }, + GetReceipts { + block_hash: B256, + response_tx: ReceiptsResponseSender, + }, + BlockWithSendersResult { + block_hash: B256, + res: ProviderResult>>, + }, + ReceiptsResult { + block_hash: B256, + res: ProviderResult>>>, + }, + EnvResult { + block_hash: B256, + res: Box>, + }, + CacheNewCanonicalChain { + chain_change: ChainChange, + }, + RemoveReorgedChain { + chain_change: ChainChange, + }, } struct BlockReceipts { diff --git a/crates/rpc/rpc-eth-types/src/fee_history.rs b/crates/rpc/rpc-eth-types/src/fee_history.rs index 08ac56845ffe..57dd276e5cf4 100644 --- a/crates/rpc/rpc-eth-types/src/fee_history.rs +++ b/crates/rpc/rpc-eth-types/src/fee_history.rs @@ -73,16 +73,16 @@ impl FeeHistoryCache { } /// Insert block data into the cache. - async fn insert_blocks(&self, blocks: I) + async fn insert_blocks<'a, I>(&self, blocks: I) where - I: IntoIterator>)>, + I: IntoIterator>)>, { let mut entries = self.inner.entries.write().await; let percentiles = self.predefined_percentiles(); // Insert all new blocks and calculate approximated rewards for (block, receipts) in blocks { - let mut fee_history_entry = FeeHistoryEntry::new(&block); + let mut fee_history_entry = FeeHistoryEntry::new(block); fee_history_entry.rewards = calculate_reward_percentiles_for_block( &percentiles, fee_history_entry.gas_used, @@ -237,7 +237,9 @@ pub async fn fee_history_cache_new_blocks_task( tokio::select! { res = &mut fetch_missing_block => { if let Ok(res) = res { - fee_history_cache.insert_blocks(res.into_iter()).await; + fee_history_cache.insert_blocks(res.as_ref() + .map(|(b, r)| (&b.block, r.clone())) + .into_iter()).await; } } event = events.next() => { @@ -245,11 +247,12 @@ pub async fn fee_history_cache_new_blocks_task( // the stream ended, we are done break; }; - let (blocks, receipts): (Vec<_>, Vec<_>) = event - .committed() + + let committed = event .committed(); + let (blocks, receipts): (Vec<_>, Vec<_>) = committed .blocks_and_receipts() .map(|(block, receipts)| { - (block.block.clone(), Arc::new(receipts.iter().flatten().cloned().collect::>())) + (&block.block, Arc::new(receipts.iter().flatten().cloned().collect::>())) }) .unzip(); fee_history_cache.insert_blocks(blocks.into_iter().zip(receipts)).await; diff --git a/crates/rpc/rpc-eth-types/src/gas_oracle.rs b/crates/rpc/rpc-eth-types/src/gas_oracle.rs index 01591bc4de38..84e7ab8306df 100644 --- a/crates/rpc/rpc-eth-types/src/gas_oracle.rs +++ b/crates/rpc/rpc-eth-types/src/gas_oracle.rs @@ -1,16 +1,16 @@ //! An implementation of the eth gas price oracle, used for providing gas price estimates based on //! previous blocks. -use std::fmt::{self, Debug, Formatter}; - use alloy_primitives::{B256, U256}; use alloy_rpc_types::BlockId; use derive_more::{Deref, DerefMut, From, Into}; +use itertools::Itertools; use reth_primitives::{constants::GWEI_TO_WEI, BlockNumberOrTag}; use reth_rpc_server_types::constants; use reth_storage_api::BlockReaderIdExt; use schnellru::{ByLength, LruMap}; use serde::{Deserialize, Serialize}; +use std::fmt::{self, Debug, Formatter}; use tokio::sync::Mutex; use tracing::warn; @@ -212,7 +212,7 @@ where limit: usize, ) -> EthResult)>> { // check the cache (this will hit the disk if the block is not cached) - let mut block = match self.cache.get_block(block_hash).await? { + let block = match self.cache.get_sealed_block_with_senders(block_hash).await? { Some(block) => block, None => return Ok(None), }; @@ -221,11 +221,15 @@ where let parent_hash = block.parent_hash; // sort the functions by ascending effective tip first - block.body.transactions.sort_by_cached_key(|tx| tx.effective_tip_per_gas(base_fee_per_gas)); + let sorted_transactions = block + .body + .transactions + .iter() + .sorted_by_cached_key(|tx| tx.effective_tip_per_gas(base_fee_per_gas)); let mut prices = Vec::with_capacity(limit); - for tx in block.body.transactions() { + for tx in sorted_transactions { let mut effective_gas_tip = None; // ignore transactions with a tip under the configured threshold if let Some(ignore_under) = self.ignore_price { diff --git a/crates/rpc/rpc-eth-types/src/logs_utils.rs b/crates/rpc/rpc-eth-types/src/logs_utils.rs index c64bbe055b79..205e2bba37be 100644 --- a/crates/rpc/rpc-eth-types/src/logs_utils.rs +++ b/crates/rpc/rpc-eth-types/src/logs_utils.rs @@ -6,8 +6,9 @@ use alloy_primitives::TxHash; use alloy_rpc_types::{FilteredParams, Log}; use reth_chainspec::ChainInfo; use reth_errors::ProviderError; -use reth_primitives::{BlockNumHash, Receipt, SealedBlock}; +use reth_primitives::{BlockNumHash, Receipt, SealedBlockWithSenders}; use reth_storage_api::BlockReader; +use std::sync::Arc; /// Returns all matching of a block's receipts when the transaction hashes are known. pub fn matching_block_logs_with_tx_hashes<'a, I>( @@ -50,8 +51,8 @@ where pub enum ProviderOrBlock<'a, P: BlockReader> { /// Provider Provider(&'a P), - /// [`SealedBlock`] - Block(SealedBlock), + /// [`SealedBlockWithSenders`] + Block(Arc), } /// Appends all matching logs of a block's receipts. diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index fbef6f7a7e0b..acf215b3b2cc 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -119,7 +119,7 @@ where env: Env::boxed( cfg.cfg_env.clone(), block_env.clone(), - Call::evm_config(this.eth_api()).tx_env(&tx), + Call::evm_config(this.eth_api()).tx_env(tx.as_signed(), tx.signer()), ), handler_cfg: cfg.handler_cfg, }; @@ -219,7 +219,7 @@ where self.trace_block( state_at.into(), - block.into_transactions_ecrecovered().collect(), + (*block).clone().into_transactions_ecrecovered().collect(), cfg, block_env, opts, @@ -245,11 +245,12 @@ where // block the transaction is included in let state_at: BlockId = block.parent_hash.into(); let block_hash = block.hash(); - let block_txs = block.into_transactions_ecrecovered(); let this = self.clone(); self.eth_api() .spawn_with_state_at_block(state_at, move |state| { + let block_txs = block.transactions_with_sender(); + // configure env for the target transaction let tx = transaction.into_recovered(); @@ -267,7 +268,7 @@ where env: Env::boxed( cfg.cfg_env.clone(), block_env, - Call::evm_config(this.eth_api()).tx_env(&tx), + Call::evm_config(this.eth_api()).tx_env(tx.as_signed(), tx.signer()), ), handler_cfg: cfg.handler_cfg, }; @@ -527,15 +528,15 @@ where if replay_block_txs { // only need to replay the transactions in the block if not all transactions are // to be replayed - let transactions = block.into_transactions_ecrecovered().take(num_txs); + let transactions = block.transactions_with_sender().take(num_txs); // Execute all transactions until index - for tx in transactions { + for (signer, tx) in transactions { let env = EnvWithHandlerCfg { env: Env::boxed( cfg.cfg_env.clone(), block_env.clone(), - Call::evm_config(this.eth_api()).tx_env(&tx), + Call::evm_config(this.eth_api()).tx_env(tx, *signer), ), handler_cfg: cfg.handler_cfg, }; @@ -613,7 +614,7 @@ where let _ = block_executor .execute_with_state_closure( - (&block.clone().unseal(), block.difficulty).into(), + (&(*block).clone().unseal(), block.difficulty).into(), |statedb: &State<_>| { codes = statedb .cache diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index 9efecf3dae7f..b136861c7961 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -19,7 +19,7 @@ use async_trait::async_trait; use jsonrpsee::{core::RpcResult, server::IdProvider}; use reth_chainspec::ChainInfo; use reth_node_api::EthApiTypes; -use reth_primitives::{Receipt, SealedBlock, TransactionSignedEcRecovered}; +use reth_primitives::{Receipt, SealedBlockWithSenders, TransactionSignedEcRecovered}; use reth_provider::{BlockIdReader, BlockReader, EvmEnvProvider, ProviderError}; use reth_rpc_eth_api::{EthFilterApiServer, FullEthApiTypes, RpcTransaction, TransactionCompat}; use reth_rpc_eth_types::{ @@ -534,7 +534,8 @@ where &self, block_num_hash: &BlockNumHash, best_number: u64, - ) -> Result>, Option)>, EthFilterError> { + ) -> Result>, Option>)>, EthFilterError> + { // The last 4 blocks are most likely cached, so we can just fetch them let cached_range = best_number.saturating_sub(4)..=best_number; let receipts_block = if cached_range.contains(&block_num_hash.number) { diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index f7b2f7ab7eac..05588d4dabea 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -118,14 +118,14 @@ where trace_types: HashSet, block_id: Option, ) -> Result { - let tx = recover_raw_transaction(tx)?; + let tx = recover_raw_transaction(tx)?.into_ecrecovered_transaction(); let (cfg, block, at) = self.eth_api().evm_env_at(block_id.unwrap_or_default()).await?; let env = EnvWithHandlerCfg::new_with_cfg_env( cfg, block, - Call::evm_config(self.eth_api()).tx_env(&tx.into_ecrecovered_transaction()), + Call::evm_config(self.eth_api()).tx_env(tx.as_signed(), tx.signer()), ); let config = TracingInspectorConfig::from_parity_config(&trace_types); @@ -372,7 +372,7 @@ where }, ); - let block = self.eth_api().block(block_id); + let block = self.eth_api().block_with_senders(block_id); let (maybe_traces, maybe_block) = futures::try_join!(traces, block)?; let mut maybe_traces = @@ -468,7 +468,9 @@ where let Some(transactions) = res else { return Ok(None) }; - let Some(block) = self.eth_api().block(block_id).await? else { return Ok(None) }; + let Some(block) = self.eth_api().block_with_senders(block_id).await? else { + return Ok(None) + }; Ok(Some(BlockOpcodeGas { block_hash: block.hash(), From 523bfb9c81ad7c2493882d83d0c9a6d0bcb2ce52 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Mon, 14 Oct 2024 17:21:41 +0400 Subject: [PATCH 67/82] feat: refactor and integrate local engine into `EngineNodeLauncher` (#11703) --- Cargo.lock | 17 +- .../consensus/beacon/src/engine/forkchoice.rs | 9 +- crates/e2e-test-utils/Cargo.toml | 1 + crates/engine/local/Cargo.toml | 24 +- crates/engine/local/src/lib.rs | 13 + crates/engine/local/src/miner.rs | 193 +++++++- crates/engine/local/src/payload.rs | 63 ++- crates/engine/local/src/service.rs | 427 +++++------------- crates/engine/service/src/service.rs | 2 +- crates/ethereum/node/tests/e2e/dev.rs | 53 ++- crates/ethereum/node/tests/e2e/utils.rs | 5 - crates/node/builder/Cargo.toml | 1 + crates/node/builder/src/launch/engine.rs | 79 +++- crates/node/events/src/node.rs | 2 +- crates/optimism/node/Cargo.toml | 2 + crates/payload/primitives/src/traits.rs | 9 +- 16 files changed, 500 insertions(+), 400 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c5639eadab0..d690cf536aa3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6971,6 +6971,7 @@ dependencies = [ "reth", "reth-chainspec", "reth-db", + "reth-engine-local", "reth-network-peers", "reth-node-builder", "reth-node-ethereum", @@ -7028,22 +7029,22 @@ dependencies = [ "alloy-rpc-types-engine", "eyre", "futures-util", + "op-alloy-rpc-types-engine", "reth-beacon-consensus", - "reth-chain-state", "reth-chainspec", - "reth-config", - "reth-db", + "reth-consensus", + "reth-engine-primitives", + "reth-engine-service", "reth-engine-tree", "reth-ethereum-engine-primitives", - "reth-exex-test-utils", - "reth-node-types", + "reth-evm", "reth-payload-builder", "reth-payload-primitives", - "reth-primitives", + "reth-payload-validator", "reth-provider", "reth-prune", + "reth-rpc-types-compat", "reth-stages-api", - "reth-tracing", "reth-transaction-pool", "tokio", "tokio-stream", @@ -7809,6 +7810,7 @@ dependencies = [ "reth-db-api", "reth-db-common", "reth-downloaders", + "reth-engine-local", "reth-engine-service", "reth-engine-tree", "reth-engine-util", @@ -8131,6 +8133,7 @@ dependencies = [ "reth-db", "reth-discv5", "reth-e2e-test-utils", + "reth-engine-local", "reth-evm", "reth-network", "reth-node-api", diff --git a/crates/consensus/beacon/src/engine/forkchoice.rs b/crates/consensus/beacon/src/engine/forkchoice.rs index 975c2ee3bc45..7e49714ba375 100644 --- a/crates/consensus/beacon/src/engine/forkchoice.rs +++ b/crates/consensus/beacon/src/engine/forkchoice.rs @@ -150,15 +150,18 @@ pub enum ForkchoiceStatus { } impl ForkchoiceStatus { - pub(crate) const fn is_valid(&self) -> bool { + /// Returns `true` if the forkchoice state is [`ForkchoiceStatus::Valid`]. + pub const fn is_valid(&self) -> bool { matches!(self, Self::Valid) } - pub(crate) const fn is_invalid(&self) -> bool { + /// Returns `true` if the forkchoice state is [`ForkchoiceStatus::Invalid`]. + pub const fn is_invalid(&self) -> bool { matches!(self, Self::Invalid) } - pub(crate) const fn is_syncing(&self) -> bool { + /// Returns `true` if the forkchoice state is [`ForkchoiceStatus::Syncing`]. + pub const fn is_syncing(&self) -> bool { matches!(self, Self::Syncing) } diff --git a/crates/e2e-test-utils/Cargo.toml b/crates/e2e-test-utils/Cargo.toml index a10162e78ff6..2742d704054e 100644 --- a/crates/e2e-test-utils/Cargo.toml +++ b/crates/e2e-test-utils/Cargo.toml @@ -13,6 +13,7 @@ workspace = true [dependencies] reth.workspace = true reth-chainspec.workspace = true +reth-engine-local.workspace = true reth-primitives.workspace = true reth-tracing.workspace = true reth-db = { workspace = true, features = ["test-utils"] } diff --git a/crates/engine/local/Cargo.toml b/crates/engine/local/Cargo.toml index d7a5d05091f1..f22ab1f8d560 100644 --- a/crates/engine/local/Cargo.toml +++ b/crates/engine/local/Cargo.toml @@ -11,15 +11,19 @@ exclude.workspace = true [dependencies] # reth reth-beacon-consensus.workspace = true -reth-chain-state.workspace = true +reth-chainspec.workspace = true +reth-consensus.workspace = true +reth-engine-primitives.workspace = true +reth-engine-service.workspace = true reth-engine-tree.workspace = true +reth-evm.workspace = true reth-ethereum-engine-primitives.workspace = true -reth-node-types.workspace = true reth-payload-builder.workspace = true reth-payload-primitives.workspace = true -reth-primitives.workspace = true +reth-payload-validator.workspace = true reth-provider.workspace = true reth-prune.workspace = true +reth-rpc-types-compat.workspace = true reth-transaction-pool.workspace = true reth-stages-api.workspace = true @@ -36,16 +40,10 @@ futures-util.workspace = true eyre.workspace = true tracing.workspace = true -[dev-dependencies] -reth-chainspec.workspace = true -reth-chain-state.workspace = true -reth-config.workspace = true -reth-db = { workspace = true, features = ["test-utils"] } -reth-ethereum-engine-primitives.workspace = true -reth-exex-test-utils.workspace = true -reth-payload-builder = { workspace = true, features = ["test-utils"] } -reth-provider = { workspace = true, features = ["test-utils"] } -reth-tracing.workspace = true +op-alloy-rpc-types-engine = { workspace = true, optional = true } [lints] workspace = true + +[features] +optimism = ["op-alloy-rpc-types-engine"] diff --git a/crates/engine/local/src/lib.rs b/crates/engine/local/src/lib.rs index 1b84c8a113f4..26c84d50c851 100644 --- a/crates/engine/local/src/lib.rs +++ b/crates/engine/local/src/lib.rs @@ -1,4 +1,17 @@ //! A local engine service that can be used to drive a dev chain. + +#![doc( + html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", + html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", + issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" +)] +#![cfg_attr(not(test), warn(unused_crate_dependencies))] +#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] + pub mod miner; pub mod payload; pub mod service; + +pub use miner::MiningMode; +pub use payload::LocalPayloadAttributesBuilder; +pub use service::LocalEngineService; diff --git a/crates/engine/local/src/miner.rs b/crates/engine/local/src/miner.rs index de3d8cb8d013..e12a2a50d036 100644 --- a/crates/engine/local/src/miner.rs +++ b/crates/engine/local/src/miner.rs @@ -1,16 +1,31 @@ //! Contains the implementation of the mining mode for the local engine. -use alloy_primitives::TxHash; +use alloy_primitives::{TxHash, B256}; +use alloy_rpc_types_engine::{CancunPayloadFields, ForkchoiceState}; +use eyre::OptionExt; use futures_util::{stream::Fuse, StreamExt}; +use reth_beacon_consensus::BeaconEngineMessage; +use reth_chainspec::EthereumHardforks; +use reth_engine_primitives::EngineTypes; +use reth_payload_builder::PayloadBuilderHandle; +use reth_payload_primitives::{ + BuiltPayload, PayloadAttributesBuilder, PayloadBuilder, PayloadTypes, +}; +use reth_provider::{BlockReader, ChainSpecProvider}; +use reth_rpc_types_compat::engine::payload::block_to_payload; use reth_transaction_pool::TransactionPool; use std::{ future::Future, pin::Pin, task::{Context, Poll}, - time::Duration, + time::{Duration, UNIX_EPOCH}, +}; +use tokio::{ + sync::{mpsc::UnboundedSender, oneshot}, + time::Interval, }; -use tokio::time::Interval; use tokio_stream::wrappers::ReceiverStream; +use tracing::error; /// A mining mode for the local dev engine. #[derive(Debug)] @@ -58,3 +73,175 @@ impl Future for MiningMode { } } } + +/// Local miner advancing the chain/ +#[derive(Debug)] +pub struct LocalMiner { + /// Provider to read the current tip of the chain. + provider: Provider, + /// The payload attribute builder for the engine + payload_attributes_builder: B, + /// Sender for events to engine. + to_engine: UnboundedSender>, + /// The mining mode for the engine + mode: MiningMode, + /// The payload builder for the engine + payload_builder: PayloadBuilderHandle, + /// Timestamp for the next block. + last_timestamp: u64, + /// Stores latest mined blocks. + last_block_hashes: Vec, +} + +impl LocalMiner +where + EngineT: EngineTypes, + Provider: BlockReader + ChainSpecProvider + 'static, + B: PayloadAttributesBuilder<::PayloadAttributes>, +{ + /// Spawns a new [`LocalMiner`] with the given parameters. + pub fn spawn_new( + provider: Provider, + payload_attributes_builder: B, + to_engine: UnboundedSender>, + mode: MiningMode, + payload_builder: PayloadBuilderHandle, + ) { + let latest_header = + provider.sealed_header(provider.best_block_number().unwrap()).unwrap().unwrap(); + + let miner = Self { + provider, + payload_attributes_builder, + to_engine, + mode, + payload_builder, + last_timestamp: latest_header.timestamp, + last_block_hashes: vec![latest_header.hash()], + }; + + // Spawn the miner + tokio::spawn(miner.run()); + } + + /// Runs the [`LocalMiner`] in a loop, polling the miner and building payloads. + async fn run(mut self) { + let mut fcu_interval = tokio::time::interval(Duration::from_secs(1)); + loop { + tokio::select! { + // Wait for the interval or the pool to receive a transaction + _ = &mut self.mode => { + if let Err(e) = self.advance().await { + error!(target: "engine::local", "Error advancing the chain: {:?}", e); + } + } + // send FCU once in a while + _ = fcu_interval.tick() => { + if let Err(e) = self.update_forkchoice_state().await { + error!(target: "engine::local", "Error updating fork choice: {:?}", e); + } + } + } + } + } + + /// Returns current forkchoice state. + fn forkchoice_state(&self) -> ForkchoiceState { + ForkchoiceState { + head_block_hash: *self.last_block_hashes.last().expect("at least 1 block exists"), + safe_block_hash: *self + .last_block_hashes + .get(self.last_block_hashes.len().saturating_sub(32)) + .expect("at least 1 block exists"), + finalized_block_hash: *self + .last_block_hashes + .get(self.last_block_hashes.len().saturating_sub(64)) + .expect("at least 1 block exists"), + } + } + + /// Sends a FCU to the engine. + async fn update_forkchoice_state(&self) -> eyre::Result<()> { + let (tx, rx) = oneshot::channel(); + self.to_engine.send(BeaconEngineMessage::ForkchoiceUpdated { + state: self.forkchoice_state(), + payload_attrs: None, + tx, + })?; + + let res = rx.await??; + if !res.forkchoice_status().is_valid() { + eyre::bail!("Invalid fork choice update") + } + + Ok(()) + } + + /// Generates payload attributes for a new block, passes them to FCU and inserts built payload + /// through newPayload. + async fn advance(&mut self) -> eyre::Result<()> { + let timestamp = std::cmp::max( + self.last_timestamp + 1, + std::time::SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("cannot be earlier than UNIX_EPOCH") + .as_secs(), + ); + + let (tx, rx) = oneshot::channel(); + self.to_engine.send(BeaconEngineMessage::ForkchoiceUpdated { + state: self.forkchoice_state(), + payload_attrs: Some(self.payload_attributes_builder.build(timestamp)), + tx, + })?; + + let res = rx.await??.await?; + if !res.payload_status.is_valid() { + eyre::bail!("Invalid payload status") + } + + let payload_id = res.payload_id.ok_or_eyre("No payload id")?; + + // wait for some time to let the payload be built + tokio::time::sleep(Duration::from_millis(200)).await; + + let Some(Ok(payload)) = self.payload_builder.best_payload(payload_id).await else { + eyre::bail!("No payload") + }; + + let block = payload.block(); + + let cancun_fields = + if self.provider.chain_spec().is_cancun_active_at_timestamp(block.timestamp) { + Some(CancunPayloadFields { + parent_beacon_block_root: block.parent_beacon_block_root.unwrap(), + versioned_hashes: block.blob_versioned_hashes().into_iter().copied().collect(), + }) + } else { + None + }; + + let (tx, rx) = oneshot::channel(); + self.to_engine.send(BeaconEngineMessage::NewPayload { + payload: block_to_payload(payload.block().clone()), + cancun_fields, + tx, + })?; + + let res = rx.await??; + + if !res.is_valid() { + eyre::bail!("Invalid payload") + } + + self.last_timestamp = timestamp; + self.last_block_hashes.push(block.hash()); + // ensure we keep at most 64 blocks + if self.last_block_hashes.len() > 64 { + self.last_block_hashes = + self.last_block_hashes.split_off(self.last_block_hashes.len() - 64); + } + + Ok(()) + } +} diff --git a/crates/engine/local/src/payload.rs b/crates/engine/local/src/payload.rs index 4fd49f53fb49..15d5ff2cf008 100644 --- a/crates/engine/local/src/payload.rs +++ b/crates/engine/local/src/payload.rs @@ -2,29 +2,62 @@ //! [`LocalEngineService`](super::service::LocalEngineService). use alloy_primitives::{Address, B256}; +use reth_chainspec::EthereumHardforks; use reth_ethereum_engine_primitives::EthPayloadAttributes; use reth_payload_primitives::PayloadAttributesBuilder; -use std::{convert::Infallible, time::UNIX_EPOCH}; +use std::sync::Arc; /// The attributes builder for local Ethereum payload. #[derive(Debug)] -pub struct EthLocalPayloadAttributesBuilder; - -impl PayloadAttributesBuilder for EthLocalPayloadAttributesBuilder { - type PayloadAttributes = EthPayloadAttributes; - type Error = Infallible; +#[non_exhaustive] +pub struct LocalPayloadAttributesBuilder { + chain_spec: Arc, +} - fn build(&self) -> Result { - let ts = std::time::SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("cannot be earlier than UNIX_EPOCH"); +impl LocalPayloadAttributesBuilder { + /// Creates a new instance of the builder. + pub const fn new(chain_spec: Arc) -> Self { + Self { chain_spec } + } +} - Ok(EthPayloadAttributes { - timestamp: ts.as_secs(), +impl PayloadAttributesBuilder + for LocalPayloadAttributesBuilder +where + ChainSpec: Send + Sync + EthereumHardforks + 'static, +{ + fn build(&self, timestamp: u64) -> EthPayloadAttributes { + EthPayloadAttributes { + timestamp, prev_randao: B256::random(), suggested_fee_recipient: Address::random(), - withdrawals: None, - parent_beacon_block_root: None, - }) + withdrawals: if self.chain_spec.is_shanghai_active_at_timestamp(timestamp) { + Some(Default::default()) + } else { + None + }, + parent_beacon_block_root: if self.chain_spec.is_cancun_active_at_timestamp(timestamp) { + Some(B256::random()) + } else { + None + }, + } + } +} + +#[cfg(feature = "optimism")] +impl PayloadAttributesBuilder + for LocalPayloadAttributesBuilder +where + ChainSpec: Send + Sync + EthereumHardforks + 'static, +{ + fn build(&self, timestamp: u64) -> op_alloy_rpc_types_engine::OpPayloadAttributes { + op_alloy_rpc_types_engine::OpPayloadAttributes { + payload_attributes: self.build(timestamp), + transactions: None, + no_tx_pool: None, + gas_limit: None, + eip_1559_params: None, + } } } diff --git a/crates/engine/local/src/service.rs b/crates/engine/local/src/service.rs index c9794ecfabb0..93a9cf11ecc4 100644 --- a/crates/engine/local/src/service.rs +++ b/crates/engine/local/src/service.rs @@ -6,357 +6,154 @@ //! with a single transaction. The `Interval` mode will initiate block //! building at a fixed interval. -use crate::miner::MiningMode; -use eyre::eyre; -use reth_beacon_consensus::EngineNodeTypes; -use reth_chain_state::{CanonicalInMemoryState, ExecutedBlock, NewCanonicalChain}; -use reth_engine_tree::persistence::PersistenceHandle; -use reth_payload_builder::PayloadBuilderHandle; -use reth_payload_primitives::{ - BuiltPayload, PayloadAttributesBuilder, PayloadBuilder, PayloadBuilderAttributes, PayloadTypes, +use core::fmt; +use std::{ + fmt::{Debug, Formatter}, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use crate::miner::{LocalMiner, MiningMode}; +use futures_util::{Stream, StreamExt}; +use reth_beacon_consensus::{BeaconConsensusEngineEvent, BeaconEngineMessage, EngineNodeTypes}; +use reth_chainspec::EthChainSpec; +use reth_consensus::Consensus; +use reth_engine_service::service::EngineMessageStream; +use reth_engine_tree::{ + chain::{ChainEvent, HandlerEvent}, + engine::{ + EngineApiKind, EngineApiRequest, EngineApiRequestHandler, EngineRequestHandler, FromEngine, + RequestHandlerEvent, + }, + persistence::PersistenceHandle, + tree::{EngineApiTreeHandler, InvalidBlockHook, TreeConfig}, }; -use reth_provider::ProviderFactory; +use reth_evm::execute::BlockExecutorProvider; +use reth_payload_builder::PayloadBuilderHandle; +use reth_payload_primitives::{PayloadAttributesBuilder, PayloadTypes}; +use reth_payload_validator::ExecutionPayloadValidator; +use reth_provider::{providers::BlockchainProvider2, ChainSpecProvider, ProviderFactory}; use reth_prune::PrunerWithFactory; use reth_stages_api::MetricEventsSender; -use tokio::sync::oneshot; -use tracing::debug; +use tokio::sync::mpsc::UnboundedSender; +use tracing::error; /// Provides a local dev service engine that can be used to drive the /// chain forward. -#[derive(Debug)] -pub struct LocalEngineService +/// +/// This service both produces and consumes [`BeaconEngineMessage`]s. This is done to allow +/// modifications of the stream +pub struct LocalEngineService where N: EngineNodeTypes, - B: PayloadAttributesBuilder::PayloadAttributes>, { - /// The payload builder for the engine - payload_builder: PayloadBuilderHandle, - /// The payload attribute builder for the engine - payload_attributes_builder: B, - /// Keep track of the Canonical chain state that isn't persisted on disk yet - canonical_in_memory_state: CanonicalInMemoryState, - /// A handle to the persistence layer - persistence_handle: PersistenceHandle, - /// The mining mode for the engine - mode: MiningMode, + /// Processes requests. + /// + /// This type is responsible for processing incoming requests. + handler: EngineApiRequestHandler>, + /// Receiver for incoming requests (from the engine API endpoint) that need to be processed. + incoming_requests: EngineMessageStream, } -impl LocalEngineService +impl LocalEngineService where N: EngineNodeTypes, - B: PayloadAttributesBuilder::PayloadAttributes>, { /// Constructor for [`LocalEngineService`]. - pub fn new( - payload_builder: PayloadBuilderHandle, - payload_attributes_builder: B, + #[allow(clippy::too_many_arguments)] + pub fn new( + consensus: Arc, + executor_factory: impl BlockExecutorProvider, provider: ProviderFactory, + blockchain_db: BlockchainProvider2, pruner: PrunerWithFactory>, - canonical_in_memory_state: CanonicalInMemoryState, + payload_builder: PayloadBuilderHandle, + tree_config: TreeConfig, + invalid_block_hook: Box, sync_metrics_tx: MetricEventsSender, + to_engine: UnboundedSender>, + from_engine: EngineMessageStream, mode: MiningMode, - ) -> Self { + payload_attributes_builder: B, + ) -> Self + where + B: PayloadAttributesBuilder<::PayloadAttributes>, + { + let chain_spec = provider.chain_spec(); + let engine_kind = + if chain_spec.is_optimism() { EngineApiKind::OpStack } else { EngineApiKind::Ethereum }; + let persistence_handle = PersistenceHandle::spawn_service(provider, pruner, sync_metrics_tx); + let payload_validator = ExecutionPayloadValidator::new(chain_spec); - Self { - payload_builder, - payload_attributes_builder, - canonical_in_memory_state, + let canonical_in_memory_state = blockchain_db.canonical_in_memory_state(); + + let (to_tree_tx, from_tree) = EngineApiTreeHandler::spawn_new( + blockchain_db.clone(), + executor_factory, + consensus, + payload_validator, persistence_handle, - mode, - } - } + payload_builder.clone(), + canonical_in_memory_state, + tree_config, + invalid_block_hook, + engine_kind, + ); - /// Spawn the [`LocalEngineService`] on a tokio green thread. The service will poll the payload - /// builder with two varying modes, [`MiningMode::Instant`] or [`MiningMode::Interval`] - /// which will respectively either execute the block as soon as it finds a - /// transaction in the pool or build the block based on an interval. - pub fn spawn_new( - payload_builder: PayloadBuilderHandle, - payload_attributes_builder: B, - provider: ProviderFactory, - pruner: PrunerWithFactory>, - canonical_in_memory_state: CanonicalInMemoryState, - sync_metrics_tx: MetricEventsSender, - mode: MiningMode, - ) { - let engine = Self::new( - payload_builder, + let handler = EngineApiRequestHandler::new(to_tree_tx, from_tree); + + LocalMiner::spawn_new( + blockchain_db, payload_attributes_builder, - provider, - pruner, - canonical_in_memory_state, - sync_metrics_tx, + to_engine, mode, + payload_builder, ); - // Spawn the engine - tokio::spawn(engine.run()); + Self { handler, incoming_requests: from_engine } } +} - /// Runs the [`LocalEngineService`] in a loop, polling the miner and building - /// payloads. - async fn run(mut self) { - loop { - // Wait for the interval or the pool to receive a transaction - (&mut self.mode).await; - - // Start a new payload building job - let executed_block = self.build_and_save_payload().await; - - if executed_block.is_err() { - debug!(target: "local_engine", err = ?executed_block.unwrap_err(), "failed payload building"); - continue - } - let block = executed_block.expect("not error"); - - let res = self.update_canonical_in_memory_state(block); - if res.is_err() { - debug!(target: "local_engine", err = ?res.unwrap_err(), "failed canonical state update"); +impl Stream for LocalEngineService +where + N: EngineNodeTypes, +{ + type Item = ChainEvent; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + + if let Poll::Ready(ev) = this.handler.poll(cx) { + return match ev { + RequestHandlerEvent::HandlerEvent(ev) => match ev { + HandlerEvent::BackfillAction(_) => { + error!(target: "engine::local", "received backfill request in local engine"); + Poll::Ready(Some(ChainEvent::FatalError)) + } + HandlerEvent::Event(ev) => Poll::Ready(Some(ChainEvent::Handler(ev))), + HandlerEvent::FatalError => Poll::Ready(Some(ChainEvent::FatalError)), + }, + RequestHandlerEvent::Download(_) => { + error!(target: "engine::local", "received download request in local engine"); + Poll::Ready(Some(ChainEvent::FatalError)) + } } } - } - - /// Builds a payload by initiating a new payload job via the [`PayloadBuilderHandle`], - /// saving the execution outcome to persistence and returning the executed block. - async fn build_and_save_payload(&self) -> eyre::Result { - let payload_attributes = self.payload_attributes_builder.build()?; - let parent = self.canonical_in_memory_state.get_canonical_head().hash(); - let payload_builder_attributes = - ::PayloadBuilderAttributes::try_new( - parent, - payload_attributes, - ) - .map_err(|_| eyre::eyre!("failed to fetch payload attributes"))?; - - let payload = self - .payload_builder - .send_and_resolve_payload(payload_builder_attributes) - .await? - .await?; - - let executed_block = - payload.executed_block().ok_or_else(|| eyre!("missing executed block"))?; - let (tx, rx) = oneshot::channel(); - - let _ = self.persistence_handle.save_blocks(vec![executed_block.clone()], tx); - - // Wait for the persistence_handle to complete - let _ = rx.await?.ok_or_else(|| eyre!("missing new head"))?; - Ok(executed_block) - } - - /// Update the canonical in memory state and send notification for a new canon state to - /// all the listeners. - fn update_canonical_in_memory_state(&self, executed_block: ExecutedBlock) -> eyre::Result<()> { - let chain = NewCanonicalChain::Commit { new: vec![executed_block] }; - let tip = chain.tip().header.clone(); - let notification = chain.to_chain_notification(); - - // Update the tracked in-memory state with the new chain - self.canonical_in_memory_state.update_chain(chain); - self.canonical_in_memory_state.set_canonical_head(tip); - - // Sends an event to all active listeners about the new canonical chain - self.canonical_in_memory_state.notify_canon_state(notification); - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use reth_chainspec::MAINNET; - use reth_config::PruneConfig; - use reth_db::test_utils::{create_test_rw_db, create_test_static_files_dir}; - use reth_ethereum_engine_primitives::EthEngineTypes; - use reth_exex_test_utils::TestNode; - use reth_node_types::NodeTypesWithDBAdapter; - use reth_payload_builder::test_utils::spawn_test_payload_service; - use reth_provider::{providers::StaticFileProvider, BlockReader, ProviderFactory}; - use reth_prune::PrunerBuilder; - use reth_transaction_pool::{ - test_utils::{testing_pool, MockTransaction}, - TransactionPool, - }; - use std::{convert::Infallible, time::Duration}; - use tokio::sync::mpsc::unbounded_channel; - - #[derive(Debug)] - struct TestPayloadAttributesBuilder; - - impl PayloadAttributesBuilder for TestPayloadAttributesBuilder { - type PayloadAttributes = alloy_rpc_types_engine::PayloadAttributes; - type Error = Infallible; - - fn build(&self) -> Result { - Ok(alloy_rpc_types_engine::PayloadAttributes { - timestamp: 0, - prev_randao: Default::default(), - suggested_fee_recipient: Default::default(), - withdrawals: None, - parent_beacon_block_root: None, - }) + // forward incoming requests to the handler + while let Poll::Ready(Some(req)) = this.incoming_requests.poll_next_unpin(cx) { + this.handler.on_event(FromEngine::Request(req.into())); } - } - - #[tokio::test] - async fn test_local_engine_service_interval() -> eyre::Result<()> { - reth_tracing::init_test_tracing(); - - // Start the provider and the pruner - let (_, static_dir_path) = create_test_static_files_dir(); - let provider = ProviderFactory::>::new( - create_test_rw_db(), - MAINNET.clone(), - StaticFileProvider::read_write(static_dir_path)?, - ); - let pruner = PrunerBuilder::new(PruneConfig::default()) - .build_with_provider_factory(provider.clone()); - - // Create an empty canonical in memory state - let canonical_in_memory_state = CanonicalInMemoryState::empty(); - - // Start the payload builder service - let payload_handle = spawn_test_payload_service::(); - - // Sync metric channel - let (sync_metrics_tx, _) = unbounded_channel(); - - // Launch the LocalEngineService in interval mode - let period = Duration::from_secs(1); - LocalEngineService::spawn_new( - payload_handle, - TestPayloadAttributesBuilder, - provider.clone(), - pruner, - canonical_in_memory_state, - sync_metrics_tx, - MiningMode::interval(period), - ); - - // Check that we have no block for now - let block = provider.block_by_number(0)?; - assert!(block.is_none()); - // Wait 4 intervals - tokio::time::sleep(2 * period).await; - - // Assert a block has been build - let block = provider.block_by_number(0)?; - assert!(block.is_some()); - - Ok(()) + Poll::Pending } +} - #[tokio::test] - async fn test_local_engine_service_instant() -> eyre::Result<()> { - reth_tracing::init_test_tracing(); - - // Start the provider and the pruner - let (_, static_dir_path) = create_test_static_files_dir(); - let provider = ProviderFactory::>::new( - create_test_rw_db(), - MAINNET.clone(), - StaticFileProvider::read_write(static_dir_path)?, - ); - let pruner = PrunerBuilder::new(PruneConfig::default()) - .build_with_provider_factory(provider.clone()); - - // Create an empty canonical in memory state - let canonical_in_memory_state = CanonicalInMemoryState::empty(); - - // Start the payload builder service - let payload_handle = spawn_test_payload_service::(); - - // Start a transaction pool - let pool = testing_pool(); - - // Sync metric channel - let (sync_metrics_tx, _) = unbounded_channel(); - - // Launch the LocalEngineService in instant mode - LocalEngineService::spawn_new( - payload_handle, - TestPayloadAttributesBuilder, - provider.clone(), - pruner, - canonical_in_memory_state, - sync_metrics_tx, - MiningMode::instant(pool.clone()), - ); - - // Wait for a small period to assert block building is - // triggered by adding a transaction to the pool - let period = Duration::from_millis(500); - tokio::time::sleep(period).await; - let block = provider.block_by_number(0)?; - assert!(block.is_none()); - - // Add a transaction to the pool - let transaction = MockTransaction::legacy().with_gas_price(10); - pool.add_transaction(Default::default(), transaction).await?; - - // Wait for block building - let period = Duration::from_secs(2); - tokio::time::sleep(period).await; - - // Assert a block has been build - let block = provider.block_by_number(0)?; - assert!(block.is_some()); - - Ok(()) - } - - #[tokio::test] - async fn test_canonical_chain_subscription() -> eyre::Result<()> { - reth_tracing::init_test_tracing(); - - // Start the provider and the pruner - let (_, static_dir_path) = create_test_static_files_dir(); - let provider = ProviderFactory::>::new( - create_test_rw_db(), - MAINNET.clone(), - StaticFileProvider::read_write(static_dir_path)?, - ); - let pruner = PrunerBuilder::new(PruneConfig::default()) - .build_with_provider_factory(provider.clone()); - - // Create an empty canonical in memory state - let canonical_in_memory_state = CanonicalInMemoryState::empty(); - let mut notifications = canonical_in_memory_state.subscribe_canon_state(); - - // Start the payload builder service - let payload_handle = spawn_test_payload_service::(); - - // Start a transaction pool - let pool = testing_pool(); - - // Sync metric channel - let (sync_metrics_tx, _) = unbounded_channel(); - - // Launch the LocalEngineService in instant mode - LocalEngineService::spawn_new( - payload_handle, - TestPayloadAttributesBuilder, - provider.clone(), - pruner, - canonical_in_memory_state, - sync_metrics_tx, - MiningMode::instant(pool.clone()), - ); - - // Add a transaction to the pool - let transaction = MockTransaction::legacy().with_gas_price(10); - pool.add_transaction(Default::default(), transaction).await?; - - // Check a notification is received for block 0 - let res = notifications.recv().await?; - - assert_eq!(res.tip().number, 0); - - Ok(()) +impl Debug for LocalEngineService { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("LocalEngineService").finish_non_exhaustive() } } diff --git a/crates/engine/service/src/service.rs b/crates/engine/service/src/service.rs index ed9c1aa1c27e..026476a8260f 100644 --- a/crates/engine/service/src/service.rs +++ b/crates/engine/service/src/service.rs @@ -31,7 +31,7 @@ use std::{ }; /// Alias for consensus engine stream. -type EngineMessageStream = Pin> + Send + Sync>>; +pub type EngineMessageStream = Pin> + Send + Sync>>; /// Alias for chain orchestrator. type EngineServiceType = ChainOrchestrator< diff --git a/crates/ethereum/node/tests/e2e/dev.rs b/crates/ethereum/node/tests/e2e/dev.rs index 2ef6e08c7e42..6b4733b6f9b4 100644 --- a/crates/ethereum/node/tests/e2e/dev.rs +++ b/crates/ethereum/node/tests/e2e/dev.rs @@ -3,29 +3,64 @@ use std::sync::Arc; use alloy_genesis::Genesis; use alloy_primitives::{b256, hex}; use futures::StreamExt; -use reth::core::rpc::eth::helpers::EthTransactions; +use reth::{args::DevArgs, core::rpc::eth::helpers::EthTransactions}; use reth_chainspec::ChainSpec; use reth_e2e_test_utils::setup; -use reth_provider::CanonStateSubscriptions; - -use crate::utils::EthNode; +use reth_node_api::{FullNodeComponents, NodeAddOns}; +use reth_node_builder::{EngineNodeLauncher, FullNode, NodeBuilder, NodeConfig, NodeHandle}; +use reth_node_ethereum::{node::EthereumAddOns, EthereumNode}; +use reth_provider::{providers::BlockchainProvider2, CanonStateSubscriptions}; +use reth_tasks::TaskManager; #[tokio::test] async fn can_run_dev_node() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let (mut nodes, _tasks, _) = setup(1, custom_chain(), true).await?; + let (mut nodes, _tasks, _) = setup::(1, custom_chain(), true).await?; + + assert_chain_advances(nodes.pop().unwrap().inner).await; + Ok(()) +} + +#[tokio::test] +async fn can_run_dev_node_new_engine() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + let tasks = TaskManager::current(); + let exec = tasks.executor(); + + let node_config = NodeConfig::test() + .with_chain(custom_chain()) + .with_dev(DevArgs { dev: true, ..Default::default() }); + let NodeHandle { node, .. } = NodeBuilder::new(node_config.clone()) + .testing_node(exec.clone()) + .with_types_and_provider::>() + .with_components(EthereumNode::components()) + .with_add_ons(EthereumAddOns::default()) + .launch_with_fn(|builder| { + let launcher = EngineNodeLauncher::new( + builder.task_executor().clone(), + builder.config().datadir(), + Default::default(), + ); + builder.launch_with(launcher) + }) + .await?; + + assert_chain_advances(node).await; - assert_chain_advances(nodes.pop().unwrap()).await; Ok(()) } -async fn assert_chain_advances(node: EthNode) { - let mut notifications = node.inner.provider.canonical_state_stream(); +async fn assert_chain_advances(node: FullNode) +where + N: FullNodeComponents, + AddOns: NodeAddOns, +{ + let mut notifications = node.provider.canonical_state_stream(); // submit tx through rpc let raw_tx = hex!("02f876820a28808477359400847735940082520894ab0840c0e43688012c1adb0f5e3fc665188f83d28a029d394a5d630544000080c080a0a044076b7e67b5deecc63f61a8d7913fab86ca365b344b5759d1fe3563b4c39ea019eab979dd000da04dfc72bb0377c092d30fd9e1cab5ae487de49586cc8b0090"); - let eth_api = node.inner.rpc_registry.eth_api(); + let eth_api = node.rpc_registry.eth_api(); let hash = eth_api.send_raw_transaction(raw_tx.into()).await.unwrap(); diff --git a/crates/ethereum/node/tests/e2e/utils.rs b/crates/ethereum/node/tests/e2e/utils.rs index 5a7950999189..6e534f5dc0ed 100644 --- a/crates/ethereum/node/tests/e2e/utils.rs +++ b/crates/ethereum/node/tests/e2e/utils.rs @@ -1,12 +1,7 @@ use alloy_primitives::{Address, B256}; use reth::rpc::types::engine::PayloadAttributes; -use reth_e2e_test_utils::NodeHelperType; -use reth_node_ethereum::{node::EthereumAddOns, EthereumNode}; use reth_payload_builder::EthPayloadBuilderAttributes; -/// Ethereum Node Helper type -pub(crate) type EthNode = NodeHelperType; - /// Helper function to create a new eth payload attributes pub(crate) fn eth_payload_attributes(timestamp: u64) -> EthPayloadBuilderAttributes { let attributes = PayloadAttributes { diff --git a/crates/node/builder/Cargo.toml b/crates/node/builder/Cargo.toml index 1bf2ba233739..53e53cd2b856 100644 --- a/crates/node/builder/Cargo.toml +++ b/crates/node/builder/Cargo.toml @@ -26,6 +26,7 @@ reth-db = { workspace = true, features = ["mdbx"], optional = true } reth-db-api.workspace = true reth-db-common.workspace = true reth-downloaders.workspace = true +reth-engine-local.workspace = true reth-engine-service.workspace = true reth-engine-tree.workspace = true reth-engine-util.workspace = true diff --git a/crates/node/builder/src/launch/engine.rs b/crates/node/builder/src/launch/engine.rs index 782cc7bbb1b0..f9e26f202fc4 100644 --- a/crates/node/builder/src/launch/engine.rs +++ b/crates/node/builder/src/launch/engine.rs @@ -9,6 +9,7 @@ use reth_beacon_consensus::{ use reth_blockchain_tree::BlockchainTreeConfig; use reth_chainspec::EthChainSpec; use reth_consensus_debug_client::{DebugConsensusClient, EtherscanBlockProvider}; +use reth_engine_local::{LocalEngineService, LocalPayloadAttributesBuilder, MiningMode}; use reth_engine_service::service::{ChainEvent, EngineService}; use reth_engine_tree::{ engine::{EngineApiRequest, EngineRequestHandler}, @@ -18,7 +19,10 @@ use reth_engine_util::EngineMessageStreamExt; use reth_exex::ExExManagerHandle; use reth_network::{NetworkSyncUpdater, SyncState}; use reth_network_api::{BlockDownloaderProvider, NetworkEventListenerProvider}; -use reth_node_api::{BuiltPayload, FullNodeTypes, NodeAddOns, NodeTypesWithEngine}; +use reth_node_api::{ + BuiltPayload, FullNodeTypes, NodeAddOns, NodeTypesWithEngine, PayloadAttributesBuilder, + PayloadTypes, +}; use reth_node_core::{ dirs::{ChainPath, DataDirPath}, exit::NodeExitFuture, @@ -80,6 +84,9 @@ where + FullEthApiServer + AddDevSigners, >, + LocalPayloadAttributesBuilder: PayloadAttributesBuilder< + <::Engine as PayloadTypes>::PayloadAttributes, + >, { type Node = NodeHandle, AO>; @@ -210,23 +217,49 @@ where let pruner_events = pruner.events(); info!(target: "reth::cli", prune_config=?ctx.prune_config().unwrap_or_default(), "Pruner initialized"); - // Configure the consensus engine - let mut eth_service = EngineService::new( - ctx.consensus(), - ctx.components().block_executor().clone(), - ctx.chain_spec(), - network_client.clone(), - Box::pin(consensus_engine_stream), - pipeline, - Box::new(ctx.task_executor().clone()), - ctx.provider_factory().clone(), - ctx.blockchain_db().clone(), - pruner, - ctx.components().payload_builder().clone(), - engine_tree_config, - ctx.invalid_block_hook()?, - ctx.sync_metrics_tx(), - ); + let mut engine_service = if ctx.is_dev() { + let mining_mode = if let Some(block_time) = ctx.node_config().dev.block_time { + MiningMode::interval(block_time) + } else { + MiningMode::instant(ctx.components().pool().clone()) + }; + let eth_service = LocalEngineService::new( + ctx.consensus(), + ctx.components().block_executor().clone(), + ctx.provider_factory().clone(), + ctx.blockchain_db().clone(), + pruner, + ctx.components().payload_builder().clone(), + engine_tree_config, + ctx.invalid_block_hook()?, + ctx.sync_metrics_tx(), + consensus_engine_tx.clone(), + Box::pin(consensus_engine_stream), + mining_mode, + LocalPayloadAttributesBuilder::new(ctx.chain_spec()), + ); + + Either::Left(eth_service) + } else { + let eth_service = EngineService::new( + ctx.consensus(), + ctx.components().block_executor().clone(), + ctx.chain_spec(), + network_client.clone(), + Box::pin(consensus_engine_stream), + pipeline, + Box::new(ctx.task_executor().clone()), + ctx.provider_factory().clone(), + ctx.blockchain_db().clone(), + pruner, + ctx.components().payload_builder().clone(), + engine_tree_config, + ctx.invalid_block_hook()?, + ctx.sync_metrics_tx(), + ); + + Either::Right(eth_service) + }; let event_sender = EventSender::default(); @@ -340,7 +373,9 @@ where ctx.task_executor().spawn_critical("consensus engine", async move { if let Some(initial_target) = initial_target { debug!(target: "reth::cli", %initial_target, "start backfill sync"); - eth_service.orchestrator_mut().start_backfill_sync(initial_target); + if let Either::Right(eth_service) = &mut engine_service { + eth_service.orchestrator_mut().start_backfill_sync(initial_target); + } } let mut res = Ok(()); @@ -351,10 +386,12 @@ where payload = built_payloads.select_next_some() => { if let Some(executed_block) = payload.executed_block() { debug!(target: "reth::cli", block=?executed_block.block().num_hash(), "inserting built payload"); - eth_service.orchestrator_mut().handler_mut().handler_mut().on_event(EngineApiRequest::InsertExecutedBlock(executed_block).into()); + if let Either::Right(eth_service) = &mut engine_service { + eth_service.orchestrator_mut().handler_mut().handler_mut().on_event(EngineApiRequest::InsertExecutedBlock(executed_block).into()); + } } } - event = eth_service.next() => { + event = engine_service.next() => { let Some(event) = event else { break }; debug!(target: "reth::cli", "Event: {event}"); match event { diff --git a/crates/node/events/src/node.rs b/crates/node/events/src/node.rs index c856c0ec9e18..e10caaee7c53 100644 --- a/crates/node/events/src/node.rs +++ b/crates/node/events/src/node.rs @@ -504,7 +504,7 @@ where } else if let Some(latest_block) = this.state.latest_block { let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); - if now - this.state.latest_block_time.unwrap_or(0) > 60 { + if now.saturating_sub(this.state.latest_block_time.unwrap_or(0)) > 60 { // Once we start receiving consensus nodes, don't emit status unless stalled for // 1 minute info!( diff --git a/crates/optimism/node/Cargo.toml b/crates/optimism/node/Cargo.toml index 3c298bea9cee..4675029b64c8 100644 --- a/crates/optimism/node/Cargo.toml +++ b/crates/optimism/node/Cargo.toml @@ -13,6 +13,7 @@ workspace = true [dependencies] # reth reth-chainspec.workspace = true +reth-engine-local.workspace = true reth-primitives.workspace = true reth-payload-builder.workspace = true reth-auto-seal-consensus.workspace = true @@ -87,6 +88,7 @@ optimism = [ "reth-revm/optimism", "reth-auto-seal-consensus/optimism", "reth-optimism-rpc/optimism", + "reth-engine-local/optimism", ] asm-keccak = ["reth-primitives/asm-keccak"] test-utils = ["reth-node-builder/test-utils"] diff --git a/crates/payload/primitives/src/traits.rs b/crates/payload/primitives/src/traits.rs index 6ae6361fdb28..494ed68aa4ee 100644 --- a/crates/payload/primitives/src/traits.rs +++ b/crates/payload/primitives/src/traits.rs @@ -160,12 +160,7 @@ impl PayloadAttributes for OpPayloadAttributes { } /// A builder that can return the current payload attribute. -pub trait PayloadAttributesBuilder: std::fmt::Debug + Send + Sync + 'static { - /// The payload attributes type returned by the builder. - type PayloadAttributes: PayloadAttributes; - /// The error type returned by [`PayloadAttributesBuilder::build`]. - type Error: core::error::Error + Send + Sync; - +pub trait PayloadAttributesBuilder: Send + Sync + 'static { /// Return a new payload attribute from the builder. - fn build(&self) -> Result; + fn build(&self, timestamp: u64) -> Attributes; } From f684dd4c4cebc9f0d0b6b1402530d45ae952e7f3 Mon Sep 17 00:00:00 2001 From: Delweng Date: Mon, 14 Oct 2024 23:45:26 +0800 Subject: [PATCH 68/82] chore(clippy): enable if_then_some_else_none lint (#11679) Signed-off-by: jsvisa Co-authored-by: Matthias Seitz --- Cargo.toml | 1 + crates/chainspec/src/spec.rs | 7 ++-- crates/consensus/auto-seal/src/lib.rs | 32 +++++++------------ crates/engine/local/src/miner.rs | 10 +++--- crates/engine/local/src/payload.rs | 18 +++++------ crates/ethereum/evm/src/lib.rs | 11 ++----- crates/net/network/src/cache.rs | 2 +- crates/net/p2p/src/test_utils/full_block.rs | 18 +++++------ crates/node/core/src/args/log.rs | 2 +- crates/optimism/evm/src/lib.rs | 11 ++----- .../src/segments/user/receipts_by_logs.rs | 6 +--- .../rpc-eth-api/src/helpers/pending_block.rs | 2 +- crates/storage/db-models/src/accounts.rs | 8 ++--- .../storage/db/src/implementation/mdbx/mod.rs | 7 ++-- .../src/providers/database/provider.rs | 14 ++++---- .../src/providers/static_file/manager.rs | 2 +- .../src/providers/static_file/writer.rs | 20 ++++++------ .../storage/provider/src/test_utils/mock.rs | 15 ++------- crates/transaction-pool/src/maintain.rs | 9 +++--- crates/trie/common/src/subnode.rs | 10 +++--- crates/trie/trie/src/trie_cursor/subnode.rs | 2 +- crates/trie/trie/src/witness.rs | 15 ++++----- 22 files changed, 87 insertions(+), 135 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 628d7d47b225..ac0ab2d39228 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -186,6 +186,7 @@ explicit_iter_loop = "warn" flat_map_option = "warn" from_iter_instead_of_collect = "warn" if_not_else = "warn" +if_then_some_else_none = "warn" implicit_clone = "warn" imprecise_flops = "warn" iter_on_empty_collections = "warn" diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index f80a20924394..e43e0bc78d86 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -297,11 +297,8 @@ impl ChainSpec { }; // If Prague is activated at genesis we set requests root to an empty trie root. - let requests_root = if self.is_prague_active_at_timestamp(self.genesis.timestamp) { - Some(EMPTY_ROOT_HASH) - } else { - None - }; + let requests_root = + self.is_prague_active_at_timestamp(self.genesis.timestamp).then_some(EMPTY_ROOT_HASH); Header { gas_limit: self.genesis.gas_limit, diff --git a/crates/consensus/auto-seal/src/lib.rs b/crates/consensus/auto-seal/src/lib.rs index f1ef64c8c0fa..261227f1074e 100644 --- a/crates/consensus/auto-seal/src/lib.rs +++ b/crates/consensus/auto-seal/src/lib.rs @@ -282,17 +282,13 @@ impl StorageInner { parent.next_block_base_fee(chain_spec.base_fee_params_at_timestamp(timestamp)) }); - let blob_gas_used = if chain_spec.is_cancun_active_at_timestamp(timestamp) { - let mut sum_blob_gas_used = 0; - for tx in transactions { - if let Some(blob_tx) = tx.transaction.as_eip4844() { - sum_blob_gas_used += blob_tx.blob_gas(); - } - } - Some(sum_blob_gas_used) - } else { - None - }; + let blob_gas_used = chain_spec.is_cancun_active_at_timestamp(timestamp).then(|| { + transactions + .iter() + .filter_map(|tx| tx.transaction.as_eip4844()) + .map(|blob_tx| blob_tx.blob_gas()) + .sum::() + }); let mut header = Header { parent_hash: self.best_hash, @@ -304,7 +300,7 @@ impl StorageInner { gas_limit: chain_spec.max_gas_limit(), timestamp, base_fee_per_gas, - blob_gas_used: blob_gas_used.map(Into::into), + blob_gas_used, requests_root: requests.map(|r| proofs::calculate_requests_root(&r.0)), ..Default::default() }; @@ -316,14 +312,10 @@ impl StorageInner { header.blob_gas_used = Some(0); let (parent_excess_blob_gas, parent_blob_gas_used) = match parent { - Some(parent_block) - if chain_spec.is_cancun_active_at_timestamp(parent_block.timestamp) => - { - ( - parent_block.excess_blob_gas.unwrap_or_default(), - parent_block.blob_gas_used.unwrap_or_default(), - ) - } + Some(parent) if chain_spec.is_cancun_active_at_timestamp(parent.timestamp) => ( + parent.excess_blob_gas.unwrap_or_default(), + parent.blob_gas_used.unwrap_or_default(), + ), _ => (0, 0), }; header.excess_blob_gas = diff --git a/crates/engine/local/src/miner.rs b/crates/engine/local/src/miner.rs index e12a2a50d036..f20d70b14893 100644 --- a/crates/engine/local/src/miner.rs +++ b/crates/engine/local/src/miner.rs @@ -212,14 +212,12 @@ where let block = payload.block(); let cancun_fields = - if self.provider.chain_spec().is_cancun_active_at_timestamp(block.timestamp) { - Some(CancunPayloadFields { + self.provider.chain_spec().is_cancun_active_at_timestamp(block.timestamp).then(|| { + CancunPayloadFields { parent_beacon_block_root: block.parent_beacon_block_root.unwrap(), versioned_hashes: block.blob_versioned_hashes().into_iter().copied().collect(), - }) - } else { - None - }; + } + }); let (tx, rx) = oneshot::channel(); self.to_engine.send(BeaconEngineMessage::NewPayload { diff --git a/crates/engine/local/src/payload.rs b/crates/engine/local/src/payload.rs index 15d5ff2cf008..5111360d5bf6 100644 --- a/crates/engine/local/src/payload.rs +++ b/crates/engine/local/src/payload.rs @@ -31,16 +31,14 @@ where timestamp, prev_randao: B256::random(), suggested_fee_recipient: Address::random(), - withdrawals: if self.chain_spec.is_shanghai_active_at_timestamp(timestamp) { - Some(Default::default()) - } else { - None - }, - parent_beacon_block_root: if self.chain_spec.is_cancun_active_at_timestamp(timestamp) { - Some(B256::random()) - } else { - None - }, + withdrawals: self + .chain_spec + .is_shanghai_active_at_timestamp(timestamp) + .then(Default::default), + parent_beacon_block_root: self + .chain_spec + .is_cancun_active_at_timestamp(timestamp) + .then(B256::random), } } } diff --git a/crates/ethereum/evm/src/lib.rs b/crates/ethereum/evm/src/lib.rs index a71f26f703e4..8ea39f93deef 100644 --- a/crates/ethereum/evm/src/lib.rs +++ b/crates/ethereum/evm/src/lib.rs @@ -134,17 +134,10 @@ impl ConfigureEvmEnv for EthEvmConfig { let spec_id = revm_spec_by_timestamp_after_merge(&self.chain_spec, attributes.timestamp); // if the parent block did not have excess blob gas (i.e. it was pre-cancun), but it is - // cancun now, we need to set the excess blob gas to the default value + // cancun now, we need to set the excess blob gas to the default value(0) let blob_excess_gas_and_price = parent .next_block_excess_blob_gas() - .or_else(|| { - if spec_id == SpecId::CANCUN { - // default excess blob gas is zero - Some(0) - } else { - None - } - }) + .or_else(|| (spec_id == SpecId::CANCUN).then_some(0)) .map(BlobExcessGasAndPrice::new); let mut basefee = parent.next_block_base_fee( diff --git a/crates/net/network/src/cache.rs b/crates/net/network/src/cache.rs index fb2daca666e4..758b49167908 100644 --- a/crates/net/network/src/cache.rs +++ b/crates/net/network/src/cache.rs @@ -42,7 +42,7 @@ impl LruCache { pub fn insert_and_get_evicted(&mut self, entry: T) -> (bool, Option) { let new = self.inner.peek(&entry).is_none(); let evicted = - if new && (self.limit as usize) <= self.inner.len() { self.remove_lru() } else { None }; + (new && (self.limit as usize) <= self.inner.len()).then(|| self.remove_lru()).flatten(); _ = self.inner.get_or_insert(entry, || ()); (new, evicted) diff --git a/crates/net/p2p/src/test_utils/full_block.rs b/crates/net/p2p/src/test_utils/full_block.rs index acc01a60ef80..8a13f69325dc 100644 --- a/crates/net/p2p/src/test_utils/full_block.rs +++ b/crates/net/p2p/src/test_utils/full_block.rs @@ -185,15 +185,15 @@ impl HeadersClient for TestFullBlockClient { .filter_map(|_| { headers.iter().find_map(|(hash, header)| { // Checks if the header matches the specified block or number. - if BlockNumHash::new(header.number, *hash).matches_block_or_num(&block) { - match request.direction { - HeadersDirection::Falling => block = header.parent_hash.into(), - HeadersDirection::Rising => block = (header.number + 1).into(), - } - Some(header.clone()) - } else { - None - } + BlockNumHash::new(header.number, *hash).matches_block_or_num(&block).then( + || { + match request.direction { + HeadersDirection::Falling => block = header.parent_hash.into(), + HeadersDirection::Rising => block = (header.number + 1).into(), + } + header.clone() + }, + ) }) }) .collect::>(); diff --git a/crates/node/core/src/args/log.rs b/crates/node/core/src/args/log.rs index aa2e0cf5f1a3..3d124fba229f 100644 --- a/crates/node/core/src/args/log.rs +++ b/crates/node/core/src/args/log.rs @@ -78,7 +78,7 @@ impl LogArgs { format, self.verbosity.directive().to_string(), filter, - if use_color { Some(self.color.to_string()) } else { None }, + use_color.then(|| self.color.to_string()), ) } diff --git a/crates/optimism/evm/src/lib.rs b/crates/optimism/evm/src/lib.rs index b220b6056de0..158ed2e8929e 100644 --- a/crates/optimism/evm/src/lib.rs +++ b/crates/optimism/evm/src/lib.rs @@ -139,17 +139,10 @@ impl ConfigureEvmEnv for OptimismEvmConfig { let spec_id = revm_spec_by_timestamp_after_bedrock(&self.chain_spec, attributes.timestamp); // if the parent block did not have excess blob gas (i.e. it was pre-cancun), but it is - // cancun now, we need to set the excess blob gas to the default value + // cancun now, we need to set the excess blob gas to the default value(0) let blob_excess_gas_and_price = parent .next_block_excess_blob_gas() - .or_else(|| { - if spec_id.is_enabled_in(SpecId::CANCUN) { - // default excess blob gas is zero - Some(0) - } else { - None - } - }) + .or_else(|| (spec_id.is_enabled_in(SpecId::CANCUN)).then_some(0)) .map(BlobExcessGasAndPrice::new); let block_env = BlockEnv { diff --git a/crates/prune/prune/src/segments/user/receipts_by_logs.rs b/crates/prune/prune/src/segments/user/receipts_by_logs.rs index 489df7e72273..05bc40b6c7b0 100644 --- a/crates/prune/prune/src/segments/user/receipts_by_logs.rs +++ b/crates/prune/prune/src/segments/user/receipts_by_logs.rs @@ -267,11 +267,7 @@ mod tests { let mut receipt = random_receipt(&mut rng, transaction, Some(1)); receipt.logs.push(random_log( &mut rng, - if txi == (block.body.transactions.len() - 1) { - Some(deposit_contract_addr) - } else { - None - }, + (txi == (block.body.transactions.len() - 1)).then_some(deposit_contract_addr), Some(1), )); receipts.push((receipts.len() as u64, receipt)); diff --git a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs index 03597289f151..85ad7fd18107 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs @@ -415,7 +415,7 @@ pub trait LoadPendingBlock: EthApiTypes { // check if cancun is activated to set eip4844 header fields correctly let blob_gas_used = - if cfg.handler_cfg.spec_id >= SpecId::CANCUN { Some(sum_blob_gas_used) } else { None }; + (cfg.handler_cfg.spec_id >= SpecId::CANCUN).then_some(sum_blob_gas_used); // note(onbjerg): the rpc spec has not been changed to include requests, so for now we just // set these to empty diff --git a/crates/storage/db-models/src/accounts.rs b/crates/storage/db-models/src/accounts.rs index e1f4773960fa..b0099d22d5fb 100644 --- a/crates/storage/db-models/src/accounts.rs +++ b/crates/storage/db-models/src/accounts.rs @@ -39,13 +39,11 @@ impl Compact for AccountBeforeTx { let address = Address::from_slice(&buf[..20]); buf.advance(20); - let info = if len - 20 > 0 { + let info = (len - 20 > 0).then(|| { let (acc, advanced_buf) = Account::from_compact(buf, len - 20); buf = advanced_buf; - Some(acc) - } else { - None - }; + acc + }); (Self { address, info }, buf) } diff --git a/crates/storage/db/src/implementation/mdbx/mod.rs b/crates/storage/db/src/implementation/mdbx/mod.rs index 1deb86ba614f..65b804e6a58d 100644 --- a/crates/storage/db/src/implementation/mdbx/mod.rs +++ b/crates/storage/db/src/implementation/mdbx/mod.rs @@ -256,10 +256,9 @@ impl DatabaseEnv { args: DatabaseArguments, ) -> Result { let _lock_file = if kind.is_rw() { - Some( - StorageLock::try_acquire(path) - .map_err(|err| DatabaseError::Other(err.to_string()))?, - ) + StorageLock::try_acquire(path) + .map_err(|err| DatabaseError::Other(err.to_string()))? + .into() } else { None }; diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 33fed1e80ce1..3b074a5afffa 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -542,18 +542,18 @@ impl DatabaseProvider { // even if empty let withdrawals = if self.chain_spec.is_shanghai_active_at_timestamp(header_ref.timestamp) { - Some( - withdrawals_cursor - .seek_exact(header_ref.number)? - .map(|(_, w)| w.withdrawals) - .unwrap_or_default(), - ) + withdrawals_cursor + .seek_exact(header_ref.number)? + .map(|(_, w)| w.withdrawals) + .unwrap_or_default() + .into() } else { None }; let requests = if self.chain_spec.is_prague_active_at_timestamp(header_ref.timestamp) { - Some(requests_cursor.seek_exact(header_ref.number)?.unwrap_or_default().1) + (requests_cursor.seek_exact(header_ref.number)?.unwrap_or_default().1) + .into() } else { None }; diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index ec76e9504d19..e233332a0e95 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -222,7 +222,7 @@ impl StaticFileProviderInner { /// Creates a new [`StaticFileProviderInner`]. fn new(path: impl AsRef, access: StaticFileAccess) -> ProviderResult { let _lock_file = if access.is_read_write() { - Some(StorageLock::try_acquire(path.as_ref())?) + StorageLock::try_acquire(path.as_ref())?.into() } else { None }; diff --git a/crates/storage/provider/src/providers/static_file/writer.rs b/crates/storage/provider/src/providers/static_file/writer.rs index 3858f1b14023..8c31c021f218 100644 --- a/crates/storage/provider/src/providers/static_file/writer.rs +++ b/crates/storage/provider/src/providers/static_file/writer.rs @@ -289,16 +289,16 @@ impl StaticFileProviderRW { // // If that expected block start is 0, then it means that there's no actual block data, and // there's no block data in static files. - let segment_max_block = match self.writer.user_header().block_range() { - Some(block_range) => Some(block_range.end()), - None => { - if self.writer.user_header().expected_block_start() > 0 { - Some(self.writer.user_header().expected_block_start() - 1) - } else { - None - } - } - }; + let segment_max_block = self + .writer + .user_header() + .block_range() + .as_ref() + .map(|block_range| block_range.end()) + .or_else(|| { + (self.writer.user_header().expected_block_start() > 0) + .then(|| self.writer.user_header().expected_block_start() - 1) + }); self.reader().update_index(self.writer.user_header().segment(), segment_max_block) } diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 3325d3ae9edb..c7c94b939ac3 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -344,13 +344,8 @@ impl TransactionsProvider for MockEthProvider { .values() .flat_map(|block| &block.body.transactions) .enumerate() - .filter_map(|(tx_number, tx)| { - if range.contains(&(tx_number as TxNumber)) { - Some(tx.clone().into()) - } else { - None - } - }) + .filter(|&(tx_number, _)| range.contains(&(tx_number as TxNumber))) + .map(|(_, tx)| tx.clone().into()) .collect(); Ok(transactions) @@ -366,11 +361,7 @@ impl TransactionsProvider for MockEthProvider { .flat_map(|block| &block.body.transactions) .enumerate() .filter_map(|(tx_number, tx)| { - if range.contains(&(tx_number as TxNumber)) { - Some(tx.recover_signer()?) - } else { - None - } + range.contains(&(tx_number as TxNumber)).then(|| tx.recover_signer()).flatten() }) .collect(); diff --git a/crates/transaction-pool/src/maintain.rs b/crates/transaction-pool/src/maintain.rs index aaf2d6d12d85..23a8d0dc66a5 100644 --- a/crates/transaction-pool/src/maintain.rs +++ b/crates/transaction-pool/src/maintain.rs @@ -455,11 +455,10 @@ impl FinalizedBlockTracker { /// Updates the tracked finalized block and returns the new finalized block if it changed fn update(&mut self, finalized_block: Option) -> Option { let finalized = finalized_block?; - if self.last_finalized_block.replace(finalized).map_or(true, |last| last < finalized) { - Some(finalized) - } else { - None - } + self.last_finalized_block + .replace(finalized) + .map_or(true, |last| last < finalized) + .then_some(finalized) } } diff --git a/crates/trie/common/src/subnode.rs b/crates/trie/common/src/subnode.rs index 98ce76a323e6..c64b2317cf30 100644 --- a/crates/trie/common/src/subnode.rs +++ b/crates/trie/common/src/subnode.rs @@ -51,16 +51,14 @@ impl Compact for StoredSubNode { buf.advance(key_len); let nibbles_exists = buf.get_u8() != 0; - let nibble = if nibbles_exists { Some(buf.get_u8()) } else { None }; + let nibble = nibbles_exists.then(|| buf.get_u8()); let node_exists = buf.get_u8() != 0; - let node = if node_exists { + let node = node_exists.then(|| { let (node, rest) = BranchNodeCompact::from_compact(buf, 0); buf = rest; - Some(node) - } else { - None - }; + node + }); (Self { key, nibble, node }, buf) } diff --git a/crates/trie/trie/src/trie_cursor/subnode.rs b/crates/trie/trie/src/trie_cursor/subnode.rs index c2ba839ebf2e..9d5a2770b268 100644 --- a/crates/trie/trie/src/trie_cursor/subnode.rs +++ b/crates/trie/trie/src/trie_cursor/subnode.rs @@ -49,7 +49,7 @@ impl From for CursorSubNode { impl From for StoredSubNode { fn from(value: CursorSubNode) -> Self { - let nibble = if value.nibble >= 0 { Some(value.nibble as u8) } else { None }; + let nibble = (value.nibble >= 0).then_some(value.nibble as u8); Self { key: value.key.to_vec(), nibble, node: value.node } } } diff --git a/crates/trie/trie/src/witness.rs b/crates/trie/trie/src/witness.rs index c042a0d8213b..582319c7fc51 100644 --- a/crates/trie/trie/src/witness.rs +++ b/crates/trie/trie/src/witness.rs @@ -111,14 +111,13 @@ where .accounts .get(&hashed_address) .ok_or(TrieWitnessError::MissingAccount(hashed_address))?; - let value = if account.is_some() || storage_multiproof.root != EMPTY_ROOT_HASH { - account_rlp.clear(); - TrieAccount::from((account.unwrap_or_default(), storage_multiproof.root)) - .encode(&mut account_rlp as &mut dyn BufMut); - Some(account_rlp.clone()) - } else { - None - }; + let value = + (account.is_some() || storage_multiproof.root != EMPTY_ROOT_HASH).then(|| { + account_rlp.clear(); + TrieAccount::from((account.unwrap_or_default(), storage_multiproof.root)) + .encode(&mut account_rlp as &mut dyn BufMut); + account_rlp.clone() + }); let key = Nibbles::unpack(hashed_address); account_trie_nodes.extend( self.target_nodes( From 600a394571e8397295256b25ed50c8c9ba6f4c8d Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Mon, 14 Oct 2024 17:52:34 +0200 Subject: [PATCH 69/82] feat: update SystemCaller (#11718) --- .../engine/invalid-block-hooks/src/witness.rs | 3 ++- crates/engine/util/src/reorg.rs | 2 +- crates/ethereum/evm/src/execute.rs | 12 +++++---- crates/ethereum/payload/src/lib.rs | 2 +- crates/evm/src/either.rs | 2 +- crates/evm/src/execute.rs | 2 +- crates/evm/src/system_calls/mod.rs | 26 +++++++++---------- crates/optimism/evm/src/execute.rs | 12 +++++---- crates/optimism/payload/src/builder.rs | 2 +- .../rpc-eth-api/src/helpers/pending_block.rs | 3 +-- crates/rpc/rpc-eth-api/src/helpers/trace.rs | 4 +-- 11 files changed, 36 insertions(+), 34 deletions(-) diff --git a/crates/engine/invalid-block-hooks/src/witness.rs b/crates/engine/invalid-block-hooks/src/witness.rs index 51978311faad..bb227e304198 100644 --- a/crates/engine/invalid-block-hooks/src/witness.rs +++ b/crates/engine/invalid-block-hooks/src/witness.rs @@ -84,7 +84,8 @@ where EnvWithHandlerCfg::new_with_cfg_env(cfg, block_env, Default::default()), ); - let mut system_caller = SystemCaller::new(&self.evm_config, self.provider.chain_spec()); + let mut system_caller = + SystemCaller::new(self.evm_config.clone(), self.provider.chain_spec()); // Apply pre-block system contract calls. system_caller.apply_pre_execution_changes(&block.clone().unseal(), &mut evm)?; diff --git a/crates/engine/util/src/reorg.rs b/crates/engine/util/src/reorg.rs index bd7b1b95642a..611095101ff3 100644 --- a/crates/engine/util/src/reorg.rs +++ b/crates/engine/util/src/reorg.rs @@ -286,7 +286,7 @@ where let mut evm = evm_config.evm_with_env(&mut state, env); // apply eip-4788 pre block contract call - let mut system_caller = SystemCaller::new(evm_config, chain_spec); + let mut system_caller = SystemCaller::new(evm_config.clone(), chain_spec); system_caller.apply_beacon_root_contract_call( reorg_target.timestamp, diff --git a/crates/ethereum/evm/src/execute.rs b/crates/ethereum/evm/src/execute.rs index 6513cf75fad5..108e1f87c455 100644 --- a/crates/ethereum/evm/src/execute.rs +++ b/crates/ethereum/evm/src/execute.rs @@ -141,10 +141,12 @@ where where DB: Database, DB::Error: Into + Display, - F: OnStateHook, + F: OnStateHook + 'static, { - let mut system_caller = - SystemCaller::new(&self.evm_config, &self.chain_spec).with_state_hook(state_hook); + let mut system_caller = SystemCaller::new(self.evm_config.clone(), &self.chain_spec); + if let Some(hook) = state_hook { + system_caller.with_state_hook(Some(Box::new(hook) as Box)); + } system_caller.apply_pre_execution_changes(block, &mut evm)?; @@ -290,7 +292,7 @@ where state_hook: Option, ) -> Result where - F: OnStateHook, + F: OnStateHook + 'static, { // 1. prepare state on new block self.on_new_block(&block.header); @@ -396,7 +398,7 @@ where state_hook: F, ) -> Result where - F: OnStateHook, + F: OnStateHook + 'static, { let BlockExecutionInput { block, total_difficulty } = input; let EthExecuteOutput { receipts, requests, gas_used } = self diff --git a/crates/ethereum/payload/src/lib.rs b/crates/ethereum/payload/src/lib.rs index 97237efa8b60..248aa3486de4 100644 --- a/crates/ethereum/payload/src/lib.rs +++ b/crates/ethereum/payload/src/lib.rs @@ -160,7 +160,7 @@ where let block_number = initialized_block_env.number.to::(); - let mut system_caller = SystemCaller::new(&evm_config, chain_spec.clone()); + let mut system_caller = SystemCaller::new(evm_config.clone(), chain_spec.clone()); // apply eip-4788 pre block contract call system_caller diff --git a/crates/evm/src/either.rs b/crates/evm/src/either.rs index 8022c68c43dc..82f84301f03a 100644 --- a/crates/evm/src/either.rs +++ b/crates/evm/src/either.rs @@ -97,7 +97,7 @@ where state_hook: F, ) -> Result where - F: OnStateHook, + F: OnStateHook + 'static, { match self { Self::Left(a) => a.execute_with_state_hook(input, state_hook), diff --git a/crates/evm/src/execute.rs b/crates/evm/src/execute.rs index 145eca29c92b..476c695e7ac6 100644 --- a/crates/evm/src/execute.rs +++ b/crates/evm/src/execute.rs @@ -56,7 +56,7 @@ pub trait Executor { state_hook: F, ) -> Result where - F: OnStateHook; + F: OnStateHook + 'static; } /// A general purpose executor that can execute multiple inputs in sequence, validate the outputs, diff --git a/crates/evm/src/system_calls/mod.rs b/crates/evm/src/system_calls/mod.rs index 43baa1c766c2..5dc3f35bd3a2 100644 --- a/crates/evm/src/system_calls/mod.rs +++ b/crates/evm/src/system_calls/mod.rs @@ -1,7 +1,7 @@ //! System contract call functions. use crate::ConfigureEvm; -use alloc::vec::Vec; +use alloc::{boxed::Box, vec::Vec}; use core::fmt::Display; use reth_chainspec::EthereumHardforks; use reth_execution_errors::BlockExecutionError; @@ -42,27 +42,26 @@ impl OnStateHook for NoopHook { /// /// This can be used to chain system transaction calls. #[allow(missing_debug_implementations)] -pub struct SystemCaller<'a, EvmConfig, Chainspec, Hook = NoopHook> { - evm_config: &'a EvmConfig, +pub struct SystemCaller { + evm_config: EvmConfig, chain_spec: Chainspec, /// Optional hook to be called after each state change. - hook: Option, + hook: Option>, } -impl<'a, EvmConfig, Chainspec> SystemCaller<'a, EvmConfig, Chainspec, NoopHook> { +impl SystemCaller { /// Create a new system caller with the given EVM config, database, and chain spec, and creates /// the EVM with the given initialized config and block environment. - pub const fn new(evm_config: &'a EvmConfig, chain_spec: Chainspec) -> Self { + pub const fn new(evm_config: EvmConfig, chain_spec: Chainspec) -> Self { Self { evm_config, chain_spec, hook: None } } + /// Installs a custom hook to be called after each state change. - pub fn with_state_hook( - self, - hook: Option, - ) -> SystemCaller<'a, EvmConfig, Chainspec, H> { - let Self { evm_config, chain_spec, .. } = self; - SystemCaller { evm_config, chain_spec, hook } + pub fn with_state_hook(&mut self, hook: Option>) -> &mut Self { + self.hook = hook; + self } + /// Convenience method to consume the type and drop borrowed fields pub fn finish(self) {} } @@ -85,11 +84,10 @@ where .build() } -impl SystemCaller<'_, EvmConfig, Chainspec, Hook> +impl SystemCaller where EvmConfig: ConfigureEvm
, Chainspec: EthereumHardforks, - Hook: OnStateHook, { /// Apply pre execution changes. pub fn apply_pre_execution_changes( diff --git a/crates/optimism/evm/src/execute.rs b/crates/optimism/evm/src/execute.rs index f4abb8c887e2..ee0d028e4251 100644 --- a/crates/optimism/evm/src/execute.rs +++ b/crates/optimism/evm/src/execute.rs @@ -121,10 +121,12 @@ where ) -> Result<(Vec, u64), BlockExecutionError> where DB: Database + Display>, - F: OnStateHook, + F: OnStateHook + 'static, { - let mut system_caller = - SystemCaller::new(&self.evm_config, &self.chain_spec).with_state_hook(state_hook); + let mut system_caller = SystemCaller::new(self.evm_config.clone(), &self.chain_spec); + if let Some(hook) = state_hook { + system_caller.with_state_hook(Some(Box::new(hook) as Box)); + } // apply pre execution changes system_caller.apply_beacon_root_contract_call( @@ -306,7 +308,7 @@ where state_hook: Option, ) -> Result<(Vec, u64), BlockExecutionError> where - F: OnStateHook, + F: OnStateHook + 'static, { // 1. prepare state on new block self.on_new_block(&block.header); @@ -410,7 +412,7 @@ where state_hook: F, ) -> Result where - F: OnStateHook, + F: OnStateHook + 'static, { let BlockExecutionInput { block, total_difficulty } = input; let (receipts, gas_used) = self.execute_without_verification_with_state_hook( diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index 2ff3d3342037..0a8dcdb12449 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -201,7 +201,7 @@ where ); // apply eip-4788 pre block contract call - let mut system_caller = SystemCaller::new(&evm_config, &chain_spec); + let mut system_caller = SystemCaller::new(evm_config.clone(), &chain_spec); system_caller .pre_block_beacon_root_contract_call( diff --git a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs index 85ad7fd18107..26e0ffc7412a 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs @@ -260,8 +260,7 @@ pub trait LoadPendingBlock: EthApiTypes { let chain_spec = self.provider().chain_spec(); - let evm_config = self.evm_config().clone(); - let mut system_caller = SystemCaller::new(&evm_config, chain_spec.clone()); + let mut system_caller = SystemCaller::new(self.evm_config().clone(), chain_spec.clone()); let parent_beacon_block_root = if origin.is_actual_pending() { // apply eip-4788 pre block contract call if we got the block from the CL with the real diff --git a/crates/rpc/rpc-eth-api/src/helpers/trace.rs b/crates/rpc/rpc-eth-api/src/helpers/trace.rs index 2dc1c5b4746d..457cbb4811f3 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/trace.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/trace.rs @@ -199,7 +199,7 @@ pub trait Trace: LoadState { // apply relevant system calls let mut system_caller = SystemCaller::new( - Trace::evm_config(&this), + Trace::evm_config(&this).clone(), LoadState::provider(&this).chain_spec(), ); system_caller @@ -332,7 +332,7 @@ pub trait Trace: LoadState { // apply relevant system calls let mut system_caller = SystemCaller::new( - Trace::evm_config(&this), + Trace::evm_config(&this).clone(), LoadState::provider(&this).chain_spec(), ); system_caller From 9f9de0fab96712761944806120b1764cf1bee0eb Mon Sep 17 00:00:00 2001 From: Delweng Date: Tue, 15 Oct 2024 01:24:20 +0800 Subject: [PATCH 70/82] fix(rpc/trace): return empty if after >= traces (#11715) Signed-off-by: jsvisa Co-authored-by: Matthias Seitz --- crates/rpc/rpc/src/trace.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index 05588d4dabea..0cd94ef15b89 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -322,13 +322,18 @@ where } } - // apply after and count to traces if specified, this allows for a pagination style. - // only consider traces after - if let Some(after) = after.map(|a| a as usize).filter(|a| *a < all_traces.len()) { - all_traces = all_traces.split_off(after); + // Skips the first `after` number of matching traces. + // If `after` is greater than or equal to the number of matched traces, it returns an empty + // array. + if let Some(after) = after.map(|a| a as usize) { + if after < all_traces.len() { + all_traces.drain(..after); + } else { + return Ok(vec![]) + } } - // at most, return count of traces + // Return at most `count` of traces if let Some(count) = count { let count = count as usize; if count < all_traces.len() { From 0dbc37463964e6985910a58008113e4c90eb506d Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Mon, 14 Oct 2024 18:38:15 -0400 Subject: [PATCH 71/82] feat(ci): add workflow for git sha container builds (#11721) --- .github/workflows/docker-git.yml | 44 ++++++++++++++++++++++++++++++++ .github/workflows/docker.yml | 1 - Makefile | 17 ++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/docker-git.yml diff --git a/.github/workflows/docker-git.yml b/.github/workflows/docker-git.yml new file mode 100644 index 000000000000..2e3ad59aea92 --- /dev/null +++ b/.github/workflows/docker-git.yml @@ -0,0 +1,44 @@ +# Publishes the Docker image, only to be used with `workflow_dispatch`. The +# images from this workflow will be tagged with the git sha of the branch used +# and will NOT tag it as `latest`. + +name: docker-git + +on: + workflow_dispatch: {} + +env: + REPO_NAME: ${{ github.repository_owner }}/reth + IMAGE_NAME: ${{ github.repository_owner }}/reth + OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth + CARGO_TERM_COLOR: always + DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth + OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth + DOCKER_USERNAME: ${{ github.actor }} + GIT_SHA: ${{ github.sha }} + +jobs: + build: + name: build and push + runs-on: ubuntu-20.04 + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - uses: taiki-e/install-action@cross + - name: Log in to Docker + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin + - name: Set up Docker builder + run: | + docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64 + docker buildx create --use --name cross-builder + - name: Build and push the git-sha-tagged reth image + run: make PROFILE=maxperf GIT_SHA=$GIT_SHA docker-build-push-git-sha + - name: Build and push the git-sha-tagged op-reth image + run: make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME GIT_SHA=$GIT_SHA PROFILE=maxperf op-docker-build-push-git-sha diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2af324a39eb7..7b6f0d51e9c9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,7 +3,6 @@ name: docker on: - workflow_dispatch: {} push: tags: - v* diff --git a/Makefile b/Makefile index 4d897c7ee482..908f1ef24daa 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ # Heavily inspired by Lighthouse: https://github.com/sigp/lighthouse/blob/693886b94176faa4cb450f024696cb69cda2fe58/Makefile .DEFAULT_GOAL := help +GIT_SHA ?= $(shell git rev-parse HEAD) GIT_TAG ?= $(shell git describe --tags --abbrev=0) BIN_DIR = "dist/bin" @@ -199,6 +200,14 @@ ef-tests: $(EF_TESTS_DIR) ## Runs Ethereum Foundation tests. docker-build-push: ## Build and push a cross-arch Docker image tagged with the latest git tag. $(call docker_build_push,$(GIT_TAG),$(GIT_TAG)) +# Note: This requires a buildx builder with emulation support. For example: +# +# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64` +# `docker buildx create --use --driver docker-container --name cross-builder` +.PHONY: docker-build-push-git-sha +docker-build-push-git-sha: ## Build and push a cross-arch Docker image tagged with the latest git sha. + $(call docker_build_push,$(GIT_SHA),$(GIT_SHA)) + # Note: This requires a buildx builder with emulation support. For example: # # `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64` @@ -243,6 +252,14 @@ endef op-docker-build-push: ## Build and push a cross-arch Docker image tagged with the latest git tag. $(call op_docker_build_push,$(GIT_TAG),$(GIT_TAG)) +# Note: This requires a buildx builder with emulation support. For example: +# +# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64` +# `docker buildx create --use --driver docker-container --name cross-builder` +.PHONY: op-docker-build-push-git-sha +op-docker-build-push-git-sha: ## Build and push a cross-arch Docker image tagged with the latest git sha. + $(call op_docker_build_push,$(GIT_SHA),$(GIT_SHA)) + # Note: This requires a buildx builder with emulation support. For example: # # `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64` From cf38ff540137692a2e1beb01500551638815c3d7 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Tue, 15 Oct 2024 10:02:00 +0200 Subject: [PATCH 72/82] primitive: introduce reth `Transaction` trait (#11728) --- crates/primitives-traits/src/lib.rs | 3 +++ crates/primitives-traits/src/transaction.rs | 26 +++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 crates/primitives-traits/src/transaction.rs diff --git a/crates/primitives-traits/src/lib.rs b/crates/primitives-traits/src/lib.rs index 9c244226b2c7..dd68607f5911 100644 --- a/crates/primitives-traits/src/lib.rs +++ b/crates/primitives-traits/src/lib.rs @@ -23,6 +23,9 @@ pub use account::{Account, Bytecode}; pub mod receipt; pub use receipt::Receipt; +pub mod transaction; +pub use transaction::Transaction; + mod integer_list; pub use integer_list::{IntegerList, IntegerListError}; diff --git a/crates/primitives-traits/src/transaction.rs b/crates/primitives-traits/src/transaction.rs new file mode 100644 index 000000000000..93645ead82e8 --- /dev/null +++ b/crates/primitives-traits/src/transaction.rs @@ -0,0 +1,26 @@ +//! Transaction abstraction + +use alloc::fmt; + +use reth_codecs::Compact; +use serde::{Deserialize, Serialize}; + +/// Helper trait that unifies all behaviour required by transaction to support full node operations. +pub trait FullTransaction: Transaction + Compact {} + +impl FullTransaction for T where T: Transaction + Compact {} + +/// Abstraction of a transaction. +pub trait Transaction: + alloy_consensus::Transaction + + Clone + + fmt::Debug + + PartialEq + + Eq + + Default + + alloy_rlp::Encodable + + alloy_rlp::Decodable + + Serialize + + for<'de> Deserialize<'de> +{ +} From e0a26ac9a2c19d4591f830958a7c9e1f336448cc Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 15 Oct 2024 10:08:45 +0200 Subject: [PATCH 73/82] feat: add node builder helpers (#11731) --- crates/node/builder/src/builder/mod.rs | 42 ++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/crates/node/builder/src/builder/mod.rs b/crates/node/builder/src/builder/mod.rs index 9f7254bad737..fb94413fe756 100644 --- a/crates/node/builder/src/builder/mod.rs +++ b/crates/node/builder/src/builder/mod.rs @@ -170,6 +170,18 @@ impl NodeBuilder<(), ChainSpec> { { f(self) } + + /// Apply a function to the builder, if the condition is `true`. + pub fn apply_if(self, cond: bool, f: F) -> Self + where + F: FnOnce(Self) -> Self, + { + if cond { + f(self) + } else { + self + } + } } impl NodeBuilder { @@ -421,6 +433,18 @@ where f(self) } + /// Apply a function to the builder, if the condition is `true`. + pub fn apply_if(self, cond: bool, f: F) -> Self + where + F: FnOnce(Self) -> Self, + { + if cond { + f(self) + } else { + self + } + } + /// Sets the hook that is run once the node's components are initialized. pub fn on_component_initialized(self, hook: F) -> Self where @@ -482,6 +506,24 @@ where } } + /// Installs an `ExEx` (Execution Extension) in the node if the condition is true. + /// + /// # Note + /// + /// The `ExEx` ID must be unique. + pub fn install_exex_if(self, cond: bool, exex_id: impl Into, exex: F) -> Self + where + F: FnOnce(ExExContext>) -> R + Send + 'static, + R: Future> + Send, + E: Future> + Send, + { + if cond { + self.install_exex(exex_id, exex) + } else { + self + } + } + /// Launches the node with the given launcher. pub async fn launch_with(self, launcher: L) -> eyre::Result where From 2a862456492e49f9d31efc65547905de70b99484 Mon Sep 17 00:00:00 2001 From: caglarkaya Date: Tue, 15 Oct 2024 11:19:23 +0300 Subject: [PATCH 74/82] feat: extend txpool remove txs utility (#11702) --- crates/transaction-pool/src/lib.rs | 14 ++ crates/transaction-pool/src/noop.rs | 14 ++ crates/transaction-pool/src/pool/mod.rs | 34 +++- crates/transaction-pool/src/pool/txpool.rs | 174 +++++++++++++++++++++ crates/transaction-pool/src/traits.rs | 18 ++- 5 files changed, 252 insertions(+), 2 deletions(-) diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index a5acd6edba5a..75465afac1a1 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -463,6 +463,20 @@ where self.pool.remove_transactions(hashes) } + fn remove_transactions_and_descendants( + &self, + hashes: Vec, + ) -> Vec>> { + self.pool.remove_transactions_and_descendants(hashes) + } + + fn remove_transactions_by_sender( + &self, + sender: Address, + ) -> Vec>> { + self.pool.remove_transactions_by_sender(sender) + } + fn retain_unknown(&self, announcement: &mut A) where A: HandleMempoolData, diff --git a/crates/transaction-pool/src/noop.rs b/crates/transaction-pool/src/noop.rs index ddab4f622741..681aa9896abb 100644 --- a/crates/transaction-pool/src/noop.rs +++ b/crates/transaction-pool/src/noop.rs @@ -183,6 +183,20 @@ impl TransactionPool for NoopTransactionPool { vec![] } + fn remove_transactions_and_descendants( + &self, + _hashes: Vec, + ) -> Vec>> { + vec![] + } + + fn remove_transactions_by_sender( + &self, + _sender: Address, + ) -> Vec>> { + vec![] + } + fn retain_unknown(&self, _announcement: &mut A) where A: HandleMempoolData, diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 090b92fb6594..44dd5a8d525f 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -195,7 +195,7 @@ where pub(crate) fn block_info(&self) -> BlockInfo { self.get_pool_data().block_info() } - /// Returns the currently tracked block + /// Sets the currently tracked block pub(crate) fn set_block_info(&self, info: BlockInfo) { self.pool.write().set_block_info(info) } @@ -715,6 +715,38 @@ where removed } + /// Removes and returns all matching transactions and their dependent transactions from the + /// pool. + pub(crate) fn remove_transactions_and_descendants( + &self, + hashes: Vec, + ) -> Vec>> { + if hashes.is_empty() { + return Vec::new() + } + let removed = self.pool.write().remove_transactions_and_descendants(hashes); + + let mut listener = self.event_listener.write(); + + removed.iter().for_each(|tx| listener.discarded(tx.hash())); + + removed + } + + pub(crate) fn remove_transactions_by_sender( + &self, + sender: Address, + ) -> Vec>> { + let sender_id = self.get_sender_id(sender); + let removed = self.pool.write().remove_transactions_by_sender(sender_id); + + let mut listener = self.event_listener.write(); + + removed.iter().for_each(|tx| listener.discarded(tx.hash())); + + removed + } + /// Removes and returns all transactions that are present in the pool. pub(crate) fn retain_unknown(&self, announcement: &mut A) where diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 10605565c85d..7aaa77df49f4 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -663,6 +663,38 @@ impl TxPool { txs } + /// Removes and returns all matching transactions and their descendants from the pool. + pub(crate) fn remove_transactions_and_descendants( + &mut self, + hashes: Vec, + ) -> Vec>> { + let mut removed = Vec::new(); + for hash in hashes { + if let Some(tx) = self.remove_transaction_by_hash(&hash) { + removed.push(tx.clone()); + self.remove_descendants(tx.id(), &mut removed); + } + } + self.update_size_metrics(); + removed + } + + /// Removes all transactions from the given sender. + pub(crate) fn remove_transactions_by_sender( + &mut self, + sender_id: SenderId, + ) -> Vec>> { + let mut removed = Vec::new(); + let txs = self.get_transactions_by_sender(sender_id); + for tx in txs { + if let Some(tx) = self.remove_transaction(tx.id()) { + removed.push(tx); + } + } + self.update_size_metrics(); + removed + } + /// Remove the transaction from the __entire__ pool. /// /// This includes the total set of transaction and the subpool it currently resides in. @@ -2963,6 +2995,148 @@ mod tests { assert_eq!(vec![v1.nonce()], pool_txs); } #[test] + fn test_remove_transactions() { + let on_chain_balance = U256::from(10_000); + let on_chain_nonce = 0; + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + let tx_0 = MockTransaction::eip1559().set_gas_price(100).inc_limit(); + let tx_1 = tx_0.next(); + let tx_2 = MockTransaction::eip1559().set_gas_price(100).inc_limit(); + let tx_3 = tx_2.next(); + + // Create 4 transactions + let v0 = f.validated(tx_0); + let v1 = f.validated(tx_1); + let v2 = f.validated(tx_2); + let v3 = f.validated(tx_3); + + // Add them to the pool + let _res = pool.add_transaction(v0.clone(), on_chain_balance, on_chain_nonce).unwrap(); + let _res = pool.add_transaction(v1.clone(), on_chain_balance, on_chain_nonce).unwrap(); + let _res = pool.add_transaction(v2.clone(), on_chain_balance, on_chain_nonce).unwrap(); + let _res = pool.add_transaction(v3.clone(), on_chain_balance, on_chain_nonce).unwrap(); + + assert_eq!(0, pool.queued_transactions().len()); + assert_eq!(4, pool.pending_transactions().len()); + + pool.remove_transactions(vec![*v0.hash(), *v2.hash()]); + + assert_eq!(0, pool.queued_transactions().len()); + assert_eq!(2, pool.pending_transactions().len()); + assert!(pool.contains(v1.hash())); + assert!(pool.contains(v3.hash())); + } + + #[test] + fn test_remove_transactions_and_descendants() { + let on_chain_balance = U256::from(10_000); + let on_chain_nonce = 0; + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + let tx_0 = MockTransaction::eip1559().set_gas_price(100).inc_limit(); + let tx_1 = tx_0.next(); + let tx_2 = MockTransaction::eip1559().set_gas_price(100).inc_limit(); + let tx_3 = tx_2.next(); + let tx_4 = tx_3.next(); + + // Create 5 transactions + let v0 = f.validated(tx_0); + let v1 = f.validated(tx_1); + let v2 = f.validated(tx_2); + let v3 = f.validated(tx_3); + let v4 = f.validated(tx_4); + + // Add them to the pool + let _res = pool.add_transaction(v0.clone(), on_chain_balance, on_chain_nonce).unwrap(); + let _res = pool.add_transaction(v1, on_chain_balance, on_chain_nonce).unwrap(); + let _res = pool.add_transaction(v2.clone(), on_chain_balance, on_chain_nonce).unwrap(); + let _res = pool.add_transaction(v3, on_chain_balance, on_chain_nonce).unwrap(); + let _res = pool.add_transaction(v4, on_chain_balance, on_chain_nonce).unwrap(); + + assert_eq!(0, pool.queued_transactions().len()); + assert_eq!(5, pool.pending_transactions().len()); + + pool.remove_transactions_and_descendants(vec![*v0.hash(), *v2.hash()]); + + assert_eq!(0, pool.queued_transactions().len()); + assert_eq!(0, pool.pending_transactions().len()); + } + #[test] + fn test_remove_descendants() { + let on_chain_balance = U256::from(10_000); + let on_chain_nonce = 0; + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + let tx_0 = MockTransaction::eip1559().set_gas_price(100).inc_limit(); + let tx_1 = tx_0.next(); + let tx_2 = tx_1.next(); + let tx_3 = tx_2.next(); + + // Create 4 transactions + let v0 = f.validated(tx_0); + let v1 = f.validated(tx_1); + let v2 = f.validated(tx_2); + let v3 = f.validated(tx_3); + + // Add them to the pool + let _res = pool.add_transaction(v0.clone(), on_chain_balance, on_chain_nonce).unwrap(); + let _res = pool.add_transaction(v1, on_chain_balance, on_chain_nonce).unwrap(); + let _res = pool.add_transaction(v2, on_chain_balance, on_chain_nonce).unwrap(); + let _res = pool.add_transaction(v3, on_chain_balance, on_chain_nonce).unwrap(); + + assert_eq!(0, pool.queued_transactions().len()); + assert_eq!(4, pool.pending_transactions().len()); + + let mut removed = Vec::new(); + pool.remove_transaction(v0.id()); + pool.remove_descendants(v0.id(), &mut removed); + + assert_eq!(0, pool.queued_transactions().len()); + assert_eq!(0, pool.pending_transactions().len()); + assert_eq!(3, removed.len()); + } + #[test] + fn test_remove_transactions_by_sender() { + let on_chain_balance = U256::from(10_000); + let on_chain_nonce = 0; + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + let tx_0 = MockTransaction::eip1559().set_gas_price(100).inc_limit(); + let tx_1 = tx_0.next(); + let tx_2 = MockTransaction::eip1559().set_gas_price(100).inc_limit(); + let tx_3 = tx_2.next(); + let tx_4 = tx_3.next(); + + // Create 5 transactions + let v0 = f.validated(tx_0); + let v1 = f.validated(tx_1); + let v2 = f.validated(tx_2); + let v3 = f.validated(tx_3); + let v4 = f.validated(tx_4); + + // Add them to the pool + let _res = pool.add_transaction(v0.clone(), on_chain_balance, on_chain_nonce).unwrap(); + let _res = pool.add_transaction(v1.clone(), on_chain_balance, on_chain_nonce).unwrap(); + let _res = pool.add_transaction(v2.clone(), on_chain_balance, on_chain_nonce).unwrap(); + let _res = pool.add_transaction(v3, on_chain_balance, on_chain_nonce).unwrap(); + let _res = pool.add_transaction(v4, on_chain_balance, on_chain_nonce).unwrap(); + + assert_eq!(0, pool.queued_transactions().len()); + assert_eq!(5, pool.pending_transactions().len()); + + pool.remove_transactions_by_sender(v2.sender_id()); + + assert_eq!(0, pool.queued_transactions().len()); + assert_eq!(2, pool.pending_transactions().len()); + assert!(pool.contains(v0.hash())); + assert!(pool.contains(v1.hash())); + } + #[test] fn wrong_best_order_of_transactions() { let on_chain_balance = U256::from(10_000); let mut on_chain_nonce = 0; diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index adae238e46b3..07fe9b9c206e 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -291,16 +291,32 @@ pub trait TransactionPool: Send + Sync + Clone { /// Consumer: RPC fn all_transactions(&self) -> AllPoolTransactions; + /// Removes all transactions corresponding to the given hashes. + /// + /// Consumer: Utility + fn remove_transactions( + &self, + hashes: Vec, + ) -> Vec>>; + /// Removes all transactions corresponding to the given hashes. /// /// Also removes all _dependent_ transactions. /// /// Consumer: Utility - fn remove_transactions( + fn remove_transactions_and_descendants( &self, hashes: Vec, ) -> Vec>>; + /// Removes all transactions from the given sender + /// + /// Consumer: Utility + fn remove_transactions_by_sender( + &self, + sender: Address, + ) -> Vec>>; + /// Retains only those hashes that are unknown to the pool. /// In other words, removes all transactions from the given set that are currently present in /// the pool. Returns hashes already known to the pool. From 3ab1f9559e2e9bb5c2c7fe1f2e2447142f8e9576 Mon Sep 17 00:00:00 2001 From: Deil Urba Date: Tue, 15 Oct 2024 09:21:01 +0100 Subject: [PATCH 75/82] feat: replace once_cell with std (#11694) Co-authored-by: Emilia Hane --- Cargo.lock | 59 +++++++++++-------- Cargo.toml | 4 +- crates/chainspec/src/lib.rs | 12 +++- crates/chainspec/src/spec.rs | 24 ++++---- crates/ethereum-forks/Cargo.toml | 7 ++- crates/ethereum-forks/src/hardfork/dev.rs | 9 ++- crates/optimism/chainspec/Cargo.toml | 11 +++- crates/optimism/chainspec/src/base.rs | 7 +-- crates/optimism/chainspec/src/base_sepolia.rs | 7 +-- crates/optimism/chainspec/src/dev.rs | 5 +- crates/optimism/chainspec/src/lib.rs | 10 +++- crates/optimism/chainspec/src/op.rs | 7 +-- crates/optimism/chainspec/src/op_sepolia.rs | 7 +-- crates/optimism/hardforks/src/dev.rs | 8 ++- crates/primitives/src/transaction/mod.rs | 10 +++- crates/rpc/rpc-eth-api/src/helpers/block.rs | 2 + crates/rpc/rpc-eth-api/src/helpers/call.rs | 1 + crates/storage/provider/Cargo.toml | 13 ++-- .../storage/provider/src/test_utils/blocks.rs | 6 +- 19 files changed, 129 insertions(+), 80 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d690cf536aa3..7cbb3e937e94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,9 +130,9 @@ dependencies = [ [[package]] name = "alloy-dyn-abi" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f95d76a38cae906fd394a5afb0736aaceee5432efe76addfd71048e623e208af" +checksum = "e6228abfc751a29cde117b0879b805a3e0b3b641358f063272c83ca459a56886" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -206,9 +206,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c66eec1acdd96b39b995b8f5ee5239bc0c871d62c527ae1ac9fd1d7fecd455" +checksum = "d46eb5871592c216d39192499c95a99f7175cb94104f88c307e6dc960676d9f1" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -283,9 +283,9 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb848c43f6b06ae3de2e4a67496cbbabd78ae87db0f1248934f15d76192c6a" +checksum = "38f35429a652765189c1c5092870d8360ee7b7769b09b06d89ebaefd34676446" dependencies = [ "alloy-rlp", "arbitrary", @@ -600,9 +600,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "661c516eb1fa3294cc7f2fb8955b3b609d639c282ac81a4eedb14d3046db503a" +checksum = "3b2395336745358cc47207442127c47c63801a7065ecc0aa928da844f8bb5576" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", @@ -614,9 +614,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-expander" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecbabb8fc3d75a0c2cea5215be22e7a267e3efde835b0f2a8922f5e3f5d47683" +checksum = "9ed5047c9a241df94327879c2b0729155b58b941eae7805a7ada2e19436e6b39" dependencies = [ "alloy-sol-macro-input", "const-hex", @@ -632,9 +632,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-input" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16517f2af03064485150d89746b8ffdcdbc9b6eeb3d536fb66efd7c2846fbc75" +checksum = "5dee02a81f529c415082235129f0df8b8e60aa1601b9c9298ffe54d75f57210b" dependencies = [ "const-hex", "dunce", @@ -647,9 +647,9 @@ dependencies = [ [[package]] name = "alloy-sol-type-parser" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c07ebb0c1674ff8cbb08378d7c2e0e27919d2a2dae07ad3bca26174deda8d389" +checksum = "f631f0bd9a9d79619b27c91b6b1ab2c4ef4e606a65192369a1ee05d40dcf81cc" dependencies = [ "serde", "winnow", @@ -657,9 +657,9 @@ dependencies = [ [[package]] name = "alloy-sol-types" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e448d879903624863f608c552d10efb0e0905ddbee98b0049412799911eb062" +checksum = "c2841af22d99e2c0f82a78fe107b6481be3dd20b89bfb067290092794734343a" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -1018,9 +1018,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "998282f8f49ccd6116b0ed8a4de0fbd3151697920e7c7533416d6e25e76434a7" +checksum = "e26a9844c659a2a293d239c7910b752f8487fe122c6c8bd1659bf85a6507c302" dependencies = [ "brotli", "flate2", @@ -1498,9 +1498,9 @@ checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" [[package]] name = "bytemuck" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae" +checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" dependencies = [ "bytemuck_derive", ] @@ -2011,6 +2011,12 @@ dependencies = [ "itertools 0.10.5", ] +[[package]] +name = "critical-section" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f64009896348fc5af4222e9cf7d7d82a95a256c634ebcf61c53e4ea461422242" + [[package]] name = "crossbeam-channel" version = "0.5.13" @@ -5131,6 +5137,10 @@ name = "once_cell" version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "oorandom" @@ -5413,9 +5423,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.13" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdbef9d1d47087a895abd220ed25eb4ad973a5e26f6a4367b038c25e28dfc2d9" +checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442" dependencies = [ "memchr", "thiserror", @@ -8393,7 +8403,6 @@ dependencies = [ "itertools 0.13.0", "metrics", "notify", - "once_cell", "parking_lot 0.12.3", "rand 0.8.5", "rayon", @@ -10256,9 +10265,9 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20e7b52ad118b2153644eea95c6fc740b6c1555b2344fdab763fc9de4075f665" +checksum = "ebfc1bfd06acc78f16d8fd3ef846bc222ee7002468d10a7dce8d703d6eab89a3" dependencies = [ "paste", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index ac0ab2d39228..2068f02bb845 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -497,7 +497,9 @@ notify = { version = "6.1.1", default-features = false, features = [ "macos_fsevent", ] } nybbles = "0.2.1" -once_cell = "1.19" +once_cell = { version = "1.19", default-features = false, features = [ + "critical-section", +] } parking_lot = "0.12" paste = "1.0" rand = "0.8.5" diff --git a/crates/chainspec/src/lib.rs b/crates/chainspec/src/lib.rs index 424b2b77c28c..81aadb834ca2 100644 --- a/crates/chainspec/src/lib.rs +++ b/crates/chainspec/src/lib.rs @@ -11,6 +11,12 @@ extern crate alloc; +use once_cell as _; +#[cfg(not(feature = "std"))] +pub(crate) use once_cell::sync::{Lazy as LazyLock, OnceCell as OnceLock}; +#[cfg(feature = "std")] +pub(crate) use std::sync::{LazyLock, OnceLock}; + /// Chain specific constants pub(crate) mod constants; pub use constants::MIN_TRANSACTION_GAS; @@ -34,9 +40,9 @@ pub use spec::{ DepositContract, ForkBaseFeeParams, DEV, HOLESKY, MAINNET, SEPOLIA, }; -/// Simple utility to create a `OnceCell` with a value set. -pub fn once_cell_set(value: T) -> once_cell::sync::OnceCell { - let once = once_cell::sync::OnceCell::new(); +/// Simple utility to create a thread-safe sync cell with a value set. +pub fn once_cell_set(value: T) -> OnceLock { + let once = OnceLock::new(); let _ = once.set(value); once } diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index e43e0bc78d86..f78fcbb61d0c 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -6,7 +6,7 @@ use alloy_genesis::Genesis; use alloy_primitives::{address, b256, Address, BlockNumber, B256, U256}; use alloy_trie::EMPTY_ROOT_HASH; use derive_more::From; -use once_cell::sync::{Lazy, OnceCell}; + use reth_ethereum_forks::{ ChainHardforks, DisplayHardforks, EthereumHardfork, EthereumHardforks, ForkCondition, ForkFilter, ForkFilterKey, ForkHash, ForkId, Hardfork, Hardforks, Head, DEV_HARDFORKS, @@ -24,10 +24,10 @@ use reth_primitives_traits::{ }; use reth_trie_common::root::state_root_ref_unhashed; -use crate::{constants::MAINNET_DEPOSIT_CONTRACT, once_cell_set, EthChainSpec}; +use crate::{constants::MAINNET_DEPOSIT_CONTRACT, once_cell_set, EthChainSpec, LazyLock, OnceLock}; /// The Ethereum mainnet spec -pub static MAINNET: Lazy> = Lazy::new(|| { +pub static MAINNET: LazyLock> = LazyLock::new(|| { let mut spec = ChainSpec { chain: Chain::mainnet(), genesis: serde_json::from_str(include_str!("../res/genesis/mainnet.json")) @@ -55,7 +55,7 @@ pub static MAINNET: Lazy> = Lazy::new(|| { }); /// The Sepolia spec -pub static SEPOLIA: Lazy> = Lazy::new(|| { +pub static SEPOLIA: LazyLock> = LazyLock::new(|| { let mut spec = ChainSpec { chain: Chain::sepolia(), genesis: serde_json::from_str(include_str!("../res/genesis/sepolia.json")) @@ -80,7 +80,7 @@ pub static SEPOLIA: Lazy> = Lazy::new(|| { }); /// The Holesky spec -pub static HOLESKY: Lazy> = Lazy::new(|| { +pub static HOLESKY: LazyLock> = LazyLock::new(|| { let mut spec = ChainSpec { chain: Chain::holesky(), genesis: serde_json::from_str(include_str!("../res/genesis/holesky.json")) @@ -106,7 +106,7 @@ pub static HOLESKY: Lazy> = Lazy::new(|| { /// /// Includes 20 prefunded accounts with `10_000` ETH each derived from mnemonic "test test test test /// test test test test test test test junk". -pub static DEV: Lazy> = Lazy::new(|| { +pub static DEV: LazyLock> = LazyLock::new(|| { ChainSpec { chain: Chain::dev(), genesis: serde_json::from_str(include_str!("../res/genesis/dev.json")) @@ -182,13 +182,13 @@ pub struct ChainSpec { /// /// This is either stored at construction time if it is known using [`once_cell_set`], or /// computed once on the first access. - pub genesis_hash: OnceCell, + pub genesis_hash: OnceLock, /// The header corresponding to the genesis block. /// /// This is either stored at construction time if it is known using [`once_cell_set`], or /// computed once on the first access. - pub genesis_header: OnceCell
, + pub genesis_header: OnceLock
, /// The block at which [`EthereumHardfork::Paris`] was activated and the final difficulty at /// this block. @@ -665,7 +665,7 @@ impl From for ChainSpec { Self { chain: genesis.config.chain_id.into(), genesis, - genesis_hash: OnceCell::new(), + genesis_hash: OnceLock::new(), hardforks: ChainHardforks::new(ordered_hardforks), paris_block_and_final_difficulty, deposit_contract, @@ -898,7 +898,7 @@ impl ChainSpecBuilder { ChainSpec { chain: self.chain.expect("The chain is required"), genesis: self.genesis.expect("The genesis is required"), - genesis_hash: OnceCell::new(), + genesis_hash: OnceLock::new(), hardforks: self.hardforks, paris_block_and_final_difficulty, deposit_contract: None, @@ -2217,7 +2217,7 @@ Post-merge hard forks (timestamp based): let spec = ChainSpec { chain: Chain::mainnet(), genesis: Genesis::default(), - genesis_hash: OnceCell::new(), + genesis_hash: OnceLock::new(), hardforks: ChainHardforks::new(vec![( EthereumHardfork::Frontier.boxed(), ForkCondition::Never, @@ -2235,7 +2235,7 @@ Post-merge hard forks (timestamp based): let spec = ChainSpec { chain: Chain::mainnet(), genesis: Genesis::default(), - genesis_hash: OnceCell::new(), + genesis_hash: OnceLock::new(), hardforks: ChainHardforks::new(vec![( EthereumHardfork::Shanghai.boxed(), ForkCondition::Never, diff --git a/crates/ethereum-forks/Cargo.toml b/crates/ethereum-forks/Cargo.toml index 7eb58b15fe59..08a0bc98dbcf 100644 --- a/crates/ethereum-forks/Cargo.toml +++ b/crates/ethereum-forks/Cargo.toml @@ -41,5 +41,10 @@ proptest.workspace = true default = ["std", "serde", "rustc-hash"] arbitrary = ["dep:arbitrary", "dep:proptest", "dep:proptest-derive"] serde = ["dep:serde"] -std = ["thiserror-no-std/std", "rustc-hash/std"] +std = [ + "alloy-chains/std", + "alloy-primitives/std", + "thiserror-no-std/std", + "rustc-hash/std", +] rustc-hash = ["dep:rustc-hash"] diff --git a/crates/ethereum-forks/src/hardfork/dev.rs b/crates/ethereum-forks/src/hardfork/dev.rs index 87abd68f230e..068e290709d2 100644 --- a/crates/ethereum-forks/src/hardfork/dev.rs +++ b/crates/ethereum-forks/src/hardfork/dev.rs @@ -1,12 +1,17 @@ use alloc::vec; use alloy_primitives::U256; -use once_cell::sync::Lazy; + +use once_cell as _; +#[cfg(not(feature = "std"))] +use once_cell::sync::Lazy as LazyLock; +#[cfg(feature = "std")] +use std::sync::LazyLock; use crate::{ChainHardforks, EthereumHardfork, ForkCondition}; /// Dev hardforks -pub static DEV_HARDFORKS: Lazy = Lazy::new(|| { +pub static DEV_HARDFORKS: LazyLock = LazyLock::new(|| { ChainHardforks::new(vec![ (EthereumHardfork::Frontier.boxed(), ForkCondition::Block(0)), (EthereumHardfork::Homestead.boxed(), ForkCondition::Block(0)), diff --git a/crates/optimism/chainspec/Cargo.toml b/crates/optimism/chainspec/Cargo.toml index e13b95056c7f..30590d4e8776 100644 --- a/crates/optimism/chainspec/Cargo.toml +++ b/crates/optimism/chainspec/Cargo.toml @@ -43,4 +43,13 @@ op-alloy-rpc-types.workspace = true [features] default = ["std"] -std = [] \ No newline at end of file +std = [ + "alloy-chains/std", + "alloy-genesis/std", + "alloy-primitives/std", + "op-alloy-rpc-types/std", + "reth-chainspec/std", + "reth-ethereum-forks/std", + "reth-primitives-traits/std", + "reth-optimism-forks/std", +] diff --git a/crates/optimism/chainspec/src/base.rs b/crates/optimism/chainspec/src/base.rs index 3d986b3bcb3a..7aa26bf9a645 100644 --- a/crates/optimism/chainspec/src/base.rs +++ b/crates/optimism/chainspec/src/base.rs @@ -1,18 +1,17 @@ //! Chain specification for the Base Mainnet network. -use alloc::sync::Arc; +use alloc::{sync::Arc, vec}; use alloy_chains::Chain; use alloy_primitives::{b256, U256}; -use once_cell::sync::Lazy; use reth_chainspec::{once_cell_set, BaseFeeParams, BaseFeeParamsKind, ChainSpec}; use reth_ethereum_forks::EthereumHardfork; use reth_optimism_forks::OptimismHardfork; -use crate::OpChainSpec; +use crate::{LazyLock, OpChainSpec}; /// The Base mainnet spec -pub static BASE_MAINNET: Lazy> = Lazy::new(|| { +pub static BASE_MAINNET: LazyLock> = LazyLock::new(|| { OpChainSpec { inner: ChainSpec { chain: Chain::base_mainnet(), diff --git a/crates/optimism/chainspec/src/base_sepolia.rs b/crates/optimism/chainspec/src/base_sepolia.rs index 5b85f5a6b02e..b992dcabaf68 100644 --- a/crates/optimism/chainspec/src/base_sepolia.rs +++ b/crates/optimism/chainspec/src/base_sepolia.rs @@ -1,18 +1,17 @@ //! Chain specification for the Base Sepolia testnet network. -use alloc::sync::Arc; +use alloc::{sync::Arc, vec}; use alloy_chains::Chain; use alloy_primitives::{b256, U256}; -use once_cell::sync::Lazy; use reth_chainspec::{once_cell_set, BaseFeeParams, BaseFeeParamsKind, ChainSpec}; use reth_ethereum_forks::EthereumHardfork; use reth_optimism_forks::OptimismHardfork; -use crate::OpChainSpec; +use crate::{LazyLock, OpChainSpec}; /// The Base Sepolia spec -pub static BASE_SEPOLIA: Lazy> = Lazy::new(|| { +pub static BASE_SEPOLIA: LazyLock> = LazyLock::new(|| { OpChainSpec { inner: ChainSpec { chain: Chain::base_sepolia(), diff --git a/crates/optimism/chainspec/src/dev.rs b/crates/optimism/chainspec/src/dev.rs index 4724e2801b3c..cb8163dfc521 100644 --- a/crates/optimism/chainspec/src/dev.rs +++ b/crates/optimism/chainspec/src/dev.rs @@ -4,18 +4,17 @@ use alloc::sync::Arc; use alloy_chains::Chain; use alloy_primitives::U256; -use once_cell::sync::Lazy; use reth_chainspec::{once_cell_set, BaseFeeParams, BaseFeeParamsKind, ChainSpec}; use reth_optimism_forks::DEV_HARDFORKS; use reth_primitives_traits::constants::DEV_GENESIS_HASH; -use crate::OpChainSpec; +use crate::{LazyLock, OpChainSpec}; /// OP dev testnet specification /// /// Includes 20 prefunded accounts with `10_000` ETH each derived from mnemonic "test test test test /// test test test test test test test junk". -pub static OP_DEV: Lazy> = Lazy::new(|| { +pub static OP_DEV: LazyLock> = LazyLock::new(|| { OpChainSpec { inner: ChainSpec { chain: Chain::dev(), diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index 5ebc18f67688..a18614240583 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -6,6 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; @@ -16,14 +17,17 @@ mod dev; mod op; mod op_sepolia; +use alloc::{vec, vec::Vec}; use alloy_chains::Chain; use alloy_genesis::Genesis; use alloy_primitives::{Parity, Signature, B256, U256}; pub use base::BASE_MAINNET; pub use base_sepolia::BASE_SEPOLIA; +use core::fmt::Display; use derive_more::{Constructor, Deref, From, Into}; pub use dev::OP_DEV; -use once_cell::sync::OnceCell; +#[cfg(not(feature = "std"))] +pub(crate) use once_cell::sync::Lazy as LazyLock; pub use op::OP_MAINNET; pub use op_sepolia::OP_SEPOLIA; use reth_chainspec::{ @@ -33,7 +37,8 @@ use reth_chainspec::{ use reth_ethereum_forks::{ChainHardforks, EthereumHardfork, ForkCondition, Hardfork}; use reth_network_peers::NodeRecord; use reth_primitives_traits::Header; -use std::fmt::Display; +#[cfg(feature = "std")] +pub(crate) use std::sync::LazyLock; /// Chain spec builder for a OP stack chain. #[derive(Debug, Default, From)] @@ -345,7 +350,6 @@ impl From for OpChainSpec { inner: ChainSpec { chain: genesis.config.chain_id.into(), genesis, - genesis_hash: OnceCell::new(), hardforks: ChainHardforks::new(ordered_hardforks), paris_block_and_final_difficulty, base_fee_params: optimism_genesis_info.base_fee_params, diff --git a/crates/optimism/chainspec/src/op.rs b/crates/optimism/chainspec/src/op.rs index 4e8b13005566..8c0da5320f94 100644 --- a/crates/optimism/chainspec/src/op.rs +++ b/crates/optimism/chainspec/src/op.rs @@ -1,19 +1,18 @@ //! Chain specification for the Optimism Mainnet network. -use alloc::sync::Arc; +use alloc::{sync::Arc, vec}; use alloy_chains::Chain; use alloy_primitives::{b256, U256}; -use once_cell::sync::Lazy; use reth_chainspec::{once_cell_set, BaseFeeParams, BaseFeeParamsKind, ChainSpec}; use reth_ethereum_forks::EthereumHardfork; use reth_optimism_forks::OptimismHardfork; use reth_primitives_traits::constants::ETHEREUM_BLOCK_GAS_LIMIT; -use crate::OpChainSpec; +use crate::{LazyLock, OpChainSpec}; /// The Optimism Mainnet spec -pub static OP_MAINNET: Lazy> = Lazy::new(|| { +pub static OP_MAINNET: LazyLock> = LazyLock::new(|| { OpChainSpec { inner: ChainSpec { chain: Chain::optimism_mainnet(), diff --git a/crates/optimism/chainspec/src/op_sepolia.rs b/crates/optimism/chainspec/src/op_sepolia.rs index 9d453c805583..d3243ebd5346 100644 --- a/crates/optimism/chainspec/src/op_sepolia.rs +++ b/crates/optimism/chainspec/src/op_sepolia.rs @@ -1,19 +1,18 @@ //! Chain specification for the Optimism Sepolia testnet network. -use alloc::sync::Arc; +use alloc::{sync::Arc, vec}; use alloy_chains::{Chain, NamedChain}; use alloy_primitives::{b256, U256}; -use once_cell::sync::Lazy; use reth_chainspec::{once_cell_set, BaseFeeParams, BaseFeeParamsKind, ChainSpec}; use reth_ethereum_forks::EthereumHardfork; use reth_optimism_forks::OptimismHardfork; use reth_primitives_traits::constants::ETHEREUM_BLOCK_GAS_LIMIT; -use crate::OpChainSpec; +use crate::{LazyLock, OpChainSpec}; /// The OP Sepolia spec -pub static OP_SEPOLIA: Lazy> = Lazy::new(|| { +pub static OP_SEPOLIA: LazyLock> = LazyLock::new(|| { OpChainSpec { inner: ChainSpec { chain: Chain::from_named(NamedChain::OptimismSepolia), diff --git a/crates/optimism/hardforks/src/dev.rs b/crates/optimism/hardforks/src/dev.rs index 07f88d3f8283..328ef501c46e 100644 --- a/crates/optimism/hardforks/src/dev.rs +++ b/crates/optimism/hardforks/src/dev.rs @@ -1,9 +1,13 @@ use alloy_primitives::U256; -use once_cell::sync::Lazy; use reth_ethereum_forks::{ChainHardforks, EthereumHardfork, ForkCondition}; +#[cfg(not(feature = "std"))] +use once_cell::sync::Lazy as LazyLock; +#[cfg(feature = "std")] +use std::sync::LazyLock; + /// Dev hardforks -pub static DEV_HARDFORKS: Lazy = Lazy::new(|| { +pub static DEV_HARDFORKS: LazyLock = LazyLock::new(|| { ChainHardforks::new(vec![ (EthereumHardfork::Frontier.boxed(), ForkCondition::Block(0)), (EthereumHardfork::Homestead.boxed(), ForkCondition::Block(0)), diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 3622b953107b..62bef16071a3 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -13,10 +13,14 @@ use alloy_primitives::{Bytes, TxHash}; use alloy_rlp::{Decodable, Encodable, Error as RlpError, Header}; use core::mem; use derive_more::{AsRef, Deref}; -use once_cell::sync::Lazy; +use once_cell as _; +#[cfg(not(feature = "std"))] +use once_cell::sync::Lazy as LazyLock; use rayon::prelude::{IntoParallelIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; use signature::{decode_with_eip155_chain_id, with_eip155_parity}; +#[cfg(feature = "std")] +use std::sync::LazyLock; pub use error::{ InvalidTransactionError, TransactionConversionError, TryFromRecoveredTransactionError, @@ -72,8 +76,8 @@ pub type TxHashOrNumber = BlockHashOrNumber; // Expected number of transactions where we can expect a speed-up by recovering the senders in // parallel. -pub(crate) static PARALLEL_SENDER_RECOVERY_THRESHOLD: Lazy = - Lazy::new(|| match rayon::current_num_threads() { +pub(crate) static PARALLEL_SENDER_RECOVERY_THRESHOLD: LazyLock = + LazyLock::new(|| match rayon::current_num_threads() { 0..=1 => usize::MAX, 2..=8 => 10, _ => 5, diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index 5d8496abdae2..9993b477a662 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -110,6 +110,7 @@ pub trait EthBlocks: LoadBlock { /// Helper function for `eth_getBlockReceipts`. /// /// Returns all transaction receipts in block, or `None` if block wasn't found. + #[allow(clippy::type_complexity)] fn block_receipts( &self, block_id: BlockId, @@ -118,6 +119,7 @@ pub trait EthBlocks: LoadBlock { Self: LoadReceipt; /// Helper method that loads a bock and all its receipts. + #[allow(clippy::type_complexity)] fn load_block_and_receipts( &self, block_id: BlockId, diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index ff6e930069f8..cb5c4e79a732 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -61,6 +61,7 @@ pub trait EthCall: Call + LoadPendingBlock { /// The transactions are packed into individual blocks. Overrides can be provided. /// /// See also: + #[allow(clippy::type_complexity)] fn simulate_v1( &self, payload: SimulatePayload, diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index 048353452c0c..7ef827a37042 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -56,13 +56,14 @@ metrics.workspace = true # misc auto_impl.workspace = true itertools.workspace = true -notify = { workspace = true, default-features = false, features = ["macos_fsevent"] } +notify = { workspace = true, default-features = false, features = [ + "macos_fsevent", +] } parking_lot.workspace = true dashmap = { workspace = true, features = ["inline"] } strum.workspace = true # test-utils -once_cell = { workspace = true, optional = true } reth-ethereum-engine-primitives = { workspace = true, optional = true } alloy-consensus = { workspace = true, optional = true } @@ -79,20 +80,22 @@ parking_lot.workspace = true tempfile.workspace = true assert_matches.workspace = true rand.workspace = true -once_cell.workspace = true eyre.workspace = true alloy-consensus.workspace = true [features] -optimism = ["reth-primitives/optimism", "reth-execution-types/optimism", "reth-optimism-primitives"] +optimism = [ + "reth-primitives/optimism", + "reth-execution-types/optimism", + "reth-optimism-primitives", +] serde = ["reth-execution-types/serde"] test-utils = [ "reth-db/test-utils", "reth-nippy-jar/test-utils", "reth-trie/test-utils", "reth-chain-state/test-utils", - "once_cell", "reth-ethereum-engine-primitives", "alloy-consensus", ] diff --git a/crates/storage/provider/src/test_utils/blocks.rs b/crates/storage/provider/src/test_utils/blocks.rs index f7928319b48c..e6a88618792d 100644 --- a/crates/storage/provider/src/test_utils/blocks.rs +++ b/crates/storage/provider/src/test_utils/blocks.rs @@ -5,7 +5,7 @@ use alloy_primitives::{ b256, hex_literal::hex, map::HashMap, Address, BlockNumber, Bytes, Log, Parity, Sealable, TxKind, B256, U256, }; -use once_cell::sync::Lazy; + use reth_db::tables; use reth_db_api::{database::Database, models::StoredBlockBodyIndices}; use reth_primitives::{ @@ -14,7 +14,7 @@ use reth_primitives::{ }; use reth_trie::root::{state_root_unhashed, storage_root_unhashed}; use revm::{db::BundleState, primitives::AccountInfo}; -use std::str::FromStr; +use std::{str::FromStr, sync::LazyLock}; /// Assert genesis block pub fn assert_genesis_block( @@ -61,7 +61,7 @@ pub fn assert_genesis_block( // StageCheckpoints is not updated in tests } -pub(crate) static TEST_BLOCK: Lazy = Lazy::new(|| SealedBlock { +pub(crate) static TEST_BLOCK: LazyLock = LazyLock::new(|| SealedBlock { header: SealedHeader::new( Header { parent_hash: hex!("c86e8cc0310ae7c531c758678ddbfd16fc51c8cef8cec650b032de9869e8b94f") From 0b6397217ca47fe7cd7f5d30963564f8e63dead8 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 15 Oct 2024 10:24:10 +0200 Subject: [PATCH 76/82] chore: rm optimism feature from chainspec (#11722) --- Cargo.lock | 1 - crates/chainspec/Cargo.toml | 4 ---- crates/chainspec/src/api.rs | 2 +- crates/chainspec/src/lib.rs | 2 +- crates/chainspec/src/spec.rs | 18 ------------------ crates/consensus/beacon/Cargo.toml | 5 ++--- crates/consensus/beacon/src/engine/mod.rs | 3 +-- crates/optimism/chainspec/Cargo.toml | 2 +- crates/optimism/chainspec/src/lib.rs | 3 +++ crates/optimism/evm/src/lib.rs | 2 +- crates/optimism/node/Cargo.toml | 1 - crates/optimism/payload/Cargo.toml | 1 - crates/optimism/rpc/src/eth/receipt.rs | 22 +--------------------- 13 files changed, 11 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7cbb3e937e94..1bbfbbd3000b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6532,7 +6532,6 @@ dependencies = [ "once_cell", "reth-ethereum-forks", "reth-network-peers", - "reth-optimism-forks", "reth-primitives-traits", "reth-trie-common", "serde_json", diff --git a/crates/chainspec/Cargo.toml b/crates/chainspec/Cargo.toml index 80636d139a1b..b44a606b65b7 100644 --- a/crates/chainspec/Cargo.toml +++ b/crates/chainspec/Cargo.toml @@ -17,9 +17,6 @@ reth-network-peers.workspace = true reth-trie-common.workspace = true reth-primitives-traits.workspace = true -# op-reth -reth-optimism-forks = { workspace = true, optional = true } - # ethereum alloy-chains = { workspace = true, features = ["serde", "rlp"] } alloy-eips = { workspace = true, features = ["serde"] } @@ -42,7 +39,6 @@ alloy-genesis.workspace = true [features] default = ["std"] -optimism = ["reth-optimism-forks"] std = [ "alloy-chains/std", "alloy-eips/std", diff --git a/crates/chainspec/src/api.rs b/crates/chainspec/src/api.rs index e481fbede70d..39a6716eedde 100644 --- a/crates/chainspec/src/api.rs +++ b/crates/chainspec/src/api.rs @@ -99,6 +99,6 @@ impl EthChainSpec for ChainSpec { } fn is_optimism(&self) -> bool { - Self::is_optimism(self) + self.chain.is_optimism() } } diff --git a/crates/chainspec/src/lib.rs b/crates/chainspec/src/lib.rs index 81aadb834ca2..2e97caba07cc 100644 --- a/crates/chainspec/src/lib.rs +++ b/crates/chainspec/src/lib.rs @@ -33,7 +33,7 @@ pub use reth_ethereum_forks::*; pub use api::EthChainSpec; pub use info::ChainInfo; -#[cfg(feature = "test-utils")] +#[cfg(any(test, feature = "test-utils"))] pub use spec::test_fork_ids; pub use spec::{ BaseFeeParams, BaseFeeParamsKind, ChainSpec, ChainSpecBuilder, ChainSpecProvider, diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index f78fcbb61d0c..e6e8a67d75d3 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -239,21 +239,6 @@ impl ChainSpec { self.chain.is_ethereum() } - /// Returns `true` if this chain contains Optimism configuration. - #[inline] - #[cfg(feature = "optimism")] - pub fn is_optimism(&self) -> bool { - self.chain.is_optimism() || - self.hardforks.get(reth_optimism_forks::OptimismHardfork::Bedrock).is_some() - } - - /// Returns `true` if this chain contains Optimism configuration. - #[inline] - #[cfg(not(feature = "optimism"))] - pub const fn is_optimism(&self) -> bool { - self.chain.is_optimism() - } - /// Returns `true` if this chain is Optimism mainnet. #[inline] pub fn is_optimism_mainnet(&self) -> bool { @@ -706,9 +691,6 @@ impl EthereumHardforks for ChainSpec { } } -#[cfg(feature = "optimism")] -impl reth_optimism_forks::OptimismHardforks for ChainSpec {} - /// A trait for reading the current chainspec. #[auto_impl::auto_impl(&, Arc)] pub trait ChainSpecProvider: Send + Sync { diff --git a/crates/consensus/beacon/Cargo.toml b/crates/consensus/beacon/Cargo.toml index f62c6fbf2a91..f1366812608c 100644 --- a/crates/consensus/beacon/Cargo.toml +++ b/crates/consensus/beacon/Cargo.toml @@ -28,6 +28,7 @@ reth-tokio-util.workspace = true reth-engine-primitives.workspace = true reth-network-p2p.workspace = true reth-node-types.workspace = true +reth-chainspec = { workspace = true, optional = true } # ethereum alloy-primitives.workspace = true @@ -48,8 +49,6 @@ thiserror.workspace = true schnellru.workspace = true itertools.workspace = true -reth-chainspec = { workspace = true, optional = true } - [dev-dependencies] # reth reth-payload-builder = { workspace = true, features = ["test-utils"] } @@ -78,8 +77,8 @@ assert_matches.workspace = true [features] optimism = [ + "reth-chainspec", "reth-primitives/optimism", "reth-provider/optimism", "reth-blockchain-tree/optimism", - "reth-chainspec" ] diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index 5774d4da26b8..ccea982bfbd9 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -462,8 +462,7 @@ where ) -> bool { // On Optimism, the proposers are allowed to reorg their own chain at will. #[cfg(feature = "optimism")] - if reth_chainspec::EthChainSpec::chain(self.blockchain.chain_spec().as_ref()).is_optimism() - { + if reth_chainspec::EthChainSpec::is_optimism(&self.blockchain.chain_spec()) { debug!( target: "consensus::engine", fcu_head_num=?header.number, diff --git a/crates/optimism/chainspec/Cargo.toml b/crates/optimism/chainspec/Cargo.toml index 30590d4e8776..c9f951c8d20d 100644 --- a/crates/optimism/chainspec/Cargo.toml +++ b/crates/optimism/chainspec/Cargo.toml @@ -13,7 +13,7 @@ workspace = true [dependencies] # reth -reth-chainspec = { workspace = true, features = ["optimism"] } +reth-chainspec.workspace = true reth-ethereum-forks.workspace = true reth-primitives-traits.workspace = true reth-network-peers.workspace = true diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index a18614240583..98c6589d1ceb 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -36,6 +36,7 @@ use reth_chainspec::{ }; use reth_ethereum_forks::{ChainHardforks, EthereumHardfork, ForkCondition, Hardfork}; use reth_network_peers::NodeRecord; +use reth_optimism_forks::OptimismHardforks; use reth_primitives_traits::Header; #[cfg(feature = "std")] pub(crate) use std::sync::LazyLock; @@ -267,6 +268,8 @@ impl EthereumHardforks for OpChainSpec { } } +impl OptimismHardforks for OpChainSpec {} + impl From for OpChainSpec { fn from(genesis: Genesis) -> Self { use reth_optimism_forks::OptimismHardfork; diff --git a/crates/optimism/evm/src/lib.rs b/crates/optimism/evm/src/lib.rs index 158ed2e8929e..63d0ee6b4f4e 100644 --- a/crates/optimism/evm/src/lib.rs +++ b/crates/optimism/evm/src/lib.rs @@ -124,7 +124,7 @@ impl ConfigureEvmEnv for OptimismEvmConfig { cfg_env.perf_analyse_created_bytecodes = AnalysisKind::Analyse; cfg_env.handler_cfg.spec_id = spec_id; - cfg_env.handler_cfg.is_optimism = self.chain_spec.is_optimism(); + cfg_env.handler_cfg.is_optimism = true; } fn next_cfg_and_block_env( diff --git a/crates/optimism/node/Cargo.toml b/crates/optimism/node/Cargo.toml index 4675029b64c8..0902d7858e41 100644 --- a/crates/optimism/node/Cargo.toml +++ b/crates/optimism/node/Cargo.toml @@ -79,7 +79,6 @@ op-alloy-consensus.workspace = true [features] optimism = [ - "reth-chainspec/optimism", "reth-primitives/optimism", "reth-provider/optimism", "reth-optimism-evm/optimism", diff --git a/crates/optimism/payload/Cargo.toml b/crates/optimism/payload/Cargo.toml index d5f8b520e822..414b2c358118 100644 --- a/crates/optimism/payload/Cargo.toml +++ b/crates/optimism/payload/Cargo.toml @@ -49,7 +49,6 @@ sha2.workspace = true [features] optimism = [ - "reth-chainspec/optimism", "reth-primitives/optimism", "reth-provider/optimism", "reth-optimism-evm/optimism", diff --git a/crates/optimism/rpc/src/eth/receipt.rs b/crates/optimism/rpc/src/eth/receipt.rs index cc685104133c..200b626d8c3a 100644 --- a/crates/optimism/rpc/src/eth/receipt.rs +++ b/crates/optimism/rpc/src/eth/receipt.rs @@ -4,7 +4,6 @@ use alloy_eips::eip2718::Encodable2718; use alloy_rpc_types::{AnyReceiptEnvelope, Log, TransactionReceipt}; use op_alloy_consensus::{OpDepositReceipt, OpDepositReceiptWithBloom, OpReceiptEnvelope}; use op_alloy_rpc_types::{receipt::L1BlockInfo, OpTransactionReceipt, OpTransactionReceiptFields}; -use reth_chainspec::ChainSpec; use reth_node_api::{FullNodeComponents, NodeTypes}; use reth_optimism_chainspec::OpChainSpec; use reth_optimism_evm::RethL1BlockInfo; @@ -55,25 +54,6 @@ where } } -impl OpEthApi -where - N: FullNodeComponents>, -{ - /// Builds a receipt w.r.t. chain spec. - pub fn build_op_receipt_meta( - &self, - tx: &TransactionSigned, - l1_block_info: revm::L1BlockInfo, - receipt: &Receipt, - ) -> Result { - Ok(OpReceiptFieldsBuilder::default() - .l1_block_info(&self.inner.provider().chain_spec(), tx, l1_block_info)? - .deposit_nonce(receipt.deposit_nonce) - .deposit_version(receipt.deposit_receipt_version) - .build()) - } -} - /// L1 fee and data gas for a non-deposit transaction, or deposit nonce and receipt version for a /// deposit transaction. #[derive(Debug, Default, Clone)] @@ -113,7 +93,7 @@ impl OpReceiptFieldsBuilder { /// Applies [`L1BlockInfo`](revm::L1BlockInfo). pub fn l1_block_info( mut self, - chain_spec: &ChainSpec, + chain_spec: &OpChainSpec, tx: &TransactionSigned, l1_block_info: revm::L1BlockInfo, ) -> Result { From e92fbaea3042172e438f9479166d0f43aff94810 Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Tue, 15 Oct 2024 10:35:47 +0200 Subject: [PATCH 77/82] chore(ci): do not run hive on legacy engine (#11733) --- .github/assets/hive/expected_failures.yaml | 25 ++------ .../hive/expected_failures_experimental.yaml | 63 ------------------- .github/workflows/hive.yml | 11 +--- 3 files changed, 8 insertions(+), 91 deletions(-) delete mode 100644 .github/assets/hive/expected_failures_experimental.yaml diff --git a/.github/assets/hive/expected_failures.yaml b/.github/assets/hive/expected_failures.yaml index 7a212a51dd5e..d4b3d2bcbd3c 100644 --- a/.github/assets/hive/expected_failures.yaml +++ b/.github/assets/hive/expected_failures.yaml @@ -20,7 +20,7 @@ rpc-compat: - eth_getBlockByNumber/get-latest (reth) - eth_getBlockByNumber/get-safe (reth) - # https://github.com/paradigmxyz/reth/issues/8732 +# https://github.com/paradigmxyz/reth/issues/8732 engine-withdrawals: - Withdrawals Fork On Genesis (Paris) (reth) - Withdrawals Fork on Block 1 (Paris) (reth) @@ -43,15 +43,7 @@ engine-withdrawals: # https://github.com/paradigmxyz/reth/issues/8305 # https://github.com/paradigmxyz/reth/issues/6217 -engine-api: - - Inconsistent Head in ForkchoiceState (Paris) (reth) - - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Paris) (reth) - - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth) - - Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth) - - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Paris) (reth) - - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth) - - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Paris) (reth) - - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth) +engine-api: [] # https://github.com/paradigmxyz/reth/issues/8305 # https://github.com/paradigmxyz/reth/issues/6217 @@ -59,17 +51,12 @@ engine-api: # https://github.com/paradigmxyz/reth/issues/7144 engine-cancun: - Blob Transaction Ordering, Multiple Clients (Cancun) (reth) - - Inconsistent Head in ForkchoiceState (Cancun) (reth) - - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Cancun) (reth) - - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) - - Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) - - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Cancun) (reth) - - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth) - - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Cancun) (reth) - - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth) - Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth) - - Invalid NewPayload, ParentBeaconBlockRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) - Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, VersionedHashes, Syncing=False, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, VersionedHashes Version, Syncing=False, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, Incomplete VersionedHashes, Syncing=False, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, Extra VersionedHashes, Syncing=False, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) # https://github.com/paradigmxyz/reth/issues/8579 sync: diff --git a/.github/assets/hive/expected_failures_experimental.yaml b/.github/assets/hive/expected_failures_experimental.yaml deleted file mode 100644 index d4b3d2bcbd3c..000000000000 --- a/.github/assets/hive/expected_failures_experimental.yaml +++ /dev/null @@ -1,63 +0,0 @@ -# https://github.com/paradigmxyz/reth/issues/7015 -# https://github.com/paradigmxyz/reth/issues/6332 -rpc-compat: - - debug_getRawBlock/get-invalid-number (reth) - - debug_getRawHeader/get-invalid-number (reth) - - debug_getRawReceipts/get-invalid-number (reth) - - debug_getRawTransaction/get-invalid-hash (reth) - - - eth_call/call-callenv (reth) - - eth_feeHistory/fee-history (reth) - - eth_getStorageAt/get-storage-invalid-key-too-large (reth) - - eth_getStorageAt/get-storage-invalid-key (reth) - - eth_getTransactionReceipt/get-access-list (reth) - - eth_getTransactionReceipt/get-blob-tx (reth) - - eth_getTransactionReceipt/get-dynamic-fee (reth) - - eth_getBlockByHash/get-block-by-hash (reth) - - eth_getBlockByNumber/get-block-n (reth) - - eth_getBlockByNumber/get-finalized (reth) - - eth_getBlockByNumber/get-genesis (reth) - - eth_getBlockByNumber/get-latest (reth) - - eth_getBlockByNumber/get-safe (reth) - -# https://github.com/paradigmxyz/reth/issues/8732 -engine-withdrawals: - - Withdrawals Fork On Genesis (Paris) (reth) - - Withdrawals Fork on Block 1 (Paris) (reth) - - Withdrawals Fork on Block 2 (Paris) (reth) - - Withdrawals Fork on Block 3 (Paris) (reth) - - Withdraw to a single account (Paris) (reth) - - Withdraw to two accounts (Paris) (reth) - - Withdraw many accounts (Paris) (reth) - - Withdraw zero amount (Paris) (reth) - - Empty Withdrawals (Paris) (reth) - - Corrupted Block Hash Payload (INVALID) (Paris) (reth) - - Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth) - - Withdrawals Fork on Block 1 - 8 Block Re-Org, Sync (Paris) (reth) - - Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth) - - Withdrawals Fork on Block 8 - 10 Block Re-Org Sync (Paris) (reth) - - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth) - - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org Sync (Paris) (reth) - - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth) - - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth) - -# https://github.com/paradigmxyz/reth/issues/8305 -# https://github.com/paradigmxyz/reth/issues/6217 -engine-api: [] - -# https://github.com/paradigmxyz/reth/issues/8305 -# https://github.com/paradigmxyz/reth/issues/6217 -# https://github.com/paradigmxyz/reth/issues/8306 -# https://github.com/paradigmxyz/reth/issues/7144 -engine-cancun: - - Blob Transaction Ordering, Multiple Clients (Cancun) (reth) - - Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth) - - Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) - - Invalid NewPayload, VersionedHashes, Syncing=False, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) - - Invalid NewPayload, VersionedHashes Version, Syncing=False, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) - - Invalid NewPayload, Incomplete VersionedHashes, Syncing=False, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) - - Invalid NewPayload, Extra VersionedHashes, Syncing=False, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) - -# https://github.com/paradigmxyz/reth/issues/8579 -sync: - - sync reth -> reth diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index 12224a951a42..6c50923d3e6b 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -82,7 +82,6 @@ jobs: strategy: fail-fast: false matrix: - engine: [regular, experimental] # ethereum/rpc to be deprecated: # https://github.com/ethereum/hive/pull/1117 scenario: @@ -181,7 +180,7 @@ jobs: needs: - prepare-reth - prepare-hive - name: run ${{ matrix.engine }} - ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }} + name: run ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }} runs-on: group: Reth permissions: @@ -218,11 +217,6 @@ jobs: ref: master path: hivetests - - name: Modify client for experimental engine - if: matrix.engine == 'experimental' - run: | - sed -ie 's/RUST_LOG=info $reth node $FLAGS/RUST_LOG=info $reth node --engine.experimental $FLAGS/' hivetests/clients/reth/reth.sh - - name: Run simulator run: | LIMIT="${{ matrix.scenario.limit }}" @@ -241,8 +235,7 @@ jobs: - name: Parse hive output run: | - FAILURE_FILE="${{ matrix.engine == 'experimental' && '.github/assets/hive/expected_failures_experimental.yaml' || '.github/assets/hive/expected_failures.yaml' }}" - find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion $FAILURE_FILE + find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.yaml - name: Print simulator output if: ${{ failure() }} From 5e386130da0b123e107f5eb5a9a3a3622bc50129 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Tue, 15 Oct 2024 10:42:15 +0200 Subject: [PATCH 78/82] primitives: impl `alloy_consensus::Transaction` for `Transaction` (#11727) --- crates/primitives/src/transaction/mod.rs | 184 ++++++++++++++++++++++- 1 file changed, 183 insertions(+), 1 deletion(-) diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 62bef16071a3..0cb860ff6b86 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -2,7 +2,7 @@ use crate::BlockHashOrNumber; use alloy_eips::eip7702::SignedAuthorization; -use alloy_primitives::{keccak256, Address, TxKind, B256, U256}; +use alloy_primitives::{keccak256, Address, ChainId, TxKind, B256, U256}; use alloy_consensus::{SignableTransaction, TxEip1559, TxEip2930, TxEip4844, TxEip7702, TxLegacy}; use alloy_eips::{ @@ -824,6 +824,188 @@ impl Encodable for Transaction { } } +impl alloy_consensus::Transaction for Transaction { + fn chain_id(&self) -> Option { + match self { + Self::Legacy(tx) => tx.chain_id(), + Self::Eip2930(tx) => tx.chain_id(), + Self::Eip1559(tx) => tx.chain_id(), + Self::Eip4844(tx) => tx.chain_id(), + Self::Eip7702(tx) => tx.chain_id(), + #[cfg(feature = "optimism")] + Self::Deposit(tx) => tx.chain_id(), + } + } + + fn nonce(&self) -> u64 { + match self { + Self::Legacy(tx) => tx.nonce(), + Self::Eip2930(tx) => tx.nonce(), + Self::Eip1559(tx) => tx.nonce(), + Self::Eip4844(tx) => tx.nonce(), + Self::Eip7702(tx) => tx.nonce(), + #[cfg(feature = "optimism")] + Self::Deposit(tx) => tx.nonce(), + } + } + + fn gas_limit(&self) -> u64 { + match self { + Self::Legacy(tx) => tx.gas_limit(), + Self::Eip2930(tx) => tx.gas_limit(), + Self::Eip1559(tx) => tx.gas_limit(), + Self::Eip4844(tx) => tx.gas_limit(), + Self::Eip7702(tx) => tx.gas_limit(), + #[cfg(feature = "optimism")] + Self::Deposit(tx) => tx.gas_limit(), + } + } + + fn gas_price(&self) -> Option { + match self { + Self::Legacy(tx) => tx.gas_price(), + Self::Eip2930(tx) => tx.gas_price(), + Self::Eip1559(tx) => tx.gas_price(), + Self::Eip4844(tx) => tx.gas_price(), + Self::Eip7702(tx) => tx.gas_price(), + #[cfg(feature = "optimism")] + Self::Deposit(tx) => tx.gas_price(), + } + } + + fn max_fee_per_gas(&self) -> u128 { + match self { + Self::Legacy(tx) => tx.max_fee_per_gas(), + Self::Eip2930(tx) => tx.max_fee_per_gas(), + Self::Eip1559(tx) => tx.max_fee_per_gas(), + Self::Eip4844(tx) => tx.max_fee_per_gas(), + Self::Eip7702(tx) => tx.max_fee_per_gas(), + #[cfg(feature = "optimism")] + Self::Deposit(tx) => tx.max_fee_per_gas(), + } + } + + fn max_priority_fee_per_gas(&self) -> Option { + match self { + Self::Legacy(tx) => tx.max_priority_fee_per_gas(), + Self::Eip2930(tx) => tx.max_priority_fee_per_gas(), + Self::Eip1559(tx) => tx.max_priority_fee_per_gas(), + Self::Eip4844(tx) => tx.max_priority_fee_per_gas(), + Self::Eip7702(tx) => tx.max_priority_fee_per_gas(), + #[cfg(feature = "optimism")] + Self::Deposit(tx) => tx.max_priority_fee_per_gas(), + } + } + + fn priority_fee_or_price(&self) -> u128 { + match self { + Self::Legacy(tx) => tx.priority_fee_or_price(), + Self::Eip2930(tx) => tx.priority_fee_or_price(), + Self::Eip1559(tx) => tx.priority_fee_or_price(), + Self::Eip4844(tx) => tx.priority_fee_or_price(), + Self::Eip7702(tx) => tx.priority_fee_or_price(), + #[cfg(feature = "optimism")] + Self::Deposit(tx) => tx.priority_fee_or_price(), + } + } + + fn max_fee_per_blob_gas(&self) -> Option { + match self { + Self::Legacy(tx) => tx.max_fee_per_blob_gas(), + Self::Eip2930(tx) => tx.max_fee_per_blob_gas(), + Self::Eip1559(tx) => tx.max_fee_per_blob_gas(), + Self::Eip4844(tx) => tx.max_fee_per_blob_gas(), + Self::Eip7702(tx) => tx.max_fee_per_blob_gas(), + #[cfg(feature = "optimism")] + Self::Deposit(tx) => tx.max_fee_per_blob_gas(), + } + } + + fn to(&self) -> TxKind { + match self { + Self::Legacy(tx) => tx.to(), + Self::Eip2930(tx) => tx.to(), + Self::Eip1559(tx) => tx.to(), + Self::Eip4844(tx) => tx.to(), + Self::Eip7702(tx) => tx.to(), + #[cfg(feature = "optimism")] + Self::Deposit(tx) => tx.to(), + } + } + + fn value(&self) -> alloy_primitives::U256 { + match self { + Self::Legacy(tx) => tx.value(), + Self::Eip2930(tx) => tx.value(), + Self::Eip1559(tx) => tx.value(), + Self::Eip4844(tx) => tx.value(), + Self::Eip7702(tx) => tx.value(), + #[cfg(feature = "optimism")] + Self::Deposit(tx) => tx.value(), + } + } + + fn input(&self) -> &[u8] { + match self { + Self::Legacy(tx) => tx.input(), + Self::Eip2930(tx) => tx.input(), + Self::Eip1559(tx) => tx.input(), + Self::Eip4844(tx) => tx.input(), + Self::Eip7702(tx) => tx.input(), + #[cfg(feature = "optimism")] + Self::Deposit(tx) => tx.input(), + } + } + + fn ty(&self) -> u8 { + match self { + Self::Legacy(tx) => tx.ty(), + Self::Eip2930(tx) => tx.ty(), + Self::Eip1559(tx) => tx.ty(), + Self::Eip4844(tx) => tx.ty(), + Self::Eip7702(tx) => tx.ty(), + #[cfg(feature = "optimism")] + Self::Deposit(tx) => tx.ty(), + } + } + + fn access_list(&self) -> Option<&AccessList> { + match self { + Self::Legacy(tx) => tx.access_list(), + Self::Eip2930(tx) => tx.access_list(), + Self::Eip1559(tx) => tx.access_list(), + Self::Eip4844(tx) => tx.access_list(), + Self::Eip7702(tx) => tx.access_list(), + #[cfg(feature = "optimism")] + Self::Deposit(tx) => tx.access_list(), + } + } + + fn blob_versioned_hashes(&self) -> Option<&[B256]> { + match self { + Self::Legacy(tx) => tx.blob_versioned_hashes(), + Self::Eip2930(tx) => tx.blob_versioned_hashes(), + Self::Eip1559(tx) => tx.blob_versioned_hashes(), + Self::Eip4844(tx) => tx.blob_versioned_hashes(), + Self::Eip7702(tx) => tx.blob_versioned_hashes(), + #[cfg(feature = "optimism")] + Self::Deposit(tx) => tx.blob_versioned_hashes(), + } + } + + fn authorization_list(&self) -> Option<&[SignedAuthorization]> { + match self { + Self::Legacy(tx) => tx.authorization_list(), + Self::Eip2930(tx) => tx.authorization_list(), + Self::Eip1559(tx) => tx.authorization_list(), + Self::Eip4844(tx) => tx.authorization_list(), + Self::Eip7702(tx) => tx.authorization_list(), + #[cfg(feature = "optimism")] + Self::Deposit(tx) => tx.authorization_list(), + } + } +} + /// Signed transaction without its Hash. Used type for inserting into the DB. /// /// This can by converted to [`TransactionSigned`] by calling [`TransactionSignedNoHash::hash`]. From 39f0ab41169ec3b8f30cdae1d138ff3a04364928 Mon Sep 17 00:00:00 2001 From: Jonathan LEI Date: Tue, 15 Oct 2024 18:32:25 +0800 Subject: [PATCH 79/82] chore: turn off `reth-revm` default features (#10215) Co-authored-by: Federico Gimenez Co-authored-by: Matthias Seitz --- Cargo.lock | 5 +++++ Cargo.toml | 4 +--- crates/ethereum/evm/src/lib.rs | 6 ++++++ crates/ethereum/node/Cargo.toml | 4 ++++ crates/ethereum/node/src/lib.rs | 3 +++ crates/evm/src/lib.rs | 6 ++++++ crates/optimism/node/Cargo.toml | 6 +++++- testing/ef-tests/Cargo.toml | 3 +++ testing/ef-tests/src/lib.rs | 3 +++ 9 files changed, 36 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1bbfbbd3000b..3fbcc7514d86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2562,7 +2562,9 @@ dependencies = [ "reth-evm-ethereum", "reth-primitives", "reth-provider", + "reth-revm", "reth-stages", + "revm", "serde", "serde_json", "thiserror", @@ -7937,10 +7939,12 @@ dependencies = [ "reth-payload-builder", "reth-primitives", "reth-provider", + "reth-revm", "reth-rpc", "reth-tasks", "reth-tracing", "reth-transaction-pool", + "revm", "serde_json", "tokio", ] @@ -8163,6 +8167,7 @@ dependencies = [ "reth-rpc-types-compat", "reth-tracing", "reth-transaction-pool", + "revm", "serde", "serde_json", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index 2068f02bb845..f734fd44c3c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -381,7 +381,7 @@ reth-primitives-traits = { path = "crates/primitives-traits", default-features = reth-provider = { path = "crates/storage/provider" } reth-prune = { path = "crates/prune/prune" } reth-prune-types = { path = "crates/prune/types" } -reth-revm = { path = "crates/revm" } +reth-revm = { path = "crates/revm", default-features = false } reth-rpc = { path = "crates/rpc/rpc" } reth-rpc-api = { path = "crates/rpc/rpc-api" } reth-rpc-api-testing-util = { path = "crates/rpc/rpc-testing-util" } @@ -412,8 +412,6 @@ reth-trie-parallel = { path = "crates/trie/parallel" } # revm revm = { version = "14.0.3", features = [ "std", - "secp256k1", - "blst", ], default-features = false } revm-inspectors = "0.8.1" revm-primitives = { version = "10.0.0", features = [ diff --git a/crates/ethereum/evm/src/lib.rs b/crates/ethereum/evm/src/lib.rs index 8ea39f93deef..cede8008b3cf 100644 --- a/crates/ethereum/evm/src/lib.rs +++ b/crates/ethereum/evm/src/lib.rs @@ -1,4 +1,10 @@ //! EVM config for vanilla ethereum. +//! +//! # Revm features +//! +//! This crate does __not__ enforce specific revm features such as `blst` or `c-kzg`, which are +//! critical for revm's evm internals, it is the responsibility of the implementer to ensure the +//! proper features are selected. #![doc( html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", diff --git a/crates/ethereum/node/Cargo.toml b/crates/ethereum/node/Cargo.toml index 145a966833cf..7a323f91d877 100644 --- a/crates/ethereum/node/Cargo.toml +++ b/crates/ethereum/node/Cargo.toml @@ -29,6 +29,10 @@ reth-rpc.workspace = true reth-node-api.workspace = true reth-chainspec.workspace = true reth-primitives.workspace = true +reth-revm = { workspace = true, features = ["std"] } + +# revm with required ethereum features +revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg"] } # misc eyre.workspace = true diff --git a/crates/ethereum/node/src/lib.rs b/crates/ethereum/node/src/lib.rs index 44ec6836c875..37ebc33c22b8 100644 --- a/crates/ethereum/node/src/lib.rs +++ b/crates/ethereum/node/src/lib.rs @@ -8,6 +8,9 @@ #![cfg_attr(not(test), warn(unused_crate_dependencies))] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +use reth_revm as _; +use revm as _; + pub use reth_ethereum_engine_primitives::EthEngineTypes; pub mod evm; diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs index e0d45f04cd31..66026a07c94e 100644 --- a/crates/evm/src/lib.rs +++ b/crates/evm/src/lib.rs @@ -1,4 +1,10 @@ //! Traits for configuring an EVM specifics. +//! +//! # Revm features +//! +//! This crate does __not__ enforce specific revm features such as `blst` or `c-kzg`, which are +//! critical for revm's evm internals, it is the responsibility of the implementer to ensure the +//! proper features are selected. #![doc( html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", diff --git a/crates/optimism/node/Cargo.toml b/crates/optimism/node/Cargo.toml index 0902d7858e41..f9e038a3d9e0 100644 --- a/crates/optimism/node/Cargo.toml +++ b/crates/optimism/node/Cargo.toml @@ -27,7 +27,7 @@ reth-provider.workspace = true reth-transaction-pool.workspace = true reth-network.workspace = true reth-evm.workspace = true -reth-revm.workspace = true +reth-revm = { workspace = true, features = ["std"] } reth-beacon-consensus.workspace = true reth-discv5.workspace = true reth-rpc-eth-types.workspace = true @@ -42,6 +42,9 @@ reth-optimism-chainspec.workspace = true reth-optimism-consensus.workspace = true reth-optimism-forks.workspace = true +# revm with required optimism features +revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg"] } + # ethereum alloy-eips.workspace = true alloy-primitives.workspace = true @@ -85,6 +88,7 @@ optimism = [ "reth-optimism-payload-builder/optimism", "reth-beacon-consensus/optimism", "reth-revm/optimism", + "revm/optimism", "reth-auto-seal-consensus/optimism", "reth-optimism-rpc/optimism", "reth-engine-local/optimism", diff --git a/testing/ef-tests/Cargo.toml b/testing/ef-tests/Cargo.toml index ca23ffcce38e..df68f5154fc4 100644 --- a/testing/ef-tests/Cargo.toml +++ b/testing/ef-tests/Cargo.toml @@ -23,6 +23,9 @@ reth-db-api.workspace = true reth-provider = { workspace = true, features = ["test-utils"] } reth-stages.workspace = true reth-evm-ethereum.workspace = true +reth-revm = { workspace = true, features = ["std"] } + +revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg"] } alloy-rlp.workspace = true alloy-primitives.workspace = true diff --git a/testing/ef-tests/src/lib.rs b/testing/ef-tests/src/lib.rs index 45f296d1f5f1..ca5e47d2d3bc 100644 --- a/testing/ef-tests/src/lib.rs +++ b/testing/ef-tests/src/lib.rs @@ -7,6 +7,9 @@ )] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +use reth_revm as _; +use revm as _; + pub mod case; pub mod result; pub mod suite; From 3cb4bf266d624b9f985fc9273540f353f34f6b94 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:53:34 +0200 Subject: [PATCH 80/82] chore(deps): bump alloy-trie 0.7 (#11362) Co-authored-by: Roman Krasiuk Co-authored-by: Alexey Shekhirin --- Cargo.lock | 12 ++++--- Cargo.toml | 2 +- crates/stages/types/src/checkpoints.rs | 2 +- crates/storage/codecs/src/alloy/trie.rs | 33 +++++++++++--------- crates/trie/common/src/hash_builder/state.rs | 12 +++---- crates/trie/trie/src/witness.rs | 8 +++-- 6 files changed, 39 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3fbcc7514d86..828444080432 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -741,13 +741,14 @@ dependencies = [ [[package]] name = "alloy-trie" -version = "0.6.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9703ce68b97f8faae6f7739d1e003fc97621b856953cbcdbb2b515743f23288" +checksum = "6fa8acead43cb238a7b7f47238c71137f4677a0b8d90e7e3be6e6ca59a28194e" dependencies = [ "alloy-primitives", "alloy-rlp", "arbitrary", + "arrayvec", "derive_arbitrary", "derive_more 1.0.0", "nybbles", @@ -992,6 +993,9 @@ name = "arrayvec" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +dependencies = [ + "serde", +] [[package]] name = "asn1_der" @@ -4460,7 +4464,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] @@ -11348,7 +11352,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f734fd44c3c4..e3ec1c1fb4a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -424,7 +424,7 @@ alloy-dyn-abi = "0.8.0" alloy-primitives = { version = "0.8.7", default-features = false } alloy-rlp = "0.3.4" alloy-sol-types = "0.8.0" -alloy-trie = { version = "0.6", default-features = false } +alloy-trie = { version = "0.7", default-features = false } alloy-consensus = { version = "0.4.2", default-features = false } alloy-eips = { version = "0.4.2", default-features = false } diff --git a/crates/stages/types/src/checkpoints.rs b/crates/stages/types/src/checkpoints.rs index e88e933d6c4a..79e896bf4d9b 100644 --- a/crates/stages/types/src/checkpoints.rs +++ b/crates/stages/types/src/checkpoints.rs @@ -8,7 +8,7 @@ use std::ops::RangeInclusive; use super::StageId; /// Saves the progress of Merkle stage. -#[derive(Default, Debug, Clone, PartialEq)] +#[derive(Default, Debug, Clone, PartialEq, Eq)] pub struct MerkleCheckpoint { /// The target block number. pub target_block: BlockNumber, diff --git a/crates/storage/codecs/src/alloy/trie.rs b/crates/storage/codecs/src/alloy/trie.rs index c89ef0bf6ec0..cc5273dd0273 100644 --- a/crates/storage/codecs/src/alloy/trie.rs +++ b/crates/storage/codecs/src/alloy/trie.rs @@ -1,15 +1,18 @@ -//! Native Compact codec impl for EIP-7685 requests. +//! Native Compact codec impl for alloy-trie types. use crate::Compact; use alloc::vec::Vec; use alloy_primitives::B256; -use alloy_trie::{hash_builder::HashBuilderValue, BranchNodeCompact, TrieMask}; +use alloy_trie::{ + hash_builder::{HashBuilderValue, HashBuilderValueRef}, + BranchNodeCompact, TrieMask, +}; use bytes::{Buf, BufMut}; -/// Identifier for [`HashBuilderValue::Hash`] +/// Identifier for [`HashBuilderValueRef::Hash`] const HASH_BUILDER_TYPE_HASH: u8 = 0; -/// Identifier for [`HashBuilderValue::Bytes`] +/// Identifier for [`HashBuilderValueRef::Bytes`] const HASH_BUILDER_TYPE_BYTES: u8 = 1; impl Compact for HashBuilderValue { @@ -17,34 +20,34 @@ impl Compact for HashBuilderValue { where B: BufMut + AsMut<[u8]>, { - match self { - Self::Hash(hash) => { + match self.as_ref() { + HashBuilderValueRef::Hash(hash) => { buf.put_u8(HASH_BUILDER_TYPE_HASH); 1 + hash.to_compact(buf) } - Self::Bytes(bytes) => { + HashBuilderValueRef::Bytes(bytes) => { buf.put_u8(HASH_BUILDER_TYPE_BYTES); 1 + bytes.to_compact(buf) } } } - // # Panics - // - // A panic will be triggered if a HashBuilderValue variant greater than 1 is passed from the - // database. fn from_compact(mut buf: &[u8], _: usize) -> (Self, &[u8]) { - match buf.get_u8() { + let mut this = Self::default(); + let buf = match buf.get_u8() { HASH_BUILDER_TYPE_HASH => { let (hash, buf) = B256::from_compact(buf, 32); - (Self::Hash(hash), buf) + this.set_from_ref(HashBuilderValueRef::Hash(&hash)); + buf } HASH_BUILDER_TYPE_BYTES => { let (bytes, buf) = Vec::from_compact(buf, 0); - (Self::Bytes(bytes), buf) + this.set_bytes_owned(bytes); + buf } _ => unreachable!("Junk data in database: unknown HashBuilderValue variant"), - } + }; + (this, buf) } } diff --git a/crates/trie/common/src/hash_builder/state.rs b/crates/trie/common/src/hash_builder/state.rs index 467931672745..c5cae21a1a3d 100644 --- a/crates/trie/common/src/hash_builder/state.rs +++ b/crates/trie/common/src/hash_builder/state.rs @@ -1,5 +1,5 @@ use crate::TrieMask; -use alloy_trie::{hash_builder::HashBuilderValue, HashBuilder}; +use alloy_trie::{hash_builder::HashBuilderValue, nodes::RlpNode, HashBuilder}; use bytes::Buf; use nybbles::Nibbles; use reth_codecs::Compact; @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; /// The hash builder state for storing in the database. /// Check the `reth-trie` crate for more info on hash builder. -#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] #[cfg_attr( feature = "arbitrary", derive(arbitrary::Arbitrary), @@ -16,10 +16,10 @@ use serde::{Deserialize, Serialize}; pub struct HashBuilderState { /// The current key. pub key: Vec, - /// The builder stack. - pub stack: Vec>, /// The current node value. pub value: HashBuilderValue, + /// The builder stack. + pub stack: Vec, /// Group masks. pub groups: Vec, @@ -112,7 +112,7 @@ impl Compact for HashBuilderState { let mut stack = Vec::with_capacity(stack_len); for _ in 0..stack_len { let item_len = buf.get_u16() as usize; - stack.push(Vec::from(&buf[..item_len])); + stack.push(RlpNode::from_raw(&buf[..item_len]).unwrap()); buf.advance(item_len); } @@ -154,7 +154,7 @@ mod tests { #[test] fn hash_builder_state_regression() { let mut state = HashBuilderState::default(); - state.stack.push(vec![]); + state.stack.push(Default::default()); let mut buf = vec![]; let len = state.clone().to_compact(&mut buf); let (decoded, _) = HashBuilderState::from_compact(&buf, len); diff --git a/crates/trie/trie/src/witness.rs b/crates/trie/trie/src/witness.rs index 582319c7fc51..971f10cfbae1 100644 --- a/crates/trie/trie/src/witness.rs +++ b/crates/trie/trie/src/witness.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeMap; - use crate::{ hashed_cursor::{HashedCursor, HashedCursorFactory}, prefix_set::TriePrefixSetsMut, @@ -19,6 +17,7 @@ use reth_primitives::constants::EMPTY_ROOT_HASH; use reth_trie_common::{ BranchNode, HashBuilder, Nibbles, StorageMultiProof, TrieAccount, TrieNode, CHILD_INDEX_RANGE, }; +use std::collections::BTreeMap; /// State transition witness for the trie. #[derive(Debug)] @@ -229,7 +228,10 @@ where TrieNode::Leaf(leaf) => { next_path.extend_from_slice(&leaf.key); if next_path != key { - trie_nodes.insert(next_path.clone(), Either::Right(leaf.value.clone())); + trie_nodes.insert( + next_path.clone(), + Either::Right(leaf.value.as_slice().to_vec()), + ); } } TrieNode::EmptyRoot => { From 161605313a567a081ea2eaf23ab1f6eeba29d19f Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Tue, 15 Oct 2024 13:12:41 +0200 Subject: [PATCH 81/82] feat: sketch composable executor (#11447) --- crates/evm/src/execute.rs | 335 +++++++++++++++++++++++++++++++++++++- 1 file changed, 331 insertions(+), 4 deletions(-) diff --git a/crates/evm/src/execute.rs b/crates/evm/src/execute.rs index 476c695e7ac6..eb77d054bdcd 100644 --- a/crates/evm/src/execute.rs +++ b/crates/evm/src/execute.rs @@ -7,11 +7,12 @@ pub use reth_execution_errors::{ pub use reth_execution_types::{BlockExecutionInput, BlockExecutionOutput, ExecutionOutcome}; pub use reth_storage_errors::provider::ProviderError; +use alloc::{boxed::Box, vec::Vec}; use alloy_primitives::BlockNumber; -use core::fmt::Display; -use reth_primitives::{BlockWithSenders, Receipt}; +use core::{fmt::Display, marker::PhantomData}; +use reth_primitives::{BlockWithSenders, Receipt, Request}; use reth_prune_types::PruneModes; -use revm::State; +use revm::{db::BundleState, State}; use revm_primitives::db::Database; use crate::system_calls::OnStateHook; @@ -163,12 +164,233 @@ pub trait BlockExecutorProvider: Send + Sync + Clone + Unpin + 'static { DB: Database + Display>; } +/// Defines the strategy for executing a single block. +pub trait BlockExecutionStrategy { + /// The error type returned by this strategy's methods. + type Error: From + core::error::Error; + + /// Applies any necessary changes before executing the block's transactions. + fn apply_pre_execution_changes(&mut self) -> Result<(), Self::Error>; + + /// Executes all transactions in the block. + fn execute_transactions( + &mut self, + block: &BlockWithSenders, + ) -> Result<(Vec, u64), Self::Error>; + + /// Applies any necessary changes after executing the block's transactions. + fn apply_post_execution_changes(&mut self) -> Result, Self::Error>; + + /// Returns a reference to the current state. + fn state_ref(&self) -> &State; + + /// Sets a hook to be called after each state change during execution. + fn with_state_hook(&mut self, hook: Option>); + + /// Returns the final bundle state. + fn finish(&self) -> BundleState; +} + +/// A strategy factory that can create block execution strategies. +pub trait BlockExecutionStrategyFactory: Send + Sync + Clone + Unpin + 'static { + /// Associated strategy type. + type Strategy + Display>>: BlockExecutionStrategy< + DB, + Error = BlockExecutionError, + >; + + /// Creates a strategy using the give database. + fn create_strategy(&self, db: DB) -> Self::Strategy + where + DB: Database + Display>; +} + +impl Clone for GenericBlockExecutorProvider +where + F: Clone, +{ + fn clone(&self) -> Self { + Self { strategy_factory: self.strategy_factory.clone() } + } +} + +/// A generic block executor provider that can create executors using a strategy factory. +#[allow(missing_debug_implementations)] +pub struct GenericBlockExecutorProvider { + strategy_factory: F, +} + +impl GenericBlockExecutorProvider { + /// Creates a new `GenericBlockExecutorProvider` with the given strategy factory. + pub const fn new(strategy_factory: F) -> Self { + Self { strategy_factory } + } +} + +impl BlockExecutorProvider for GenericBlockExecutorProvider +where + F: BlockExecutionStrategyFactory, +{ + type Executor + Display>> = + GenericBlockExecutor, DB>; + + type BatchExecutor + Display>> = + GenericBatchExecutor, DB>; + + fn executor(&self, db: DB) -> Self::Executor + where + DB: Database + Display>, + { + let strategy = self.strategy_factory.create_strategy(db); + GenericBlockExecutor::new(strategy) + } + + fn batch_executor(&self, db: DB) -> Self::BatchExecutor + where + DB: Database + Display>, + { + let strategy = self.strategy_factory.create_strategy(db); + GenericBatchExecutor::new(strategy) + } +} + +/// A generic block executor that uses a [`BlockExecutionStrategy`] to +/// execute blocks. +#[allow(missing_debug_implementations, dead_code)] +pub struct GenericBlockExecutor +where + S: BlockExecutionStrategy, +{ + strategy: S, + _phantom: PhantomData, +} + +impl GenericBlockExecutor +where + S: BlockExecutionStrategy, +{ + /// Creates a new `GenericBlockExecutor` with the given strategy. + pub const fn new(strategy: S) -> Self { + Self { strategy, _phantom: PhantomData } + } +} + +impl Executor for GenericBlockExecutor +where + S: BlockExecutionStrategy, + DB: Database + Display>, +{ + type Input<'a> = BlockExecutionInput<'a, BlockWithSenders>; + type Output = BlockExecutionOutput; + type Error = S::Error; + + fn execute(mut self, input: Self::Input<'_>) -> Result { + let BlockExecutionInput { block, total_difficulty: _ } = input; + + self.strategy.apply_pre_execution_changes()?; + let (receipts, gas_used) = self.strategy.execute_transactions(block)?; + let requests = self.strategy.apply_post_execution_changes()?; + let state = self.strategy.finish(); + + Ok(BlockExecutionOutput { state, receipts, requests, gas_used }) + } + + fn execute_with_state_closure( + mut self, + input: Self::Input<'_>, + mut state: F, + ) -> Result + where + F: FnMut(&State), + { + let BlockExecutionInput { block, total_difficulty: _ } = input; + + self.strategy.apply_pre_execution_changes()?; + let (receipts, gas_used) = self.strategy.execute_transactions(block)?; + let requests = self.strategy.apply_post_execution_changes()?; + + state(self.strategy.state_ref()); + + let state = self.strategy.finish(); + + Ok(BlockExecutionOutput { state, receipts, requests, gas_used }) + } + + fn execute_with_state_hook( + mut self, + input: Self::Input<'_>, + state_hook: H, + ) -> Result + where + H: OnStateHook + 'static, + { + let BlockExecutionInput { block, total_difficulty: _ } = input; + + self.strategy.with_state_hook(Some(Box::new(state_hook))); + + self.strategy.apply_pre_execution_changes()?; + let (receipts, gas_used) = self.strategy.execute_transactions(block)?; + let requests = self.strategy.apply_post_execution_changes()?; + + let state = self.strategy.finish(); + + Ok(BlockExecutionOutput { state, receipts, requests, gas_used }) + } +} + +/// A generic batch executor that uses a [`BlockExecutionStrategy`] to +/// execute batches. +#[allow(missing_debug_implementations)] +pub struct GenericBatchExecutor { + _strategy: S, + _phantom: PhantomData, +} + +impl GenericBatchExecutor { + /// Creates a new `GenericBatchExecutor` with the given strategy. + pub const fn new(_strategy: S) -> Self { + Self { _strategy, _phantom: PhantomData } + } +} + +impl BatchExecutor for GenericBatchExecutor +where + S: BlockExecutionStrategy, + DB: Database + Display>, +{ + type Input<'a> = BlockExecutionInput<'a, BlockWithSenders>; + type Output = ExecutionOutcome; + type Error = BlockExecutionError; + + fn execute_and_verify_one(&mut self, _input: Self::Input<'_>) -> Result<(), Self::Error> { + todo!() + } + + fn finalize(self) -> Self::Output { + todo!() + } + + fn set_tip(&mut self, _tip: BlockNumber) { + todo!() + } + + fn set_prune_modes(&mut self, _prune_modes: PruneModes) { + todo!() + } + + fn size_hint(&self) -> Option { + None + } +} + #[cfg(test)] mod tests { use super::*; + use alloy_eips::eip6110::DepositRequest; use alloy_primitives::U256; + use reth_chainspec::{ChainSpec, MAINNET}; use revm::db::{CacheDB, EmptyDBTyped}; - use std::marker::PhantomData; + use std::sync::Arc; #[derive(Clone, Default)] struct TestExecutorProvider; @@ -252,6 +474,83 @@ mod tests { } } + struct TestExecutorStrategy { + // chain spec and evm config here only to illustrate how the strategy + // factory can use them in a real use case. + _chain_spec: Arc, + _evm_config: EvmConfig, + state: State, + execute_transactions_result: (Vec, u64), + apply_post_execution_changes_result: Vec, + finish_result: BundleState, + } + + #[derive(Clone)] + struct TestExecutorStrategyFactory { + execute_transactions_result: (Vec, u64), + apply_post_execution_changes_result: Vec, + finish_result: BundleState, + } + + impl BlockExecutionStrategyFactory for TestExecutorStrategyFactory { + type Strategy + Display>> = + TestExecutorStrategy; + + fn create_strategy(&self, db: DB) -> Self::Strategy + where + DB: Database + Display>, + { + let state = State::builder() + .with_database(db) + .with_bundle_update() + .without_state_clear() + .build(); + + TestExecutorStrategy { + _chain_spec: MAINNET.clone(), + _evm_config: TestEvmConfig {}, + execute_transactions_result: self.execute_transactions_result.clone(), + apply_post_execution_changes_result: self + .apply_post_execution_changes_result + .clone(), + finish_result: self.finish_result.clone(), + state, + } + } + } + + impl BlockExecutionStrategy for TestExecutorStrategy { + type Error = BlockExecutionError; + + fn apply_pre_execution_changes(&mut self) -> Result<(), Self::Error> { + Ok(()) + } + + fn execute_transactions( + &mut self, + _block: &BlockWithSenders, + ) -> Result<(Vec, u64), Self::Error> { + Ok(self.execute_transactions_result.clone()) + } + + fn apply_post_execution_changes(&mut self) -> Result, Self::Error> { + Ok(self.apply_post_execution_changes_result.clone()) + } + + fn state_ref(&self) -> &State { + &self.state + } + + fn with_state_hook(&mut self, _hook: Option>) {} + + fn finish(&self) -> BundleState { + self.finish_result.clone() + } + } + + #[derive(Clone)] + struct TestEvmConfig {} + #[test] fn test_provider() { let provider = TestExecutorProvider; @@ -259,4 +558,32 @@ mod tests { let executor = provider.executor(db); let _ = executor.execute(BlockExecutionInput::new(&Default::default(), U256::ZERO)); } + + #[test] + fn test_strategy() { + let expected_gas_used = 10; + let expected_receipts = vec![Receipt::default()]; + let expected_execute_transactions_result = (expected_receipts.clone(), expected_gas_used); + let expected_apply_post_execution_changes_result = + vec![Request::DepositRequest(DepositRequest::default())]; + let expected_finish_result = BundleState::default(); + + let strategy_factory = TestExecutorStrategyFactory { + execute_transactions_result: expected_execute_transactions_result, + apply_post_execution_changes_result: expected_apply_post_execution_changes_result + .clone(), + finish_result: expected_finish_result.clone(), + }; + let provider = GenericBlockExecutorProvider::new(strategy_factory); + let db = CacheDB::>::default(); + let executor = provider.executor(db); + let result = executor.execute(BlockExecutionInput::new(&Default::default(), U256::ZERO)); + + assert!(result.is_ok()); + let block_execution_output = result.unwrap(); + assert_eq!(block_execution_output.gas_used, expected_gas_used); + assert_eq!(block_execution_output.receipts, expected_receipts); + assert_eq!(block_execution_output.requests, expected_apply_post_execution_changes_result); + assert_eq!(block_execution_output.state, expected_finish_result); + } } From 900d10e54379ae05695b76ac7c4b453bd0314da5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Garamv=C3=B6lgyi?= Date: Tue, 15 Oct 2024 15:08:30 +0200 Subject: [PATCH 82/82] update fork base commit --- fork.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fork.yaml b/fork.yaml index b8c77208f4ab..93920105fe2a 100644 --- a/fork.yaml +++ b/fork.yaml @@ -4,7 +4,7 @@ footer: | base: name: reth url: https://github.com/paradigmxyz/reth - hash: b787d9e521fad1cb28a372637474ae4ec4986bf3 + hash: 161605313a567a081ea2eaf23ab1f6eeba29d19f fork: name: scroll-reth url: https://github.com/scroll-tech/reth