From 1d1a777cb265ae8cadc7b198eb9563f92aac7034 Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Thu, 24 Aug 2023 23:55:46 +0000 Subject: [PATCH 1/6] setup: Add `trie` and stubs --- pyproject.toml | 3 +++ setup.cfg | 1 + stubs/trie/__init__.pyi | 3 +++ stubs/trie/hexary.pyi | 9 +++++++++ whitelist.txt | 1 + 5 files changed, 17 insertions(+) create mode 100644 stubs/trie/__init__.pyi create mode 100644 stubs/trie/hexary.pyi diff --git a/pyproject.toml b/pyproject.toml index d3c96929e5..c87d4ba623 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,3 +9,6 @@ line_length = 99 [tool.black] line-length = 99 + +[tool.mypy] +mypy_path = "$MYPY_CONFIG_FILE_DIR/stubs" diff --git a/setup.cfg b/setup.cfg index 534b5feda1..af7cc10de0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,6 +31,7 @@ install_requires = pytest==7.3.2 pytest-xdist>=3.3.1,<4 coincurve==17.0.0 + trie==2.1.1 [options.package_data] ethereum_test_tools = diff --git a/stubs/trie/__init__.pyi b/stubs/trie/__init__.pyi new file mode 100644 index 0000000000..c7b52113ff --- /dev/null +++ b/stubs/trie/__init__.pyi @@ -0,0 +1,3 @@ +from .hexary import HexaryTrie as HexaryTrie + +__all__ = ("HexaryTrie",) diff --git a/stubs/trie/hexary.pyi b/stubs/trie/hexary.pyi new file mode 100644 index 0000000000..254cef4041 --- /dev/null +++ b/stubs/trie/hexary.pyi @@ -0,0 +1,9 @@ +from typing import Dict + + +class HexaryTrie: + db: Dict + root_hash: bytes + + def __init__(self, db: Dict) -> None: ... + def set(self, key: bytes, value: bytes) -> None: ... diff --git a/whitelist.txt b/whitelist.txt index f4da2db31c..d25fa894b4 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -102,6 +102,7 @@ glightbox gwei hash32 hasher +hexary hexsha homebrew html From 9d10e0d1e1e629a1619ce02cd1dedc071773e3ba Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Thu, 24 Aug 2023 23:59:34 +0000 Subject: [PATCH 2/6] types: add withdrawals root calculation --- src/ethereum_test_tools/common/__init__.py | 2 + src/ethereum_test_tools/common/types.py | 11 +++ .../spec/blockchain_test.py | 9 +-- src/ethereum_test_tools/spec/state_test.py | 10 +-- src/ethereum_test_tools/tests/test_types.py | 68 ++++++++++++++++++- 5 files changed, 84 insertions(+), 16 deletions(-) diff --git a/src/ethereum_test_tools/common/__init__.py b/src/ethereum_test_tools/common/__init__.py index f9438f1b0b..a8a175169c 100644 --- a/src/ethereum_test_tools/common/__init__.py +++ b/src/ethereum_test_tools/common/__init__.py @@ -52,6 +52,7 @@ serialize_transactions, str_or_none, to_json, + withdrawals_root, ) __all__ = ( @@ -101,4 +102,5 @@ "to_hash_bytes", "to_hash", "to_json", + "withdrawals_root", ) diff --git a/src/ethereum_test_tools/common/types.py b/src/ethereum_test_tools/common/types.py index 7fd03bffa4..f31633aa46 100644 --- a/src/ethereum_test_tools/common/types.py +++ b/src/ethereum_test_tools/common/types.py @@ -25,6 +25,7 @@ from ethereum import rlp as eth_rlp from ethereum.base_types import Uint from ethereum.crypto.hash import keccak256 +from trie import HexaryTrie from ethereum_test_forks import Fork from evm_transition_tool import TransitionTool @@ -814,6 +815,16 @@ def to_serializable_list(self) -> List[Any]: ] +def withdrawals_root(withdrawals: List[Withdrawal]) -> bytes: + """ + Returns the withdrawals root of a list of withdrawals. + """ + t = HexaryTrie(db={}) + for i, w in enumerate(withdrawals): + t.set(eth_rlp.encode(Uint(i)), eth_rlp.encode(w.to_serializable_list())) + return t.root_hash + + @dataclass(kw_only=True) class FixtureWithdrawal(Withdrawal): """ diff --git a/src/ethereum_test_tools/spec/blockchain_test.py b/src/ethereum_test_tools/spec/blockchain_test.py index 94cffbd837..06839a2af5 100644 --- a/src/ethereum_test_tools/spec/blockchain_test.py +++ b/src/ethereum_test_tools/spec/blockchain_test.py @@ -25,6 +25,7 @@ Number, ZeroPaddedHexNumber, to_json, + withdrawals_root, ) from ..common.constants import EmptyOmmersRoot from .base_test import BaseTest, verify_post_alloc, verify_transactions @@ -86,13 +87,7 @@ def make_genesis( blob_gas_used=ZeroPaddedHexNumber.or_none(env.blob_gas_used), excess_blob_gas=ZeroPaddedHexNumber.or_none(env.excess_blob_gas), withdrawals_root=Hash.or_none( - t8n.calc_withdrawals_root( - withdrawals=env.withdrawals, - fork=fork, - debug_output_path=self.get_next_transition_tool_output_path(), - ) - if env.withdrawals is not None - else None + withdrawals_root(env.withdrawals) if env.withdrawals is not None else None ), beacon_root=Hash.or_none(env.beacon_root), ) diff --git a/src/ethereum_test_tools/spec/state_test.py b/src/ethereum_test_tools/spec/state_test.py index c2670cfdc1..30d1dae357 100644 --- a/src/ethereum_test_tools/spec/state_test.py +++ b/src/ethereum_test_tools/spec/state_test.py @@ -24,6 +24,7 @@ Transaction, ZeroPaddedHexNumber, to_json, + withdrawals_root, ) from ..common.constants import EmptyOmmersRoot, EngineAPIError from .base_test import BaseTest, verify_post_alloc, verify_transactions @@ -73,7 +74,6 @@ def make_genesis( fork=fork, debug_output_path=self.get_next_transition_tool_output_path(), ) - genesis = FixtureHeader( parent_hash=Hash(0), ommers_hash=Hash(EmptyOmmersRoot), @@ -94,13 +94,7 @@ def make_genesis( blob_gas_used=ZeroPaddedHexNumber.or_none(env.blob_gas_used), excess_blob_gas=ZeroPaddedHexNumber.or_none(env.excess_blob_gas), withdrawals_root=Hash.or_none( - t8n.calc_withdrawals_root( - withdrawals=env.withdrawals, - fork=fork, - debug_output_path=self.get_next_transition_tool_output_path(), - ) - if env.withdrawals is not None - else None + withdrawals_root(env.withdrawals) if env.withdrawals is not None else None ), beacon_root=Hash.or_none(env.beacon_root), ) diff --git a/src/ethereum_test_tools/tests/test_types.py b/src/ethereum_test_tools/tests/test_types.py index ebdc2c40a3..24dd67572e 100644 --- a/src/ethereum_test_tools/tests/test_types.py +++ b/src/ethereum_test_tools/tests/test_types.py @@ -2,7 +2,7 @@ Test suite for `ethereum_test` module. """ -from typing import Any, Dict +from typing import Any, Dict, List import pytest @@ -15,6 +15,7 @@ Transaction, Withdrawal, to_json, + withdrawals_root, ) from ..common.constants import TestPrivateKey from ..common.types import ( @@ -1194,3 +1195,68 @@ def test_transaction_post_init_defaults(tx_args, expected_attributes_and_values) for attr, val in expected_attributes_and_values: assert hasattr(tx, attr) assert getattr(tx, attr) == val + + +@pytest.mark.parametrize( + ["withdrawals", "expected_root"], + [ + pytest.param( + [], + bytes.fromhex("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), + id="empty-withdrawals", + ), + pytest.param( + [ + Withdrawal( + index=0, + validator=1, + address=0x1234, + amount=2, + ) + ], + bytes.fromhex("dc3ead883fc17ea3802cd0f8e362566b07b223f82e52f94c76cf420444b8ff81"), + id="single-withdrawal", + ), + pytest.param( + [ + Withdrawal( + index=0, + validator=1, + address=0x1234, + amount=2, + ), + Withdrawal( + index=1, + validator=2, + address=0xABCD, + amount=0, + ), + ], + bytes.fromhex("069ab71e5d228db9b916880f02670c85682c46641bb9c95df84acc5075669e01"), + id="multiple-withdrawals", + ), + pytest.param( + [ + Withdrawal( + index=0, + validator=0, + address=0x100, + amount=0, + ), + Withdrawal( + index=0, + validator=0, + address=0x200, + amount=0, + ), + ], + bytes.fromhex("daacd8fe889693f7d20436d9c0c044b5e92cc17b57e379997273fc67fd2eb7b8"), + id="multiple-withdrawals", + ), + ], +) +def test_withdrawals_root(withdrawals: List[Withdrawal], expected_root: bytes): + """ + Test that withdrawals_root returns the expected hash. + """ + assert withdrawals_root(withdrawals) == expected_root From 6e3fed66bdcd1b9ca5253ad73a562bb142bf19a6 Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Fri, 25 Aug 2023 00:00:48 +0000 Subject: [PATCH 3/6] transition_tool: remove withdrawals root --- src/evm_transition_tool/transition_tool.py | 55 ---------------------- 1 file changed, 55 deletions(-) diff --git a/src/evm_transition_tool/transition_tool.py b/src/evm_transition_tool/transition_tool.py index a9f0cfd986..3dbd1474ef 100644 --- a/src/evm_transition_tool/transition_tool.py +++ b/src/evm_transition_tool/transition_tool.py @@ -387,58 +387,3 @@ def calc_state_root( if state_root is None or not isinstance(state_root, str): raise Exception("Unable to calculate state root") return new_alloc, bytes.fromhex(state_root[2:]) - - def calc_withdrawals_root( - self, *, withdrawals: Any, fork: Fork, debug_output_path: str = "" - ) -> bytes: - """ - Calculate the state root for the given `alloc`. - """ - if isinstance(withdrawals, list) and len(withdrawals) == 0: - # Optimize returning the empty root immediately - return bytes.fromhex( - "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" - ) - - env: Dict[str, Any] = { - "currentCoinbase": "0x0000000000000000000000000000000000000000", - "currentDifficulty": "0x0", - "currentGasLimit": "0x0", - "currentNumber": "0", - "currentTimestamp": "0", - "withdrawals": withdrawals, - } - - if fork.header_base_fee_required(0, 0): - env["currentBaseFee"] = "7" - - if fork.header_prev_randao_required(0, 0): - env["currentRandom"] = "0" - - if fork.header_excess_blob_gas_required(0, 0): - env["currentExcessBlobGas"] = "0" - - if fork.header_beacon_root_required(0, 0): - env[ - "parentBeaconBlockRoot" - ] = "0x0000000000000000000000000000000000000000000000000000000000000000" - - _, result = self.evaluate( - alloc={}, - txs=[], - env=env, - fork_name=fork.fork(block_number=0, timestamp=0), - debug_output_path=debug_output_path, - ) - withdrawals_root = result.get("withdrawalsRoot") - if withdrawals_root is None: - raise Exception( - "Unable to calculate withdrawals root: no value returned from transition tool" - ) - if not isinstance(withdrawals_root, str): - raise Exception( - "Unable to calculate withdrawals root: " - + "incorrect type returned from transition tool: " - + f"{withdrawals_root}" - ) - return bytes.fromhex(withdrawals_root[2:]) From 672c1d4b9c3c521ce5e28767099eaa58c8a1d766 Mon Sep 17 00:00:00 2001 From: Dan Date: Fri, 25 Aug 2023 18:54:52 +0200 Subject: [PATCH 4/6] framework: test the withdrawals root in t8n's output result (#9) --- src/ethereum_test_tools/spec/base_test.py | 23 ++++++++++++++++++- .../spec/blockchain_test.py | 3 ++- src/ethereum_test_tools/spec/state_test.py | 3 ++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/ethereum_test_tools/spec/base_test.py b/src/ethereum_test_tools/spec/base_test.py index f8d099c8a3..76c7b95c3e 100644 --- a/src/ethereum_test_tools/spec/base_test.py +++ b/src/ethereum_test_tools/spec/base_test.py @@ -10,7 +10,19 @@ from ethereum_test_forks import Fork from evm_transition_tool import TransitionTool -from ..common import Account, Address, Alloc, Bytes, FixtureBlock, FixtureHeader, Hash, Transaction +from ..common import ( + Account, + Address, + Alloc, + Bytes, + Environment, + FixtureBlock, + FixtureHeader, + Hash, + Transaction, + withdrawals_root, +) +from ..common.conversions import to_hex def verify_transactions(txs: List[Transaction] | None, result) -> List[int]: @@ -59,6 +71,15 @@ def verify_post_alloc(expected_post: Mapping, got_alloc: Mapping): raise Exception(f"expected account not found: {address}") +def verify_result(result: Mapping, env: Environment): + """ + Verify that values in the t8n result match the expected values. + Raises exception on unexpected values. + """ + if env.withdrawals is not None: + assert result["withdrawalsRoot"] == to_hex(withdrawals_root(env.withdrawals)) + + @dataclass(kw_only=True) class BaseTestConfig: """ diff --git a/src/ethereum_test_tools/spec/blockchain_test.py b/src/ethereum_test_tools/spec/blockchain_test.py index 06839a2af5..0dba43661f 100644 --- a/src/ethereum_test_tools/spec/blockchain_test.py +++ b/src/ethereum_test_tools/spec/blockchain_test.py @@ -28,7 +28,7 @@ withdrawals_root, ) from ..common.constants import EmptyOmmersRoot -from .base_test import BaseTest, verify_post_alloc, verify_transactions +from .base_test import BaseTest, verify_post_alloc, verify_result, verify_transactions from .debugging import print_traces @@ -164,6 +164,7 @@ def make_block( ) try: rejected_txs = verify_transactions(txs, result) + verify_result(result, env) except Exception as e: print_traces(t8n.get_traces()) pprint(result) diff --git a/src/ethereum_test_tools/spec/state_test.py b/src/ethereum_test_tools/spec/state_test.py index 30d1dae357..9b1d4988ca 100644 --- a/src/ethereum_test_tools/spec/state_test.py +++ b/src/ethereum_test_tools/spec/state_test.py @@ -27,7 +27,7 @@ withdrawals_root, ) from ..common.constants import EmptyOmmersRoot, EngineAPIError -from .base_test import BaseTest, verify_post_alloc, verify_transactions +from .base_test import BaseTest, verify_post_alloc, verify_result, verify_transactions from .debugging import print_traces @@ -148,6 +148,7 @@ def make_blocks( try: verify_post_alloc(self.post, alloc) + verify_result(result, env) except Exception as e: print_traces(traces=t8n.get_traces()) raise e From 665d1c4480ee7045de08cc9786f395bbbb20f3d3 Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Fri, 25 Aug 2023 17:37:44 +0000 Subject: [PATCH 5/6] state_test: genesis environment fixes --- src/ethereum_test_tools/common/types.py | 8 ++-- src/ethereum_test_tools/spec/state_test.py | 50 ++++++++++++++-------- 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/src/ethereum_test_tools/common/types.py b/src/ethereum_test_tools/common/types.py index f31633aa46..dcbd7586ec 100644 --- a/src/ethereum_test_tools/common/types.py +++ b/src/ethereum_test_tools/common/types.py @@ -1070,13 +1070,13 @@ def apply_new_parent(self, new_parent: "FixtureHeader") -> "Environment": env.block_hashes[new_parent.number] = new_parent.hash if new_parent.hash is not None else 0 return env - def set_fork_requirements(self, fork: Fork) -> "Environment": + def set_fork_requirements(self, fork: Fork, in_place: bool = False) -> "Environment": """ Fills the required fields in an environment depending on the fork. """ - res = copy(self) - number = Number(self.number) - timestamp = Number(self.timestamp) + res = self if in_place else copy(self) + number = Number(res.number) + timestamp = Number(res.timestamp) if fork.header_prev_randao_required(number, timestamp) and res.prev_randao is None: res.prev_randao = 0 diff --git a/src/ethereum_test_tools/spec/state_test.py b/src/ethereum_test_tools/spec/state_test.py index 9b1d4988ca..64476e6b10 100644 --- a/src/ethereum_test_tools/spec/state_test.py +++ b/src/ethereum_test_tools/spec/state_test.py @@ -24,7 +24,6 @@ Transaction, ZeroPaddedHexNumber, to_json, - withdrawals_root, ) from ..common.constants import EmptyOmmersRoot, EngineAPIError from .base_test import BaseTest, verify_post_alloc, verify_result, verify_transactions @@ -59,15 +58,26 @@ def make_genesis( """ Create a genesis block from the state test definition. """ - env = copy(self.env) - - # Remove fields that should not be present in the genesis block. - env.withdrawals = None - env.beacon_root = None - - env = env.set_fork_requirements(fork) - - pre_alloc = Alloc(fork.pre_allocation(block_number=0, timestamp=Number(env.timestamp))) + # The genesis environment is similar to the block 1 environment specified by the test + # with some slight differences, so make a copy here + genesis_env = copy(self.env) + + # Modify values to the proper values for the genesis block + genesis_env.withdrawals = None + genesis_env.beacon_root = None + genesis_env.number = Number(genesis_env.number) - 1 + assert ( + genesis_env.number >= 0 + ), "genesis block number cannot be negative, set state test env.number to 1" + + # Set the fork requirements to the genesis environment in-place + genesis_env.set_fork_requirements(fork, in_place=True) + + pre_alloc = Alloc( + fork.pre_allocation( + block_number=genesis_env.number, timestamp=Number(genesis_env.timestamp) + ) + ) new_alloc, state_root = t8n.calc_state_root( alloc=to_json(Alloc.merge(pre_alloc, Alloc(self.pre))), @@ -82,27 +92,29 @@ def make_genesis( transactions_root=Hash(EmptyTrieRoot), receipt_root=Hash(EmptyTrieRoot), bloom=Bloom(0), - difficulty=ZeroPaddedHexNumber(0x20000 if env.difficulty is None else env.difficulty), - number=ZeroPaddedHexNumber(Number(env.number) - 1), - gas_limit=ZeroPaddedHexNumber(env.gas_limit), + difficulty=ZeroPaddedHexNumber( + 0x20000 if genesis_env.difficulty is None else genesis_env.difficulty + ), + number=ZeroPaddedHexNumber(genesis_env.number), + gas_limit=ZeroPaddedHexNumber(genesis_env.gas_limit), gas_used=0, timestamp=0, extra_data=Bytes([0]), mix_digest=Hash(0), nonce=HeaderNonce(0), - base_fee=ZeroPaddedHexNumber.or_none(env.base_fee), - blob_gas_used=ZeroPaddedHexNumber.or_none(env.blob_gas_used), - excess_blob_gas=ZeroPaddedHexNumber.or_none(env.excess_blob_gas), + base_fee=ZeroPaddedHexNumber.or_none(genesis_env.base_fee), + blob_gas_used=ZeroPaddedHexNumber.or_none(genesis_env.blob_gas_used), + excess_blob_gas=ZeroPaddedHexNumber.or_none(genesis_env.excess_blob_gas), withdrawals_root=Hash.or_none( - withdrawals_root(env.withdrawals) if env.withdrawals is not None else None + EmptyTrieRoot if genesis_env.withdrawals is not None else None ), - beacon_root=Hash.or_none(env.beacon_root), + beacon_root=Hash.or_none(genesis_env.beacon_root), ) genesis_rlp, genesis.hash = genesis.build( txs=[], ommers=[], - withdrawals=env.withdrawals, + withdrawals=genesis_env.withdrawals, ) return Alloc(new_alloc), genesis_rlp, genesis From 5194983630d24bee6c562f990d96f2d138b17c67 Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Fri, 25 Aug 2023 17:48:38 +0000 Subject: [PATCH 6/6] blockchain_test: sanity check genesis values --- src/ethereum_test_tools/spec/blockchain_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/ethereum_test_tools/spec/blockchain_test.py b/src/ethereum_test_tools/spec/blockchain_test.py index 0dba43661f..7ee063d3da 100644 --- a/src/ethereum_test_tools/spec/blockchain_test.py +++ b/src/ethereum_test_tools/spec/blockchain_test.py @@ -60,6 +60,10 @@ def make_genesis( Create a genesis block from the state test definition. """ env = self.genesis_environment.set_fork_requirements(fork) + if env.withdrawals is not None: + assert len(env.withdrawals) == 0, "withdrawals must be empty at genesis" + if env.beacon_root is not None: + assert Hash(env.beacon_root) == Hash(0), "beacon_root must be empty at genesis" pre_alloc = Alloc(fork.pre_allocation(block_number=0, timestamp=Number(env.timestamp))) new_alloc, state_root = t8n.calc_state_root(