all: Replace t8n call with internal withdrawals root calculation

pyproject.toml

@@ -9,3 +9,6 @@ line_length = 99
 
 [tool.black]
 line-length = 99
+
+[tool.mypy]
+mypy_path = "$MYPY_CONFIG_FILE_DIR/stubs"

setup.cfg

@@ -31,6 +31,7 @@ install_requires =
     pytest==7.3.2
     pytest-xdist>=3.3.1,<4
     coincurve==17.0.0
+    trie==2.1.1
 
 [options.package_data]
 ethereum_test_tools =

src/ethereum_test_tools/common/__init__.py

@@ -52,6 +52,7 @@
     serialize_transactions,
     str_or_none,
     to_json,
+    withdrawals_root,
 )
 
 __all__ = (
@@ -101,4 +102,5 @@
     "to_hash_bytes",
     "to_hash",
     "to_json",
+    "withdrawals_root",
 )

src/ethereum_test_tools/common/types.py

@@ -25,6 +25,7 @@
 from ethereum import rlp as eth_rlp
 from ethereum.base_types import Uint
 from ethereum.crypto.hash import keccak256
+from trie import HexaryTrie
 
 from ethereum_test_forks import Fork
 from evm_transition_tool import TransitionTool
@@ -814,6 +815,16 @@ def to_serializable_list(self) -> List[Any]:
         ]
 
 
+def withdrawals_root(withdrawals: List[Withdrawal]) -> bytes:
+    """
+    Returns the withdrawals root of a list of withdrawals.
+    """
+    t = HexaryTrie(db={})
+    for i, w in enumerate(withdrawals):
+        t.set(eth_rlp.encode(Uint(i)), eth_rlp.encode(w.to_serializable_list()))
+    return t.root_hash
+
+
 @dataclass(kw_only=True)
 class FixtureWithdrawal(Withdrawal):
     """
@@ -1059,13 +1070,13 @@ def apply_new_parent(self, new_parent: "FixtureHeader") -> "Environment":
         env.block_hashes[new_parent.number] = new_parent.hash if new_parent.hash is not None else 0
         return env
 
-    def set_fork_requirements(self, fork: Fork) -> "Environment":
+    def set_fork_requirements(self, fork: Fork, in_place: bool = False) -> "Environment":
         """
         Fills the required fields in an environment depending on the fork.
         """
-        res = copy(self)
-        number = Number(self.number)
-        timestamp = Number(self.timestamp)
+        res = self if in_place else copy(self)
+        number = Number(res.number)
+        timestamp = Number(res.timestamp)
         if fork.header_prev_randao_required(number, timestamp) and res.prev_randao is None:
             res.prev_randao = 0
 

src/ethereum_test_tools/spec/base_test.py

@@ -10,7 +10,19 @@
 from ethereum_test_forks import Fork
 from evm_transition_tool import TransitionTool
 
-from ..common import Account, Address, Alloc, Bytes, FixtureBlock, FixtureHeader, Hash, Transaction
+from ..common import (
+    Account,
+    Address,
+    Alloc,
+    Bytes,
+    Environment,
+    FixtureBlock,
+    FixtureHeader,
+    Hash,
+    Transaction,
+    withdrawals_root,
+)
+from ..common.conversions import to_hex
 
 
 def verify_transactions(txs: List[Transaction] | None, result) -> List[int]:
@@ -59,6 +71,15 @@ def verify_post_alloc(expected_post: Mapping, got_alloc: Mapping):
                     raise Exception(f"expected account not found: {address}")
 
 
+def verify_result(result: Mapping, env: Environment):
+    """
+    Verify that values in the t8n result match the expected values.
+    Raises exception on unexpected values.
+    """
+    if env.withdrawals is not None:
+        assert result["withdrawalsRoot"] == to_hex(withdrawals_root(env.withdrawals))
+
+
 @dataclass(kw_only=True)
 class BaseTestConfig:
     """

src/ethereum_test_tools/spec/blockchain_test.py

@@ -25,9 +25,10 @@
     Number,
     ZeroPaddedHexNumber,
     to_json,
+    withdrawals_root,
 )
 from ..common.constants import EmptyOmmersRoot
-from .base_test import BaseTest, verify_post_alloc, verify_transactions
+from .base_test import BaseTest, verify_post_alloc, verify_result, verify_transactions
 from .debugging import print_traces
 
 
@@ -59,6 +60,10 @@ def make_genesis(
         Create a genesis block from the state test definition.
         """
         env = self.genesis_environment.set_fork_requirements(fork)
+        if env.withdrawals is not None:
+            assert len(env.withdrawals) == 0, "withdrawals must be empty at genesis"
+        if env.beacon_root is not None:
+            assert Hash(env.beacon_root) == Hash(0), "beacon_root must be empty at genesis"
 
         pre_alloc = Alloc(fork.pre_allocation(block_number=0, timestamp=Number(env.timestamp)))
         new_alloc, state_root = t8n.calc_state_root(
@@ -86,13 +91,7 @@ def make_genesis(
             blob_gas_used=ZeroPaddedHexNumber.or_none(env.blob_gas_used),
             excess_blob_gas=ZeroPaddedHexNumber.or_none(env.excess_blob_gas),
             withdrawals_root=Hash.or_none(
-                t8n.calc_withdrawals_root(
-                    withdrawals=env.withdrawals,
-                    fork=fork,
-                    debug_output_path=self.get_next_transition_tool_output_path(),
-                )
-                if env.withdrawals is not None
-                else None
+                withdrawals_root(env.withdrawals) if env.withdrawals is not None else None
             ),
             beacon_root=Hash.or_none(env.beacon_root),
         )
@@ -169,6 +168,7 @@ def make_block(
             )
             try:
                 rejected_txs = verify_transactions(txs, result)
+                verify_result(result, env)
             except Exception as e:
                 print_traces(t8n.get_traces())
                 pprint(result)

src/ethereum_test_tools/spec/state_test.py

@@ -26,7 +26,7 @@
     to_json,
 )
 from ..common.constants import EmptyOmmersRoot, EngineAPIError
-from .base_test import BaseTest, verify_post_alloc, verify_transactions
+from .base_test import BaseTest, verify_post_alloc, verify_result, verify_transactions
 from .debugging import print_traces
 
 
@@ -58,22 +58,32 @@ def make_genesis(
         """
         Create a genesis block from the state test definition.
         """
-        env = copy(self.env)
-
-        # Remove fields that should not be present in the genesis block.
-        env.withdrawals = None
-        env.beacon_root = None
-
-        env = env.set_fork_requirements(fork)
-
-        pre_alloc = Alloc(fork.pre_allocation(block_number=0, timestamp=Number(env.timestamp)))
+        # The genesis environment is similar to the block 1 environment specified by the test
+        # with some slight differences, so make a copy here
+        genesis_env = copy(self.env)
+
+        # Modify values to the proper values for the genesis block
+        genesis_env.withdrawals = None
+        genesis_env.beacon_root = None
+        genesis_env.number = Number(genesis_env.number) - 1
+        assert (
+            genesis_env.number >= 0
+        ), "genesis block number cannot be negative, set state test env.number to 1"
+
+        # Set the fork requirements to the genesis environment in-place
+        genesis_env.set_fork_requirements(fork, in_place=True)
+
+        pre_alloc = Alloc(
+            fork.pre_allocation(
+                block_number=genesis_env.number, timestamp=Number(genesis_env.timestamp)
+            )
+        )
 
         new_alloc, state_root = t8n.calc_state_root(
             alloc=to_json(Alloc.merge(pre_alloc, Alloc(self.pre))),
             fork=fork,
             debug_output_path=self.get_next_transition_tool_output_path(),
         )
-
         genesis = FixtureHeader(
             parent_hash=Hash(0),
             ommers_hash=Hash(EmptyOmmersRoot),
@@ -82,33 +92,29 @@ def make_genesis(
             transactions_root=Hash(EmptyTrieRoot),
             receipt_root=Hash(EmptyTrieRoot),
             bloom=Bloom(0),
-            difficulty=ZeroPaddedHexNumber(0x20000 if env.difficulty is None else env.difficulty),
-            number=ZeroPaddedHexNumber(Number(env.number) - 1),
-            gas_limit=ZeroPaddedHexNumber(env.gas_limit),
+            difficulty=ZeroPaddedHexNumber(
+                0x20000 if genesis_env.difficulty is None else genesis_env.difficulty
+            ),
+            number=ZeroPaddedHexNumber(genesis_env.number),
+            gas_limit=ZeroPaddedHexNumber(genesis_env.gas_limit),
             gas_used=0,
             timestamp=0,
             extra_data=Bytes([0]),
             mix_digest=Hash(0),
             nonce=HeaderNonce(0),
-            base_fee=ZeroPaddedHexNumber.or_none(env.base_fee),
-            blob_gas_used=ZeroPaddedHexNumber.or_none(env.blob_gas_used),
-            excess_blob_gas=ZeroPaddedHexNumber.or_none(env.excess_blob_gas),
+            base_fee=ZeroPaddedHexNumber.or_none(genesis_env.base_fee),
+            blob_gas_used=ZeroPaddedHexNumber.or_none(genesis_env.blob_gas_used),
+            excess_blob_gas=ZeroPaddedHexNumber.or_none(genesis_env.excess_blob_gas),
             withdrawals_root=Hash.or_none(
-                t8n.calc_withdrawals_root(
-                    withdrawals=env.withdrawals,
-                    fork=fork,
-                    debug_output_path=self.get_next_transition_tool_output_path(),
-                )
-                if env.withdrawals is not None
-                else None
+                EmptyTrieRoot if genesis_env.withdrawals is not None else None
             ),
-            beacon_root=Hash.or_none(env.beacon_root),
+            beacon_root=Hash.or_none(genesis_env.beacon_root),
         )
 
         genesis_rlp, genesis.hash = genesis.build(
             txs=[],
             ommers=[],
-            withdrawals=env.withdrawals,
+            withdrawals=genesis_env.withdrawals,
         )
 
         return Alloc(new_alloc), genesis_rlp, genesis
@@ -154,6 +160,7 @@ def make_blocks(
 
         try:
             verify_post_alloc(self.post, alloc)
+            verify_result(result, env)
         except Exception as e:
             print_traces(traces=t8n.get_traces())
             raise e

src/ethereum_test_tools/tests/test_types.py

@@ -2,7 +2,7 @@
 Test suite for `ethereum_test` module.
 """
 
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 import pytest
 
@@ -15,6 +15,7 @@
     Transaction,
     Withdrawal,
     to_json,
+    withdrawals_root,
 )
 from ..common.constants import TestPrivateKey
 from ..common.types import (
@@ -1194,3 +1195,68 @@ def test_transaction_post_init_defaults(tx_args, expected_attributes_and_values)
     for attr, val in expected_attributes_and_values:
         assert hasattr(tx, attr)
         assert getattr(tx, attr) == val
+
+
+@pytest.mark.parametrize(
+    ["withdrawals", "expected_root"],
+    [
+        pytest.param(
+            [],
+            bytes.fromhex("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"),
+            id="empty-withdrawals",
+        ),
+        pytest.param(
+            [
+                Withdrawal(
+                    index=0,
+                    validator=1,
+                    address=0x1234,
+                    amount=2,
+                )
+            ],
+            bytes.fromhex("dc3ead883fc17ea3802cd0f8e362566b07b223f82e52f94c76cf420444b8ff81"),
+            id="single-withdrawal",
+        ),
+        pytest.param(
+            [
+                Withdrawal(
+                    index=0,
+                    validator=1,
+                    address=0x1234,
+                    amount=2,
+                ),
+                Withdrawal(
+                    index=1,
+                    validator=2,
+                    address=0xABCD,
+                    amount=0,
+                ),
+            ],
+            bytes.fromhex("069ab71e5d228db9b916880f02670c85682c46641bb9c95df84acc5075669e01"),
+            id="multiple-withdrawals",
+        ),
+        pytest.param(
+            [
+                Withdrawal(
+                    index=0,
+                    validator=0,
+                    address=0x100,
+                    amount=0,
+                ),
+                Withdrawal(
+                    index=0,
+                    validator=0,
+                    address=0x200,
+                    amount=0,
+                ),
+            ],
+            bytes.fromhex("daacd8fe889693f7d20436d9c0c044b5e92cc17b57e379997273fc67fd2eb7b8"),
+            id="multiple-withdrawals",
+        ),
+    ],
+)
+def test_withdrawals_root(withdrawals: List[Withdrawal], expected_root: bytes):
+    """
+    Test that withdrawals_root returns the expected hash.
+    """
+    assert withdrawals_root(withdrawals) == expected_root