diff --git a/docs/getting_started/debugging_t8n_tools.md b/docs/getting_started/debugging_t8n_tools.md new file mode 100644 index 0000000000..908397cf56 --- /dev/null +++ b/docs/getting_started/debugging_t8n_tools.md @@ -0,0 +1,95 @@ +# Debugging Transition Tools + +The `--t8n-dump-dir` flag can be used to dump the inputs and outputs of every call made to the `t8n` command to help debugging or simply understand how a test is interacting with the EVM. + +In particular, a script `t8n.sh` is generated for each call to the `t8n` command which can be used to reproduce the call to trigger errors or attach a debugger without the need to execute Python. + +For example, running: + +```console +fill tests/berlin/eip2930_access_list/ --fork Berlin \ + --t8n-dump-dir=/tmp/evm-t8n-dump +``` + +will produce the directory structure: + +```text +📁 /tmp/evm-t8n-dump/ +└─╴📁 test_access_list_fork_Berlin + ├── 📁 0 + │ ├── 📄 args.py + │ ├── 📁 input + │ │ ├── 📄 alloc.json + │ │ ├── 📄 env.json + │ │ └── 📄 txs.json + │ ├── 📁 output + │ │ ├── 📄 alloc.json + │ │ ├── 📄 result.json + │ │ └── 📄 txs.rlp + │ ├── 📄 returncode.txt + │ ├── 📄 stderr.txt + │ ├── 📄 stdin.txt + │ ├── 📄 stdout.txt + │ └── 📄 t8n.sh + └── 📁 1 + ├── 📄 args.py + ├── 📁 input + │ ├── 📄 alloc.json + │ ├── 📄 env.json + │ └── 📄 txs.json + ├── 📁 output + │ ├── 📄 alloc.json + │ ├── 📄 result.json + │ └── 📄 txs.rlp + ├── 📄 returncode.txt + ├── 📄 stderr.txt + ├── 📄 stdin.txt + ├── 📄 stdout.txt + └── 📄 t8n.sh +``` + +where the directories `0` and `1` correspond to the different calls made to the `t8n` tool executed during the test: + +- `0` corresponds to the call used to calculate the state root of the test's initial alloc (which is why it has an empty transaction list). +- `1` corresponds to the call used to execute the first transaction or block from the test. + +Note, there may be more directories present `2`, `3`, `4`,... if the test executes more transactions/blocks. + +Each directory contains files containing information corresponding to the call, for example, the `args.py` file contains the arguments passed to the `t8n` command and the `output/alloc.json` file contains the output of the `t8n` command's `--output-alloc` flag. + +## The `t8n.sh` Script + +The `t8n.sh` script written to the debug directory can be used to reproduce any call made to the `t8n` command, for example, if a Besu `t8n-server` has been started on port `3001`, the request made by the test for first transaction can be reproduced as: + +```console +/tmp/besu/test_access_list_fork_Berlin/1/t8n.sh 3001 +``` + +which writes the response the from the `t8n-server` to the console output: + +```json +{ + "alloc" : { + "0x000000000000000000000000000000000000aaaa" : { + "code" : "0x5854505854", + "balance" : "0x4", + "nonce" : "0x1" + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba" : { + "balance" : "0x1bc16d674ecb26ce" + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" : { + "balance" : "0x2cd931", + "nonce" : "0x1" + } + }, + "body" : "0xf8a0b89e01f89b0180078304ef0094000000000000000000000000000000000000aaaa0180f838f7940000000000000000000000000000000000000000e1a0000000000000000000000000000000000000000000000000000000000000000001a02e16eb72206c93c471b5894800495ee9c64ae2d9823bcc4d6adeb5d9d9af0dd4a03be6691e933a0816c59d059a556c27c6753e6ce76d1e357b9201865c80b28df3", + "result" : { + "stateRoot" : "0x51799508f764047aee6606bc6a00863856f83ee5b91555f00c8a3cbdfbec5acb", + ... + ... + } +} +``` + +The `t8n.sh` is written to the debug directory for all [supported t8n tools](../index.md#transition-tool-support). diff --git a/docs/getting_started/executing_tests_command_line.md b/docs/getting_started/executing_tests_command_line.md index 1d8911eaa4..500cbbde56 100644 --- a/docs/getting_started/executing_tests_command_line.md +++ b/docs/getting_started/executing_tests_command_line.md @@ -90,47 +90,7 @@ fill tests/shanghai/eip3651_warm_coinbase/test_warm_coinbase.py::test_warm_coinb ## Debugging the `t8n` Command -The `--t8n-dump-dir` flag can be used to dump the inputs and outputs of every call made to the `t8n` command for debugging purposes: - -```console -fill --t8n-dump-dir=/tmp/evm-t8n-dump -``` - -For example, running: - -```console -fill tests/berlin/eip2930_access_list/ --fork Berlin \ - --t8n-dump-dir=/tmp/evm-t8n-dump -``` - -will produce the directory structure: - -```console -/tmp/evm-t8n-dump/ -└── test_access_list_fork_Berlin - ├── 0 - │   ├── alloc - │   ├── args - │   ├── env - │   ├── output_alloc - │   ├── output_result - │   ├── returncode - │   ├── stderr - │   ├── stdout - │   └── txs - └── 1 - ├── alloc - ├── args - ├── env - ├── output_alloc - ├── output_result - ├── returncode - ├── stderr - ├── stdout - └── txs -``` - -where the directories `0` and `1` correspond to the different calls made to the `t8n` tool executed during the test. Each directory then contain files containing information corresponding to the call, for example, the `args` file contains the arguments passed to the `t8n` command and the `output_alloc` file contains the output of the `t8n` command's `--output-alloc` flag. Note, the first call is used to calculate the state root of the starting alloc and therefore has an empty transaction list. +The `--t8n-dump-dir` flag can be used to dump the inputs and outputs of every call made to the `t8n` command for debugging purposes, see [Debugging Transition Tools](./debugging_t8n_tools.md). ## Other Useful Pytest Command-Line Options diff --git a/docs/navigation.md b/docs/navigation.md index 9f9cb84b2d..88d14d3b21 100644 --- a/docs/navigation.md +++ b/docs/navigation.md @@ -7,6 +7,7 @@ * [Executing Tests at a Prompt](getting_started/executing_tests_command_line.md) * [Executing Tests in VS Code](getting_started/executing_tests_vs_code.md) * [Executing Tests for Features Under Development](getting_started/executing_tests_dev_fork.md) + * [Debugging Transition Tools](getting_started/debugging_t8n_tools.md) * [Writing Tests](writing_tests/index.md) * [Code Standards](writing_tests/code_standards.md) * [Types of Test](writing_tests/types_of_tests.md) @@ -20,8 +21,9 @@ * [Developer Doc](dev/index.md) * [Documentation](dev/docs.md) * [Coding Style](dev/coding_style.md) + * [Enabling Precommit Checks](dev/precommit.md) * [Library Reference](library/index.md) * [EVM Transition Tool Package](library/evm_transition_tool.md) * [Ethereum Test Tools Package](library/ethereum_test_tools.md) * [Ethereum Test Forks Package](library/ethereum_test_forks.md) - * [Pytest Plugins](library/pytest_plugins/) + * [Pytest Plugins](library/pytest_plugins/index.md) diff --git a/src/evm_transition_tool/besu.py b/src/evm_transition_tool/besu.py index 564cc1dafe..8546d2d5c8 100644 --- a/src/evm_transition_tool/besu.py +++ b/src/evm_transition_tool/besu.py @@ -2,8 +2,10 @@ Hyperledger Besu Transition tool frontend. """ +import json import re import subprocess +import textwrap from pathlib import Path from re import compile from typing import Any, Dict, List, Optional, Tuple @@ -109,23 +111,36 @@ def evaluate( "chainid": chain_id, "reward": reward, } + + post_data = {"state": state_json, "input": input_json} + if debug_output_path: + post_data_string = json.dumps(post_data, indent=4) + additional_indent = " " * 16 # for pretty indentation in t8n.sh + indented_post_data_string = "{\n" + "\n".join( + additional_indent + line for line in post_data_string[1:].splitlines() + ) + t8n_script = textwrap.dedent( + f"""\ + #!/bin/bash + # Use $1 as t8n-server port if provided, else default to 3000 + PORT=${{1:-3000}} + curl http://localhost:${{PORT}}/ -X POST -H "Content-Type: application/json" \\ + --data '{indented_post_data_string}' + """ + ) dump_files_to_directory( debug_output_path, - input_json - | { - "state": state_json, + { + "state.json": state_json, + "input/alloc.json": input_json["alloc"], + "input/env.json": input_json["env"], + "input/txs.json": input_json["txs"], + "t8n.sh+x": t8n_script, }, ) - response = requests.post( - self.server_url, - json={ - "state": state_json, - "input": input_json, - }, - timeout=5, - ) + response = requests.post(self.server_url, json=post_data, timeout=5) response.raise_for_status() # exception visible in pytest failure output output = response.json() @@ -133,8 +148,30 @@ def evaluate( dump_files_to_directory( debug_output_path, { - "output_alloc": output["alloc"], - "output_result": output["result"], + "response.txt": response.text, + "status_code.txt": response.status_code, + "time_elapsed_seconds.txt": response.elapsed.total_seconds(), + }, + ) + + if response.status_code != 200: + raise Exception( + f"t8n-server returned status code {response.status_code}, " + f"response: {response.text}" + ) + if not all([x in output for x in ["alloc", "result", "body"]]): + raise Exception( + "Malformed t8n output: missing 'alloc', 'result' or 'body', server response: " + f"{response.text}" + ) + + if debug_output_path: + dump_files_to_directory( + debug_output_path, + { + "output/alloc.json": output["alloc"], + "output/result.json": output["result"], + "output/txs.rlp": output["body"], }, ) diff --git a/src/evm_transition_tool/evmone.py b/src/evm_transition_tool/evmone.py index b6c92a4a69..1b912d9b5e 100644 --- a/src/evm_transition_tool/evmone.py +++ b/src/evm_transition_tool/evmone.py @@ -6,6 +6,7 @@ import shutil import subprocess import tempfile +import textwrap from pathlib import Path from re import compile from typing import Any, Dict, List, Optional, Tuple @@ -62,18 +63,25 @@ def evaluate( fork_name = "+".join([fork_name] + [str(eip) for eip in eips]) temp_dir = tempfile.TemporaryDirectory() + os.mkdir(os.path.join(temp_dir.name, "input")) + os.mkdir(os.path.join(temp_dir.name, "output")) input_contents = { "alloc": alloc, "env": env, "txs": txs, } + input_paths = { - k: os.path.join(temp_dir.name, f"input_{k}.json") for k in input_contents.keys() + k: os.path.join(temp_dir.name, "input", f"{k}.json") for k in input_contents.keys() } - for key, val in input_contents.items(): - file_path = os.path.join(temp_dir.name, f"input_{key}.json") - write_json_file(val, file_path) + for key, file_path in input_paths.items(): + write_json_file(input_contents[key], file_path) + + output_paths = { + output: os.path.join("output", f"{output}.json") for output in ["alloc", "result"] + } + output_paths["body"] = os.path.join("output", "txs.rlp") # Construct args for evmone-t8n binary args = [ @@ -89,11 +97,11 @@ def evaluate( "--output.basedir", temp_dir.name, "--output.result", - "output_result.json", + output_paths["result"], "--output.alloc", - "output_alloc.json", + output_paths["alloc"], "--output.body", - "txs.rlp", + output_paths["body"], "--state.reward", str(reward), "--state.chainid", @@ -110,63 +118,56 @@ def evaluate( ) if debug_output_path: + if os.path.exists(debug_output_path): + shutil.rmtree(debug_output_path) + shutil.copytree(temp_dir.name, debug_output_path) + t8n_output_base_dir = os.path.join(debug_output_path, "t8n.sh.out") + t8n_call = " ".join(args) + for file_path in input_paths.values(): # update input paths + t8n_call = t8n_call.replace( + os.path.dirname(file_path), os.path.join(debug_output_path, "input") + ) + t8n_call = t8n_call.replace( # use a new output path for basedir and outputs + temp_dir.name, + t8n_output_base_dir, + ) + t8n_script = textwrap.dedent( + f"""\ + #!/bin/bash + rm -rf {debug_output_path}/t8n.sh.out # hard-coded to avoid surprises + mkdir -p {debug_output_path}/t8n.sh.out/output + {t8n_call} + """ + ) dump_files_to_directory( debug_output_path, - input_contents - | { - "args": args, - "stdout": result.stdout.decode(), - "stderr": result.stderr.decode(), - "returncode": result.returncode, + { + "args.py": args, + "returncode.txt": result.returncode, + "stdout.txt": result.stdout.decode(), + "stderr.txt": result.stderr.decode(), + "t8n.sh+x": t8n_script, }, ) if result.returncode != 0: raise Exception("failed to evaluate: " + result.stderr.decode()) - output_paths = { - "alloc": os.path.join(temp_dir.name, "output_alloc.json"), - "result": os.path.join(temp_dir.name, "output_result.json"), - } + for key, file_path in output_paths.items(): + output_paths[key] = os.path.join(temp_dir.name, file_path) output_contents = {} for key, file_path in output_paths.items(): + if "txs.rlp" in file_path: + continue with open(file_path, "r+") as file: - contents = json.load(file) - file.seek(0) - json.dump(contents, file, ensure_ascii=False, indent=4) - file.truncate() - output_contents[key] = contents + output_contents[key] = json.load(file) if self.trace: - receipts: List[Any] = output_contents["result"]["receipts"] - traces: List[List[Dict]] = [] - for i, r in enumerate(receipts): - h = r["transactionHash"] - trace_file_name = f"trace-{i}-{h}.jsonl" - if debug_output_path: - shutil.copy( - os.path.join(temp_dir.name, trace_file_name), - os.path.join(debug_output_path, trace_file_name), - ) - with open(os.path.join(temp_dir.name, trace_file_name), "r") as trace_file: - tx_traces: List[Dict] = [] - for trace_line in trace_file.readlines(): - tx_traces.append(json.loads(trace_line)) - traces.append(tx_traces) - self.append_traces(traces) + self.collect_traces(output_contents["result"]["receipts"], temp_dir, debug_output_path) temp_dir.cleanup() - if debug_output_path: - dump_files_to_directory( - debug_output_path, - { - "output_alloc": output_contents["alloc"], - "output_result": output_contents["result"], - }, - ) - return output_contents["alloc"], output_contents["result"] def is_fork_supported(self, fork: Fork) -> bool: diff --git a/src/evm_transition_tool/execution_specs.py b/src/evm_transition_tool/execution_specs.py index a8064a5031..bd4c3cc000 100644 --- a/src/evm_transition_tool/execution_specs.py +++ b/src/evm_transition_tool/execution_specs.py @@ -7,11 +7,14 @@ from pathlib import Path from re import compile -from ethereum_test_forks import ConstantinopleFix, Fork +from ethereum_test_forks import Constantinople, ConstantinopleFix, Fork from .geth import GethTransitionTool -UNSUPPORTED_FORKS = (ConstantinopleFix,) +UNSUPPORTED_FORKS = ( + Constantinople, + ConstantinopleFix, +) class ExecutionSpecsTransitionTool(GethTransitionTool): diff --git a/src/evm_transition_tool/transition_tool.py b/src/evm_transition_tool/transition_tool.py index a9f0cfd986..80a85f0f9f 100644 --- a/src/evm_transition_tool/transition_tool.py +++ b/src/evm_transition_tool/transition_tool.py @@ -39,11 +39,16 @@ def dump_files_to_directory(output_path: str, files: Dict[str, Any]) -> None: Dump the files to the given directory. """ os.makedirs(output_path, exist_ok=True) - for file_name_flags, file_contents in files.items(): - file_name, flags = ( - file_name_flags.split("+") if "+" in file_name_flags else (file_name_flags, "") + for file_rel_path_flags, file_contents in files.items(): + file_rel_path, flags = ( + file_rel_path_flags.split("+") + if "+" in file_rel_path_flags + else (file_rel_path_flags, "") ) - file_path = os.path.join(output_path, file_name) + rel_path = os.path.dirname(file_rel_path) + if rel_path: + os.makedirs(os.path.join(output_path, rel_path), exist_ok=True) + file_path = os.path.join(output_path, file_rel_path) with open(file_path, "w") as f: if isinstance(file_contents, str): f.write(file_contents) @@ -225,6 +230,31 @@ def get_traces(self) -> List[List[List[Dict]]] | None: """ return self.traces + def collect_traces( + self, + receipts: List[Any], + temp_dir: tempfile.TemporaryDirectory, + debug_output_path: str = "", + ) -> None: + """ + Collect the traces from the t8n tool output and store them in the traces list. + """ + traces: List[List[Dict]] = [] + for i, r in enumerate(receipts): + h = r["transactionHash"] + trace_file_name = f"trace-{i}-{h}.jsonl" + if debug_output_path: + shutil.copy( + os.path.join(temp_dir.name, trace_file_name), + os.path.join(debug_output_path, trace_file_name), + ) + with open(os.path.join(temp_dir.name, trace_file_name), "r") as trace_file: + tx_traces: List[Dict] = [] + for trace_line in trace_file.readlines(): + tx_traces.append(json.loads(trace_line)) + traces.append(tx_traces) + self.append_traces(traces) + def evaluate( self, *, @@ -246,8 +276,6 @@ def evaluate( if eips is not None: fork_name = "+".join([fork_name] + [str(eip) for eip in eips]) - temp_dir = tempfile.TemporaryDirectory() - if int(env["currentNumber"], 0) == 0: reward = -1 @@ -261,15 +289,22 @@ def evaluate( "--input.env=stdin", "--output.result=stdout", "--output.alloc=stdout", - "--output.body=txs.rlp", - f"--output.basedir={temp_dir.name}", + "--output.body=stdout", f"--state.fork={fork_name}", f"--state.chainid={chain_id}", f"--state.reward={reward}", ] if self.trace: + if str(self.default_binary) == "ethereum-spec-evm": + raise Exception( + "`ethereum-spec-evm` tracing is not currently implemented in " + "execution-spec-tests, see " + "https://github.com/ethereum/execution-spec-tests/issues/267." + ) + temp_dir = tempfile.TemporaryDirectory() args.append("--trace") + args.append(f"--output.basedir={temp_dir.name}") stdin = { "alloc": alloc, @@ -286,23 +321,30 @@ def evaluate( ) if debug_output_path: + t8n_call = " ".join(args) + t8n_output_base_dir = os.path.join(debug_output_path, "t8n.sh.out") + if self.trace: + t8n_call = t8n_call.replace(temp_dir.name, t8n_output_base_dir) t8n_script = textwrap.dedent( f"""\ #!/bin/bash - mkdir {temp_dir.name} - {' '.join(args)} < {debug_output_path}/stdin + rm -rf {debug_output_path}/t8n.sh.out # hard-coded to avoid surprises + mkdir {debug_output_path}/t8n.sh.out # unused if tracing is not enabled + {t8n_call} < {debug_output_path}/stdin.txt """ ) dump_files_to_directory( debug_output_path, - stdin - | { - "args": args, + { + "args.py": args, + "input/alloc.json": stdin["alloc"], + "input/env.json": stdin["env"], + "input/txs.json": stdin["txs"], + "returncode.txt": result.returncode, + "stdin.txt": stdin, + "stdout.txt": result.stdout.decode(), + "stderr.txt": result.stderr.decode(), "t8n.sh+x": t8n_script, - "stdin": stdin, - "stdout": result.stdout.decode(), - "stderr": result.stderr.decode(), - "returncode": result.returncode, }, ) @@ -311,38 +353,23 @@ def evaluate( output = json.loads(result.stdout) - if "alloc" not in output or "result" not in output: - raise Exception("malformed result") - - if self.trace: - receipts: List[Any] = output["result"]["receipts"] - traces: List[List[Dict]] = [] - for i, r in enumerate(receipts): - h = r["transactionHash"] - trace_file_name = f"trace-{i}-{h}.jsonl" - if debug_output_path: - shutil.copy( - os.path.join(temp_dir.name, trace_file_name), - os.path.join(debug_output_path, trace_file_name), - ) - with open(os.path.join(temp_dir.name, trace_file_name), "r") as trace_file: - tx_traces: List[Dict] = [] - for trace_line in trace_file.readlines(): - tx_traces.append(json.loads(trace_line)) - traces.append(tx_traces) - self.append_traces(traces) - - temp_dir.cleanup() + if not all([x in output for x in ["alloc", "result", "body"]]): + raise Exception("Malformed t8n output: missing 'alloc', 'result' or 'body'.") if debug_output_path: dump_files_to_directory( debug_output_path, { - "output_alloc": output["alloc"], - "output_result": output["result"], + "output/alloc.json": output["alloc"], + "output/result.json": output["result"], + "output/txs.rlp": output["body"], }, ) + if self.trace: + self.collect_traces(output["result"]["receipts"], temp_dir, debug_output_path) + temp_dir.cleanup() + return output["alloc"], output["result"] def calc_state_root( diff --git a/tests/shanghai/eip4895_withdrawals/test_withdrawals.py b/tests/shanghai/eip4895_withdrawals/test_withdrawals.py index 43c1aeacc2..bf6671e279 100644 --- a/tests/shanghai/eip4895_withdrawals/test_withdrawals.py +++ b/tests/shanghai/eip4895_withdrawals/test_withdrawals.py @@ -22,6 +22,7 @@ to_hash, ) from ethereum_test_tools.vm.opcode import Opcodes as Op +from evm_transition_tool import TransitionTool REFERENCE_SPEC_GIT_PATH = "EIPS/eip-4895.md" REFERENCE_SPEC_VERSION = "81af3b60b632bc9c03513d1d137f25410e3f4d34" @@ -727,11 +728,13 @@ def test_large_amount(blockchain_test: BlockchainTestFiller): @pytest.mark.parametrize("amount", [0, 1]) @pytest.mark.with_all_precompiles def test_withdrawing_to_precompiles( - blockchain_test: BlockchainTestFiller, precompile: int, amount: int + blockchain_test: BlockchainTestFiller, precompile: int, amount: int, t8n: TransitionTool ): """ Test withdrawing to all precompiles for a given fork. """ + if precompile == 3 and str(t8n.default_binary) == "ethereum-spec-evm": + pytest.xfail("ethereum-spec-evm doesn't support hash type ripemd160") pre: Dict = { TestAddress: Account(balance=1000000000000000000000, nonce=0), } diff --git a/whitelist.txt b/whitelist.txt index f4da2db31c..7885f85a02 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -269,8 +269,10 @@ addoption addinivalue argname autouse +basedir callspec collectonly +copytree dedent dest exc @@ -305,6 +307,7 @@ readline regexes reportinfo ret +ripemd rjust runpytest runtest