From 8cb9338c0cb4425750b8ba174c5ee0433084c174 Mon Sep 17 00:00:00 2001 From: saratnt Date: Tue, 18 Jun 2024 17:42:05 +0200 Subject: [PATCH 1/6] Bump version to 0.13.0-SNAPSHOT --- README.md | 2 +- ci/run_sc.sh | 2 +- examples/README.md | 10 +++++----- examples/account/evmapp/pom.xml | 4 ++-- examples/account/evmapp_sctool/pom.xml | 6 +++--- examples/mc_sc_workflow_example.md | 16 ++++++++-------- examples/utxo/simpleapp/pom.xml | 4 ++-- examples/utxo/utxoapp_sctool/pom.xml | 6 +++--- pom.xml | 2 +- qa/SidechainTestFramework/scutil.py | 2 +- sdk/pom.xml | 2 +- tools/dbtool/pom.xml | 4 ++-- tools/sctool/pom.xml | 4 ++-- tools/sidechains-sdk-account_sctools/pom.xml | 4 ++-- tools/sidechains-sdk-utxo_sctools/pom.xml | 4 ++-- tools/signingtool/pom.xml | 4 ++-- 16 files changed, 38 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index bc26d9a288..0d873ef4ba 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ While we keep monitoring the memory footprint of the proofs generation process, - After the installation, just run `export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` before starting the sidechain node, or run the sidechain node adding `LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` at the beginning of the java command line as follows: ``` -LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.12.0.jar:./target/lib/* io.horizen.examples.SimpleApp +LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.13.0-SNAPSHOT.jar:./target/lib/* io.horizen.examples.SimpleApp ``` - In the folder `ci` you will find the script `run_sc.sh` to automatically check and use jemalloc library while starting the sidechain node. diff --git a/ci/run_sc.sh b/ci/run_sc.sh index df2a03e4d0..5413152761 100755 --- a/ci/run_sc.sh +++ b/ci/run_sc.sh @@ -2,7 +2,7 @@ set -eo pipefail -SIMPLE_APP_VERSION="${SIMPLE_APP_VERSION:-0.12.0}" +SIMPLE_APP_VERSION="${SIMPLE_APP_VERSION:-0.13.0-SNAPSHOT}" if [ -d "$1" ] && [ -f "$2" ]; then path_to_jemalloc="$(ldconfig -p | grep "$(arch)" | grep 'libjemalloc\.so\.1$' | tr -d ' ' | cut -d '>' -f 2)" diff --git a/examples/README.md b/examples/README.md index 0b17158b16..f3935adc95 100644 --- a/examples/README.md +++ b/examples/README.md @@ -44,24 +44,24 @@ Otherwise, to run an Example App outside the IDE: * (Windows) ``` cd Sidechains-SDK\examples\simpleapp - java -cp ./target/sidechains-sdk-simpleapp-0.12.0.jar;./target/lib/* io.horizen.examples.SimpleApp + java -cp ./target/sidechains-sdk-simpleapp-0.13.0-SNAPSHOT.jar;./target/lib/* io.horizen.examples.SimpleApp ``` * (Linux) ``` cd ./Sidechains-SDK/examples/utxo/simpleapp - java -cp ./target/sidechains-sdk-simpleapp-0.12.0.jar:./target/lib/\* io.horizen.examples.SimpleApp + java -cp ./target/sidechains-sdk-simpleapp-0.13.0-SNAPSHOT.jar:./target/lib/\* io.horizen.examples.SimpleApp ``` **Model: Account** * (Windows) ``` cd Sidechains-SDK\examples\evmapp - java -cp ./target/sidechains-sdk-evmapp-0.12.0.jar;./target/lib/* io.horizen.examples.EvmApp + java -cp ./target/sidechains-sdk-evmapp-0.13.0-SNAPSHOT.jar;./target/lib/* io.horizen.examples.EvmApp ``` * (Linux) ``` cd ./Sidechains-SDK/examples/account/evmapp - java -cp ./target/sidechains-evmapp-0.12.0.jar:./target/lib/\* io.horizen.examples.EvmApp + java -cp ./target/sidechains-evmapp-0.13.0-SNAPSHOT.jar:./target/lib/\* io.horizen.examples.EvmApp ``` On some Linux OSs during backward transfers certificates proofs generation an extremely large RAM consumption may happen, that will lead to the process being force killed by the OS. @@ -74,7 +74,7 @@ While we keep monitoring the memory footprint of the proofs generation process, - After the installation, just run `export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` before starting the sidechain node, or run the sidechain node adding `LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` at the beginning of the java command line as follows: ``` - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.12.0.jar:./target/lib/* io.horizen.examples.SimpleApp + LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.13.0-SNAPSHOT.jar:./target/lib/* io.horizen.examples.SimpleApp ``` - In the folder `ci` you will find the script `run_sc.sh` to automatically check and use jemalloc library while starting the sidechain node. diff --git a/examples/account/evmapp/pom.xml b/examples/account/evmapp/pom.xml index 7e32c62b65..8d3bd05bab 100644 --- a/examples/account/evmapp/pom.xml +++ b/examples/account/evmapp/pom.xml @@ -3,7 +3,7 @@ 4.0.0 io.horizen sidechains-sdk-evmapp - 0.12.0 + 0.13.0-SNAPSHOT 2022 UTF-8 @@ -15,7 +15,7 @@ io.horizen sidechains-sdk - 0.12.0 + 0.13.0-SNAPSHOT junit diff --git a/examples/account/evmapp_sctool/pom.xml b/examples/account/evmapp_sctool/pom.xml index 29f4627f37..dcf1cb8ecf 100644 --- a/examples/account/evmapp_sctool/pom.xml +++ b/examples/account/evmapp_sctool/pom.xml @@ -3,7 +3,7 @@ 4.0.0 io.horizen sidechains-sdk-evmapp_sctool - 0.12.0 + 0.13.0-SNAPSHOT 2022 11 @@ -14,13 +14,13 @@ io.horizen sidechains-sdk-scbootstrappingtools - 0.12.0 + 0.13.0-SNAPSHOT compile io.horizen sidechains-sdk-evmapp - 0.12.0 + 0.13.0-SNAPSHOT compile diff --git a/examples/mc_sc_workflow_example.md b/examples/mc_sc_workflow_example.md index 3c0da83f82..35bd48ba19 100644 --- a/examples/mc_sc_workflow_example.md +++ b/examples/mc_sc_workflow_example.md @@ -32,8 +32,8 @@ Build SDK components by using a command (in the root of the Sidechains-SDK folde Run Bootstrapping tool using the command depending on the sidechain model: -- account: `java -jar tools/sidechains-sdk-account_sctools/target/sidechains-sdk-account_sctools-0.12.0.jar` -- utxo: `java -jar tools/sidechains-sdk-utxo_sctools/target/sidechains-sdk-utxo_sctools-0.12.0.jar` +- account: `java -jar tools/sidechains-sdk-account_sctools/target/sidechains-sdk-account_sctools-0.13.0-SNAPSHOT.jar` +- utxo: `java -jar tools/sidechains-sdk-utxo_sctools/target/sidechains-sdk-utxo_sctools-0.13.0-SNAPSHOT.jar` All other commands are performed as commands for Bootstrapping tool in the next format: `"command name" "parameters for command in JSON format"`. For any help, you could use the command `help`, for the exit just print `exit` @@ -507,30 +507,30 @@ Run an Example App with the `my_settings.conf`: * For Windows: ``` - java -cp ./examples/utxo/simpleapp/target/sidechains-sdk-simpleapp-0.12.0.jar;./examples/simpleapp/target/lib/* io.horizen.examples.SimpleApp ./examples/my_settings.conf + java -cp ./examples/utxo/simpleapp/target/sidechains-sdk-simpleapp-0.13.0-SNAPSHOT.jar;./examples/simpleapp/target/lib/* io.horizen.examples.SimpleApp ./examples/my_settings.conf ``` * For Linux (Glibc): ``` - java -cp ./examples/utxo/simpleapp/target/sidechains-sdk-simpleapp-0.12.0.jar:./examples/simpleapp/target/lib/* io.horizen.examples.SimpleApp ./examples/my_settings.conf + java -cp ./examples/utxo/simpleapp/target/sidechains-sdk-simpleapp-0.13.0-SNAPSHOT.jar:./examples/simpleapp/target/lib/* io.horizen.examples.SimpleApp ./examples/my_settings.conf ``` * For Linux (Jemalloc): ``` - LD_PRELOAD=/libjemalloc.so.1 java -cp ./examples/utxo/simpleapp/target/sidechains-sdk-simpleapp-0.12.0.jar:./examples/simpleapp/target/lib/* io.horizen.examples.SimpleApp ./examples/my_settings.conf + LD_PRELOAD=/libjemalloc.so.1 java -cp ./examples/utxo/simpleapp/target/sidechains-sdk-simpleapp-0.13.0-SNAPSHOT.jar:./examples/simpleapp/target/lib/* io.horizen.examples.SimpleApp ./examples/my_settings.conf ``` **Model: Account** * For Windows: ``` - java -cp ./examples/account/evmapp/target/sidechains-sdk-evmapp-0.12.0.jar;./examples/evmapp/target/lib/* io.horizen.examples.EvmApp ./examples/my_settings.conf + java -cp ./examples/account/evmapp/target/sidechains-sdk-evmapp-0.13.0-SNAPSHOT.jar;./examples/evmapp/target/lib/* io.horizen.examples.EvmApp ./examples/my_settings.conf ``` * For Linux (Glibc): ``` - java -cp ./examples/account/evmapp/target/sidechains-sdk-evmapp-0.12.0.jar:./examples/evmapp/target/lib/* io.horizen.examples.EvmApp ./examples/my_settings.conf + java -cp ./examples/account/evmapp/target/sidechains-sdk-evmapp-0.13.0-SNAPSHOT.jar:./examples/evmapp/target/lib/* io.horizen.examples.EvmApp ./examples/my_settings.conf ``` * For Linux (Jemalloc): ``` - LD_PRELOAD=/libjemalloc.so.1 java -cp ./examples/account/evmapp/target/sidechains-sdk-evmapp-0.12.0.jar:./examples/evmapp/target/lib/* io.horizen.examples.EvmApp ./examples/my_settings.conf + LD_PRELOAD=/libjemalloc.so.1 java -cp ./examples/account/evmapp/target/sidechains-sdk-evmapp-0.13.0-SNAPSHOT.jar:./examples/evmapp/target/lib/* io.horizen.examples.EvmApp ./examples/my_settings.conf ``` diff --git a/examples/utxo/simpleapp/pom.xml b/examples/utxo/simpleapp/pom.xml index 5cedc1b27d..068aeb5e05 100644 --- a/examples/utxo/simpleapp/pom.xml +++ b/examples/utxo/simpleapp/pom.xml @@ -3,7 +3,7 @@ 4.0.0 io.horizen sidechains-sdk-simpleapp - 0.12.0 + 0.13.0-SNAPSHOT 2018 UTF-8 @@ -15,7 +15,7 @@ io.horizen sidechains-sdk - 0.12.0 + 0.13.0-SNAPSHOT junit diff --git a/examples/utxo/utxoapp_sctool/pom.xml b/examples/utxo/utxoapp_sctool/pom.xml index 6b2a7dcdf8..d1c9220445 100644 --- a/examples/utxo/utxoapp_sctool/pom.xml +++ b/examples/utxo/utxoapp_sctool/pom.xml @@ -3,7 +3,7 @@ 4.0.0 io.horizen sidechains-sdk-utxoapp_sctool - 0.12.0 + 0.13.0-SNAPSHOT 2018 11 @@ -14,13 +14,13 @@ io.horizen sidechains-sdk-scbootstrappingtools - 0.12.0 + 0.13.0-SNAPSHOT compile io.horizen sidechains-sdk-simpleapp - 0.12.0 + 0.13.0-SNAPSHOT compile diff --git a/pom.xml b/pom.xml index 58046d80f5..dbeac5fc10 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 io.horizen Sidechains - 0.12.0 + 0.13.0-SNAPSHOT 2018 UTF-8 diff --git a/qa/SidechainTestFramework/scutil.py b/qa/SidechainTestFramework/scutil.py index bda7ba9ac0..2ab77cadf7 100755 --- a/qa/SidechainTestFramework/scutil.py +++ b/qa/SidechainTestFramework/scutil.py @@ -19,7 +19,7 @@ WAIT_CONST = 1 -SNAPSHOT_VERSION_TAG = "0.12.0" +SNAPSHOT_VERSION_TAG = "0.13.0-SNAPSHOT" # log levels of the log4j trace system used by java applications APP_LEVEL_OFF = "off" diff --git a/sdk/pom.xml b/sdk/pom.xml index bcd3165c0e..5d13a65b59 100644 --- a/sdk/pom.xml +++ b/sdk/pom.xml @@ -3,7 +3,7 @@ 4.0.0 io.horizen sidechains-sdk - 0.12.0 + 0.13.0-SNAPSHOT ${project.groupId}:${project.artifactId} Zendoo is a unique sidechain and scaling solution developed by Horizen. The Zendoo ${project.artifactId} is a framework that supports the creation of sidechains and their custom business logic, with the Horizen public blockchain as the mainchain. https://github.com/${project.github.organization}/${project.artifactId} diff --git a/tools/dbtool/pom.xml b/tools/dbtool/pom.xml index 34ad305db3..3b74c95315 100644 --- a/tools/dbtool/pom.xml +++ b/tools/dbtool/pom.xml @@ -3,7 +3,7 @@ 4.0.0 io.horizen sidechains-sdk-dbtools - 0.12.0 + 0.13.0-SNAPSHOT 2022 UTF-8 @@ -15,7 +15,7 @@ io.horizen sidechains-sdk - 0.12.0 + 0.13.0-SNAPSHOT junit diff --git a/tools/sctool/pom.xml b/tools/sctool/pom.xml index 35b13e8dc2..d83a67f902 100644 --- a/tools/sctool/pom.xml +++ b/tools/sctool/pom.xml @@ -3,7 +3,7 @@ 4.0.0 io.horizen sidechains-sdk-scbootstrappingtools - 0.12.0 + 0.13.0-SNAPSHOT ${project.groupId}:${project.artifactId} This module offers a way to create a sidechain's configuration file and some utilities. https://github.com/${project.github.organization}/${project.artifactId} @@ -49,7 +49,7 @@ io.horizen sidechains-sdk - 0.12.0 + 0.13.0-SNAPSHOT compile diff --git a/tools/sidechains-sdk-account_sctools/pom.xml b/tools/sidechains-sdk-account_sctools/pom.xml index 9b133a6a1d..72b6b98b73 100644 --- a/tools/sidechains-sdk-account_sctools/pom.xml +++ b/tools/sidechains-sdk-account_sctools/pom.xml @@ -3,7 +3,7 @@ 4.0.0 io.horizen sidechains-sdk-account_sctools - 0.12.0 + 0.13.0-SNAPSHOT ${project.groupId}:${project.artifactId} This module offers a way to create a sidechain's configuration file and some utilities (account model). https://github.com/${project.github.organization}/${project.artifactId} @@ -48,7 +48,7 @@ io.horizen sidechains-sdk-scbootstrappingtools - 0.12.0 + 0.13.0-SNAPSHOT compile diff --git a/tools/sidechains-sdk-utxo_sctools/pom.xml b/tools/sidechains-sdk-utxo_sctools/pom.xml index 38f311fe4a..747e5d7e38 100644 --- a/tools/sidechains-sdk-utxo_sctools/pom.xml +++ b/tools/sidechains-sdk-utxo_sctools/pom.xml @@ -3,7 +3,7 @@ 4.0.0 io.horizen sidechains-sdk-utxo_sctools - 0.12.0 + 0.13.0-SNAPSHOT ${project.groupId}:${project.artifactId} This module offers a way to create a sidechain's configuration file and some utilities (utxo model). https://github.com/${project.github.organization}/${project.artifactId} @@ -48,7 +48,7 @@ io.horizen sidechains-sdk-scbootstrappingtools - 0.12.0 + 0.13.0-SNAPSHOT compile diff --git a/tools/signingtool/pom.xml b/tools/signingtool/pom.xml index 04b1cb0090..cc630f4df7 100644 --- a/tools/signingtool/pom.xml +++ b/tools/signingtool/pom.xml @@ -3,7 +3,7 @@ 4.0.0 io.horizen sidechains-sdk-signingtools - 0.12.0 + 0.13.0-SNAPSHOT 2022 UTF-8 @@ -15,7 +15,7 @@ io.horizen sidechains-sdk - 0.12.0 + 0.13.0-SNAPSHOT junit From 33fe5e7fe447b8b0141a63335bf294f94d1520cc Mon Sep 17 00:00:00 2001 From: saratnt Date: Wed, 26 Jun 2024 10:18:29 +0200 Subject: [PATCH 2/6] Added zen_dump method --- qa/run_sc_tests.sh | 1 + qa/sc_evm_dump.py | 206 ++++++++++++++++++ sdk/pom.xml | 2 +- .../horizen/account/AccountSidechainApp.scala | 3 +- .../account/api/rpc/service/EthService.scala | 25 ++- .../state/StateDbAccountStateView.scala | 2 + .../api/rpc/service/EthServiceTest.scala | 16 +- 7 files changed, 246 insertions(+), 9 deletions(-) create mode 100755 qa/sc_evm_dump.py diff --git a/qa/run_sc_tests.sh b/qa/run_sc_tests.sh index 1008526abd..7d3331dd0d 100755 --- a/qa/run_sc_tests.sh +++ b/qa/run_sc_tests.sh @@ -152,6 +152,7 @@ testScriptsEvm=( 'sc_evm_forger_reward_address.py' 'sc_evm_native_forger_v2.py' 'sc_evm_forger_v2_register.py' + 'sc_evm_dump.py' ); testScriptsUtxo=( diff --git a/qa/sc_evm_dump.py b/qa/sc_evm_dump.py new file mode 100755 index 0000000000..d15d861fb5 --- /dev/null +++ b/qa/sc_evm_dump.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +import json +import os +import pprint +from decimal import Decimal +from stat import S_IREAD, S_IRGRP, S_IROTH + +from eth_utils import add_0x_prefix + +from SidechainTestFramework.account.ac_chain_setup import AccountChainSetup +from SidechainTestFramework.account.ac_use_smart_contract import SmartContract +from SidechainTestFramework.account.ac_utils import rpc_get_balance +from SidechainTestFramework.account.httpCalls.transaction.createEIP1559Transaction import createEIP1559Transaction +from SidechainTestFramework.account.utils import WITHDRAWAL_REQ_SMART_CONTRACT_ADDRESS, \ + FORGER_STAKE_SMART_CONTRACT_ADDRESS +from SidechainTestFramework.sc_boostrap_info import SCNodeConfiguration, MCConnectionInfo, SCNetworkConfiguration, \ + SCCreationInfo +from SidechainTestFramework.scutil import generate_next_block, \ + connect_sc_nodes, assert_equal, assert_true, bootstrap_sidechain_nodes, AccountModel, EVM_APP_SLOT_TIME +from test_framework.util import forward_transfer_to_sidechain, websocket_port_by_mc_node_index + +""" +Test zen_dump rpc method. + +Configuration: + - 2 SC nodes connected with each other, one of them has dump enabled. + - 1 MC node + +Test: + - Try zen_dump on node with dump disabled: it should not be allowed + - try dump on different blocks + - try with wrong params + - load test with lots of accounts (only if uncommented) + +""" + + +def deploy_smart_contract(node, smart_contract, from_address, *args, + nonce=None): + _, address = smart_contract.deploy(node, *args, + fromAddress=from_address, + gasLimit=200000, + nonce=nonce) + + return address + + +class SCEvmDump(AccountChainSetup): + + def __init__(self): + super().__init__(number_of_sidechain_nodes=2, connect_nodes=False, + block_timestamp_rewind=1500 * EVM_APP_SLOT_TIME * 1000) + + def sc_setup_chain(self): + mc_node = self.nodes[0] + sc_node_configuration = [ + SCNodeConfiguration( + MCConnectionInfo( + address="ws://{0}:{1}".format(mc_node.hostname, websocket_port_by_mc_node_index(0))), + api_key=self.API_KEY, + remote_keys_manager_enabled=self.remote_keys_manager_enabled, + max_nonce_gap=110, + max_account_slots=110, + evm_state_dump_enabled=False), + SCNodeConfiguration( + MCConnectionInfo( + address="ws://{0}:{1}".format(mc_node.hostname, websocket_port_by_mc_node_index(0))), + api_key=self.API_KEY, + remote_keys_manager_enabled=self.remote_keys_manager_enabled, + max_nonce_gap=110, + max_account_slots=110, + evm_state_dump_enabled=True) + ] + + network = SCNetworkConfiguration(SCCreationInfo(mc_node, self.forward_amount, self.withdrawalEpochLength), + *sc_node_configuration) + self.sc_nodes_bootstrap_info = bootstrap_sidechain_nodes(self.options, network, + block_timestamp_rewind=self.block_timestamp_rewind, + model=AccountModel) + + def run_test(self): + mc_node = self.nodes[0] + sc_node_without_dump = self.sc_nodes[0] + sc_node_with_dump = self.sc_nodes[1] + connect_sc_nodes(sc_node_without_dump, 1) + + # transfer some fund from MC to SC1 at a new evm address, then mine mc block + evm_address_sc1 = sc_node_without_dump.wallet_createPrivateKeySecp256k1()["result"]["proposition"]["address"] + evm_address_sc1_prefix = add_0x_prefix(evm_address_sc1) + + ft_amount_in_zen = Decimal('500.0') + + mc_return_address = mc_node.getnewaddress() + forward_transfer_to_sidechain(self.sc_nodes_bootstrap_info.sidechain_id, + mc_node, + evm_address_sc1, + ft_amount_in_zen, + mc_return_address=mc_return_address, + generate_block=True) + + self.sync_all() + + generate_next_block(sc_node_without_dump, "first node") + self.sc_sync_all() + + # Create a transaction on node 1. Verify that the tx is in node 3 mempool but seeder node mempool is empty + nonce_addr_1 = 0 + createEIP1559Transaction(sc_node_without_dump, fromAddress=evm_address_sc1, toAddress=evm_address_sc1, + nonce=nonce_addr_1, gasLimit=230000, maxPriorityFeePerGas=900000000, + maxFeePerGas=900000000, value=1) + nonce_addr_1 += 1 + + # Generate a block in order to clean the mempool + block_id_1 = generate_next_block(sc_node_without_dump, "first node") + self.sc_sync_all() + + dump_file = os.path.join(self.options.tmpdir, "dump.json") + + contract = SmartContract("Storage") + contract_address = add_0x_prefix(deploy_smart_contract(sc_node_without_dump, contract, evm_address_sc1, + 15)).lower() + nonce_addr_1 += 1 + + block_id_2 = generate_next_block(sc_node_without_dump, "first node") + self.sc_sync_all() + + # Try the dump + sc_node_with_dump.rpc_zen_dump(add_0x_prefix(block_id_2), dump_file) + + with open(dump_file, 'r') as file: + dump_data = json.load(file) + accounts = dump_data["accounts"] + assert_equal(4, len(accounts)) + assert_true(add_0x_prefix(WITHDRAWAL_REQ_SMART_CONTRACT_ADDRESS) in accounts) + assert_true(add_0x_prefix(FORGER_STAKE_SMART_CONTRACT_ADDRESS) in accounts) + assert_true(evm_address_sc1_prefix in accounts) + evm_address_sc1_account = accounts[evm_address_sc1_prefix] + assert_equal(nonce_addr_1, evm_address_sc1_account["nonce"]) + evm_address_sc1_balance = rpc_get_balance(sc_node_with_dump, evm_address_sc1_prefix) + assert_equal(str(evm_address_sc1_balance), evm_address_sc1_account["balance"]) + assert_true(contract_address in accounts) + assert_equal(1, accounts[contract_address]["nonce"]) + + response = sc_node_without_dump.rpc_zen_dump(add_0x_prefix(block_id_1), dump_file) + self.check_rpc_not_allowed(response) + + # Try the dump at an old block + sc_node_with_dump.rpc_zen_dump(add_0x_prefix(block_id_1), dump_file) + + with open(dump_file, 'r') as file: + dump_data = json.load(file) + + accounts = dump_data["accounts"] + assert_equal(3, len(dump_data["accounts"])) + assert_true(add_0x_prefix(WITHDRAWAL_REQ_SMART_CONTRACT_ADDRESS) in accounts) + assert_true(add_0x_prefix(FORGER_STAKE_SMART_CONTRACT_ADDRESS) in accounts) + assert_true(evm_address_sc1_prefix in accounts) + evm_address_sc1_account = accounts[evm_address_sc1_prefix] + assert_equal(nonce_addr_1 - 1, evm_address_sc1_account["nonce"]) + eip1898_block_hash = { + "blockHash": add_0x_prefix(block_id_1) + } + + res = sc_node_with_dump.rpc_eth_getBalance(evm_address_sc1_prefix, eip1898_block_hash) + evm_address_sc1_balance = int(res['result'], 16) + assert_equal(str(evm_address_sc1_balance), evm_address_sc1_account["balance"]) + + # Make the dump file readonly and check that the dump will fail + os.chmod(dump_file, S_IREAD | S_IRGRP | S_IROTH) + response = sc_node_with_dump.rpc_zen_dump(add_0x_prefix(block_id_1), dump_file) + assert_true("error" in response) + + # # Uncomment to do the load test + # # Load test: creates a lot of accounts and tries dump + # for i in range(0, 1000): + # pprint.pprint("Iteration {}".format(i)) + # for k in range(0, 100): + # deploy_smart_contract(sc_node_without_dump, contract, evm_address_sc1, k, nonce=nonce_addr_1) + # nonce_addr_1 += 1 + # + # generate_next_block(sc_node_without_dump, "first node") + # # Generate a second block, so the base fee remains low + # block_id = generate_next_block(sc_node_without_dump, "first node") + # mc_node.generate(1) + # self.sc_sync_all() + # + # self.sc_sync_all() + # + # dump_file = os.path.join(self.options.tmpdir, "load_dump.json") + # + # sc_node_with_dump.rpc_zen_dump(add_0x_prefix(block_id), dump_file) + # with open(dump_file, 'r') as file: + # dump_data = json.load(file) + # accounts = dump_data["accounts"] + # # There are 100005 instead of 100004 because the redistribution of the fees creates the account related to the default delegator + # assert_equal(100005, len(accounts)) + + @staticmethod + def check_rpc_not_allowed(response): + assert_true("error" in response) + assert_equal("Action not allowed", response['error']['message']) + assert_equal(2, response['error']['code']) + + +if __name__ == "__main__": + SCEvmDump().main() diff --git a/sdk/pom.xml b/sdk/pom.xml index 5d13a65b59..299325f5fd 100644 --- a/sdk/pom.xml +++ b/sdk/pom.xml @@ -356,7 +356,7 @@ io.horizen libevm - 1.1.0-snapshot1 + 1.2.0-snapshot at.favre.lib diff --git a/sdk/src/main/scala/io/horizen/account/AccountSidechainApp.scala b/sdk/src/main/scala/io/horizen/account/AccountSidechainApp.scala index e7bc76137a..7a4f8f3818 100644 --- a/sdk/src/main/scala/io/horizen/account/AccountSidechainApp.scala +++ b/sdk/src/main/scala/io/horizen/account/AccountSidechainApp.scala @@ -175,7 +175,8 @@ class AccountSidechainApp @Inject() RpcUtils.getClientVersion(appVersion), sidechainTransactionActorRef, syncStatusActorRef, - sidechainTransactionsCompanion + sidechainTransactionsCompanion, + sidechainSettings.evmStateDump.enabled ) ) //Initialize RpcProcessor object with the rpcHandler diff --git a/sdk/src/main/scala/io/horizen/account/api/rpc/service/EthService.scala b/sdk/src/main/scala/io/horizen/account/api/rpc/service/EthService.scala index 0b36f3e011..3b23db8bd9 100644 --- a/sdk/src/main/scala/io/horizen/account/api/rpc/service/EthService.scala +++ b/sdk/src/main/scala/io/horizen/account/api/rpc/service/EthService.scala @@ -53,7 +53,7 @@ import scala.collection.JavaConverters.seqAsJavaListConverter import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable` import scala.collection.mutable.ListBuffer import scala.compat.java8.OptionConverters.RichOptionalGeneric -import scala.concurrent.duration.FiniteDuration +import scala.concurrent.duration.{FiniteDuration, MINUTES} import scala.concurrent.{Await, Future, TimeoutException} import scala.language.postfixOps import scala.util.{Failure, Success, Try} @@ -68,7 +68,8 @@ class EthService( rpcClientVersion: String, sidechainTransactionActorRef: ActorRef, syncStatusActorRef: ActorRef, - transactionsCompanion: SidechainAccountTransactionsCompanion + transactionsCompanion: SidechainAccountTransactionsCompanion, + isEvmDumpEnabled: Boolean = false ) extends RpcService with ClosableResourceHandler with SparkzLogging { @@ -936,7 +937,6 @@ class EthService( throw new RpcException(RpcError.fromCode(RpcCode.InvalidParams, s"transaction not found: $transactionHash")) ) - applyOnAccountView { nodeView => getStateViewAtTag(nodeView, (blockNumber - 1).toString) { (tagStateView, _) => @@ -1184,4 +1184,23 @@ class EthService( new Hash(Keccak256.hash(data)) } + + @RpcMethod("zen_dump") + def dump(blockHashOrNumber: String, dumpFile: String): Unit = { + if (!isEvmDumpEnabled) + throw new RpcException(RpcError.fromCode(RpcCode.ActionNotAllowed)) + applyOnAccountView ({ nodeView => + // get block info + val blockInfo = getBlockInfoById(nodeView, getBlockIdByHashOrNumber(nodeView, blockHashOrNumber)) + + // get state at selected block + getStateViewAtTag(nodeView, blockInfo.height.toString) { + (tagStateView, _) => + tagStateView.dump(dumpFile) + } + }, new FiniteDuration(5, MINUTES) + ) + } + + } diff --git a/sdk/src/main/scala/io/horizen/account/state/StateDbAccountStateView.scala b/sdk/src/main/scala/io/horizen/account/state/StateDbAccountStateView.scala index f668d4006f..85fef95b3a 100644 --- a/sdk/src/main/scala/io/horizen/account/state/StateDbAccountStateView.scala +++ b/sdk/src/main/scala/io/horizen/account/state/StateDbAccountStateView.scala @@ -389,6 +389,8 @@ class StateDbAccountStateView( override def getBalance(address: Address): BigInteger = stateDb.getBalance(address) + def dump(dumpFile: String): Unit = stateDb.dump(dumpFile) + override def getCodeHash(address: Address): Array[Byte] = stateDb.getCodeHash(address).toBytes override def getCode(address: Address): Array[Byte] = stateDb.getCode(address) diff --git a/sdk/src/test/scala/io/horizen/account/api/rpc/service/EthServiceTest.scala b/sdk/src/test/scala/io/horizen/account/api/rpc/service/EthServiceTest.scala index c71d9f9593..7065fea02a 100644 --- a/sdk/src/test/scala/io/horizen/account/api/rpc/service/EthServiceTest.scala +++ b/sdk/src/test/scala/io/horizen/account/api/rpc/service/EthServiceTest.scala @@ -5,6 +5,7 @@ import akka.testkit.{TestActor, TestProbe} import com.fasterxml.jackson.databind.ObjectMapper import io.horizen.account.api.rpc.handler.RpcException import io.horizen.account.api.rpc.request.RpcRequest +import io.horizen.account.api.rpc.utils.RpcCode import io.horizen.account.block.AccountBlock import io.horizen.account.chain.AccountFeePaymentsInfo import io.horizen.account.fork.GasFeeFork.DefaultGasFeeFork @@ -31,8 +32,9 @@ import io.horizen.network.SyncStatusActor.ReceivableMessages.GetSyncStatus import io.horizen.params.RegTestParams import io.horizen.utils.{BytesUtils, TimeToEpochUtils} import io.horizen.{EthServiceSettings, SidechainTypes} +import org.junit.Assert.assertEquals import org.junit.{Before, Test} -import org.mockito.ArgumentMatchers.{any, anyString} +import org.mockito.ArgumentMatchers.any import org.mockito.Mockito import org.mockito.Mockito.when import org.scalatest.prop.TableDrivenPropertyChecks @@ -45,7 +47,7 @@ import sparkz.core.bytesToId import sparkz.core.network.NetworkController.ReceivableMessages.GetConnectedPeers import sparkz.core.network.NodeViewSynchronizer.ReceivableMessages.SuccessfulTransaction import sparkz.crypto.hash.Keccak256 -import sparkz.util.{ByteArrayBuilder, ModifierId} +import sparkz.util.ByteArrayBuilder import sparkz.util.serialization.VLQByteBufferWriter import java.math.BigInteger @@ -596,9 +598,9 @@ class EthServiceTest extends JUnitSuite with MockitoSugar with ReceiptFixture wi val invalidCases = Table( ("Block tag", "Transaction index"), - // the "safe" block is not available here (mininum block height > 100) + // the "safe" block is not available here (minimum block height > 100) ("safe", "0"), - // the "finalized" block is not available here (mininum block height > 100) + // the "finalized" block is not available here (minimum block height > 100) ("finalized", "0"), // invalid block tag ("aaaa", "0"), @@ -1483,4 +1485,10 @@ class EthServiceTest extends JUnitSuite with MockitoSugar with ReceiptFixture wi rpc("eth_getLogs", Map("fromBlock" -> "1", "toBlock" -> "10002")) } } + + @Test + def zen_dump(): Unit = { + assertEquals(RpcCode.ActionNotAllowed.code, intercept[RpcException] { rpc("zen_dump", "latest", "/tmp/file.json")}.error.code) + } + } From bf4bbed88d4b1450991b607c71f7744db7b117b0 Mon Sep 17 00:00:00 2001 From: saratnt Date: Wed, 26 Jun 2024 10:51:01 +0200 Subject: [PATCH 3/6] Updated release docs --- CHANGELOG.md | 1 + doc/release/0.13.0.md | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 515e5fce30..a34672a164 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ **0.13.0** 1. Enabling preimages configuration +2. [eth RPC endpoint]: added zen_dump rpc method for requesting an EVM state dump on a json file **0.12.0** 1. Sparkz dependency updated to 2.4.0 diff --git a/doc/release/0.13.0.md b/doc/release/0.13.0.md index 1ad48e7cba..f3bcf2ed97 100644 --- a/doc/release/0.13.0.md +++ b/doc/release/0.13.0.md @@ -13,6 +13,12 @@ How to specify it: enabled = "true" +### New zen_dump rpc method +This method executes a dump on a json file of the EVM state. It takes 2 input parameters: +- blockHashOrNumber: hash or height of the block where the dump needs to be executed +- fileName: fully-qualified name of the json file where the dump will be saved + +It can be called only on nodes where preimages were enabled. --- ## Bug Fixes From 38d1370ad21866e01e700f0bd0ef52943a889f19 Mon Sep 17 00:00:00 2001 From: saratnt Date: Fri, 28 Jun 2024 14:30:48 +0200 Subject: [PATCH 4/6] Changes after review comments --- .../account/api/rpc/service/EthService.scala | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/sdk/src/main/scala/io/horizen/account/api/rpc/service/EthService.scala b/sdk/src/main/scala/io/horizen/account/api/rpc/service/EthService.scala index 3b23db8bd9..488825e9de 100644 --- a/sdk/src/main/scala/io/horizen/account/api/rpc/service/EthService.scala +++ b/sdk/src/main/scala/io/horizen/account/api/rpc/service/EthService.scala @@ -53,7 +53,7 @@ import scala.collection.JavaConverters.seqAsJavaListConverter import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable` import scala.collection.mutable.ListBuffer import scala.compat.java8.OptionConverters.RichOptionalGeneric -import scala.concurrent.duration.{FiniteDuration, MINUTES} +import scala.concurrent.duration.{FiniteDuration, SECONDS} import scala.concurrent.{Await, Future, TimeoutException} import scala.language.postfixOps import scala.util.{Failure, Success, Try} @@ -1186,19 +1186,24 @@ class EthService( @RpcMethod("zen_dump") - def dump(blockHashOrNumber: String, dumpFile: String): Unit = { + @RpcOptionalParameters(1) + def dump(blockHashOrNumber: String, dumpFile: String, timeoutInSeconds: Int): Unit = { if (!isEvmDumpEnabled) throw new RpcException(RpcError.fromCode(RpcCode.ActionNotAllowed)) + + log.info(s"Dump timeout $timeoutInSeconds") + val dumpTimeout = if (timeoutInSeconds == 0) 300 else timeoutInSeconds applyOnAccountView ({ nodeView => // get block info val blockInfo = getBlockInfoById(nodeView, getBlockIdByHashOrNumber(nodeView, blockHashOrNumber)) - // get state at selected block getStateViewAtTag(nodeView, blockInfo.height.toString) { (tagStateView, _) => + log.error(s"Starting dump at block $blockHashOrNumber to file $dumpFile") tagStateView.dump(dumpFile) + log.error("Dump completed successfully") } - }, new FiniteDuration(5, MINUTES) + }, new FiniteDuration(dumpTimeout, SECONDS) ) } From e38587c631816dbf115d72ee8bb38186bb4ba062 Mon Sep 17 00:00:00 2001 From: saratnt Date: Mon, 1 Jul 2024 11:04:29 +0200 Subject: [PATCH 5/6] Changed log --- .../scala/io/horizen/account/api/rpc/service/EthService.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/src/main/scala/io/horizen/account/api/rpc/service/EthService.scala b/sdk/src/main/scala/io/horizen/account/api/rpc/service/EthService.scala index 488825e9de..18dd6f9403 100644 --- a/sdk/src/main/scala/io/horizen/account/api/rpc/service/EthService.scala +++ b/sdk/src/main/scala/io/horizen/account/api/rpc/service/EthService.scala @@ -1191,8 +1191,8 @@ class EthService( if (!isEvmDumpEnabled) throw new RpcException(RpcError.fromCode(RpcCode.ActionNotAllowed)) - log.info(s"Dump timeout $timeoutInSeconds") val dumpTimeout = if (timeoutInSeconds == 0) 300 else timeoutInSeconds + log.info(s"Dump timeout $dumpTimeout") applyOnAccountView ({ nodeView => // get block info val blockInfo = getBlockInfoById(nodeView, getBlockIdByHashOrNumber(nodeView, blockHashOrNumber)) From 289952783b779d556a8470f7b337da0188be79e2 Mon Sep 17 00:00:00 2001 From: paolocappelletti Date: Wed, 3 Jul 2024 10:16:46 +0200 Subject: [PATCH 6/6] changed tag to uppercase --- sdk/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/pom.xml b/sdk/pom.xml index 299325f5fd..40a4365c3e 100644 --- a/sdk/pom.xml +++ b/sdk/pom.xml @@ -356,7 +356,7 @@ io.horizen libevm - 1.2.0-snapshot + 1.2.0-SNAPSHOT at.favre.lib