diff --git a/.github/workflows/sync-supported-chains.yml b/.github/workflows/sync-supported-chains.yml index 5456b896195..1c68f74cdce 100644 --- a/.github/workflows/sync-supported-chains.yml +++ b/.github/workflows/sync-supported-chains.yml @@ -160,6 +160,7 @@ jobs: --network $stripped_network \ --consensus-url $CONSENSUS_URL \ --execution-api-url $EXECUTION_URL \ + --el-op-extra-flag Sync.VerifyTrieOnStateSyncFinished=true \ $extra_param else ./build/sedge generate \ @@ -176,6 +177,7 @@ jobs: --el-extra-flag Sync.DownloadBodiesInFastSync=false \ --el-extra-flag Sync.DownloadReceiptsInFastSync=false \ --el-extra-flag JsonRpc.EnabledModules=[Eth,Subscribe,Trace,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Debug] \ + --el-extra-flag Sync.VerifyTrieOnStateSyncFinished=true \ --el-extra-flag Sync.SnapSync=true \ --checkpoint-sync-url=${{ matrix.config.checkpoint-sync-url }} fi @@ -190,12 +192,14 @@ jobs: declare -A good_logs declare -A required_count - bad_logs["Corrupt"]=1 bad_logs["Exception"]=1 + bad_logs["Missing node found!"]=1 good_logs["Processed"]=0 + good_logs["Stats after finishing state"]=0 required_count["Processed"]=20 + required_count["Stats after finishing state"]=1 network="${{ matrix.config.network }}" if [[ "$network" != "joc-mainnet" && "$network" != "joc-testnet" && "$network" != "linea-mainnet" && "$network" != "linea-sepolia" ]]; then diff --git a/scripts/dev-setup.sh b/scripts/dev-setup.sh index 40a7953666b..ee84104d15f 100644 --- a/scripts/dev-setup.sh +++ b/scripts/dev-setup.sh @@ -34,11 +34,11 @@ echo ======================================================= echo -e "\033[32m"; read -e -p "Which configuration/s (space separated) you wish to run? " -i "mainnet" config -for cfg in $config; do cp nethermind/configs/$cfg.cfg ~; done +for cfg in $config; do cp nethermind/configs/$cfg.json ~; done echo -e "\033[00m"; echo ======================================================= echo To run the node type: echo 1. screen -S node -echo 2. ./infra.sh config-name.cfg +echo 2. ./infra.sh config-name.json echo ======================================================= diff --git a/scripts/infra.sh b/scripts/infra.sh index aad2014ac36..9d8d0f13c27 100644 --- a/scripts/infra.sh +++ b/scripts/infra.sh @@ -17,5 +17,5 @@ mkdir ~/nethermind_$CONFIG/keystore cp ~/$CONFIG.key ~/nethermind_$CONFIG/keystore/node.key.plain DB_PATH="/root/db/$CONFIG" echo "DB PATH: " $DB_PATH -cat ~/$CONFIG.cfg | jq '.Init.BaseDbPath = "'$DB_PATH'"' | sponge ~/$CONFIG.cfg -dotnet nethermind.dll -c ../$CONFIG.cfg +cat ~/$CONFIG.json | jq '.Init.BaseDbPath = "'$DB_PATH'"' | sponge ~/$CONFIG.json +dotnet nethermind.dll -c ../$CONFIG.json diff --git a/scripts/private-networking/clique-validators.sh b/scripts/private-networking/clique-validators.sh index 1a8775772ca..cce07bc0119 100644 --- a/scripts/private-networking/clique-validators.sh +++ b/scripts/private-networking/clique-validators.sh @@ -127,7 +127,7 @@ docker-compose up #END of main function writeNethermindConfig() { -cat < node_$1/configs/config.cfg +cat < node_$1/configs/config.json { "Init": { "WebSocketsEnabled": false, @@ -196,7 +196,7 @@ cat <> docker-compose.yml command: --config config volumes: - ./genesis:/config/genesis - - ./node_$1/configs/config.cfg:/nethermind/configs/config.cfg + - ./node_$1/configs/config.json:/nethermind/configs/config.json - ./static-nodes.json:/nethermind/Data/static-nodes.json - ./node_$1/db/clique:/nethermind/nethermind_db/clique - ./node_$1/keystore:/nethermind/keystore @@ -233,4 +233,4 @@ function clearDbs() { done } -main \ No newline at end of file +main diff --git a/scripts/syncSettings.py b/scripts/syncSettings.py index d137de19890..730b3122511 100644 --- a/scripts/syncSettings.py +++ b/scripts/syncSettings.py @@ -121,12 +121,12 @@ def fastBlocksSettings(configuration, apiUrl, blockReduced, multiplierRequiremen print(configuration + 'PivotHash: ' + str(pivotHash)) print(configuration + 'PivotTotalDifficulty: ' + str(pivotTotalDifficulty)) data = {} - with open(f'{configsPath}/{configuration}.cfg', 'r') as mainnetCfg: + with open(f'{configsPath}/{configuration}.json', 'r') as mainnetCfg: data = json.load(mainnetCfg) data['Sync']['PivotNumber'] = baseBlock data['Sync']['PivotHash'] = pivotHash data['Sync']['PivotTotalDifficulty'] = str(pivotTotalDifficulty) - with open(f'{configsPath}/{configuration}.cfg', 'w') as mainnetCfgChanged: + with open(f'{configsPath}/{configuration}.json', 'w') as mainnetCfgChanged: json.dump(data, mainnetCfgChanged, indent=2) for config, value in configs.items(): diff --git a/src/Nethermind/Chains/kovan.json b/src/Nethermind/Chains/kovan.json deleted file mode 100644 index aa1efddb802..00000000000 --- a/src/Nethermind/Chains/kovan.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "name": "Kovan Testnet", - "dataDir": "kovan", - "engine": { - "authorityRound": { - "params": { - "stepDuration": "0x4", - "blockReward": "0x4563918244F40000", - "validators": { - "multi": { - "0": { - "list": [ - "0x00D6Cc1BA9cf89BD2e58009741f4F7325BAdc0ED", - "0x00427feae2419c15b89d1c21af10d1b6650a4d3d", - "0x4Ed9B08e6354C70fE6F8CB0411b0d3246b424d6c", - "0x0020ee4Be0e2027d76603cB751eE069519bA81A1", - "0x0010f94b296a852aaac52ea6c5ac72e03afd032d", - "0x007733a1FE69CF3f2CF989F81C7b4cAc1693387A", - "0x00E6d2b931F55a3f1701c7389d592a7778897879", - "0x00e4a10650e5a6D6001C38ff8E64F97016a1645c", - "0x00a0a24b9f0e5ec7aa4c7389b8302fd0123194de" - ] - }, - "10960440": { - "list": [ - "0x00D6Cc1BA9cf89BD2e58009741f4F7325BAdc0ED", - "0x0010f94b296a852aaac52ea6c5ac72e03afd032d", - "0x00a0a24b9f0e5ec7aa4c7389b8302fd0123194de" - ] - }, - "10960500": { - "safeContract": "0xaE71807C1B0a093cB1547b682DC78316D945c9B8" - } - } - }, - "validateScoreTransition": "0x41a3c4", - "validateStepTransition": "0x16e360", - "maximumUncleCountTransition": "0x4d50f8", - "maximumUncleCount": "0x0" - } - } - }, - "params": { - "gasLimitBoundDivisor": "0x400", - "registrar": "0xfAb104398BBefbd47752E7702D9fE23047E1Bca3", - "maximumExtraDataSize": "0x20", - "minGasLimit": "0x1388", - "networkID": "0x2A", - "forkBlock": "0x9766dc", - "forkCanonHash": "0xf2fa4bcc417ad374100c2035aa865ff60fb568a83db1b6f6cb8fb52cfebc28b5", - "eip155Transition": "0xf4240", - "maxCodeSize": "0x6000", - "maxCodeSizeTransition": "0x64b540", - "validateChainIdTransition": "0xf4240", - "validateReceiptsTransition": "0xf4240", - "eip98Transition": "0x0", - "eip140Transition": "0x4d50f8", - "eip211Transition": "0x4d50f8", - "eip214Transition": "0x4d50f8", - "eip658Transition": "0x4d50f8", - "wasmActivationTransition": "0x64b540", - "wasmDisableTransition": "0x198096c", - "eip145Transition": "0x8c6180", - "eip1014Transition": "0x8c6180", - "eip1052Transition": "0x8c6180", - "eip1283Transition": "0x8c6180", - "eip1283DisableTransition": "0x9c7b61", - "eip1283ReenableTransition": "0xd751a5", - "eip1344Transition": "0xd751a5", - "eip1706Transition": "0xd751a5", - "eip1884Transition": "0xd751a5", - "eip2028Transition": "0xd751a5", - "eip2929Transition": "0x179f954", - "eip2930Transition": "0x179f954", - "eip1559Transition": "0x198096c", - "eip3198Transition": "0x198096c", - "eip3541Transition": "0x198096c", - "eip3529Transition": "0x198096c", - "eip1559BaseFeeMaxChangeDenominator": "0x8", - "eip1559ElasticityMultiplier": "0x2", - "eip1559BaseFeeInitialValue": "0x3B9ACA00", - "kip4Transition": "0x8c6180", - "kip6Transition": "0x8c6180" - }, - "genesis": { - "seal": { - "authorityRound": { - "step": "0x0", - "signature": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" - } - }, - "difficulty": "0x20000", - "gasLimit": "0x5B8D80" - }, - "accounts": { - "0x0000000000000000000000000000000000000001": { - "balance": "0x1", - "builtin": { - "name": "ecrecover", - "pricing": { - "0": { - "price": { - "linear": { - "base": 3000, - "word": 0 - } - } - } - } - } - }, - "0x0000000000000000000000000000000000000002": { - "balance": "0x1", - "builtin": { - "name": "sha256", - "pricing": { - "0": { - "price": { - "linear": { - "base": 60, - "word": 12 - } - } - } - } - } - }, - "0x0000000000000000000000000000000000000003": { - "balance": "0x1", - "builtin": { - "name": "ripemd160", - "pricing": { - "0": { - "price": { - "linear": { - "base": 600, - "word": 120 - } - } - } - } - } - }, - "0x0000000000000000000000000000000000000004": { - "balance": "0x1", - "builtin": { - "name": "identity", - "pricing": { - "0": { - "price": { - "linear": { - "base": 15, - "word": 3 - } - } - } - } - } - }, - "0x0000000000000000000000000000000000000005": { - "builtin": { - "name": "modexp", - "pricing": { - "0x4d50f8": { - "price": { - "modexp": { - "divisor": 20 - } - } - }, - "0x179f954": { - "info": "EIP-2565: ModExp Gas Cost. Berlin hardfork (24_770_900)", - "price": { - "modexp2565": {} - } - } - } - } - }, - "0x0000000000000000000000000000000000000006": { - "builtin": { - "name": "alt_bn128_add", - "pricing": { - "0x4d50f8": { - "price": { - "alt_bn128_const_operations": { - "price": 500 - } - } - }, - "0xd751a5": { - "price": { - "alt_bn128_const_operations": { - "price": 150 - } - } - } - } - } - }, - "0x0000000000000000000000000000000000000007": { - "builtin": { - "name": "alt_bn128_mul", - "pricing": { - "0x4d50f8": { - "price": { - "alt_bn128_const_operations": { - "price": 40000 - } - } - }, - "0xd751a5": { - "price": { - "alt_bn128_const_operations": { - "price": 6000 - } - } - } - } - } - }, - "0x0000000000000000000000000000000000000008": { - "builtin": { - "name": "alt_bn128_pairing", - "pricing": { - "0x4d50f8": { - "price": { - "alt_bn128_pairing": { - "base": 100000, - "pair": 80000 - } - } - }, - "0xd751a5": { - "price": { - "alt_bn128_pairing": { - "base": 45000, - "pair": 34000 - } - } - } - } - } - }, - "0x0000000000000000000000000000000000000009": { - "builtin": { - "name": "blake2_f", - "pricing": { - "0xd751a5": { - "price": { - "blake2_f": { - "gas_per_round": 1 - } - } - } - } - } - }, - "0x00521965e7bd230323c423d96c657db5b79d099f": { - "balance": "1606938044258990275541962092341162602522202993782792835301376" - } - }, - "nodes": [ - "enode://30499bde23362f7d310a34518a2a6ff765921870bf0c3e63d21153cfa7ba9cf39cc7c8e54e9dad2f2b3c07288b3e91b220656833cc2d843a54875c229f3f959a@8.9.8.175:30303", - "enode://16898006ba2cd4fa8bf9a3dfe32684c178fa861df144bfc21fe800dc4838a03e342056951fa9fd533dcb0be1219e306106442ff2cf1f7e9f8faa5f2fc1a3aa45@116.203.116.241:30303", - "enode://49a0e1aa38caa12cbf31222cb4e31cef1e8794cb4dd38012f84498ac867b19584e29bbf6d53201d7dfd3b5eb0998a4d908d096ed4ddb5f9102c623852cd331ec@54.87.247.5:30303" - ] -} \ No newline at end of file diff --git a/src/Nethermind/Directory.Packages.props b/src/Nethermind/Directory.Packages.props index a19efb10615..7d129079f0f 100644 --- a/src/Nethermind/Directory.Packages.props +++ b/src/Nethermind/Directory.Packages.props @@ -13,7 +13,6 @@ - @@ -37,7 +36,6 @@ - @@ -76,6 +74,7 @@ + @@ -84,4 +83,4 @@ - \ No newline at end of file + diff --git a/src/Nethermind/Ethereum.Test.Base/GeneralTestBase.cs b/src/Nethermind/Ethereum.Test.Base/GeneralTestBase.cs index 47487e7e212..0a53e1ab450 100644 --- a/src/Nethermind/Ethereum.Test.Base/GeneralTestBase.cs +++ b/src/Nethermind/Ethereum.Test.Base/GeneralTestBase.cs @@ -4,14 +4,11 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using Nethermind.Blockchain; -using Nethermind.Consensus.Ethash; using Nethermind.Consensus.Validators; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Core.Specs; -using Nethermind.Core.Test.Builders; using Nethermind.Crypto; using Nethermind.Db; using Nethermind.Int256; @@ -89,7 +86,7 @@ protected EthereumTestResult RunTest(GeneralStateTest test, ITxTracer txTracer) codeInfoRepository, _logManager); - InitializeTestState(test, stateProvider, specProvider); + InitializeTestState(test.Pre, stateProvider, specProvider); BlockHeader header = new( test.PreviousHash, @@ -135,8 +132,6 @@ protected EthereumTestResult RunTest(GeneralStateTest test, ITxTracer txTracer) header.ExcessBlobGas = BlobGasCalculator.CalculateExcessBlobGas(parent, spec); } - Block block = Build.A.Block.WithTransactions(test.Transaction).WithHeader(header).TestObject; - ValidationResult txIsValid = _txValidator.IsWellFormed(test.Transaction, spec); if (txIsValid) @@ -172,9 +167,9 @@ protected EthereumTestResult RunTest(GeneralStateTest test, ITxTracer txTracer) return testResult; } - private static void InitializeTestState(GeneralStateTest test, WorldState stateProvider, ISpecProvider specProvider) + private static void InitializeTestState(Dictionary preState, WorldState stateProvider, ISpecProvider specProvider) { - foreach (KeyValuePair accountState in test.Pre) + foreach (KeyValuePair accountState in preState) { foreach (KeyValuePair storageItem in accountState.Value.Storage) { @@ -192,22 +187,6 @@ private static void InitializeTestState(GeneralStateTest test, WorldState stateP stateProvider.Reset(); } - private bool IsValidBlock(Block block, ISpecProvider specProvider) - { - IBlockTree blockTree = Build.A.BlockTree() - .WithSpecProvider(specProvider) - .WithoutSettingHead - .TestObject; - - var difficultyCalculator = new EthashDifficultyCalculator(specProvider); - var sealer = new EthashSealValidator(_logManager, difficultyCalculator, new CryptoRandom(), new Ethash(_logManager), Timestamper.Default); - IHeaderValidator headerValidator = new HeaderValidator(blockTree, sealer, specProvider, _logManager); - IUnclesValidator unclesValidator = new UnclesValidator(blockTree, headerValidator, _logManager); - IBlockValidator blockValidator = new BlockValidator(_txValidator, headerValidator, unclesValidator, specProvider, _logManager); - - return blockValidator.ValidateOrphanedBlock(block, out _); - } - private List RunAssertions(GeneralStateTest test, IWorldState stateProvider) { List differences = []; diff --git a/src/Nethermind/Ethereum.Test.Base/JsonToEthereumTest.cs b/src/Nethermind/Ethereum.Test.Base/JsonToEthereumTest.cs index fb9c6769bcc..fac8ee33244 100644 --- a/src/Nethermind/Ethereum.Test.Base/JsonToEthereumTest.cs +++ b/src/Nethermind/Ethereum.Test.Base/JsonToEthereumTest.cs @@ -26,8 +26,8 @@ public static IReleaseSpec ParseSpec(string network) network = network.Replace("EIP150", "TangerineWhistle"); network = network.Replace("EIP158", "SpuriousDragon"); network = network.Replace("DAO", "Dao"); - network = network.Replace("Merged", "GrayGlacier"); - network = network.Replace("Merge", "GrayGlacier"); + network = network.Replace("Merged", "Paris"); + network = network.Replace("Merge", "Paris"); network = network.Replace("London+3540+3670", "Shanghai"); network = network.Replace("GrayGlacier+3540+3670", "Shanghai"); network = network.Replace("GrayGlacier+3860", "Shanghai"); @@ -55,6 +55,7 @@ public static IReleaseSpec ParseSpec(string network) "Istanbul" => Istanbul.Instance, "Berlin" => Berlin.Instance, "London" => London.Instance, + "ArrowGlacier" => ArrowGlacier.Instance, "GrayGlacier" => GrayGlacier.Instance, "Shanghai" => Shanghai.Instance, "Cancun" => Cancun.Instance, diff --git a/src/Nethermind/Nethermind.Api.Test/PluginLoaderTests.cs b/src/Nethermind/Nethermind.Api.Test/PluginLoaderTests.cs index 9671c959d0f..252b02cede9 100644 --- a/src/Nethermind/Nethermind.Api.Test/PluginLoaderTests.cs +++ b/src/Nethermind/Nethermind.Api.Test/PluginLoaderTests.cs @@ -23,9 +23,9 @@ public class PluginLoaderTests public void full_lexicographical_order() { IFileSystem fileSystem = Substitute.For(); - IPluginLoader loader = new PluginLoader("", fileSystem, typeof(AuRaPlugin), typeof(CliquePlugin), - typeof(EthashPlugin), typeof(NethDevPlugin), typeof(HivePlugin), typeof(TestPlugin)); - loader.Load(new TestLogManager()); + IPluginLoader loader = new PluginLoader(string.Empty, fileSystem, new TestLogManager().GetClassLogger(), + typeof(AuRaPlugin), typeof(CliquePlugin), typeof(EthashPlugin), typeof(NethDevPlugin), typeof(HivePlugin), typeof(TestPlugin)); + loader.Load(); loader.OrderPlugins(new PluginConfig { PluginOrder = Array.Empty() }); var expected = new List { @@ -43,11 +43,11 @@ public void full_lexicographical_order() public void full_order() { IFileSystem fileSystem = Substitute.For(); - IPluginLoader loader = new PluginLoader("", fileSystem, typeof(AuRaPlugin), typeof(CliquePlugin), - typeof(EthashPlugin), typeof(NethDevPlugin), typeof(HivePlugin), typeof(TestPlugin)); - loader.Load(new TestLogManager()); + IPluginLoader loader = new PluginLoader(string.Empty, fileSystem, new TestLogManager().GetClassLogger(), + typeof(AuRaPlugin), typeof(CliquePlugin), typeof(EthashPlugin), typeof(NethDevPlugin), typeof(HivePlugin), typeof(TestPlugin)); + loader.Load(); IPluginConfig pluginConfig = - new PluginConfig { PluginOrder = new[] { "Hive", "TestPlugin", "NethDev", "Ethash", "Clique", "Aura" } }; + new PluginConfig { PluginOrder = ["Hive", "TestPlugin", "NethDev", "Ethash", "Clique", "Aura"] }; loader.OrderPlugins(pluginConfig); var expected = new List @@ -66,11 +66,11 @@ public void full_order() public void partial_lexicographical_order() { IFileSystem fileSystem = Substitute.For(); - IPluginLoader loader = new PluginLoader("", fileSystem, typeof(AuRaPlugin), typeof(CliquePlugin), - typeof(EthashPlugin), typeof(NethDevPlugin), typeof(HivePlugin), typeof(TestPlugin)); - loader.Load(new TestLogManager()); + IPluginLoader loader = new PluginLoader(string.Empty, fileSystem, new TestLogManager().GetClassLogger(), + typeof(AuRaPlugin), typeof(CliquePlugin), typeof(EthashPlugin), typeof(NethDevPlugin), typeof(HivePlugin), typeof(TestPlugin)); + loader.Load(); IPluginConfig pluginConfig = - new PluginConfig() { PluginOrder = new[] { "Hive", "NethDev", "Ethash" } }; + new PluginConfig() { PluginOrder = ["Hive", "NethDev", "Ethash"] }; loader.OrderPlugins(pluginConfig); var expected = new List @@ -89,10 +89,9 @@ public void partial_lexicographical_order() public void default_config() { IFileSystem fileSystem = Substitute.For(); - IPluginLoader loader = new PluginLoader("", fileSystem, - typeof(EthashPlugin), typeof(NethDevPlugin), typeof(HivePlugin), typeof(HealthChecksPlugin), - typeof(MergePlugin)); - loader.Load(new TestLogManager()); + IPluginLoader loader = new PluginLoader(string.Empty, fileSystem, new TestLogManager().GetClassLogger(), + typeof(EthashPlugin), typeof(NethDevPlugin), typeof(HivePlugin), typeof(HealthChecksPlugin), typeof(MergePlugin)); + loader.Load(); IPluginConfig pluginConfig = new PluginConfig(); loader.OrderPlugins(pluginConfig); diff --git a/src/Nethermind/Nethermind.Api.Test/SinglePluginLoaderTests.cs b/src/Nethermind/Nethermind.Api.Test/SinglePluginLoaderTests.cs index ed735e2e5f2..f8ea82485de 100644 --- a/src/Nethermind/Nethermind.Api.Test/SinglePluginLoaderTests.cs +++ b/src/Nethermind/Nethermind.Api.Test/SinglePluginLoaderTests.cs @@ -4,23 +4,21 @@ using System.Linq; using FluentAssertions; using Nethermind.Api.Extensions; -using Nethermind.Logging; using NUnit.Framework; -namespace Nethermind.Api.Test +namespace Nethermind.Api.Test; + +public class SinglePluginLoaderTests { - public class SinglePluginLoaderTests + [Test] + public void Can_load() { - [Test] - public void Can_load() - { - SinglePluginLoader.Instance.Load(LimboLogs.Instance); - } + SinglePluginLoader.Instance.Load(); + } - [Test] - public void Returns_correct_plugin() - { - SinglePluginLoader.Instance.PluginTypes.FirstOrDefault().Should().Be(typeof(TestPlugin)); - } + [Test] + public void Returns_correct_plugin() + { + SinglePluginLoader.Instance.PluginTypes.FirstOrDefault().Should().Be(typeof(TestPlugin)); } } diff --git a/src/Nethermind/Nethermind.Api/Extensions/IPluginLoader.cs b/src/Nethermind/Nethermind.Api/Extensions/IPluginLoader.cs index 033246a1ba1..23145817eac 100644 --- a/src/Nethermind/Nethermind.Api/Extensions/IPluginLoader.cs +++ b/src/Nethermind/Nethermind.Api/Extensions/IPluginLoader.cs @@ -3,16 +3,14 @@ using System; using System.Collections.Generic; -using Nethermind.Logging; -namespace Nethermind.Api.Extensions +namespace Nethermind.Api.Extensions; + +public interface IPluginLoader { - public interface IPluginLoader - { - IEnumerable PluginTypes { get; } + IEnumerable PluginTypes { get; } - void Load(ILogManager logManager); + void Load(); - public void OrderPlugins(IPluginConfig pluginConfig); - } + public void OrderPlugins(IPluginConfig pluginConfig); } diff --git a/src/Nethermind/Nethermind.Api/Extensions/PluginLoader.cs b/src/Nethermind/Nethermind.Api/Extensions/PluginLoader.cs index 5cddfabf5b5..8a4f3741934 100644 --- a/src/Nethermind/Nethermind.Api/Extensions/PluginLoader.cs +++ b/src/Nethermind/Nethermind.Api/Extensions/PluginLoader.cs @@ -10,127 +10,119 @@ using System.Runtime.Loader; using Nethermind.Logging; -namespace Nethermind.Api.Extensions +namespace Nethermind.Api.Extensions; + +public class PluginLoader(string pluginPath, IFileSystem fileSystem, ILogger logger, params Type[] embedded) : IPluginLoader { - public class PluginLoader : IPluginLoader - { - private readonly List _pluginTypes = new(); - private readonly IFileSystem _fileSystem; - private readonly Type[] _embedded; - private readonly string _pluginsDirectory; + private readonly ILogger _logger = logger; + private readonly List _pluginTypes = []; + private readonly IFileSystem _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + private readonly Type[] _embedded = embedded; + private readonly string _pluginsDirectory = pluginPath ?? throw new ArgumentNullException(nameof(pluginPath)); - public IEnumerable PluginTypes => _pluginTypes; + public IEnumerable PluginTypes => _pluginTypes; - public PluginLoader(string pluginPath, IFileSystem fileSystem, params Type[] embedded) + public void Load() + { + if (_logger.IsInfo) _logger.Info("Loading embedded plugins"); + foreach (Type embeddedPlugin in _embedded) { - _pluginsDirectory = pluginPath ?? throw new ArgumentNullException(nameof(pluginPath)); - _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); - _embedded = embedded; + if (_logger.IsInfo) _logger.Info($" Found plugin type {embeddedPlugin}"); + _pluginTypes.Add(embeddedPlugin); } - public void Load(ILogManager logManager) + string baseDir = string.Empty.GetApplicationResourcePath(); + string pluginAssembliesDir = _pluginsDirectory.GetApplicationResourcePath(); + if (!_fileSystem.Directory.Exists(pluginAssembliesDir)) { - ILogger logger = logManager.GetClassLogger(); - if (logger.IsInfo) logger.Info("Loading embedded plugins"); - foreach (Type embeddedPlugin in _embedded) - { - if (logger.IsInfo) logger.Info($" Found plugin type {embeddedPlugin}"); - _pluginTypes.Add(embeddedPlugin); - } + if (_logger.IsWarn) _logger.Warn($"Plugin assemblies folder {pluginAssembliesDir} was not found. Skipping."); + return; + } - string baseDir = string.Empty.GetApplicationResourcePath(); - string pluginAssembliesDir = _pluginsDirectory.GetApplicationResourcePath(); - if (!_fileSystem.Directory.Exists(pluginAssembliesDir)) - { - if (logger.IsWarn) logger.Warn($"Plugin assemblies folder {pluginAssembliesDir} was not found. Skipping."); - return; - } + string[] assemblies = _fileSystem.Directory.GetFiles(pluginAssembliesDir, "*.dll"); + if (assemblies.Length > 0) + { + if (_logger.IsInfo) _logger.Info($"Loading {assemblies.Length} assemblies from {pluginAssembliesDir}"); + } - string[] assemblies = _fileSystem.Directory.GetFiles(pluginAssembliesDir, "*.dll"); - if (assemblies.Length > 0) - { - if (logger.IsInfo) logger.Info($"Loading {assemblies.Length} assemblies from {pluginAssembliesDir}"); - } + foreach (string assemblyName in assemblies) + { + string pluginAssembly = _fileSystem.Path.GetFileNameWithoutExtension(assemblyName); - foreach (string assemblyName in assemblies) + try { - string pluginAssembly = _fileSystem.Path.GetFileNameWithoutExtension(assemblyName); - - try + if (_logger.IsInfo) _logger.Warn($"Loading assembly {pluginAssembly}"); + string assemblyPath = _fileSystem.Path.Combine(pluginAssembliesDir, assemblyName); + Assembly assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(assemblyPath); + AssemblyLoadContext.Default.Resolving += (_, name) => { - if (logger.IsInfo) logger.Warn($"Loading assembly {pluginAssembly}"); - string assemblyPath = _fileSystem.Path.Combine(pluginAssembliesDir, assemblyName); - Assembly assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(assemblyPath); - AssemblyLoadContext.Default.Resolving += (_, name) => + string fileName = name.Name + ".dll"; + try { - string fileName = name.Name + ".dll"; - try - { - return AssemblyLoadContext.Default.LoadFromAssemblyPath(_fileSystem.Path.Combine(pluginAssembliesDir, fileName)); - } - catch (FileNotFoundException) - { - return AssemblyLoadContext.Default.LoadFromAssemblyPath(_fileSystem.Path.Combine(baseDir, fileName)); - } - }; + return AssemblyLoadContext.Default.LoadFromAssemblyPath(_fileSystem.Path.Combine(pluginAssembliesDir, fileName)); + } + catch (FileNotFoundException) + { + return AssemblyLoadContext.Default.LoadFromAssemblyPath(_fileSystem.Path.Combine(baseDir, fileName)); + } + }; - foreach (Type type in assembly.GetExportedTypes().Where(t => !t.IsInterface)) + foreach (Type type in assembly.GetExportedTypes().Where(t => !t.IsInterface)) + { + if (typeof(INethermindPlugin).IsAssignableFrom(type)) { - if (typeof(INethermindPlugin).IsAssignableFrom(type)) + if (!PluginTypes.Contains(type)) { - if (!PluginTypes.Contains(type)) - { - if (logger.IsInfo) logger.Warn($" Found plugin type {pluginAssembly}"); - _pluginTypes.Add(type); - } + if (_logger.IsInfo) _logger.Warn($" Found plugin type {pluginAssembly}"); + _pluginTypes.Add(type); } } } - catch (Exception e) - { - logger.Error($"Failed to load plugin {pluginAssembly}", e); - } + } + catch (Exception e) + { + _logger.Error($"Failed to load plugin {pluginAssembly}", e); } } + } - public void OrderPlugins(IPluginConfig pluginConfig) + public void OrderPlugins(IPluginConfig pluginConfig) + { + List order = pluginConfig.PluginOrder.Select(s => s.ToLower() + "plugin").ToList(); + _pluginTypes.Sort((f, s) => { - List order = pluginConfig.PluginOrder.Select(s => s.ToLower() + "plugin").ToList(); - _pluginTypes.Sort((f, s) => + bool fIsConsensus = typeof(IConsensusPlugin).IsAssignableFrom(f); + bool sIsConsensus = typeof(IConsensusPlugin).IsAssignableFrom(s); + + // Consensus plugins always at front + if (fIsConsensus && !sIsConsensus) { - bool fIsConsensus = typeof(IConsensusPlugin).IsAssignableFrom(f); - bool sIsConsensus = typeof(IConsensusPlugin).IsAssignableFrom(s); + return -1; + } - // Consensus plugins always at front - if (fIsConsensus && !sIsConsensus) - { - return -1; - } + if (sIsConsensus && !fIsConsensus) + { + return 1; + } - if (sIsConsensus && !fIsConsensus) + int fPos = order.IndexOf(f.Name.ToLower()); + int sPos = order.IndexOf(s.Name.ToLower()); + if (fPos == -1) + { + if (sPos == -1) { - return 1; + return f.Name.CompareTo(s.Name); } - int fPos = order.IndexOf(f.Name.ToLower()); - int sPos = order.IndexOf(s.Name.ToLower()); - if (fPos == -1) - { - if (sPos == -1) - { - return f.Name.CompareTo(s.Name); - } - - return 1; - } + return 1; + } - if (sPos == -1) - { - return -1; - } + if (sPos == -1) + { + return -1; + } - return fPos.CompareTo(sPos); - }); - } + return fPos.CompareTo(sPos); + }); } } diff --git a/src/Nethermind/Nethermind.Api/Extensions/SinglePluginLoader.cs b/src/Nethermind/Nethermind.Api/Extensions/SinglePluginLoader.cs index def9267ed72..845bd2a4279 100644 --- a/src/Nethermind/Nethermind.Api/Extensions/SinglePluginLoader.cs +++ b/src/Nethermind/Nethermind.Api/Extensions/SinglePluginLoader.cs @@ -4,25 +4,23 @@ using System; using System.Collections.Generic; using System.Linq; -using Nethermind.Logging; -namespace Nethermind.Api.Extensions +namespace Nethermind.Api.Extensions; + +/// +/// This class is introduced for easier testing of the plugins under construction - it allows to load a plugin +/// directly from the current solution +/// +/// Type of the plugin to load +public class SinglePluginLoader : IPluginLoader where T : INethermindPlugin { - /// - /// This class is introduced for easier testing of the plugins under construction - it allows to load a plugin - /// directly from the current solution - /// - /// Type of the plugin to load - public class SinglePluginLoader : IPluginLoader where T : INethermindPlugin - { - private SinglePluginLoader() { } + private SinglePluginLoader() { } - public static IPluginLoader Instance { get; } = new SinglePluginLoader(); + public static IPluginLoader Instance { get; } = new SinglePluginLoader(); - public IEnumerable PluginTypes => Enumerable.Repeat(typeof(T), 1); + public IEnumerable PluginTypes => Enumerable.Repeat(typeof(T), 1); - public void Load(ILogManager logManager) { } + public void Load() { } - public void OrderPlugins(IPluginConfig pluginConfig) { } - } + public void OrderPlugins(IPluginConfig pluginConfig) { } } diff --git a/src/Nethermind/Nethermind.Api/IApiWithStores.cs b/src/Nethermind/Nethermind.Api/IApiWithStores.cs index 09142b2c0ed..459b92e3339 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithStores.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithStores.cs @@ -4,7 +4,6 @@ using Autofac; using Nethermind.Blockchain; using Nethermind.Blockchain.Blocks; -using Nethermind.Blockchain.Find; using Nethermind.Blockchain.Receipts; using Nethermind.Consensus; using Nethermind.Core; diff --git a/src/Nethermind/Nethermind.Api/NethermindApi.cs b/src/Nethermind/Nethermind.Api/NethermindApi.cs index bc1288d8529..b6a3574c212 100644 --- a/src/Nethermind/Nethermind.Api/NethermindApi.cs +++ b/src/Nethermind/Nethermind.Api/NethermindApi.cs @@ -77,12 +77,13 @@ public NethermindApi(IConfigProvider configProvider, IJsonSerializer jsonSeriali public IBlockchainBridge CreateBlockchainBridge() { ReadOnlyBlockTree readOnlyTree = BlockTree!.AsReadOnly(); + OverridableWorldStateManager overridableWorldStateManager = new(DbProvider!, WorldStateManager!.TrieStore, LogManager); // TODO: reuse the same trie cache here - ReadOnlyTxProcessingEnv readOnlyTxProcessingEnv = new( - WorldStateManager!, + OverridableTxProcessingEnv txProcessingEnv = new( + overridableWorldStateManager, readOnlyTree, - SpecProvider, + SpecProvider!, LogManager); SimulateReadOnlyBlocksProcessingEnvFactory simulateReadOnlyBlocksProcessingEnvFactory = @@ -97,7 +98,7 @@ public IBlockchainBridge CreateBlockchainBridge() IBlocksConfig blocksConfig = ConfigProvider.GetConfig(); return new BlockchainBridge( - readOnlyTxProcessingEnv, + txProcessingEnv, simulateReadOnlyBlocksProcessingEnvFactory, TxPool, ReceiptFinder, diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Receipts/PersistentReceiptStorageTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Receipts/PersistentReceiptStorageTests.cs index b4e21bae898..fbe5413a0ad 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Receipts/PersistentReceiptStorageTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Receipts/PersistentReceiptStorageTests.cs @@ -7,6 +7,7 @@ using System.Threading; using System.Threading.Tasks; using FluentAssertions; +using FluentAssertions.Equivalency; using Nethermind.Blockchain.Blocks; using Nethermind.Blockchain.Receipts; using Nethermind.Core; @@ -18,6 +19,7 @@ using Nethermind.Db; using Nethermind.Serialization.Rlp; using Nethermind.Specs; +using Nethermind.Specs.Forks; using NSubstitute; using NSubstitute.Core; using NUnit.Framework; @@ -29,6 +31,7 @@ namespace Nethermind.Blockchain.Test.Receipts; [TestFixture(false)] public class PersistentReceiptStorageTests { + private TestSpecProvider _specProvider = new TestSpecProvider(Byzantium.Instance); private TestMemColumnsDb _receiptsDb = null!; private ReceiptsRecovery _receiptsRecovery = null!; private IBlockTree _blockTree = null!; @@ -46,10 +49,9 @@ public PersistentReceiptStorageTests(bool useCompactReceipts) [SetUp] public void SetUp() { - MainnetSpecProvider specProvider = MainnetSpecProvider.Instance; - EthereumEcdsa ethereumEcdsa = new(specProvider.ChainId); + EthereumEcdsa ethereumEcdsa = new(_specProvider.ChainId); _receiptConfig = new ReceiptConfig(); - _receiptsRecovery = new(ethereumEcdsa, specProvider); + _receiptsRecovery = new(ethereumEcdsa, _specProvider); _receiptsDb = new TestMemColumnsDb(); _receiptsDb.GetColumnDb(ReceiptsColumns.Blocks).Set(Keccak.Zero, Array.Empty()); _blockTree = Substitute.For(); @@ -68,7 +70,7 @@ private void CreateStorage() _decoder = new ReceiptArrayStorageDecoder(_useCompactReceipts); _storage = new PersistentReceiptStorage( _receiptsDb, - MainnetSpecProvider.Instance, + _specProvider, _receiptsRecovery, _blockTree, _blockStore, @@ -111,9 +113,10 @@ public void Adds_and_retrieves_receipts_for_block() { var (block, receipts) = InsertBlock(); - _storage.Get(block).Should().BeEquivalentTo(receipts); + _storage.ClearCache(); + _storage.Get(block).Should().BeEquivalentTo(receipts, ReceiptCompareOpt); // second should be from cache - _storage.Get(block).Should().BeEquivalentTo(receipts); + _storage.Get(block).Should().BeEquivalentTo(receipts, ReceiptCompareOpt); } [Test] @@ -128,6 +131,38 @@ public void Adds_should_prefix_key_with_blockNumber() _receiptsDb.GetColumnDb(ReceiptsColumns.Blocks)[blockNumPrefixed].Should().NotBeNull(); } + [Test] + public void Adds_should_forward_write_flags() + { + (Block block, _) = InsertBlock(writeFlags: WriteFlags.DisableWAL); + + Span blockNumPrefixed = stackalloc byte[40]; + block.Number.ToBigEndianByteArray().CopyTo(blockNumPrefixed); // TODO: We don't need to create an array here... + block.Hash!.Bytes.CopyTo(blockNumPrefixed[8..]); + + TestMemDb blockDb = (TestMemDb)_receiptsDb.GetColumnDb(ReceiptsColumns.Blocks); + + blockDb.KeyWasWrittenWithFlags(blockNumPrefixed.ToArray(), WriteFlags.DisableWAL); + } + + [Test, MaxTime(Timeout.MaxTestTime)] + public void Get_receipts_for_block_without_recovering_sender() + { + var (block, receipts) = InsertBlock(); + foreach (Transaction tx in block.Transactions) + { + tx.SenderAddress = null; + } + + _storage.ClearCache(); + _storage.Get(block, recoverSender: false).Should().BeEquivalentTo(receipts, ReceiptCompareOpt); + + foreach (Transaction tx in block.Transactions) + { + tx.SenderAddress.Should().BeNull(); + } + } + [Test] public void Adds_should_attempt_hash_key_first_if_inserted_with_hashkey() { @@ -455,12 +490,18 @@ public void When_NewHeadBlock_ClearOldTxIndex() return (block, receipts); } - private (Block block, TxReceipt[] receipts) InsertBlock(Block? block = null, bool isFinalized = false, long? headNumber = null) + private (Block block, TxReceipt[] receipts) InsertBlock(Block? block = null, bool isFinalized = false, long? headNumber = null, WriteFlags writeFlags = WriteFlags.None) { (block, TxReceipt[] receipts) = PrepareBlock(block, isFinalized, headNumber); - _storage.Insert(block, receipts); + _storage.Insert(block, receipts, writeFlags: writeFlags); _receiptsRecovery.TryRecover(new ReceiptRecoveryBlock(block), receipts); return (block, receipts); } + + private EquivalencyAssertionOptions ReceiptCompareOpt(EquivalencyAssertionOptions opts) + { + return opts + .Excluding(su => su.Error); + } } diff --git a/src/Nethermind/Nethermind.Blockchain/BlockTree.cs b/src/Nethermind/Nethermind.Blockchain/BlockTree.cs index 9cf43a62c25..f649be65a65 100644 --- a/src/Nethermind/Nethermind.Blockchain/BlockTree.cs +++ b/src/Nethermind/Nethermind.Blockchain/BlockTree.cs @@ -446,6 +446,8 @@ public AddBlockResult SuggestBlock(Block block, BlockTreeSuggestOptions options public Hash256? FindBlockHash(long blockNumber) => GetBlockHashOnMainOrBestDifficultyHash(blockNumber); + public bool HasBlock(long blockNumber, Hash256 blockHash) => _blockStore.HasBlock(blockNumber, blockHash); + public BlockHeader? FindHeader(Hash256? blockHash, BlockTreeLookupOptions options, long? blockNumber = null) { if (blockHash is null || blockHash == Keccak.Zero) diff --git a/src/Nethermind/Nethermind.Blockchain/BlockTreeOverlay.cs b/src/Nethermind/Nethermind.Blockchain/BlockTreeOverlay.cs index c16834f63a8..214a0ada99e 100644 --- a/src/Nethermind/Nethermind.Blockchain/BlockTreeOverlay.cs +++ b/src/Nethermind/Nethermind.Blockchain/BlockTreeOverlay.cs @@ -231,6 +231,9 @@ public void UpdateBeaconMainChain(BlockInfo[]? blockInfos, long clearBeaconMainC public Block? FindBlock(long blockNumber, BlockTreeLookupOptions options) => _overlayTree.FindBlock(blockNumber, options) ?? _baseTree.FindBlock(blockNumber, options); + public bool HasBlock(long blockNumber, Hash256 blockHash) => + _overlayTree.HasBlock(blockNumber, blockHash) || _baseTree.HasBlock(blockNumber, blockHash); + public BlockHeader? FindHeader(Hash256 blockHash, BlockTreeLookupOptions options, long? blockNumber = null) => _overlayTree.FindHeader(blockHash, options, blockNumber) ?? _baseTree.FindHeader(blockHash, options, blockNumber); diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs index 98bb1030fff..d4de86e653c 100644 --- a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs @@ -30,6 +30,13 @@ public void SetMetadata(byte[] key, byte[] value) return blockDb.Get(key); } + public bool HasBlock(long blockNumber, Hash256 blockHash) + { + Span dbKey = stackalloc byte[40]; + KeyValueStoreExtensions.GetBlockNumPrefixedKey(blockNumber, blockHash, dbKey); + return blockDb.KeyExists(dbKey); + } + public void Insert(Block block, WriteFlags writeFlags = WriteFlags.None) { if (block.Hash is null) diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs index 05bfec926de..d34dde4ace8 100644 --- a/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs @@ -24,4 +24,5 @@ public interface IBlockStore // These two are used by blocktree. Try not to use them... void SetMetadata(byte[] key, byte[] value); byte[]? GetMetadata(byte[] key); + bool HasBlock(long blockNumber, Hash256 blockHash); } diff --git a/src/Nethermind/Nethermind.Blockchain/Find/IBlockFinder.cs b/src/Nethermind/Nethermind.Blockchain/Find/IBlockFinder.cs index e65b57f8c49..16788b8c901 100644 --- a/src/Nethermind/Nethermind.Blockchain/Find/IBlockFinder.cs +++ b/src/Nethermind/Nethermind.Blockchain/Find/IBlockFinder.cs @@ -25,6 +25,8 @@ public interface IBlockFinder Block? FindBlock(long blockNumber, BlockTreeLookupOptions options); + bool HasBlock(long blockNumber, Hash256 blockHash); + /// Find a header. blockNumber is optional, but specifying it can improve performance. BlockHeader? FindHeader(Hash256 blockHash, BlockTreeLookupOptions options, long? blockNumber = null); diff --git a/src/Nethermind/Nethermind.Blockchain/ReadOnlyBlockTree.cs b/src/Nethermind/Nethermind.Blockchain/ReadOnlyBlockTree.cs index 506973f7194..45aa4969678 100644 --- a/src/Nethermind/Nethermind.Blockchain/ReadOnlyBlockTree.cs +++ b/src/Nethermind/Nethermind.Blockchain/ReadOnlyBlockTree.cs @@ -94,6 +94,8 @@ public void UpdateHeadBlock(Hash256 blockHash) public Block FindBlock(Hash256 blockHash, BlockTreeLookupOptions options, long? blockNumber = null) => _wrapped.FindBlock(blockHash, options, blockNumber); + public bool HasBlock(long blockNumber, Hash256 blockHash) => _wrapped.HasBlock(blockNumber, blockHash); + public BlockHeader FindHeader(Hash256 blockHash, BlockTreeLookupOptions options, long? blockNumber = null) => _wrapped.FindHeader(blockHash, options, blockNumber: blockNumber); public BlockHeader FindHeader(long blockNumber, BlockTreeLookupOptions options) => _wrapped.FindHeader(blockNumber, options); diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/FullInfoReceiptFinder.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/FullInfoReceiptFinder.cs index b28cca1c291..c0da2b210c0 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/FullInfoReceiptFinder.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/FullInfoReceiptFinder.cs @@ -23,10 +23,10 @@ public FullInfoReceiptFinder(IReceiptStorage receiptStorage, IReceiptsRecovery r public Hash256 FindBlockHash(Hash256 txHash) => _receiptStorage.FindBlockHash(txHash); - public TxReceipt[] Get(Block block, bool recover = true) + public TxReceipt[] Get(Block block, bool recover = true, bool recoverSender = true) { var receipts = _receiptStorage.Get(block); - if (recover && _receiptsRecovery.TryRecover(block, receipts) == ReceiptsRecoveryResult.NeedReinsert) + if (recover && _receiptsRecovery.TryRecover(block, receipts, recoverSender) == ReceiptsRecoveryResult.NeedReinsert) { _receiptStorage.Insert(block, receipts); } @@ -41,7 +41,7 @@ public TxReceipt[] Get(Hash256 blockHash, bool recover = true) if (recover && _receiptsRecovery.NeedRecover(receipts)) { var block = _blockFinder.FindBlock(blockHash, BlockTreeLookupOptions.TotalDifficultyNotNeeded); - if (_receiptsRecovery.TryRecover(block, receipts) == ReceiptsRecoveryResult.NeedReinsert) + if (_receiptsRecovery.TryRecover(block, receipts, forceRecoverSender: false) == ReceiptsRecoveryResult.NeedReinsert) { _receiptStorage.Insert(block, receipts); } diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptFinder.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptFinder.cs index 9c6000ba15a..8ac067a8dc1 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptFinder.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptFinder.cs @@ -9,7 +9,7 @@ namespace Nethermind.Blockchain.Receipts public interface IReceiptFinder { Hash256? FindBlockHash(Hash256 txHash); - TxReceipt[] Get(Block block, bool recover = true); + TxReceipt[] Get(Block block, bool recover = true, bool recoverSender = true); TxReceipt[] Get(Hash256 blockHash, bool recover = true); bool CanGetReceiptsByHash(long blockNumber); bool TryGetReceiptsIterator(long blockNumber, Hash256 blockHash, out ReceiptsIterator iterator); diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs index 236540cbe48..dc59a7ddc25 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs @@ -10,7 +10,7 @@ namespace Nethermind.Blockchain.Receipts public interface IReceiptStorage : IReceiptFinder { void Insert(Block block, params TxReceipt[]? txReceipts) => Insert(block, txReceipts, true); - void Insert(Block block, TxReceipt[]? txReceipts, bool ensureCanonical); + void Insert(Block block, TxReceipt[]? txReceipts, bool ensureCanonical, WriteFlags writeFlags = WriteFlags.None); long? LowestInsertedReceiptBlockNumber { get; set; } long MigratedBlockNumber { get; set; } bool HasBlock(long blockNumber, Hash256 hash); diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs index 1cc3b17e9fa..4290af14f2b 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs @@ -40,7 +40,7 @@ public Hash256 FindBlockHash(Hash256 txHash) return receipt?.BlockHash; } - public TxReceipt[] Get(Block block, bool recover = true) => Get(block.Hash); + public TxReceipt[] Get(Block block, bool recover = true, bool recoverSender = true) => Get(block.Hash); public TxReceipt[] Get(Hash256 blockHash, bool recover = true) => _receipts.TryGetValue(blockHash, out TxReceipt[] receipts) ? receipts : Array.Empty(); @@ -62,7 +62,7 @@ public bool TryGetReceiptsIterator(long blockNumber, Hash256 blockHash, out Rece } } - public void Insert(Block block, TxReceipt[] txReceipts, bool ensureCanonical = true) + public void Insert(Block block, TxReceipt[] txReceipts, bool ensureCanonical = true, WriteFlags writeFlags = WriteFlags.None) { _receipts[block.Hash] = txReceipts; if (ensureCanonical) diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs index dadb03df866..d5f05b1a952 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs @@ -21,9 +21,9 @@ private NullReceiptStorage() { } - public void Insert(Block block, TxReceipt[] txReceipts, bool ensureCanonical) { } + public void Insert(Block block, TxReceipt[] txReceipts, bool ensureCanonical, WriteFlags writeFlags) { } - public TxReceipt[] Get(Block block, bool recover = true) => Array.Empty(); + public TxReceipt[] Get(Block block, bool recover = true, bool recoverSender = false) => Array.Empty(); public TxReceipt[] Get(Hash256 blockHash, bool recover = true) => Array.Empty(); public bool CanGetReceiptsByHash(long blockNumber) => true; diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs index 732aaad4db0..4edeedb56c2 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs @@ -137,7 +137,7 @@ private TxReceipt DeserializeReceiptObsolete(Hash256 hash, Span receiptDat return null; } - public TxReceipt[] Get(Block block, bool recover = true) + public TxReceipt[] Get(Block block, bool recover = true, bool recoverSender = true) { if (block.ReceiptsRoot == Keccak.EmptyTreeHash) { @@ -164,7 +164,7 @@ public TxReceipt[] Get(Block block, bool recover = true) if (recover) { - _receiptsRecovery.TryRecover(block, receipts); + _receiptsRecovery.TryRecover(block, receipts, forceRecoverSender: recoverSender); _receiptsCache.Set(blockHash, receipts); } @@ -223,7 +223,7 @@ public TxReceipt[] Get(Hash256 blockHash, bool recover = true) { Block? block = _blockTree.FindBlock(blockHash); if (block is null) return Array.Empty(); - return Get(block, recover); + return Get(block, recover, false); } public bool CanGetReceiptsByHash(long blockNumber) => blockNumber >= MigratedBlockNumber; @@ -263,7 +263,7 @@ public bool TryGetReceiptsIterator(long blockNumber, Hash256 blockHash, out Rece } [SkipLocalsInit] - public void Insert(Block block, TxReceipt[]? txReceipts, bool ensureCanonical = true) + public void Insert(Block block, TxReceipt[]? txReceipts, bool ensureCanonical = true, WriteFlags writeFlags = WriteFlags.None) { txReceipts ??= Array.Empty(); int txReceiptsLength = txReceipts.Length; @@ -286,7 +286,7 @@ public void Insert(Block block, TxReceipt[]? txReceipts, bool ensureCanonical = Span blockNumPrefixed = stackalloc byte[40]; GetBlockNumPrefixedKey(blockNumber, block.Hash!, blockNumPrefixed); - _blocksDb.PutSpan(blockNumPrefixed, stream.AsSpan()); + _blocksDb.PutSpan(blockNumPrefixed, stream.AsSpan(), writeFlags); } if (blockNumber < MigratedBlockNumber) diff --git a/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs b/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs index 24cca9a46d5..958eb673a67 100644 --- a/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs +++ b/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs @@ -146,4 +146,7 @@ public interface ISyncConfig : IConfig [ConfigItem(Description = "_Technical._ MultiSyncModeSelector will wait for header to completely sync first.", DefaultValue = "false", HiddenFromDocs = true)] bool NeedToWaitForHeader { get; set; } + + [ConfigItem(Description = "_Technical._ Run verify trie on state sync is finished.", DefaultValue = "false", HiddenFromDocs = true)] + bool VerifyTrieOnStateSyncFinished { get; } } diff --git a/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs b/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs index c467650ca7a..546d5017286 100644 --- a/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs +++ b/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs @@ -66,6 +66,7 @@ public string? PivotHash public bool? SnapServingEnabled { get; set; } = null; public int MultiSyncModeSelectorLoopTimerMs { get; set; } = 1000; public bool NeedToWaitForHeader { get; set; } + public bool VerifyTrieOnStateSyncFinished { get; set; } public bool TrieHealing { get; set; } = true; public override string ToString() diff --git a/src/Nethermind/Nethermind.Cli/Nethermind.Cli.csproj b/src/Nethermind/Nethermind.Cli/Nethermind.Cli.csproj index 6e9f05573db..b67d8aca117 100644 --- a/src/Nethermind/Nethermind.Cli/Nethermind.Cli.csproj +++ b/src/Nethermind/Nethermind.Cli/Nethermind.Cli.csproj @@ -10,8 +10,8 @@ - + diff --git a/src/Nethermind/Nethermind.Cli/Program.cs b/src/Nethermind/Nethermind.Cli/Program.cs index efadd38771c..848183c4c01 100644 --- a/src/Nethermind/Nethermind.Cli/Program.cs +++ b/src/Nethermind/Nethermind.Cli/Program.cs @@ -3,12 +3,12 @@ using System; using System.Collections.Generic; +using System.CommandLine; using System.IO; using System.IO.Abstractions; using System.Runtime.CompilerServices; using System.Text; using Jint.Native; -using McMaster.Extensions.CommandLineUtils; using Nethermind.Cli.Console; using Nethermind.Cli.Modules; using Nethermind.Config; @@ -16,161 +16,160 @@ using Nethermind.Serialization.Json; [assembly: InternalsVisibleTo("Nethermind.Cli.Test")] -namespace Nethermind.Cli +namespace Nethermind.Cli; + +public static class Program { - public static class Program + private static readonly Dictionary _availableColorSchemes = new(){ {"basic", BasicColorScheme.Instance }, + {"dracula", DraculaColorScheme.Instance }}; + private static readonly IJsonSerializer Serializer = new EthereumJsonSerializer(); + + public static void Main(string[] args) { - private static readonly Dictionary _availableColorSchemes = new(){ {"basic", BasicColorScheme.Instance }, - {"dracula", DraculaColorScheme.Instance }}; - private static readonly IJsonSerializer Serializer = new EthereumJsonSerializer(); + CliOption colorSchemeOption = new("--colorScheme", "-cs") + { + Description = "Color Scheme. Possible values: Basic|Dracula", + HelpName = "colorScheme" + }; + CliOption nodeAddressOption = new("--address", "-a") + { + Description = "Node Address", + HelpName = "address" + }; + CliRootCommand rootCommand = [colorSchemeOption, nodeAddressOption]; - public static void Main(string[] args) + rootCommand.SetAction(parseResult => { - CommandLineApplication app = new() { Name = "Nethermind.Cli" }; - _ = app.HelpOption("-?|-h|--help"); + string? colorSchemeValue = parseResult.GetValue(colorSchemeOption); + ColorScheme? cs; + ICliConsole cliConsole = colorSchemeValue is not null && (cs = MapColorScheme(colorSchemeValue)) is not null + ? new ColorfulCliConsole(cs) + : new CliConsole(); + + var historyManager = new StatementHistoryManager(cliConsole, new FileSystem()); + ILogManager logManager = new OneLoggerLogManager(new(new CliLogger(cliConsole))); + ICliEngine engine = new CliEngine(cliConsole); + INodeManager nodeManager = new NodeManager(engine, Serializer, cliConsole, logManager); + var moduleLoader = new CliModuleLoader(engine, nodeManager, cliConsole); + + engine.JintEngine.SetValue("serialize", new Action(v => + { + string text = Serializer.Serialize(v.ToObject(), true); + cliConsole.WriteGood(text); + })); - var colorSchemeOption = app.Option("-cs|--colorScheme ", "Color Scheme. Possible values: Basic|Dracula", CommandOptionType.SingleValue); - var nodeAddressOption = app.Option("-a|--address
", "Node Address", CommandOptionType.SingleValue); + moduleLoader.DiscoverAndLoadModules(); + ReadLine.AutoCompletionHandler = new AutoCompletionHandler(moduleLoader); - app.OnExecute(() => - { - ColorScheme? cs; - ICliConsole cliConsole = colorSchemeOption.HasValue() && (cs = MapColorScheme(colorSchemeOption.Value()!)) is not null - ? new ColorfulCliConsole(cs) - : new CliConsole(); - - var historyManager = new StatementHistoryManager(cliConsole, new FileSystem()); - ILogManager logManager = new OneLoggerLogManager(new(new CliLogger(cliConsole))); - ICliEngine engine = new CliEngine(cliConsole); - INodeManager nodeManager = new NodeManager(engine, Serializer, cliConsole, logManager); - var moduleLoader = new CliModuleLoader(engine, nodeManager, cliConsole); - - engine.JintEngine.SetValue("serialize", new Action(v => - { - string text = Serializer.Serialize(v.ToObject(), true); - cliConsole.WriteGood(text); - })); - - moduleLoader.DiscoverAndLoadModules(); - ReadLine.AutoCompletionHandler = new AutoCompletionHandler(moduleLoader); - - string nodeAddress = nodeAddressOption.HasValue() - ? nodeAddressOption.Value()! - : "http://localhost:8545"; - nodeManager.SwitchUri(new Uri(nodeAddress)); - historyManager.Init(); - TestConnection(nodeManager, engine, cliConsole); - cliConsole.WriteLine(); - RunEvalLoop(engine, historyManager, cliConsole); - - cliConsole.ResetColor(); - return ExitCodes.Ok; - }); + string nodeAddress = parseResult.GetValue(nodeAddressOption) ?? "http://localhost:8545"; + nodeManager.SwitchUri(new Uri(nodeAddress)); + historyManager.Init(); + TestConnection(nodeManager, engine, cliConsole); + cliConsole.WriteLine(); + RunEvalLoop(engine, historyManager, cliConsole); - try - { - app.Execute(args); - } - catch (CommandParsingException) - { - app.ShowHelp(); - } - } + cliConsole.ResetColor(); + return ExitCodes.Ok; + }); + + CliConfiguration cli = new(rootCommand); - private static void TestConnection(INodeManager nodeManager, ICliEngine cliEngine, ICliConsole cliConsole) + cli.Invoke(args); + } + + private static void TestConnection(INodeManager nodeManager, ICliEngine cliEngine, ICliConsole cliConsole) + { + cliConsole.WriteLine($"Connecting to {nodeManager.CurrentUri}"); + JsValue result = cliEngine.Execute("web3.clientVersion"); + if (result != JsValue.Null) { - cliConsole.WriteLine($"Connecting to {nodeManager.CurrentUri}"); - JsValue result = cliEngine.Execute("web3.clientVersion"); - if (result != JsValue.Null) - { - cliConsole.WriteGood("Connected"); - } + cliConsole.WriteGood("Connected"); } + } - internal static string RemoveDangerousCharacters(string statement) + internal static string RemoveDangerousCharacters(string statement) + { + if (string.IsNullOrWhiteSpace(statement)) { - if (string.IsNullOrWhiteSpace(statement)) - { - return ""; - } + return ""; + } - StringBuilder cleaned = new(); - for (int i = 0; i < statement.Length; i++) + StringBuilder cleaned = new(); + for (int i = 0; i < statement.Length; i++) + { + switch (statement[i]) { - switch (statement[i]) - { - case '\x0008': - if (cleaned.Length != 0) - { - cleaned.Remove(cleaned.Length - 1, 1); - } - break; - case '\x0000': - return cleaned.ToString(); - default: - cleaned.Append(statement[i]); - break; - } + case '\x0008': + if (cleaned.Length != 0) + { + cleaned.Remove(cleaned.Length - 1, 1); + } + break; + case '\x0000': + return cleaned.ToString(); + default: + cleaned.Append(statement[i]); + break; } - - return cleaned.ToString(); } - private static void RunEvalLoop(ICliEngine engine, StatementHistoryManager historyManager, ICliConsole console) + return cleaned.ToString(); + } + + private static void RunEvalLoop(ICliEngine engine, StatementHistoryManager historyManager, ICliConsole console) + { + while (true) { - while (true) + try { - try + const int bufferSize = 1024 * 16; + string statement; + using (Stream inStream = System.Console.OpenStandardInput(bufferSize)) { - const int bufferSize = 1024 * 16; - string statement; - using (Stream inStream = System.Console.OpenStandardInput(bufferSize)) - { - Colorful.Console.SetIn(new StreamReader(inStream, Colorful.Console.InputEncoding, false, bufferSize)); - console.WriteLessImportant("nethermind> "); - statement = console.Terminal == Terminal.Cygwin ? Colorful.Console.ReadLine() : ReadLine.Read(); - statement = RemoveDangerousCharacters(statement); + Colorful.Console.SetIn(new StreamReader(inStream, Colorful.Console.InputEncoding, false, bufferSize)); + console.WriteLessImportant("nethermind> "); + statement = console.Terminal == Terminal.Cygwin ? Colorful.Console.ReadLine() : ReadLine.Read(); + statement = RemoveDangerousCharacters(statement); - historyManager.UpdateHistory(statement); - } - - if (statement == "exit") - { - break; - } - - JsValue result = engine.Execute(statement); - WriteResult(console, result); + historyManager.UpdateHistory(statement); } - catch (Exception e) + + if (statement == "exit") { - console.WriteException(e); + break; } - } - } - private static void WriteResult(ICliConsole cliConsole, JsValue result) - { - if (result.IsObject() && result.AsObject().Class == "Function") - { - cliConsole.WriteGood(result.ToString()); - cliConsole.WriteLine(); + JsValue result = engine.Execute(statement); + WriteResult(console, result); } - else if (!result.IsNull()) + catch (Exception e) { - string text = Serializer.Serialize(result.ToObject(), true); - cliConsole.WriteGood(text); - } - else - { - cliConsole.WriteLessImportant("null"); - cliConsole.WriteLine(); + console.WriteException(e); } } + } - private static ColorScheme? MapColorScheme(string colorSchemeOption) + private static void WriteResult(ICliConsole cliConsole, JsValue result) + { + if (result.IsObject() && result.AsObject().Class == "Function") { - return _availableColorSchemes.TryGetValue(colorSchemeOption.ToLower(), out var scheme) ? scheme : null; + cliConsole.WriteGood(result.ToString()); + cliConsole.WriteLine(); } + else if (!result.IsNull()) + { + string text = Serializer.Serialize(result.ToObject(), true); + cliConsole.WriteGood(text); + } + else + { + cliConsole.WriteLessImportant("null"); + cliConsole.WriteLine(); + } + } + + private static ColorScheme? MapColorScheme(string colorSchemeOption) + { + return _availableColorSchemes.TryGetValue(colorSchemeOption.ToLower(), out var scheme) ? scheme : null; } } diff --git a/src/Nethermind/Nethermind.Config.Test/ConfigProvider_FindIncorrectSettings_Tests.cs b/src/Nethermind/Nethermind.Config.Test/ConfigProvider_FindIncorrectSettings_Tests.cs index d4352462177..d6c489d6eb9 100644 --- a/src/Nethermind/Nethermind.Config.Test/ConfigProvider_FindIncorrectSettings_Tests.cs +++ b/src/Nethermind/Nethermind.Config.Test/ConfigProvider_FindIncorrectSettings_Tests.cs @@ -6,141 +6,113 @@ using NSubstitute; using NUnit.Framework; -namespace Nethermind.Config.Test +namespace Nethermind.Config.Test; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class ConfigProvider_FindIncorrectSettings_Tests { - [TestFixture] - [Parallelizable(ParallelScope.All)] - public class ConfigProvider_FindIncorrectSettings_Tests + [Test] + public void CorrectSettingNames_CaseInsensitive() + { + JsonConfigSource? jsonSource = new("SampleJson/CorrectSettingNames.json"); + + IEnvironment? env = Substitute.For(); + env.GetEnvironmentVariables().Returns(new Dictionary() { { "NETHERMIND_NETWORKCONFIG_MAXCANDIDATEPEERCOUNT", "500" } }); + EnvConfigSource? envSource = new(env); + + ArgsConfigSource? argsSource = new(new Dictionary() { + { "DiscoveryConfig.BucketSize", "10" }, + { "NetworkConfig.DiscoveryPort", "30301" } }); + + ConfigProvider? configProvider = new(); + configProvider.AddSource(jsonSource); + configProvider.AddSource(envSource); + configProvider.AddSource(argsSource); + + configProvider.Initialize(); + + + (string ErrorMsg, IList<(IConfigSource Source, string Category, string Name)> Errors) res = configProvider.FindIncorrectSettings(); + + Assert.That(res.Errors.Count, Is.EqualTo(0)); + } + + [Test] + public void NoCategorySettings() { - [Test] - public void CorrectSettingNames_CaseInsensitive() - { - JsonConfigSource? jsonSource = new("SampleJson/CorrectSettingNames.cfg"); - - IEnvironment? env = Substitute.For(); - env.GetEnvironmentVariables().Returns(new Dictionary() { { "NETHERMIND_NETWORKCONFIG_MAXCANDIDATEPEERCOUNT", "500" } }); - EnvConfigSource? envSource = new(env); - - ArgsConfigSource? argsSource = new(new Dictionary() { - { "DiscoveryConfig.BucketSize", "10" }, - { "NetworkConfig.DiscoveryPort", "30301" } }); - - ConfigProvider? configProvider = new(); - configProvider.AddSource(jsonSource); - configProvider.AddSource(envSource); - configProvider.AddSource(argsSource); - - configProvider.Initialize(); - - - (string ErrorMsg, IList<(IConfigSource Source, string Category, string Name)> Errors) res = configProvider.FindIncorrectSettings(); - - Assert.That(res.Errors.Count, Is.EqualTo(0)); - } - - [Test] - public void NoCategorySettings() - { - IEnvironment? env = Substitute.For(); - env.GetEnvironmentVariables().Returns(new Dictionary() { - { "NETHERMIND_CLI_SWITCH_LOCAL", "http://localhost:80" }, - { "NETHERMIND_MONITORING_JOB", "nethermindJob" }, - { "NETHERMIND_MONITORING_GROUP", "nethermindGroup" }, - { "NETHERMIND_ENODE_IPADDRESS", "1.2.3.4" }, - { "NETHERMIND_HIVE_ENABLED", "true" }, - { "NETHERMIND_URL", "http://test:80" }, - { "NETHERMIND_CORS_ORIGINS", "*" }, - { "NETHERMIND_CONFIG", "test2.cfg" }, - { "NETHERMIND_XYZ", "xyz" }, // not existing, should get error - { "QWER", "qwerty" } // not Nethermind setting, no error - }); - EnvConfigSource? envSource = new(env); - - ArgsConfigSource? argsSource = new(new Dictionary() { - { "config", "test.cfg" }, - { "datadir", "Data" }, - { "ConfigsDirectory", "ConfDir" }, - { "baseDbPath", "DB" }, - { "log", "info" }, - { "loggerConfigSource", "logSource" }, - { "pluginsDirectory", "Plugins" }, - { "Abc", "abc" } // not existing, should get error - }); - - ConfigProvider? configProvider = new(); - configProvider.AddSource(envSource); - configProvider.AddSource(argsSource); - - configProvider.Initialize(); - - (string ErrorMsg, IList<(IConfigSource Source, string Category, string Name)> Errors) res = configProvider.FindIncorrectSettings(); - - Assert.That(res.Errors.Count, Is.EqualTo(2)); - Assert.That(res.Errors[0].Name, Is.EqualTo("XYZ")); - Assert.That(res.Errors[1].Name, Is.EqualTo("Abc")); - Assert.That(res.ErrorMsg, Is.EqualTo($"ConfigType:EnvironmentVariable(NETHERMIND_*)|Category:|Name:XYZ{Environment.NewLine}ConfigType:RuntimeOption|Category:|Name:Abc")); - - } - - [Test] - public void SettingWithTypos() - { - JsonConfigSource? jsonSource = new("SampleJson/ConfigWithTypos.cfg"); - - IEnvironment? env = Substitute.For(); - env.GetEnvironmentVariables().Returns(new Dictionary() { - { "NETHERMIND_NETWORKCONFIG_MAXCANDIDATEPERCOUNT", "500" } // incorrect, should be NETHERMIND_NETWORKCONFIG_MAXCANDIDATEPEERCOUNT - }); - EnvConfigSource? envSource = new(env); - - ArgsConfigSource? argsSource = new(new Dictionary() { - { "DiscoveryConfig.BucketSize", "10" }, - { "NetworkConfig.DiscoverPort", "30301" }, // incorrect, should be NetworkConfig.DiscoveryPort - { "Network.P2PPort", "30301" } }); - - ConfigProvider? configProvider = new(); - configProvider.AddSource(jsonSource); - configProvider.AddSource(envSource); - configProvider.AddSource(argsSource); - - configProvider.Initialize(); - - (string ErrorMsg, IList<(IConfigSource Source, string Category, string Name)> Errors) res = configProvider.FindIncorrectSettings(); - - Assert.That(res.Errors.Count, Is.EqualTo(4)); - Assert.That(res.Errors[0].Name, Is.EqualTo("Concurrenc")); - Assert.That(res.Errors[1].Category, Is.EqualTo("BlomConfig")); - Assert.That(res.Errors[2].Name, Is.EqualTo("MAXCANDIDATEPERCOUNT")); - Assert.That(res.Errors[3].Name, Is.EqualTo("DiscoverPort")); - Assert.That(res.ErrorMsg, Is.EqualTo($"ConfigType:JsonConfigFile|Category:DiscoveRyConfig|Name:Concurrenc{Environment.NewLine}ConfigType:JsonConfigFile|Category:BlomConfig|Name:IndexLevelBucketSizes{Environment.NewLine}ConfigType:EnvironmentVariable(NETHERMIND_*)|Category:NETWORKCONFIG|Name:MAXCANDIDATEPERCOUNT{Environment.NewLine}ConfigType:RuntimeOption|Category:NetworkConfig|Name:DiscoverPort")); - } - - [Test] - public void IncorrectFormat() - { - IEnvironment? env = Substitute.For(); - env.GetEnvironmentVariables().Returns(new Dictionary() { - { "NETHERMIND_NETWORKCONFIGMAXCANDIDATEPEERCOUNT", "500" } // incorrect, should be NETHERMIND_NETWORKCONFIG_MAXCANDIDATEPEERCOUNT - }); - EnvConfigSource? envSource = new(env); - - ArgsConfigSource? argsSource = new(new Dictionary() { - { "DiscoveryConfig.BucketSize", "10" }, - { "NetworkConfigP2PPort", "30301" } }); // incorrect, should be Network.P2PPort - - ConfigProvider? configProvider = new(); - configProvider.AddSource(envSource); - configProvider.AddSource(argsSource); - - configProvider.Initialize(); - - (string ErrorMsg, IList<(IConfigSource Source, string Category, string Name)> Errors) res = configProvider.FindIncorrectSettings(); - - Assert.That(res.Errors.Count, Is.EqualTo(2)); - Assert.That(res.Errors[0].Name, Is.EqualTo("NETWORKCONFIGMAXCANDIDATEPEERCOUNT")); - Assert.That(res.Errors[1].Name, Is.EqualTo("NetworkConfigP2PPort")); - Assert.That(res.ErrorMsg, Is.EqualTo($"ConfigType:EnvironmentVariable(NETHERMIND_*)|Category:|Name:NETWORKCONFIGMAXCANDIDATEPEERCOUNT{Environment.NewLine}ConfigType:RuntimeOption|Category:|Name:NetworkConfigP2PPort")); - } + IEnvironment? env = Substitute.For(); + env.GetEnvironmentVariables().Returns(new Dictionary() { + { "NETHERMIND_CLI_SWITCH_LOCAL", "http://localhost:80" }, + { "NETHERMIND_MONITORING_JOB", "nethermindJob" }, + { "NETHERMIND_MONITORING_GROUP", "nethermindGroup" }, + { "NETHERMIND_ENODE_IPADDRESS", "1.2.3.4" }, + { "NETHERMIND_URL", "http://test:80" }, + { "NETHERMIND_CORS_ORIGINS", "*" }, + { "NETHERMIND_CONFIG", "test2.json" }, + { "NETHERMIND_XYZ", "xyz" }, // not existing, should get error + { "QWER", "qwerty" } // not Nethermind setting, no error + }); + EnvConfigSource? envSource = new(env); + + ConfigProvider? configProvider = new(); + configProvider.AddSource(envSource); + + configProvider.Initialize(); + + (string ErrorMsg, IList<(IConfigSource Source, string Category, string Name)> Errors) res = configProvider.FindIncorrectSettings(); + + Assert.That(res.Errors.Count, Is.EqualTo(1)); + Assert.That(res.Errors[0].Name, Is.EqualTo("XYZ")); + Assert.That(res.ErrorMsg, Is.EqualTo($"ConfigType:EnvironmentVariable(NETHERMIND_*)|Category:|Name:XYZ")); } + + [Test] + public void SettingWithTypos() + { + JsonConfigSource? jsonSource = new("SampleJson/ConfigWithTypos.json"); + + IEnvironment? env = Substitute.For(); + env.GetEnvironmentVariables().Returns(new Dictionary() { + { "NETHERMIND_NETWORKCONFIG_MAXCANDIDATEPERCOUNT", "500" } // incorrect, should be NETHERMIND_NETWORKCONFIG_MAXCANDIDATEPEERCOUNT + }); + EnvConfigSource? envSource = new(env); + + ConfigProvider? configProvider = new(); + configProvider.AddSource(jsonSource); + configProvider.AddSource(envSource); + + configProvider.Initialize(); + + (string ErrorMsg, IList<(IConfigSource Source, string Category, string Name)> Errors) res = configProvider.FindIncorrectSettings(); + + Assert.That(res.Errors.Count, Is.EqualTo(3)); + Assert.That(res.Errors[0].Name, Is.EqualTo("Concurrenc")); + Assert.That(res.Errors[1].Category, Is.EqualTo("BlomConfig")); + Assert.That(res.Errors[2].Name, Is.EqualTo("MAXCANDIDATEPERCOUNT")); + Assert.That(res.ErrorMsg, Is.EqualTo($"ConfigType:JsonConfigFile|Category:DiscoveRyConfig|Name:Concurrenc{Environment.NewLine}ConfigType:JsonConfigFile|Category:BlomConfig|Name:IndexLevelBucketSizes{Environment.NewLine}ConfigType:EnvironmentVariable(NETHERMIND_*)|Category:NETWORKCONFIG|Name:MAXCANDIDATEPERCOUNT")); + } + + [Test] + public void IncorrectFormat() + { + IEnvironment? env = Substitute.For(); + env.GetEnvironmentVariables().Returns(new Dictionary() { + { "NETHERMIND_NETWORKCONFIGMAXCANDIDATEPEERCOUNT", "500" } // incorrect, should be NETHERMIND_NETWORKCONFIG_MAXCANDIDATEPEERCOUNT + }); + EnvConfigSource? envSource = new(env); + + ConfigProvider? configProvider = new(); + configProvider.AddSource(envSource); + + configProvider.Initialize(); + + (string ErrorMsg, IList<(IConfigSource Source, string Category, string Name)> Errors) res = configProvider.FindIncorrectSettings(); + + Assert.That(res.Errors.Count, Is.EqualTo(1)); + Assert.That(res.Errors[0].Name, Is.EqualTo("NETWORKCONFIGMAXCANDIDATEPEERCOUNT")); + Assert.That(res.ErrorMsg, Is.EqualTo($"ConfigType:EnvironmentVariable(NETHERMIND_*)|Category:|Name:NETWORKCONFIGMAXCANDIDATEPEERCOUNT")); + } + } diff --git a/src/Nethermind/Nethermind.Config.Test/JsonConfigProviderTests.cs b/src/Nethermind/Nethermind.Config.Test/JsonConfigProviderTests.cs index 5017671f906..5878c6d80a8 100644 --- a/src/Nethermind/Nethermind.Config.Test/JsonConfigProviderTests.cs +++ b/src/Nethermind/Nethermind.Config.Test/JsonConfigProviderTests.cs @@ -31,7 +31,7 @@ public void Initialize() JsonRpcConfig jsonRpcConfig = new(); StatsParameters statsConfig = StatsParameters.Instance; - _configProvider = new JsonConfigProvider("SampleJson/SampleJsonConfig.cfg"); + _configProvider = new JsonConfigProvider("SampleJson/SampleJsonConfig.json"); } [TestCase(12ul, typeof(BlocksConfig), nameof(BlocksConfig.SecondsPerSlot))] @@ -48,7 +48,7 @@ public void Test_getDefaultValue(T expected, Type type, string propName) [Test] public void Provides_helpful_error_message_when_file_does_not_exist() { - Assert.Throws(() => _configProvider = new JsonConfigProvider("SampleJson.cfg")); + Assert.Throws(() => _configProvider = new JsonConfigProvider("SampleJson.json")); } [Test] diff --git a/src/Nethermind/Nethermind.Config.Test/Nethermind.Config.Test.csproj b/src/Nethermind/Nethermind.Config.Test/Nethermind.Config.Test.csproj index e81ac159db0..298ab14cf31 100644 --- a/src/Nethermind/Nethermind.Config.Test/Nethermind.Config.Test.csproj +++ b/src/Nethermind/Nethermind.Config.Test/Nethermind.Config.Test.csproj @@ -28,16 +28,15 @@ - - - - + + + PreserveNewest - + PreserveNewest - + PreserveNewest diff --git a/src/Nethermind/Nethermind.Config.Test/SampleJson/ConfigWithTypos.cfg b/src/Nethermind/Nethermind.Config.Test/SampleJson/ConfigWithTypos.json similarity index 100% rename from src/Nethermind/Nethermind.Config.Test/SampleJson/ConfigWithTypos.cfg rename to src/Nethermind/Nethermind.Config.Test/SampleJson/ConfigWithTypos.json diff --git a/src/Nethermind/Nethermind.Config.Test/SampleJson/CorrectSettingNames.cfg b/src/Nethermind/Nethermind.Config.Test/SampleJson/CorrectSettingNames.json similarity index 100% rename from src/Nethermind/Nethermind.Config.Test/SampleJson/CorrectSettingNames.cfg rename to src/Nethermind/Nethermind.Config.Test/SampleJson/CorrectSettingNames.json diff --git a/src/Nethermind/Nethermind.Config.Test/SampleJson/SampleJsonConfig.cfg b/src/Nethermind/Nethermind.Config.Test/SampleJson/SampleJsonConfig.json similarity index 100% rename from src/Nethermind/Nethermind.Config.Test/SampleJson/SampleJsonConfig.cfg rename to src/Nethermind/Nethermind.Config.Test/SampleJson/SampleJsonConfig.json diff --git a/src/Nethermind/Nethermind.Config/ConfigProvider.cs b/src/Nethermind/Nethermind.Config/ConfigProvider.cs index bbc2ca7ad21..ce5b5100184 100644 --- a/src/Nethermind/Nethermind.Config/ConfigProvider.cs +++ b/src/Nethermind/Nethermind.Config/ConfigProvider.cs @@ -15,10 +15,10 @@ public class ConfigProvider : IConfigProvider { private readonly ConcurrentDictionary _instances = new(); - private readonly List _configSource = new(); + private readonly List _configSource = []; private Dictionary Categories { get; set; } = new(StringComparer.InvariantCultureIgnoreCase); - private readonly Dictionary _implementations = new(); + private readonly Dictionary _implementations = []; public T GetConfig() where T : IConfig { @@ -114,23 +114,24 @@ public void Initialize() Initialize(); } - HashSet propertySet = _instances.Values + var propertySet = _instances.Values .SelectMany(i => i.GetType() .GetProperties() .Select(p => GetKey(i.GetType().Name, p.Name))) .ToHashSet(StringComparer.OrdinalIgnoreCase); - List<(IConfigSource Source, string Category, string Name)> incorrectSettings = new(); + List<(IConfigSource Source, string Category, string Name)> incorrectSettings = []; - foreach (var source in _configSource) + // Skip the validation for ArgsConfigSource items as they are already validated by the CLI parser + foreach (IConfigSource source in _configSource.Where(s => s is not ArgsConfigSource)) { var configs = source.GetConfigKeys(); - foreach (var conf in configs) + foreach ((string category, string name) in configs) { - if (!propertySet.Contains(GetKey(conf.Category, conf.Name))) + if (!propertySet.Contains(GetKey(category, name))) { - incorrectSettings.Add((source, conf.Category, conf.Name)); + incorrectSettings.Add((source, category, name)); } } } @@ -155,10 +156,10 @@ static string GetKey(string category, string name) } else if (!category.EndsWith("config", StringComparison.OrdinalIgnoreCase)) { - category += "Config"; + category = $"{category}Config"; } - return category + '.' + name; + return $"{category}.{name}"; } } } diff --git a/src/Nethermind/Nethermind.Config/INoCategoryConfig.cs b/src/Nethermind/Nethermind.Config/INoCategoryConfig.cs index e40fa35577f..d1b038857f9 100644 --- a/src/Nethermind/Nethermind.Config/INoCategoryConfig.cs +++ b/src/Nethermind/Nethermind.Config/INoCategoryConfig.cs @@ -6,27 +6,9 @@ namespace Nethermind.Config; [ConfigCategory(HiddenFromDocs = true)] public interface INoCategoryConfig : IConfig { - [ConfigItem(Description = "Parent directory or path for BaseDbPath, KeyStoreDirectory, LogDirectory configurations.")] - public string DataDir { get; set; } - - [ConfigItem(Description = "Path to the JSON configuration file.")] + [ConfigItem(Description = "Path to the configuration file.")] public string Config { get; set; } - [ConfigItem(Description = "Path or directory for configuration files.", DefaultValue = "configs")] - public string ConfigsDirectory { get; set; } - - [ConfigItem(Description = "Path or directory for database files.", DefaultValue = "db")] - public string BaseDbPath { get; set; } - - [ConfigItem(Description = "Log level override. Possible values: OFF|TRACE|DEBUG|INFO|WARN|ERROR")] - public string Log { get; set; } - - [ConfigItem(Description = "Path to the NLog config file")] - public string LoggerConfigSource { get; set; } - - [ConfigItem(Description = "Plugins directory")] - public string PluginsDirectory { get; set; } - [ConfigItem(Description = "Sets the job name for metrics monitoring.", EnvironmentVariable = "NETHERMIND_MONITORING_JOB")] public string MonitoringJob { get; set; } @@ -36,9 +18,6 @@ public interface INoCategoryConfig : IConfig [ConfigItem(Description = "Sets the external IP for the node.", EnvironmentVariable = "NETHERMIND_ENODE_IPADDRESS")] public string EnodeIpAddress { get; set; } - [ConfigItem(Description = "Enables Hive plugin used for executing Hive Ethereum Tests.", EnvironmentVariable = "NETHERMIND_HIVE_ENABLED", DefaultValue = "false")] - public bool HiveEnabled { get; set; } - [ConfigItem(Description = "Defines default URL for JSON RPC.", EnvironmentVariable = "NETHERMIND_URL")] public string Url { get; set; } diff --git a/src/Nethermind/Nethermind.Config/IProcessExitSource.cs b/src/Nethermind/Nethermind.Config/IProcessExitSource.cs index ea447bd4c22..22945eb257e 100644 --- a/src/Nethermind/Nethermind.Config/IProcessExitSource.cs +++ b/src/Nethermind/Nethermind.Config/IProcessExitSource.cs @@ -16,14 +16,14 @@ public interface IProcessExitSource public class ProcessExitSource : IProcessExitSource { - private CancellationTokenSource _cancellationTokenSource = new(); - public int ExitCode { get; set; } = ExitCodes.Ok; - + private CancellationTokenSource _cancellationTokenSource; private readonly TaskCompletionSource _exitResult = new(); - public Task ExitTask => _exitResult.Task; - - public CancellationToken Token => _cancellationTokenSource!.Token; + public ProcessExitSource(CancellationToken cancellationToken) + { + _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + _cancellationTokenSource.Token.Register(() => Exit(ExitCodes.SigInt)); + } public void Exit(int exitCode) { @@ -33,4 +33,10 @@ public void Exit(int exitCode) _exitResult.SetResult(); } } + + public int ExitCode { get; set; } = ExitCodes.Ok; + + public Task ExitTask => _exitResult.Task; + + public CancellationToken Token => _cancellationTokenSource.Token; } diff --git a/src/Nethermind/Nethermind.Config/JsonConfigSource.cs b/src/Nethermind/Nethermind.Config/JsonConfigSource.cs index 9e9f49a18b3..2f647f104f1 100644 --- a/src/Nethermind/Nethermind.Config/JsonConfigSource.cs +++ b/src/Nethermind/Nethermind.Config/JsonConfigSource.cs @@ -8,153 +8,152 @@ using System.Text.Json; using System.Linq; -namespace Nethermind.Config +namespace Nethermind.Config; + +public class JsonConfigSource : IConfigSource { - public class JsonConfigSource : IConfigSource + public JsonConfigSource(string configFilePath) { - public JsonConfigSource(string configFilePath) - { - LoadJsonConfig(configFilePath); - } + LoadJsonConfig(configFilePath); + } - private void ApplyJsonConfig(string jsonContent) + private void ApplyJsonConfig(string jsonContent) + { + try { - try + using var json = JsonDocument.Parse(jsonContent); + foreach (var moduleEntry in json.RootElement.EnumerateObject()) { - using var json = JsonDocument.Parse(jsonContent); - foreach (var moduleEntry in json.RootElement.EnumerateObject()) - { - LoadModule(moduleEntry.Name, moduleEntry.Value); - } - } - catch (JsonException e) - { - throw new System.Configuration.ConfigurationErrorsException($"Config is not correctly formed JSON. See inner exception for details.", e); + LoadModule(moduleEntry.Name, moduleEntry.Value); } } + catch (JsonException e) + { + throw new System.Configuration.ConfigurationErrorsException($"Config is not correctly formed JSON. See inner exception for details.", e); + } + } - private void LoadJsonConfig(string configFilePath) + private void LoadJsonConfig(string configFilePath) + { + if (!File.Exists(configFilePath)) { - if (!File.Exists(configFilePath)) + StringBuilder missingConfigFileMessage = new($"Config file {configFilePath} does not exist."); + try { - StringBuilder missingConfigFileMessage = new($"Config file {configFilePath} does not exist."); - try - { - string directory = Path.GetDirectoryName(configFilePath); - directory = Path.IsPathRooted(configFilePath) - ? directory - : Path.Combine(AppDomain.CurrentDomain.BaseDirectory, directory); + string directory = Path.GetDirectoryName(configFilePath); + directory = Path.IsPathRooted(configFilePath) + ? directory + : Path.Combine(AppDomain.CurrentDomain.BaseDirectory, directory); - missingConfigFileMessage.AppendLine().AppendLine($"Search directory: {directory}"); + missingConfigFileMessage.AppendLine().AppendLine($"Search directory: {directory}"); - string[] configFiles = Directory.GetFiles(directory, "*.cfg"); - if (configFiles.Length > 0) + string[] configFiles = Directory.GetFiles(directory, "*.json"); + if (configFiles.Length > 0) + { + missingConfigFileMessage.AppendLine("Found the following config files:"); + for (int i = 0; i < configFiles.Length; i++) { - missingConfigFileMessage.AppendLine("Found the following config files:"); - for (int i = 0; i < configFiles.Length; i++) - { - missingConfigFileMessage.AppendLine($" * {configFiles[i]}"); - } + missingConfigFileMessage.AppendLine($" * {configFiles[i]}"); } } - catch (Exception) - { - // do nothing - the lines above just give extra info and config is loaded at the beginning so unlikely we have any catastrophic errors here - } - - throw new IOException(missingConfigFileMessage.ToString()); + } + catch (Exception) + { + // do nothing - the lines above just give extra info and config is loaded at the beginning so unlikely we have any catastrophic errors here } - ApplyJsonConfig(File.ReadAllText(configFilePath)); + throw new IOException(missingConfigFileMessage.ToString()); } - private void LoadModule(string moduleName, JsonElement configItems) - { - var itemsDict = new Dictionary(StringComparer.InvariantCultureIgnoreCase); + ApplyJsonConfig(File.ReadAllText(configFilePath)); + } + + private void LoadModule(string moduleName, JsonElement configItems) + { + var itemsDict = new Dictionary(StringComparer.InvariantCultureIgnoreCase); - foreach (var configItem in configItems.EnumerateObject()) + foreach (var configItem in configItems.EnumerateObject()) + { + var key = configItem.Name; + if (!itemsDict.ContainsKey(key)) { - var key = configItem.Name; - if (!itemsDict.ContainsKey(key)) + var value = configItem.Value; + if (value.ValueKind == JsonValueKind.Number) { - var value = configItem.Value; - if (value.ValueKind == JsonValueKind.Number) - { - itemsDict[key] = value.GetInt64().ToString(); - } - else if (value.ValueKind == JsonValueKind.True) - { - itemsDict[key] = "true"; - } - else if (value.ValueKind == JsonValueKind.False) - { - itemsDict[key] = "false"; - } - else - { - itemsDict[key] = configItem.Value.ToString(); - } + itemsDict[key] = value.GetInt64().ToString(); + } + else if (value.ValueKind == JsonValueKind.True) + { + itemsDict[key] = "true"; + } + else if (value.ValueKind == JsonValueKind.False) + { + itemsDict[key] = "false"; } else { - throw new System.Configuration.ConfigurationErrorsException($"Duplicated config value: {key}, module: {moduleName}"); + itemsDict[key] = configItem.Value.ToString(); } } - - ApplyConfigValues(moduleName, itemsDict); + else + { + throw new System.Configuration.ConfigurationErrorsException($"Duplicated config value: {key}, module: {moduleName}"); + } } - private readonly Dictionary> _values = new(StringComparer.InvariantCultureIgnoreCase); - - private readonly Dictionary> _parsedValues = new(StringComparer.InvariantCultureIgnoreCase); + ApplyConfigValues(moduleName, itemsDict); + } - private void ApplyConfigValues(string configModule, Dictionary items) - { - if (!configModule.EndsWith("Config")) - { - configModule += "Config"; - } + private readonly Dictionary> _values = new(StringComparer.InvariantCultureIgnoreCase); - _values[configModule] = items; - _parsedValues[configModule] = new Dictionary(StringComparer.InvariantCultureIgnoreCase); - } + private readonly Dictionary> _parsedValues = new(StringComparer.InvariantCultureIgnoreCase); - private void ParseValue(Type type, string category, string name) + private void ApplyConfigValues(string configModule, Dictionary items) + { + if (!configModule.EndsWith("Config")) { - string valueString = _values[category][name]; - _parsedValues[category][name] = ConfigSourceHelper.ParseValue(type, valueString, category, name); + configModule += "Config"; } - public (bool IsSet, object Value) GetValue(Type type, string category, string name) - { - (bool isSet, _) = GetRawValue(category, name); - if (isSet) - { - if (!_parsedValues[category].ContainsKey(name)) - { - ParseValue(type, category, name); - } - - return (true, _parsedValues[category][name]); - } + _values[configModule] = items; + _parsedValues[configModule] = new Dictionary(StringComparer.InvariantCultureIgnoreCase); + } - return (false, ConfigSourceHelper.GetDefault(type)); - } + private void ParseValue(Type type, string category, string name) + { + string valueString = _values[category][name]; + _parsedValues[category][name] = ConfigSourceHelper.ParseValue(type, valueString, category, name); + } - public (bool IsSet, string Value) GetRawValue(string category, string name) + public (bool IsSet, object Value) GetValue(Type type, string category, string name) + { + (bool isSet, _) = GetRawValue(category, name); + if (isSet) { - if (string.IsNullOrEmpty(category) || string.IsNullOrEmpty(name)) + if (!_parsedValues[category].ContainsKey(name)) { - return (false, null); + ParseValue(type, category, name); } - bool isSet = _values.ContainsKey(category) && _values[category].ContainsKey(name); - return (isSet, isSet ? _values[category][name] : null); + return (true, _parsedValues[category][name]); } - public IEnumerable<(string Category, string Name)> GetConfigKeys() + return (false, ConfigSourceHelper.GetDefault(type)); + } + + public (bool IsSet, string Value) GetRawValue(string category, string name) + { + if (string.IsNullOrEmpty(category) || string.IsNullOrEmpty(name)) { - return _values.SelectMany(m => m.Value.Keys.Select(n => (m.Key, n))); + return (false, null); } + + bool isSet = _values.ContainsKey(category) && _values[category].ContainsKey(name); + return (isSet, isSet ? _values[category][name] : null); + } + + public IEnumerable<(string Category, string Name)> GetConfigKeys() + { + return _values.SelectMany(m => m.Value.Keys.Select(n => (m.Key, n))); } } diff --git a/src/Nethermind/Nethermind.Config/NoCategoryConfig.cs b/src/Nethermind/Nethermind.Config/NoCategoryConfig.cs index 56a73d77ac4..e231c770f24 100644 --- a/src/Nethermind/Nethermind.Config/NoCategoryConfig.cs +++ b/src/Nethermind/Nethermind.Config/NoCategoryConfig.cs @@ -1,23 +1,15 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -namespace Nethermind.Config +namespace Nethermind.Config; + +public class NoCategoryConfig : INoCategoryConfig { - public class NoCategoryConfig : INoCategoryConfig - { - public string Config { get; set; } = null; - public string DataDir { get; set; } - public string ConfigsDirectory { get; set; } - public string BaseDbPath { get; set; } - public string Log { get; set; } - public string LoggerConfigSource { get; set; } - public string PluginsDirectory { get; set; } - public string MonitoringJob { get; set; } - public string MonitoringGroup { get; set; } - public string EnodeIpAddress { get; set; } - public bool HiveEnabled { get; set; } - public string Url { get; set; } - public string CorsOrigins { get; set; } - public string CliSwitchLocal { get; set; } - } + public string Config { get; set; } = null; + public string MonitoringJob { get; set; } + public string MonitoringGroup { get; set; } + public string EnodeIpAddress { get; set; } + public string Url { get; set; } + public string CorsOrigins { get; set; } + public string CliSwitchLocal { get; set; } } diff --git a/src/Nethermind/Nethermind.Consensus.Clique/CliqueRpcModule.cs b/src/Nethermind/Nethermind.Consensus.Clique/CliqueRpcModule.cs index cc5ec1da3af..51361017ee6 100644 --- a/src/Nethermind/Nethermind.Consensus.Clique/CliqueRpcModule.cs +++ b/src/Nethermind/Nethermind.Consensus.Clique/CliqueRpcModule.cs @@ -54,9 +54,13 @@ public void UncastVote(Address signer) cliqueBlockProducer.UncastVote(signer); } - public Snapshot GetSnapshot() + public Snapshot GetSnapshot(long? number = null) { Block head = _blockTree.Head; + if (number is not null && head.Number != number) + { + head = _blockTree.FindBlock(number.Value); + } return _snapshotManager.GetOrCreateSnapshot(head.Number, head.Hash); } @@ -105,7 +109,7 @@ public string[] GetSignersAnnotated(Hash256 hash) public ResultWrapper> clique_proposals() => ResultWrapper>.Success(cliqueBlockProducer?.GetProposals() ?? new Dictionary()); - public ResultWrapper clique_getSnapshot() => ResultWrapper.Success(GetSnapshot()); + public ResultWrapper clique_getSnapshot(long? number) => ResultWrapper.Success(GetSnapshot(number)); public ResultWrapper clique_getSnapshotAtHash(Hash256 hash) => ResultWrapper.Success(GetSnapshot(hash)); diff --git a/src/Nethermind/Nethermind.Consensus.Clique/ICliqueRpcModule.cs b/src/Nethermind/Nethermind.Consensus.Clique/ICliqueRpcModule.cs index 53440be3fd9..45b690ae68a 100644 --- a/src/Nethermind/Nethermind.Consensus.Clique/ICliqueRpcModule.cs +++ b/src/Nethermind/Nethermind.Consensus.Clique/ICliqueRpcModule.cs @@ -16,7 +16,7 @@ public interface ICliqueRpcModule : IRpcModule ResultWrapper clique_getBlockSigner(Hash256? hash); [JsonRpcMethod(Description = "Retrieves a snapshot of all clique state at a given block.", IsImplemented = true)] - ResultWrapper clique_getSnapshot(); + ResultWrapper clique_getSnapshot(long? number = null); [JsonRpcMethod(Description = "Retrieves the state snapshot at a given block.", IsImplemented = true)] ResultWrapper clique_getSnapshotAtHash(Hash256 hash); diff --git a/src/Nethermind/Nethermind.Consensus.Clique/Snapshot.cs b/src/Nethermind/Nethermind.Consensus.Clique/Snapshot.cs index d2f07492a65..d1703a9d247 100644 --- a/src/Nethermind/Nethermind.Consensus.Clique/Snapshot.cs +++ b/src/Nethermind/Nethermind.Consensus.Clique/Snapshot.cs @@ -14,7 +14,7 @@ public class Snapshot : ICloneable public Hash256 Hash { get; set; } public SortedList Signers { get; } public List Votes { get; init; } - internal Dictionary Tally { get; } + public Dictionary Tally { get; } internal Snapshot(long number, Hash256 hash, SortedList signers, Dictionary tally) { diff --git a/src/Nethermind/Nethermind.Consensus.Clique/Tally.cs b/src/Nethermind/Nethermind.Consensus.Clique/Tally.cs index 211c1af532f..7f8949a2baa 100644 --- a/src/Nethermind/Nethermind.Consensus.Clique/Tally.cs +++ b/src/Nethermind/Nethermind.Consensus.Clique/Tally.cs @@ -3,7 +3,7 @@ namespace Nethermind.Consensus.Clique { - internal class Tally + public class Tally { public bool Authorize { get; } public int Votes { get; set; } diff --git a/src/Nethermind/Nethermind.Consensus.Ethash/EthashDifficultyCalculator.cs b/src/Nethermind/Nethermind.Consensus.Ethash/EthashDifficultyCalculator.cs index aedb99cc03c..9cd8ba25f6c 100644 --- a/src/Nethermind/Nethermind.Consensus.Ethash/EthashDifficultyCalculator.cs +++ b/src/Nethermind/Nethermind.Consensus.Ethash/EthashDifficultyCalculator.cs @@ -10,6 +10,7 @@ [assembly: InternalsVisibleTo("Ethereum.Test.Base")] [assembly: InternalsVisibleTo("Ethereum.Difficulty.Test")] +[assembly: InternalsVisibleTo("Evm")] namespace Nethermind.Consensus.Ethash { diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs index 5e4ed8b7f2d..59c816ad211 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs @@ -630,8 +630,10 @@ private ProcessingBranch PrepareProcessingBranch(Block suggestedBlock, Processin break; } - branchingPoint = _blockTree.FindParentHeader(toBeProcessed.Header, - BlockTreeLookupOptions.TotalDifficultyNotNeeded); + branchingPoint = options.ContainsFlag(ProcessingOptions.ForceSameBlock) + ? toBeProcessed.Header + : _blockTree.FindParentHeader(toBeProcessed.Header, BlockTreeLookupOptions.TotalDifficultyNotNeeded); + if (branchingPoint is null) { // genesis block diff --git a/src/Nethermind/Nethermind.Consensus/Processing/OverridableTxProcessingEnv.cs b/src/Nethermind/Nethermind.Consensus/Processing/OverridableTxProcessingEnv.cs new file mode 100644 index 00000000000..da0f252c160 --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Processing/OverridableTxProcessingEnv.cs @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using Nethermind.Blockchain; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using Nethermind.Evm; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Logging; +using Nethermind.State; + +namespace Nethermind.Consensus.Processing; + +public class OverridableTxProcessingEnv : ReadOnlyTxProcessingEnvBase, IOverridableTxProcessorSource +{ + private readonly Lazy _transactionProcessorLazy; + + protected new OverridableWorldState StateProvider { get; } + protected OverridableWorldStateManager WorldStateManager { get; } + protected OverridableCodeInfoRepository CodeInfoRepository { get; } + protected IVirtualMachine Machine { get; } + protected ITransactionProcessor TransactionProcessor => _transactionProcessorLazy.Value; + + public OverridableTxProcessingEnv( + OverridableWorldStateManager worldStateManager, + IReadOnlyBlockTree readOnlyBlockTree, + ISpecProvider specProvider, + ILogManager? logManager, + IWorldState? worldStateToWarmUp = null + ) : base(worldStateManager, readOnlyBlockTree, specProvider, logManager, worldStateToWarmUp) + { + WorldStateManager = worldStateManager; + StateProvider = (OverridableWorldState)base.StateProvider; + CodeInfoRepository = new(new CodeInfoRepository((worldStateToWarmUp as IPreBlockCaches)?.Caches.PrecompileCache)); + Machine = new VirtualMachine(BlockhashProvider, specProvider, CodeInfoRepository, logManager); + _transactionProcessorLazy = new(CreateTransactionProcessor); + } + + protected virtual ITransactionProcessor CreateTransactionProcessor() => + new TransactionProcessor(SpecProvider, StateProvider, Machine, CodeInfoRepository, LogManager); + + IOverridableTxProcessingScope IOverridableTxProcessorSource.Build(Hash256 stateRoot) => Build(stateRoot); + + public OverridableTxProcessingScope Build(Hash256 stateRoot) + { + Hash256 originalStateRoot = StateProvider.StateRoot; + StateProvider.StateRoot = stateRoot; + return new(CodeInfoRepository, TransactionProcessor, StateProvider, originalStateRoot); + } + + IOverridableTxProcessingScope IOverridableTxProcessorSource.BuildAndOverride(BlockHeader header, Dictionary? stateOverride) + { + OverridableTxProcessingScope scope = Build(header.StateRoot ?? throw new ArgumentException($"Block {header.Hash} state root is null", nameof(header))); + if (stateOverride != null) + { + scope.WorldState.ApplyStateOverrides(scope.CodeInfoRepository, stateOverride, SpecProvider.GetSpec(header), header.Number); + header.StateRoot = scope.WorldState.StateRoot; + } + return scope; + } +} diff --git a/src/Nethermind/Nethermind.Consensus/Processing/OverridableTxProcessingScope.cs b/src/Nethermind/Nethermind.Consensus/Processing/OverridableTxProcessingScope.cs new file mode 100644 index 00000000000..b168b1c1e93 --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Processing/OverridableTxProcessingScope.cs @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Crypto; +using Nethermind.Evm; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.State; + +namespace Nethermind.Consensus.Processing; + +public class OverridableTxProcessingScope( + IOverridableCodeInfoRepository codeInfoRepository, + ITransactionProcessor transactionProcessor, + OverridableWorldState worldState, + Hash256 originalStateRoot +) : IOverridableTxProcessingScope +{ + public IOverridableCodeInfoRepository CodeInfoRepository => codeInfoRepository; + public ITransactionProcessor TransactionProcessor => transactionProcessor; + public IWorldState WorldState => worldState; + + public void Dispose() + { + worldState.StateRoot = originalStateRoot; + worldState.Reset(); + worldState.ResetOverrides(); + codeInfoRepository.ResetOverrides(); + } +} diff --git a/src/Nethermind/Nethermind.Consensus/Processing/ProcessingOptions.cs b/src/Nethermind/Nethermind.Consensus/Processing/ProcessingOptions.cs index 555e1c9e1dd..82799e5578d 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/ProcessingOptions.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/ProcessingOptions.cs @@ -51,7 +51,10 @@ public enum ProcessingOptions /// MarkAsProcessed = 128, - All = 255, + /// + /// Forces to run on top of the specified block state, instead of reverting to the previous one. + /// + ForceSameBlock = 1 << 8, /// /// Combination of switches for block producers when they preprocess block for state root calculation. @@ -63,6 +66,12 @@ public enum ProcessingOptions /// Trace = ForceProcessing | ReadOnlyChain | DoNotVerifyNonce | NoValidation, + /// + /// EVM tracing needs to process one or more transactions on top of the specified block (instead of the previous one) + /// without storing the data on chain. + /// + TraceTransactions = Trace | ForceSameBlock, + /// /// Processing options for engine_NewPayload /// diff --git a/src/Nethermind/Nethermind.Consensus/Tracing/GethStyleTracer.cs b/src/Nethermind/Nethermind.Consensus/Tracing/GethStyleTracer.cs index 7ff20c4d203..60392ad5c81 100644 --- a/src/Nethermind/Nethermind.Consensus/Tracing/GethStyleTracer.cs +++ b/src/Nethermind/Nethermind.Consensus/Tracing/GethStyleTracer.cs @@ -16,6 +16,7 @@ using Nethermind.Core.Extensions; using Nethermind.Core.Specs; using Nethermind.Crypto; +using Nethermind.Evm; using Nethermind.Evm.Tracing; using Nethermind.Evm.Tracing.GethStyle; using Nethermind.Evm.Tracing.GethStyle.Custom.JavaScript; @@ -36,6 +37,7 @@ public class GethStyleTracer : IGethStyleTracer private readonly IWorldState _worldState; private readonly IReceiptStorage _receiptStorage; private readonly IFileSystem _fileSystem; + private readonly IOverridableTxProcessorSource _env; public GethStyleTracer(IBlockchainProcessor processor, IWorldState worldState, @@ -44,7 +46,8 @@ public GethStyleTracer(IBlockchainProcessor processor, IBadBlockStore badBlockStore, ISpecProvider specProvider, ChangeableTransactionProcessorAdapter transactionProcessorAdapter, - IFileSystem fileSystem) + IFileSystem fileSystem, + IOverridableTxProcessorSource env) { _processor = processor ?? throw new ArgumentNullException(nameof(processor)); _worldState = worldState; @@ -54,6 +57,7 @@ public GethStyleTracer(IBlockchainProcessor processor, _specProvider = specProvider; _transactionProcessorAdapter = transactionProcessorAdapter; _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + _env = env ?? throw new ArgumentNullException(nameof(env)); } public GethLikeTxTrace Trace(Hash256 blockHash, int txIndex, GethTraceOptions options, CancellationToken cancellationToken) @@ -82,7 +86,10 @@ public GethLikeTxTrace Trace(Hash256 blockHash, int txIndex, GethTraceOptions op try { - return Trace(block, tx.Hash, cancellationToken, options); + Dictionary? stateOverride = options.StateOverrides; + using IOverridableTxProcessingScope? scope = stateOverride != null ? _env.BuildAndOverride(block.Header, stateOverride) : null; + + return Trace(block, tx.Hash, cancellationToken, options, ProcessingOptions.TraceTransactions); } finally { @@ -189,7 +196,8 @@ public IEnumerable TraceBadBlockToFile(Hash256 blockHash, GethTraceOptio return tracer.FileNames; } - private GethLikeTxTrace? Trace(Block block, Hash256? txHash, CancellationToken cancellationToken, GethTraceOptions options) + private GethLikeTxTrace? Trace(Block block, Hash256? txHash, CancellationToken cancellationToken, GethTraceOptions options, + ProcessingOptions processingOptions = ProcessingOptions.Trace) { ArgumentNullException.ThrowIfNull(txHash); @@ -197,7 +205,7 @@ public IEnumerable TraceBadBlockToFile(Hash256 blockHash, GethTraceOptio try { - _processor.Process(block, ProcessingOptions.Trace, tracer.WithCancellation(cancellationToken)); + _processor.Process(block, processingOptions, tracer.WithCancellation(cancellationToken)); return tracer.BuildResult().SingleOrDefault(); } catch diff --git a/src/Nethermind/Nethermind.Consensus/Tracing/Tracer.cs b/src/Nethermind/Nethermind.Consensus/Tracing/Tracer.cs index 00c2a85147d..df6c3c721bd 100644 --- a/src/Nethermind/Nethermind.Consensus/Tracing/Tracer.cs +++ b/src/Nethermind/Nethermind.Consensus/Tracing/Tracer.cs @@ -16,18 +16,20 @@ public class Tracer : ITracer private readonly IWorldState _stateProvider; private readonly IBlockchainProcessor _traceProcessor; private readonly IBlockchainProcessor _executeProcessor; - private readonly ProcessingOptions _processingOptions; + private readonly ProcessingOptions _executeOptions; + private readonly ProcessingOptions _traceOptions; public Tracer(IWorldState stateProvider, IBlockchainProcessor traceProcessor, IBlockchainProcessor executeProcessor, - ProcessingOptions processingOptions = ProcessingOptions.Trace) + ProcessingOptions executeOptions = ProcessingOptions.Trace, ProcessingOptions traceOptions = ProcessingOptions.Trace) { _traceProcessor = traceProcessor; _executeProcessor = executeProcessor; _stateProvider = stateProvider; - _processingOptions = processingOptions; + _executeOptions = executeOptions; + _traceOptions = traceOptions; } - private void Process(Block block, IBlockTracer blockTracer, IBlockchainProcessor processor) + private void Process(Block block, IBlockTracer blockTracer, IBlockchainProcessor processor, ProcessingOptions options) { /* We force process since we want to process a block that has already been processed in the past and normally it would be ignored. We also want to make it read only so the state is not modified persistently in any way. */ @@ -36,7 +38,7 @@ We also want to make it read only so the state is not modified persistently in a try { - processor.Process(block, _processingOptions, blockTracer); + processor.Process(block, options, blockTracer); } catch (Exception) { @@ -47,9 +49,9 @@ We also want to make it read only so the state is not modified persistently in a blockTracer.EndBlockTrace(); } - public void Trace(Block block, IBlockTracer tracer) => Process(block, tracer, _traceProcessor); + public void Trace(Block block, IBlockTracer tracer) => Process(block, tracer, _traceProcessor, _traceOptions); - public void Execute(Block block, IBlockTracer tracer) => Process(block, tracer, _executeProcessor); + public void Execute(Block block, IBlockTracer tracer) => Process(block, tracer, _executeProcessor, _executeOptions); public void Accept(ITreeVisitor visitor, Hash256 stateRoot) { diff --git a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs index d9f198e6e5b..982c9190a8c 100644 --- a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs +++ b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs @@ -77,6 +77,7 @@ public IBlockFinder BlockFinder public IWorldState State { get; set; } = null!; public IReadOnlyStateProvider ReadOnlyState { get; private set; } = null!; public IDb StateDb => DbProvider.StateDb; + public IDb BlocksDb => DbProvider.BlocksDb; public TrieStore TrieStore { get; set; } = null!; public IBlockProducer BlockProducer { get; private set; } = null!; public IBlockProducerRunner BlockProducerRunner { get; protected set; } = null!; diff --git a/src/Nethermind/Nethermind.Core.Test/Builders/TransactionBuilder.cs b/src/Nethermind/Nethermind.Core.Test/Builders/TransactionBuilder.cs index 2246e7c319d..6710cb896e7 100644 --- a/src/Nethermind/Nethermind.Core.Test/Builders/TransactionBuilder.cs +++ b/src/Nethermind/Nethermind.Core.Test/Builders/TransactionBuilder.cs @@ -251,12 +251,12 @@ public TransactionBuilder Signed(IEthereumEcdsa ecdsa, PrivateKey privateKey, return this; } - public TransactionBuilder Signed(PrivateKey? privateKey = null) + public TransactionBuilder Signed(PrivateKey? privateKey = null, bool isEip155Enabled = true) { privateKey ??= TestItem.IgnoredPrivateKey; EthereumEcdsa ecdsa = new(TestObjectInternal.ChainId ?? TestBlockchainIds.ChainId); - return Signed(ecdsa, privateKey, isEip155Enabled: true); + return Signed(ecdsa, privateKey, isEip155Enabled); } // TODO: auto create ecdsa here diff --git a/src/Nethermind/Nethermind.Core.Test/Threading/ConcurrencyControllerTests.cs b/src/Nethermind/Nethermind.Core.Test/Threading/ConcurrencyControllerTests.cs new file mode 100644 index 00000000000..8757425bfa1 --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/Threading/ConcurrencyControllerTests.cs @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Core.Threading; +using NUnit.Framework; + +namespace Nethermind.Core.Test.Threading; + +public class ConcurrencyControllerTests +{ + [Test] + public void ThreadLimiterWillLimit() + { + ConcurrencyController.Slot returner; + ConcurrencyController limiter = new ConcurrencyController(3); + + limiter.TryTakeSlot(out returner).Should().Be(true); + limiter.TryTakeSlot(out returner).Should().Be(true); + limiter.TryTakeSlot(out returner).Should().Be(false); + + returner.Dispose(); + + limiter.TryTakeSlot(out returner).Should().Be(true); + limiter.TryTakeSlot(out returner).Should().Be(false); + } +} diff --git a/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs b/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs index 1c30a830b55..a64ba5d833a 100644 --- a/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs +++ b/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Reflection; using Autofac; +using Autofac.Core; using Autofac.Features.AttributeFilters; namespace Nethermind.Core; @@ -115,6 +116,13 @@ public static ContainerBuilder RegisterNamedComponentInItsOwnLifetime(this Co return builder; } + + public static ContainerBuilder AddModule(this ContainerBuilder builder, IModule module) + { + builder.RegisterModule(module); + + return builder; + } } /// diff --git a/src/Nethermind/Nethermind.Core/Threading/ConcurrencyController.cs b/src/Nethermind/Nethermind.Core/Threading/ConcurrencyController.cs new file mode 100644 index 00000000000..e841b44cd23 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Threading/ConcurrencyController.cs @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading; + +namespace Nethermind.Core.Threading; + +/// +/// Encapsulate the pattern of checking if new task can be spawned based on a predefined limit. +/// Used in multithreaded tree visit where we don't know if we can spawn task or not and spawning task itself +/// is not a cheap operation. +/// +/// Yes, I don't like the name. Give me a good one. +/// +/// Desired concurrency which include the calling thread. So slot is slot-1. +public class ConcurrencyController(int concurrency) +{ + private int _slots = concurrency; + + public bool TryTakeSlot(out Slot returner) + { + returner = new Slot(this); + int newSlot = Volatile.Read(ref _slots); + if (newSlot < 2) + { + return false; + } + + newSlot = Interlocked.Decrement(ref _slots); + if (newSlot < 1) + { + Interlocked.Increment(ref _slots); + return false; + } + + return true; + } + + private void ReturnSlot() + { + Interlocked.Increment(ref _slots); + } + + public readonly struct Slot(ConcurrencyController limiter) : IDisposable + { + public void Dispose() + { + limiter.ReturnSlot(); + } + } +} diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs index 62944818adf..73d6ecd1547 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs @@ -67,6 +67,7 @@ public class DbConfig : IDbConfig public ulong? ReceiptsDbCompactionReadAhead { get; set; } public ulong ReceiptsDbTargetFileSizeBase { get; set; } = (ulong)64.MiB(); public double ReceiptsDbCompressibilityHint { get; set; } = 0.35; + public bool ReceiptsDbOptimizeFiltersForHits { get; set; } = false; public string? ReceiptsDbAdditionalRocksDbOptions { get; set; } = "compaction_pri=kOldestLargestSeqFirst"; public ulong BlocksDbWriteBufferSize { get; set; } = (ulong)64.MiB(); @@ -79,6 +80,7 @@ public class DbConfig : IDbConfig public bool? BlocksDbUseDirectReads { get; set; } public bool? BlocksDbUseDirectIoForFlushAndCompactions { get; set; } public ulong? BlocksDbCompactionReadAhead { get; set; } + public bool BlocksDbOptimizeFiltersForHits { get; set; } = false; public string? BlocksDbAdditionalRocksDbOptions { get; set; } = "compaction_pri=kOldestLargestSeqFirst"; public ulong HeadersDbWriteBufferSize { get; set; } = (ulong)8.MiB(); diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs index a1d6af06eb6..0af4a0e66e0 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs @@ -68,6 +68,7 @@ public interface IDbConfig : IConfig ulong? ReceiptsDbCompactionReadAhead { get; set; } ulong ReceiptsDbTargetFileSizeBase { get; set; } double ReceiptsDbCompressibilityHint { get; set; } + bool ReceiptsDbOptimizeFiltersForHits { get; set; } string? ReceiptsDbAdditionalRocksDbOptions { get; set; } ulong BlocksDbWriteBufferSize { get; set; } @@ -80,6 +81,7 @@ public interface IDbConfig : IConfig bool? BlocksDbUseDirectReads { get; set; } bool? BlocksDbUseDirectIoForFlushAndCompactions { get; set; } ulong? BlocksDbCompactionReadAhead { get; set; } + bool BlocksDbOptimizeFiltersForHits { get; set; } string? BlocksDbAdditionalRocksDbOptions { get; set; } ulong HeadersDbWriteBufferSize { get; set; } diff --git a/src/Nethermind/Nethermind.Evm.Test/Tracing/GethLikeTxMemoryTracerTests.cs b/src/Nethermind/Nethermind.Evm.Test/Tracing/GethLikeTxMemoryTracerTests.cs index 878e1496570..8ca428a2820 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Tracing/GethLikeTxMemoryTracerTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/Tracing/GethLikeTxMemoryTracerTests.cs @@ -298,13 +298,13 @@ public void Storage_is_cleared_and_restored_when_moving_between_call_levels() }; */ Assert.That(trace.Entries[0].Storage.Count, Is.EqualTo(0), "BEGIN 1"); - Assert.That(trace.Entries[13].Storage.Count, Is.EqualTo(2), "CALL FROM 1"); + Assert.That(trace.Entries[13].Storage.Count, Is.EqualTo(0), "CALL FROM 1"); Assert.That(trace.Entries[14].Storage.Count, Is.EqualTo(0), "BEGIN 2"); - Assert.That(trace.Entries[26].Storage.Count, Is.EqualTo(1), "CREATE FROM 2"); + Assert.That(trace.Entries[26].Storage.Count, Is.EqualTo(0), "CREATE FROM 2"); Assert.That(trace.Entries[27].Storage.Count, Is.EqualTo(0), "BEGIN 3"); Assert.That(trace.Entries[32].Storage.Count, Is.EqualTo(0), "END 3"); - Assert.That(trace.Entries[33].Storage.Count, Is.EqualTo(1), "END 2"); - Assert.That(trace.Entries[34].Storage.Count, Is.EqualTo(2), "END 1"); + Assert.That(trace.Entries[33].Storage.Count, Is.EqualTo(0), "END 2"); + Assert.That(trace.Entries[34].Storage.Count, Is.EqualTo(0), "END 1"); } [Test] diff --git a/src/Nethermind/Nethermind.Evm.Test/VirtualMachineTests.cs b/src/Nethermind/Nethermind.Evm.Test/VirtualMachineTests.cs index 19fb39617f3..b626cf96a55 100644 --- a/src/Nethermind/Nethermind.Evm.Test/VirtualMachineTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/VirtualMachineTests.cs @@ -42,7 +42,7 @@ public void Trace() Assert.That(entry.GasCost, Is.EqualTo(GasCostOf.VeryLow), nameof(entry.GasCost)); Assert.That(entry.Memory.Count, Is.EqualTo(0), nameof(entry.Memory)); Assert.That(entry.Stack.Count, Is.EqualTo(1), nameof(entry.Stack)); - Assert.That(trace.Entries[4].Storage.Count, Is.EqualTo(1), nameof(entry.Storage)); + Assert.That(trace.Entries[4].Storage.Count, Is.EqualTo(0), nameof(entry.Storage)); Assert.That(entry.ProgramCounter, Is.EqualTo(2), nameof(entry.ProgramCounter)); Assert.That(entry.Opcode, Is.EqualTo("PUSH1"), nameof(entry.Opcode)); } @@ -135,7 +135,7 @@ public void Trace_each_tx_separate() Assert.That(entry.GasCost, Is.EqualTo(GasCostOf.VeryLow), nameof(entry.GasCost)); Assert.That(entry.Memory.Count, Is.EqualTo(0), nameof(entry.Memory)); Assert.That(entry.Stack.Count, Is.EqualTo(1), nameof(entry.Stack)); - Assert.That(trace.Entries[4].Storage.Count, Is.EqualTo(1), nameof(entry.Storage)); + Assert.That(trace.Entries[4].Storage.Count, Is.EqualTo(0), nameof(entry.Storage)); Assert.That(entry.ProgramCounter, Is.EqualTo(2), nameof(entry.ProgramCounter)); Assert.That(entry.Opcode, Is.EqualTo("PUSH1"), nameof(entry.Opcode)); } diff --git a/src/Nethermind/Nethermind.Facade/Proxy/Models/AccountOverride.cs b/src/Nethermind/Nethermind.Evm/AccountOverride.cs similarity index 94% rename from src/Nethermind/Nethermind.Facade/Proxy/Models/AccountOverride.cs rename to src/Nethermind/Nethermind.Evm/AccountOverride.cs index 5b8a26ab431..91fe22d9f40 100644 --- a/src/Nethermind/Nethermind.Facade/Proxy/Models/AccountOverride.cs +++ b/src/Nethermind/Nethermind.Evm/AccountOverride.cs @@ -6,7 +6,7 @@ using Nethermind.Core.Crypto; using Nethermind.Int256; -namespace Nethermind.Facade.Proxy.Models; +namespace Nethermind.Evm; public class AccountOverride { diff --git a/src/Nethermind/Nethermind.Evm/IOverridableCodeInfoRepository.cs b/src/Nethermind/Nethermind.Evm/IOverridableCodeInfoRepository.cs new file mode 100644 index 00000000000..19e14f63b35 --- /dev/null +++ b/src/Nethermind/Nethermind.Evm/IOverridableCodeInfoRepository.cs @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Core.Specs; +using Nethermind.Evm.CodeAnalysis; +using Nethermind.State; + +namespace Nethermind.Evm; + +public interface IOverridableCodeInfoRepository : ICodeInfoRepository +{ + void SetCodeOverwrite(IWorldState worldState, IReleaseSpec vmSpec, Address key, CodeInfo value, Address? redirectAddress = null); + public void ResetOverrides(); +} diff --git a/src/Nethermind/Nethermind.Facade/OverridableCodeInfoRepository.cs b/src/Nethermind/Nethermind.Evm/OverridableCodeInfoRepository.cs similarity index 93% rename from src/Nethermind/Nethermind.Facade/OverridableCodeInfoRepository.cs rename to src/Nethermind/Nethermind.Evm/OverridableCodeInfoRepository.cs index 082d5ad57fe..0342586f261 100644 --- a/src/Nethermind/Nethermind.Facade/OverridableCodeInfoRepository.cs +++ b/src/Nethermind/Nethermind.Evm/OverridableCodeInfoRepository.cs @@ -7,13 +7,12 @@ using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Specs; -using Nethermind.Evm; using Nethermind.Evm.CodeAnalysis; using Nethermind.State; -namespace Nethermind.Facade; +namespace Nethermind.Evm; -public class OverridableCodeInfoRepository(ICodeInfoRepository codeInfoRepository) : ICodeInfoRepository +public class OverridableCodeInfoRepository(ICodeInfoRepository codeInfoRepository) : IOverridableCodeInfoRepository { private readonly Dictionary _codeOverwrites = new(); @@ -51,4 +50,6 @@ public bool TryGetDelegation(IReadOnlyStateProvider worldState, Address address, public ValueHash256 GetExecutableCodeHash(IWorldState worldState, Address address) => codeInfoRepository.GetExecutableCodeHash(worldState, address); + + public void ResetOverrides() => _codeOverwrites.Clear(); } diff --git a/src/Nethermind/Nethermind.Facade/StateOverridesExtensions.cs b/src/Nethermind/Nethermind.Evm/StateOverridesExtensions.cs similarity index 94% rename from src/Nethermind/Nethermind.Facade/StateOverridesExtensions.cs rename to src/Nethermind/Nethermind.Evm/StateOverridesExtensions.cs index 32d3d947b7c..d34ef57f238 100644 --- a/src/Nethermind/Nethermind.Facade/StateOverridesExtensions.cs +++ b/src/Nethermind/Nethermind.Evm/StateOverridesExtensions.cs @@ -7,17 +7,16 @@ using Nethermind.Core.Extensions; using Nethermind.Core.Specs; using Nethermind.Evm.CodeAnalysis; -using Nethermind.Facade.Proxy.Models; using Nethermind.Int256; using Nethermind.State; -namespace Nethermind.Facade; +namespace Nethermind.Evm; public static class StateOverridesExtensions { public static void ApplyStateOverrides( this IWorldState state, - OverridableCodeInfoRepository overridableCodeInfoRepository, + IOverridableCodeInfoRepository overridableCodeInfoRepository, Dictionary? overrides, IReleaseSpec spec, long blockNumber) @@ -69,13 +68,15 @@ void ApplyState(Dictionary diff) private static void UpdateCode( this IWorldState stateProvider, - OverridableCodeInfoRepository overridableCodeInfoRepository, + IOverridableCodeInfoRepository overridableCodeInfoRepository, IReleaseSpec currentSpec, AccountOverride accountOverride, Address address) { if (accountOverride.Code is not null) { + stateProvider.InsertCode(address, accountOverride.Code, currentSpec); + overridableCodeInfoRepository.SetCodeOverwrite( stateProvider, currentSpec, diff --git a/src/Nethermind/Nethermind.Evm/Tracing/GethStyle/GethTraceOptions.cs b/src/Nethermind/Nethermind.Evm/Tracing/GethStyle/GethTraceOptions.cs index 3a9504af097..950d084634f 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/GethStyle/GethTraceOptions.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/GethStyle/GethTraceOptions.cs @@ -2,9 +2,10 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Collections.Generic; using System.Text.Json; using System.Text.Json.Serialization; - +using Nethermind.Core; using Nethermind.Core.Crypto; namespace Nethermind.Evm.Tracing.GethStyle; @@ -36,5 +37,8 @@ public record GethTraceOptions [JsonPropertyName("tracerConfig")] public JsonElement? TracerConfig { get; init; } + [JsonPropertyName("stateOverrides")] + public Dictionary? StateOverrides { get; init; } + public static GethTraceOptions Default { get; } = new(); } diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/IOverridableTxProcessingScope.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/IOverridableTxProcessingScope.cs new file mode 100644 index 00000000000..9f0ed17a089 --- /dev/null +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/IOverridableTxProcessingScope.cs @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Evm.TransactionProcessing; + +public interface IOverridableTxProcessingScope : IReadOnlyTxProcessingScope +{ + IOverridableCodeInfoRepository CodeInfoRepository { get; } +} diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/IOverridableTxProcessorSource.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/IOverridableTxProcessorSource.cs new file mode 100644 index 00000000000..324fff8d409 --- /dev/null +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/IOverridableTxProcessorSource.cs @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using Nethermind.Core; +using Nethermind.Core.Crypto; + +namespace Nethermind.Evm.TransactionProcessing; + +public interface IOverridableTxProcessorSource +{ + IOverridableTxProcessingScope Build(Hash256 stateRoot); + IOverridableTxProcessingScope BuildAndOverride(BlockHeader header, Dictionary? stateOverride); +} diff --git a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs index 3a8677da6be..101b88d120c 100644 --- a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs +++ b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs @@ -2679,19 +2679,19 @@ private EvmExceptionType InstructionSStore valueToStore = newIsZero ? BytesZero.AsSpan() : bytes; - byte[] storageBytes = new byte[32]; // do not stackalloc here - storageCell.Index.ToBigEndian(storageBytes); - _txTracer.ReportStorageChange(storageBytes, valueToStore); - } + if (typeof(TTracingInstructions) == typeof(IsTracing)) + { + ReadOnlySpan valueToStore = newIsZero ? BytesZero.AsSpan() : bytes; + byte[] storageBytes = new byte[32]; // do not stackalloc here + storageCell.Index.ToBigEndian(storageBytes); + _txTracer.ReportStorageChange(storageBytes, valueToStore); + } - if (typeof(TTracingStorage) == typeof(IsTracing)) - { - _txTracer.SetOperationStorage(storageCell.Address, result, bytes, currentValue); + if (typeof(TTracingStorage) == typeof(IsTracing)) + { + _txTracer.SetOperationStorage(storageCell.Address, result, bytes, currentValue); + } } return EvmExceptionType.None; diff --git a/src/Nethermind/Nethermind.Facade.Test/BlockchainBridgeTests.cs b/src/Nethermind/Nethermind.Facade.Test/BlockchainBridgeTests.cs index c698909a89d..7232165c584 100644 --- a/src/Nethermind/Nethermind.Facade.Test/BlockchainBridgeTests.cs +++ b/src/Nethermind/Nethermind.Facade.Test/BlockchainBridgeTests.cs @@ -4,12 +4,10 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Nethermind.Blockchain; using Nethermind.Blockchain.Filters; -using Nethermind.Blockchain.Find; using Nethermind.Blockchain.Receipts; using Nethermind.Consensus.Processing; using Nethermind.Core; @@ -37,7 +35,7 @@ namespace Nethermind.Facade.Test { public class BlockchainBridgeTests { - private BlockchainBridge _blockchainBridge; + private IBlockchainBridge _blockchainBridge; private IBlockTree _blockTree; private ITxPool _txPool; private IReceiptStorage _receiptStorage; @@ -49,18 +47,15 @@ public class BlockchainBridgeTests private ISpecProvider _specProvider; private IDbProvider _dbProvider; - private class TestReadOnlyTxProcessingEnv : ReadOnlyTxProcessingEnv + private class TestReadOnlyTxProcessingEnv( + OverridableWorldStateManager worldStateManager, + IReadOnlyBlockTree blockTree, + ISpecProvider specProvider, + ILogManager logManager, + ITransactionProcessor transactionProcessor) + : OverridableTxProcessingEnv(worldStateManager, blockTree, specProvider, logManager) { - public TestReadOnlyTxProcessingEnv( - IWorldStateManager worldStateManager, - IBlockTree blockTree, - ISpecProvider specProvider, - ILogManager logManager, - ITransactionProcessor transactionProcessor) : - base(worldStateManager, blockTree, specProvider, logManager) - { - _transactionProcessor = transactionProcessor; - } + protected override ITransactionProcessor CreateTransactionProcessor() => transactionProcessor; } [SetUp] @@ -77,22 +72,20 @@ public async Task SetUp() _ethereumEcdsa = Substitute.For(); _specProvider = MainnetSpecProvider.Instance; - ReadOnlyDbProvider dbProvider = new ReadOnlyDbProvider(_dbProvider, false); IReadOnlyTrieStore trieStore = new TrieStore(_dbProvider.StateDb, LimboLogs.Instance).AsReadOnly(); - IWorldStateManager readOnlyWorldStateManager = - new ReadOnlyWorldStateManager(dbProvider, trieStore, LimboLogs.Instance); + OverridableWorldStateManager worldStateManager = new(_dbProvider, trieStore, LimboLogs.Instance); IReadOnlyBlockTree readOnlyBlockTree = _blockTree.AsReadOnly(); - ReadOnlyTxProcessingEnv processingEnv = new TestReadOnlyTxProcessingEnv( - readOnlyWorldStateManager, + TestReadOnlyTxProcessingEnv processingEnv = new( + worldStateManager, readOnlyBlockTree, _specProvider, LimboLogs.Instance, _transactionProcessor); SimulateReadOnlyBlocksProcessingEnvFactory simulateProcessingEnvFactory = new SimulateReadOnlyBlocksProcessingEnvFactory( - readOnlyWorldStateManager, + worldStateManager, readOnlyBlockTree, new ReadOnlyDbProvider(_dbProvider, true), _specProvider, @@ -157,7 +150,7 @@ public void Call_uses_valid_post_merge_and_random_value() Transaction tx = Build.A.Transaction.TestObject; - _blockchainBridge.Call(header, tx, CancellationToken.None); + _blockchainBridge.Call(header, tx); _transactionProcessor.Received().CallAndRestore( tx, Arg.Is(blkCtx => @@ -173,7 +166,7 @@ public void Call_uses_valid_block_number() BlockHeader header = Build.A.BlockHeader.WithNumber(10).TestObject; Transaction tx = new() { GasLimit = Transaction.BaseTxGasCost }; - _blockchainBridge.Call(header, tx, CancellationToken.None); + _blockchainBridge.Call(header, tx); _transactionProcessor.Received().CallAndRestore( tx, Arg.Is(blkCtx => blkCtx.Header.Number == 10), @@ -188,7 +181,7 @@ public void Call_uses_valid_mix_hash() BlockHeader header = Build.A.BlockHeader.WithMixHash(TestItem.KeccakA).TestObject; Transaction tx = new() { GasLimit = Transaction.BaseTxGasCost }; - _blockchainBridge.Call(header, tx, CancellationToken.None); + _blockchainBridge.Call(header, tx); _transactionProcessor.Received().CallAndRestore( tx, Arg.Is(blkCtx => blkCtx.Header.MixHash == TestItem.KeccakA), @@ -203,7 +196,7 @@ public void Call_uses_valid_beneficiary() BlockHeader header = Build.A.BlockHeader.WithBeneficiary(TestItem.AddressB).TestObject; Transaction tx = new() { GasLimit = Transaction.BaseTxGasCost }; - _blockchainBridge.Call(header, tx, CancellationToken.None); + _blockchainBridge.Call(header, tx); _transactionProcessor.Received().CallAndRestore( tx, Arg.Is(blkCtx => blkCtx.Header.Beneficiary == TestItem.AddressB), @@ -214,20 +207,19 @@ public void Call_uses_valid_beneficiary() [TestCase(0)] public void Bridge_head_is_correct(long headNumber) { - ReadOnlyDbProvider dbProvider = new ReadOnlyDbProvider(_dbProvider, false); IReadOnlyTrieStore trieStore = new TrieStore(_dbProvider.StateDb, LimboLogs.Instance).AsReadOnly(); - IWorldStateManager readOnlyWorldStateManager = - new ReadOnlyWorldStateManager(dbProvider, trieStore, LimboLogs.Instance); + OverridableWorldStateManager worldStateManager = + new(_dbProvider, trieStore, LimboLogs.Instance); IReadOnlyBlockTree roBlockTree = _blockTree.AsReadOnly(); - ReadOnlyTxProcessingEnv processingEnv = new( - readOnlyWorldStateManager, + OverridableTxProcessingEnv processingEnv = new( + worldStateManager, roBlockTree, _specProvider, LimboLogs.Instance); SimulateReadOnlyBlocksProcessingEnvFactory simulateProcessingEnv = new SimulateReadOnlyBlocksProcessingEnvFactory( - readOnlyWorldStateManager, + worldStateManager, roBlockTree, new ReadOnlyDbProvider(_dbProvider, true), _specProvider, diff --git a/src/Nethermind/Nethermind.Facade/BlockchainBridge.cs b/src/Nethermind/Nethermind.Facade/BlockchainBridge.cs index 50621ebb666..42e3c884e2f 100644 --- a/src/Nethermind/Nethermind.Facade/BlockchainBridge.cs +++ b/src/Nethermind/Nethermind.Facade/BlockchainBridge.cs @@ -41,7 +41,7 @@ public interface IBlockchainBridgeFactory [Todo(Improve.Refactor, "I want to remove BlockchainBridge, split it into something with logging, state and tx processing. Then we can start using independent modules.")] public class BlockchainBridge : IBlockchainBridge { - private readonly IReadOnlyTxProcessorSource _processingEnv; + private readonly IOverridableTxProcessorSource _processingEnv; private readonly IBlockTree _blockTree; private readonly IStateReader _stateReader; private readonly ITxPool _txPool; @@ -55,7 +55,7 @@ public class BlockchainBridge : IBlockchainBridge private readonly IBlocksConfig _blocksConfig; private readonly SimulateBridgeHelper _simulateBridgeHelper; - public BlockchainBridge(ReadOnlyTxProcessingEnv processingEnv, + public BlockchainBridge(OverridableTxProcessingEnv processingEnv, SimulateReadOnlyBlocksProcessingEnvFactory simulateProcessingEnvFactory, ITxPool? txPool, IReceiptFinder? receiptStorage, @@ -148,11 +148,14 @@ private bool TryGetCanonicalTransaction( return blockHash is not null ? _receiptFinder.Get(blockHash).ForTransaction(txHash) : null; } - public CallOutput Call(BlockHeader header, Transaction tx, CancellationToken cancellationToken) + public CallOutput Call(BlockHeader header, Transaction tx, Dictionary? stateOverride, CancellationToken cancellationToken) { + using IOverridableTxProcessingScope scope = _processingEnv.BuildAndOverride(header, stateOverride); + CallOutputTracer callOutputTracer = new(); - TransactionResult tryCallResult = TryCallAndRestore(header, tx, false, + TransactionResult tryCallResult = TryCallAndRestore(scope, header, tx, false, callOutputTracer.WithCancellation(cancellationToken)); + return new CallOutput { Error = tryCallResult.Success ? callOutputTracer.Error : tryCallResult.Error, @@ -175,6 +178,10 @@ public SimulateOutput Simulate(BlockHeader header, SimulatePayload? stateOverride, CancellationToken cancellationToken) { - using IReadOnlyTxProcessingScope scope = _processingEnv.Build(header.StateRoot!); + using IOverridableTxProcessingScope scope = _processingEnv.BuildAndOverride(header, stateOverride); EstimateGasTracer estimateGasTracer = new(); - TransactionResult tryCallResult = TryCallAndRestore( - header, - tx, - true, + TransactionResult tryCallResult = TryCallAndRestore(scope, header, tx, true, estimateGasTracer.WithCancellation(cancellationToken)); - GasEstimator gasEstimator = new(scope.TransactionProcessor, scope.WorldState, - _specProvider, _blocksConfig); + GasEstimator gasEstimator = new(scope.TransactionProcessor, scope.WorldState, _specProvider, _blocksConfig); long estimate = gasEstimator.Estimate(tx, header, estimateGasTracer, errorMargin, cancellationToken); return new CallOutput @@ -209,13 +212,13 @@ public CallOutput EstimateGas(BlockHeader header, Transaction tx, int errorMargi public CallOutput CreateAccessList(BlockHeader header, Transaction tx, CancellationToken cancellationToken, bool optimize) { - CallOutputTracer callOutputTracer = new(); AccessTxTracer accessTxTracer = optimize ? new(tx.SenderAddress, tx.GetRecipient(tx.IsContractCreation ? _stateReader.GetNonce(header.StateRoot, tx.SenderAddress) : 0), header.GasBeneficiary) : new(header.GasBeneficiary); - TransactionResult tryCallResult = TryCallAndRestore(header, tx, false, + CallOutputTracer callOutputTracer = new(); + TransactionResult tryCallResult = TryCallAndRestore(_processingEnv.Build(header.StateRoot!), header, tx, false, new CompositeTxTracer(callOutputTracer, accessTxTracer).WithCancellation(cancellationToken)); return new CallOutput @@ -229,6 +232,7 @@ public CallOutput CreateAccessList(BlockHeader header, Transaction tx, Cancellat } private TransactionResult TryCallAndRestore( + IOverridableTxProcessingScope scope, BlockHeader blockHeader, Transaction transaction, bool treatBlockHeaderAsParentBlock, @@ -236,7 +240,7 @@ private TransactionResult TryCallAndRestore( { try { - return CallAndRestore(blockHeader, transaction, treatBlockHeaderAsParentBlock, tracer); + return CallAndRestore(blockHeader, transaction, treatBlockHeaderAsParentBlock, tracer, scope); } catch (InsufficientBalanceException ex) { @@ -248,12 +252,11 @@ private TransactionResult CallAndRestore( BlockHeader blockHeader, Transaction transaction, bool treatBlockHeaderAsParentBlock, - ITxTracer tracer) + ITxTracer tracer, + IOverridableTxProcessingScope scope) { transaction.SenderAddress ??= Address.SystemUser; - Hash256 stateRoot = blockHeader.StateRoot!; - using IReadOnlyTxProcessingScope scope = _processingEnv.Build(stateRoot); if (transaction.Nonce == 0) { diff --git a/src/Nethermind/Nethermind.Facade/BlockchainBridgeContract.cs b/src/Nethermind/Nethermind.Facade/BlockchainBridgeContract.cs index 438bb096bd0..abe697c2540 100644 --- a/src/Nethermind/Nethermind.Facade/BlockchainBridgeContract.cs +++ b/src/Nethermind/Nethermind.Facade/BlockchainBridgeContract.cs @@ -36,7 +36,7 @@ public ConstantBridgeContract(Contract contract, IBlockchainBridge blockchainBri public override object[] Call(CallInfo callInfo) { var transaction = GenerateTransaction(callInfo); - var result = _blockchainBridge.Call(callInfo.ParentHeader, transaction, CancellationToken.None); + var result = _blockchainBridge.Call(callInfo.ParentHeader, transaction); if (!string.IsNullOrEmpty(result.Error)) { throw new AbiException(result.Error); diff --git a/src/Nethermind/Nethermind.Facade/IBlockchainBridge.cs b/src/Nethermind/Nethermind.Facade/IBlockchainBridge.cs index b3bce3accf9..459dd3f3de1 100644 --- a/src/Nethermind/Nethermind.Facade/IBlockchainBridge.cs +++ b/src/Nethermind/Nethermind.Facade/IBlockchainBridge.cs @@ -27,9 +27,9 @@ public interface IBlockchainBridge : ILogFinder TxReceipt GetReceipt(Hash256 txHash); (TxReceipt? Receipt, TxGasInfo? GasInfo, int LogIndexStart) GetReceiptAndGasInfo(Hash256 txHash); (TxReceipt? Receipt, Transaction? Transaction, UInt256? baseFee) GetTransaction(Hash256 txHash, bool checkTxnPool = true); - CallOutput Call(BlockHeader header, Transaction tx, CancellationToken cancellationToken); + CallOutput Call(BlockHeader header, Transaction tx, Dictionary? stateOverride = null, CancellationToken cancellationToken = default); SimulateOutput Simulate(BlockHeader header, SimulatePayload payload, CancellationToken cancellationToken); - CallOutput EstimateGas(BlockHeader header, Transaction tx, int errorMarginBasisPoints, CancellationToken cancellationToken); + CallOutput EstimateGas(BlockHeader header, Transaction tx, int errorMarginBasisPoints, Dictionary? stateOverride = null, CancellationToken cancellationToken = default); CallOutput CreateAccessList(BlockHeader header, Transaction tx, CancellationToken cancellationToken, bool optimize); ulong GetChainId(); diff --git a/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/BlockStateCall.cs b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/BlockStateCall.cs index db4efb549ba..3900900bd7c 100644 --- a/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/BlockStateCall.cs +++ b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/BlockStateCall.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using Nethermind.Core; +using Nethermind.Evm; namespace Nethermind.Facade.Proxy.Models.Simulate; diff --git a/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/Error.cs b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/Error.cs index a53bdd9a9e5..6819d2772e4 100644 --- a/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/Error.cs +++ b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/Error.cs @@ -7,5 +7,5 @@ public class Error { public int Code { get; set; } public string Message { get; set; } - public string Data { get; set; } + public string? Data { get; set; } } diff --git a/src/Nethermind/Nethermind.Facade/Simulate/SimulateDictionaryBlockStore.cs b/src/Nethermind/Nethermind.Facade/Simulate/SimulateDictionaryBlockStore.cs index c3825373730..05a4fe994e3 100644 --- a/src/Nethermind/Nethermind.Facade/Simulate/SimulateDictionaryBlockStore.cs +++ b/src/Nethermind/Nethermind.Facade/Simulate/SimulateDictionaryBlockStore.cs @@ -81,4 +81,9 @@ public void SetMetadata(byte[] key, byte[] value) { return _metadataDict.TryGetValue(key, out var value) ? value : readonlyBaseBlockStore.GetMetadata(key); } + + public bool HasBlock(long blockNumber, Hash256 blockHash) + { + return _blockNumDict.ContainsKey(blockNumber); + } } diff --git a/src/Nethermind/Nethermind.Hive/HivePlugin.cs b/src/Nethermind/Nethermind.Hive/HivePlugin.cs index 62aaef0a766..cc60b017556 100644 --- a/src/Nethermind/Nethermind.Hive/HivePlugin.cs +++ b/src/Nethermind/Nethermind.Hive/HivePlugin.cs @@ -8,72 +8,71 @@ using Nethermind.Api.Extensions; using Nethermind.Logging; -namespace Nethermind.Hive +namespace Nethermind.Hive; + +public class HivePlugin : INethermindPlugin { - public class HivePlugin : INethermindPlugin - { - private INethermindApi _api = null!; - private IHiveConfig _hiveConfig = null!; - private ILogger _logger; - private readonly CancellationTokenSource _disposeCancellationToken = new(); + private INethermindApi _api = null!; + private IHiveConfig _hiveConfig = null!; + private ILogger _logger; + private readonly CancellationTokenSource _disposeCancellationToken = new(); - public ValueTask DisposeAsync() - { - _disposeCancellationToken.Cancel(); - _disposeCancellationToken.Dispose(); - return ValueTask.CompletedTask; - } + public ValueTask DisposeAsync() + { + _disposeCancellationToken.Cancel(); + _disposeCancellationToken.Dispose(); + return ValueTask.CompletedTask; + } - public string Name => "Hive"; + public string Name => "Hive"; - public string Description => "Plugin used for executing Hive Ethereum Tests"; + public string Description => "Plugin used for executing Hive Ethereum Tests"; - public string Author => "Nethermind"; + public string Author => "Nethermind"; - public Task Init(INethermindApi api) - { - _api = api ?? throw new ArgumentNullException(nameof(api)); - _hiveConfig = _api.ConfigProvider.GetConfig(); - _logger = _api.LogManager.GetClassLogger(); + public Task Init(INethermindApi api) + { + _api = api ?? throw new ArgumentNullException(nameof(api)); + _hiveConfig = _api.ConfigProvider.GetConfig(); + _logger = _api.LogManager.GetClassLogger(); - Enabled = Environment.GetEnvironmentVariable("NETHERMIND_HIVE_ENABLED")?.ToLowerInvariant() == "true" || _hiveConfig.Enabled; + Enabled = _hiveConfig.Enabled; - return Task.CompletedTask; - } - - public async Task InitNetworkProtocol() - { - if (Enabled) - { - if (_api.BlockTree is null) throw new ArgumentNullException(nameof(_api.BlockTree)); - if (_api.BlockProcessingQueue is null) throw new ArgumentNullException(nameof(_api.BlockProcessingQueue)); - if (_api.ConfigProvider is null) throw new ArgumentNullException(nameof(_api.ConfigProvider)); - if (_api.LogManager is null) throw new ArgumentNullException(nameof(_api.LogManager)); - if (_api.FileSystem is null) throw new ArgumentNullException(nameof(_api.FileSystem)); - if (_api.BlockValidator is null) throw new ArgumentNullException(nameof(_api.BlockValidator)); - - _api.TxGossipPolicy.Policies.Clear(); - - HiveRunner hiveRunner = new( - _api.BlockTree, - _api.BlockProcessingQueue, - _api.ConfigProvider, - _api.LogManager.GetClassLogger(), - _api.FileSystem, - _api.BlockValidator - ); - - if (_logger.IsInfo) _logger.Info("Hive is starting"); - - await hiveRunner.Start(_disposeCancellationToken.Token); - } - } + return Task.CompletedTask; + } - public Task InitRpcModules() + public async Task InitNetworkProtocol() + { + if (Enabled) { - return Task.CompletedTask; + if (_api.BlockTree is null) throw new ArgumentNullException(nameof(_api.BlockTree)); + if (_api.BlockProcessingQueue is null) throw new ArgumentNullException(nameof(_api.BlockProcessingQueue)); + if (_api.ConfigProvider is null) throw new ArgumentNullException(nameof(_api.ConfigProvider)); + if (_api.LogManager is null) throw new ArgumentNullException(nameof(_api.LogManager)); + if (_api.FileSystem is null) throw new ArgumentNullException(nameof(_api.FileSystem)); + if (_api.BlockValidator is null) throw new ArgumentNullException(nameof(_api.BlockValidator)); + + _api.TxGossipPolicy.Policies.Clear(); + + HiveRunner hiveRunner = new( + _api.BlockTree, + _api.BlockProcessingQueue, + _api.ConfigProvider, + _api.LogManager.GetClassLogger(), + _api.FileSystem, + _api.BlockValidator + ); + + if (_logger.IsInfo) _logger.Info("Hive is starting"); + + await hiveRunner.Start(_disposeCancellationToken.Token); } + } - private bool Enabled { get; set; } + public Task InitRpcModules() + { + return Task.CompletedTask; } + + private bool Enabled { get; set; } } diff --git a/src/Nethermind/Nethermind.Init/InitializeStateDb.cs b/src/Nethermind/Nethermind.Init/InitializeStateDb.cs index 8f53c42a517..4fd09fe8e97 100644 --- a/src/Nethermind/Nethermind.Init/InitializeStateDb.cs +++ b/src/Nethermind/Nethermind.Init/InitializeStateDb.cs @@ -198,20 +198,10 @@ public Task Execute(CancellationToken cancellationToken) if (_api.Config().DiagnosticMode == DiagnosticMode.VerifyTrie) { - Task.Run(() => - { - try - { - _logger!.Info("Collecting trie stats and verifying that no nodes are missing..."); - Hash256 stateRoot = getApi.BlockTree!.Head?.StateRoot ?? Keccak.EmptyTreeHash; - TrieStats stats = stateManager.GlobalStateReader.CollectStats(stateRoot, getApi.DbProvider.CodeDb, _api.LogManager); - _logger.Info($"Starting from {getApi.BlockTree.Head?.Number} {getApi.BlockTree.Head?.StateRoot}{Environment.NewLine}" + stats); - } - catch (Exception ex) - { - _logger!.Error(ex.ToString()); - } - }); + _logger!.Info("Collecting trie stats and verifying that no nodes are missing..."); + Hash256 stateRoot = getApi.BlockTree!.Head?.StateRoot ?? Keccak.EmptyTreeHash; + TrieStats stats = stateManager.GlobalStateReader.CollectStats(stateRoot, getApi.DbProvider.CodeDb, _api.LogManager, _api.ProcessExit!.Token); + _logger.Info($"Starting from {getApi.BlockTree.Head?.Number} {getApi.BlockTree.Head?.StateRoot}{Environment.NewLine}" + stats); } // Init state if we need system calls before actual processing starts diff --git a/src/Nethermind/Nethermind.Init/Steps/RegisterRpcModules.cs b/src/Nethermind/Nethermind.Init/Steps/RegisterRpcModules.cs index bc77defb9d5..5bc4f2ffc9a 100644 --- a/src/Nethermind/Nethermind.Init/Steps/RegisterRpcModules.cs +++ b/src/Nethermind/Nethermind.Init/Steps/RegisterRpcModules.cs @@ -99,7 +99,7 @@ public virtual async Task Execute(CancellationToken cancellationToken) rpcModuleProvider.RegisterBounded(proofModuleFactory, 2, _jsonRpcConfig.Timeout); DebugModuleFactory debugModuleFactory = new( - _api.WorldStateManager, + _api.WorldStateManager.TrieStore, _api.DbProvider, _api.BlockTree, _jsonRpcConfig, @@ -250,13 +250,15 @@ protected virtual void RegisterEthRpcModule(IRpcModuleProvider rpcModuleProvider protected ModuleFactoryBase CreateTraceModuleFactory() { StepDependencyException.ThrowIfNull(_api.WorldStateManager); + StepDependencyException.ThrowIfNull(_api.DbProvider); StepDependencyException.ThrowIfNull(_api.BlockTree); StepDependencyException.ThrowIfNull(_api.RewardCalculatorSource); StepDependencyException.ThrowIfNull(_api.ReceiptStorage); StepDependencyException.ThrowIfNull(_api.SpecProvider); return new TraceModuleFactory( - _api.WorldStateManager, + _api.WorldStateManager.TrieStore, + _api.DbProvider, _api.BlockTree, _jsonRpcConfig, _api.BlockPreprocessor, diff --git a/src/Nethermind/Nethermind.JsonRpc.Benchmark/EthModuleBenchmarks.cs b/src/Nethermind/Nethermind.JsonRpc.Benchmark/EthModuleBenchmarks.cs index 875acb33763..817b5d0be02 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Benchmark/EthModuleBenchmarks.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Benchmark/EthModuleBenchmarks.cs @@ -66,7 +66,7 @@ public void GlobalSetup() stateProvider.Commit(spec); stateProvider.CommitTree(0); - WorldStateManager stateManager = new WorldStateManager(stateProvider, trieStore, dbProvider, LimboLogs.Instance); + OverridableWorldStateManager stateManager = new(dbProvider, trieStore.AsReadOnly(), LimboLogs.Instance); StateReader stateReader = new(trieStore, codeDb, LimboLogs.Instance); @@ -135,7 +135,7 @@ TransactionProcessor transactionProcessor new ReceiptsRecovery(ecdsa, specProvider)); BlockchainBridge bridge = new( - new ReadOnlyTxProcessingEnv( + new OverridableTxProcessingEnv( stateManager, new ReadOnlyBlockTree(blockTree), specProvider, diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Data/SerializationTestBase.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Data/SerializationTestBase.cs index 9481c0e77d7..af8d325612c 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Data/SerializationTestBase.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Data/SerializationTestBase.cs @@ -69,7 +69,7 @@ protected void TestToJson(T item, string expectedResult, params JsonConverter IJsonSerializer serializer = BuildSerializer(converters); string result = serializer.Serialize(item); - Assert.That(result, Is.EqualTo(expectedResult.Replace("+", "\\u002B")), result.Replace("\"", "\\\"")); + Assert.That(result, Is.EqualTo(expectedResult), result.Replace("\"", "\\\"")); } private static IJsonSerializer BuildSerializer(params JsonConverter[] converters) => new EthereumJsonSerializer(converters); diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/DebugModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/DebugModuleTests.cs index 15038cf51b9..ef4111628be 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/DebugModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/DebugModuleTests.cs @@ -31,6 +31,7 @@ namespace Nethermind.JsonRpc.Test.Modules; +// Tests with mocked IDebugBridge [Parallelizable(ParallelScope.Self)] public class DebugModuleTests { diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/DebugRpcModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/DebugRpcModuleTests.cs new file mode 100644 index 00000000000..2c78136b61b --- /dev/null +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/DebugRpcModuleTests.cs @@ -0,0 +1,213 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.IO.Abstractions; +using System.Text.Json; +using System.Threading.Tasks; +using FluentAssertions; +using FluentAssertions.Execution; +using FluentAssertions.Json; +using Nethermind.Blockchain.Blocks; +using Nethermind.Blockchain.Receipts; +using Nethermind.Config; +using Nethermind.Consensus.Rewards; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Specs; +using Nethermind.Core.Test.Builders; +using Nethermind.Evm.Tracing.GethStyle; +using Nethermind.Int256; +using Nethermind.JsonRpc.Modules.DebugModule; +using Nethermind.Synchronization.ParallelSync; +using Newtonsoft.Json.Linq; +using NSubstitute; +using NUnit.Framework; + +namespace Nethermind.JsonRpc.Test.Modules; + +[Parallelizable(ParallelScope.Self)] +public class DebugRpcModuleTests +{ + private class Context : IDisposable + { + public IDebugRpcModule DebugRpcModule { get; } + public TestRpcBlockchain Blockchain { get; } + + private Context(TestRpcBlockchain blockchain, IDebugRpcModule debugRpcModule) + { + DebugRpcModule = debugRpcModule; + Blockchain = blockchain; + } + + public static async Task Create(ISpecProvider? specProvider = null, bool isAura = false) + { + TestRpcBlockchain blockchain = await TestRpcBlockchain.ForTest(isAura ? SealEngineType.AuRa : SealEngineType.NethDev).Build(specProvider); + + IConfigProvider configProvider = Substitute.For(); + IReceiptsMigration receiptsMigration = Substitute.For(); + ISyncModeSelector syncModeSelector = Substitute.For(); + var factory = new DebugModuleFactory( + blockchain.WorldStateManager.TrieStore, + blockchain.DbProvider, + blockchain.BlockTree, + blockchain.RpcConfig, + blockchain.BlockValidator, + blockchain.BlockPreprocessorStep, + NoBlockRewards.Instance, + blockchain.ReceiptStorage, + receiptsMigration, + configProvider, + blockchain.SpecProvider, + syncModeSelector, + new BadBlockStore(blockchain.BlocksDb, 100), + new FileSystem(), + blockchain.LogManager + ); + + IDebugRpcModule debugRpcModule = factory.Create(); + + return new(blockchain, debugRpcModule); + } + + public void Dispose() => Blockchain.Dispose(); + } + + [Test] + public async Task Debug_traceCall_fails_when_not_enough_balance() + { + using Context ctx = await Context.Create(); + + Address address = Build.An.Address.TestObject; + UInt256 balance = 100.Ether(), send = balance / 2; + + JsonRpcResponse response = await RpcTest.TestRequest(ctx.DebugRpcModule, "debug_traceCall", + new { from = $"{address}", to = $"{TestItem.AddressC}", value = send.ToString("X") } + ); + + response.Should().BeOfType() + .Which.Error?.Message?.Should().Contain("insufficient funds"); + } + + [Test] + public async Task Debug_traceCall_runs_on_top_of_specified_block() + { + using Context ctx = await Context.Create(); + TestRpcBlockchain blockchain = ctx.Blockchain; + + Address address = Build.An.Address.TestObject; + UInt256 balance = 100.Ether(); + + await blockchain.AddFunds(address, balance / 2); + await blockchain.AddFunds(address, balance / 2); + Hash256 lastBlockHash = blockchain.BlockTree.Head!.Hash!; + + JsonRpcResponse response = await RpcTest.TestRequest(ctx.DebugRpcModule, "debug_traceCall", + new { from = $"{address}", to = $"{TestItem.AddressC}", value = balance.ToString("X") }, + $"{lastBlockHash}" + ); + + response.Should().BeOfType(); + } + + [TestCase( + "Nonce override doesn't cause failure", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000"}""", + """{"0x7f554713be84160fdf0178cc8df86f5aabd33397":{"nonce":"0x123"}}""" + )] + [TestCase( + "Uses account balance from state override", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000","value":"0x100"}""", + """{"0x7f554713be84160fdf0178cc8df86f5aabd33397":{"balance":"0x100"}}""" + )] + [TestCase( + "Executes code from state override", + """{"from":"0xb7705ae4c6f81b66cdb323c65f4e8133690fc099","to":"0xc200000000000000000000000000000000000000","input":"0xf8b2cb4f000000000000000000000000b7705ae4c6f81b66cdb323c65f4e8133690fc099"}""", + """{"0xc200000000000000000000000000000000000000":{"code":"0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063f8b2cb4f14610030575b600080fd5b61004a600480360381019061004591906100e4565b610060565b604051610057919061012a565b60405180910390f35b60008173ffffffffffffffffffffffffffffffffffffffff16319050919050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b60006100b182610086565b9050919050565b6100c1816100a6565b81146100cc57600080fd5b50565b6000813590506100de816100b8565b92915050565b6000602082840312156100fa576100f9610081565b5b6000610108848285016100cf565b91505092915050565b6000819050919050565b61012481610111565b82525050565b600060208201905061013f600083018461011b565b9291505056fea2646970667358221220172c443a163d8a43e018c339d1b749c312c94b6de22835953d960985daf228c764736f6c63430008120033"}}""", + "00000000000000000000000000000000000000000000003635c9adc5de9f09e5" + )] + [TestCase( + "Executes precompile using overriden address", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000","input":"0xB6E16D27AC5AB427A7F68900AC5559CE272DC6C37C82B3E052246C82244C50E4000000000000000000000000000000000000000000000000000000000000001C7B8B1991EB44757BC688016D27940DF8FB971D7C87F77A6BC4E938E3202C44037E9267B0AEAA82FA765361918F2D8ABD9CDD86E64AA6F2B81D3C4E0B69A7B055"}""", + """{"0x0000000000000000000000000000000000000001":{"movePrecompileToAddress":"0xc200000000000000000000000000000000000000", "code": "0x"}}""", + "000000000000000000000000b7705ae4c6f81b66cdb323c65f4e8133690fc099" + )] + public async Task Debug_traceCall_with_state_override(string name, string transactionJson, string stateOverrideJson, string? expectedValue = null) + { + var transaction = JsonSerializer.Deserialize(transactionJson); + var stateOverride = JsonSerializer.Deserialize(stateOverrideJson); + + using Context ctx = await Context.Create(); + + JsonRpcResponse response = await RpcTest.TestRequest(ctx.DebugRpcModule, "debug_traceCall", + transaction, null, new { stateOverrides = stateOverride } + ); + + GethLikeTxTrace trace = response.Should().BeOfType() + .Which.Result.Should().BeOfType() + .Subject; + + if (expectedValue != null) + Convert.ToHexString(trace.ReturnValue).Should().BeEquivalentTo(expectedValue); + } + + [TestCase( + "When balance is overriden", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xbe5c953dd0ddb0ce033a98f36c981f1b74d3b33f","value":"0x100"}""", + """{"0x7f554713be84160fdf0178cc8df86f5aabd33397":{"balance":"0x100"}}""" + )] + [TestCase( + "When address code is overriden", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000","input":"0xf8b2cb4f000000000000000000000000b7705ae4c6f81b66cdb323c65f4e8133690fc099"}""", + """{"0xc200000000000000000000000000000000000000":{"code":"0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063f8b2cb4f14610030575b600080fd5b61004a600480360381019061004591906100e4565b610060565b604051610057919061012a565b60405180910390f35b60008173ffffffffffffffffffffffffffffffffffffffff16319050919050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b60006100b182610086565b9050919050565b6100c1816100a6565b81146100cc57600080fd5b50565b6000813590506100de816100b8565b92915050565b6000602082840312156100fa576100f9610081565b5b6000610108848285016100cf565b91505092915050565b6000819050919050565b61012481610111565b82525050565b600060208201905061013f600083018461011b565b9291505056fea2646970667358221220172c443a163d8a43e018c339d1b749c312c94b6de22835953d960985daf228c764736f6c63430008120033"}}""" + )] + [TestCase( + "When precompile address is changed", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000","input":"0xB6E16D27AC5AB427A7F68900AC5559CE272DC6C37C82B3E052246C82244C50E4000000000000000000000000000000000000000000000000000000000000001C7B8B1991EB44757BC688016D27940DF8FB971D7C87F77A6BC4E938E3202C44037E9267B0AEAA82FA765361918F2D8ABD9CDD86E64AA6F2B81D3C4E0B69A7B055"}""", + """{"0x0000000000000000000000000000000000000001":{"movePrecompileToAddress":"0xc200000000000000000000000000000000000000", "code": "0x"}}""" + )] + public async Task Debug_traceCall_with_state_override_does_not_affect_other_calls(string name, string transactionJson, string stateOverrideJson) + { + var transaction = JsonSerializer.Deserialize(transactionJson); + var stateOverride = JsonSerializer.Deserialize(stateOverrideJson); + + using Context ctx = await Context.Create(); + + var resultOverrideBefore = await RpcTest.TestSerializedRequest(ctx.DebugRpcModule, "debug_traceCall", transaction, null, new + { + stateOverrides = stateOverride, + enableMemory = false, + disableStorage = true, + disableStack = true, + tracer = "callTracer", + tracerConfig = new { withLog = false } + }); + + var resultNoOverride = await RpcTest.TestSerializedRequest(ctx.DebugRpcModule, "debug_traceCall", transaction, null, new + { + // configuration to minimize number of fields being compared + enableMemory = false, + disableStorage = true, + disableStack = true, + tracer = "callTracer", + tracerConfig = new { withLog = false } + }); + + var resultOverrideAfter = await RpcTest.TestSerializedRequest(ctx.DebugRpcModule, "debug_traceCall", transaction, null, new + { + stateOverrides = stateOverride, + enableMemory = false, + disableStorage = true, + disableStack = true, + tracer = "callTracer", + tracerConfig = new { withLog = false } + }); + + using (new AssertionScope()) + { + JToken.Parse(resultOverrideBefore).Should().BeEquivalentTo(resultOverrideAfter); + JToken.Parse(resultNoOverride).Should().NotBeEquivalentTo(resultOverrideAfter); + } + } +} diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EstimateGas.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EstimateGas.cs index 41b8804aec2..181e0fd06fc 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EstimateGas.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EstimateGas.cs @@ -2,8 +2,11 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Text.Json; using System.Threading.Tasks; using FluentAssertions; +using FluentAssertions.Execution; +using FluentAssertions.Json; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; @@ -258,4 +261,76 @@ public async Task should_estimate_transaction_with_deployed_code_when_eip3607_en await ctx.Test.TestEthRpc("eth_estimateGas", transaction, "latest"); Assert.That(serialized, Is.EqualTo("{\"jsonrpc\":\"2.0\",\"result\":\"0x5208\",\"id\":67}")); } + + [TestCase( + "Nonce override doesn't cause failure", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000"}""", + """{"0x7f554713be84160fdf0178cc8df86f5aabd33397":{"nonce":"0x123"}}""", + """{"jsonrpc":"2.0","result":"0x5208","id":67}""" // ETH transfer (intrinsic transaction cost) + )] + [TestCase( + "Uses account balance from state override", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000","value":"0x100"}""", + """{"0x7f554713be84160fdf0178cc8df86f5aabd33397":{"balance":"0x100"}}""", + """{"jsonrpc":"2.0","result":"0x5208","id":67}""" // ETH transfer (intrinsic transaction cost) + )] + [TestCase( + "Executes code from state override", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000","input":"0x60fe47b1112233445566778899001122334455667788990011223344556677889900112233445566778899001122"}""", + """{"0xc200000000000000000000000000000000000000":{"code":"0x6080604052348015600e575f80fd5b50600436106030575f3560e01c80632a1afcd914603457806360fe47b114604d575b5f80fd5b603b5f5481565b60405190815260200160405180910390f35b605c6058366004605e565b5f55565b005b5f60208284031215606d575f80fd5b503591905056fea2646970667358221220fd4e5f3894be8e57fc7460afebb5c90d96c3486d79bf47b00c2ed666ab2f82b364736f6c634300081a0033"}}""", + """{"jsonrpc":"2.0","result":"0xabdd","id":67}""" // Store uint256 (cold access) + few other light instructions + intrinsic transaction cost + )] + [TestCase( + "Executes precompile using overriden address", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000","input":"0xB6E16D27AC5AB427A7F68900AC5559CE272DC6C37C82B3E052246C82244C50E4000000000000000000000000000000000000000000000000000000000000001C7B8B1991EB44757BC688016D27940DF8FB971D7C87F77A6BC4E938E3202C44037E9267B0AEAA82FA765361918F2D8ABD9CDD86E64AA6F2B81D3C4E0B69A7B055"}""", + """{"0x0000000000000000000000000000000000000001":{"movePrecompileToAddress":"0xc200000000000000000000000000000000000000", "code": "0x"}}""", + """{"jsonrpc":"2.0","result":"0x6440","id":67}""" // EcRecover call + intrinsic transaction cost + )] + public async Task Estimate_gas_with_state_override(string name, string transactionJson, string stateOverrideJson, string expectedResult) + { + var transaction = JsonSerializer.Deserialize(transactionJson); + var stateOverride = JsonSerializer.Deserialize(stateOverrideJson); + + TestSpecProvider specProvider = new(Prague.Instance); + using Context ctx = await Context.Create(specProvider); + + string serialized = await ctx.Test.TestEthRpc("eth_estimateGas", transaction, "latest", stateOverride); + + JToken.Parse(serialized).Should().BeEquivalentTo(expectedResult); + } + + [TestCase( + "When balance and nonce is overriden", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000","value":"0x123"}""", + """{"0x7f554713be84160fdf0178cc8df86f5aabd33397":{"balance":"0x123", "nonce": "0x123"}}""" + )] + [TestCase( + "When address code is overriden", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000","input":"0x60fe47b1112233445566778899001122334455667788990011223344556677889900112233445566778899001122"}""", + """{"0xc200000000000000000000000000000000000000":{"code":"0x6080604052348015600e575f80fd5b50600436106030575f3560e01c80632a1afcd914603457806360fe47b114604d575b5f80fd5b603b5f5481565b60405190815260200160405180910390f35b605c6058366004605e565b5f55565b005b5f60208284031215606d575f80fd5b503591905056fea2646970667358221220fd4e5f3894be8e57fc7460afebb5c90d96c3486d79bf47b00c2ed666ab2f82b364736f6c634300081a0033"}}""" + )] + [TestCase( + "When precompile address is changed", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000","input":"0xB6E16D27AC5AB427A7F68900AC5559CE272DC6C37C82B3E052246C82244C50E4000000000000000000000000000000000000000000000000000000000000001C7B8B1991EB44757BC688016D27940DF8FB971D7C87F77A6BC4E938E3202C44037E9267B0AEAA82FA765361918F2D8ABD9CDD86E64AA6F2B81D3C4E0B69A7B055"}""", + """{"0x0000000000000000000000000000000000000001":{"movePrecompileToAddress":"0xc200000000000000000000000000000000000000", "code": "0x"}}""" + )] + public async Task Estimate_gas_with_state_override_does_not_affect_other_calls(string name, string transactionJson, string stateOverrideJson) + { + var transaction = JsonSerializer.Deserialize(transactionJson); + var stateOverride = JsonSerializer.Deserialize(stateOverrideJson); + + using Context ctx = await Context.Create(); + + var resultOverrideBefore = await ctx.Test.TestEthRpc("eth_estimateGas", transaction, "latest", stateOverride); + + var resultNoOverride = await ctx.Test.TestEthRpc("eth_estimateGas", transaction, "latest"); + + var resultOverrideAfter = await ctx.Test.TestEthRpc("eth_estimateGas", transaction, "latest", stateOverride); + + using (new AssertionScope()) + { + JToken.Parse(resultOverrideBefore).Should().BeEquivalentTo(resultOverrideAfter); + JToken.Parse(resultNoOverride).Should().NotBeEquivalentTo(resultOverrideAfter); + } + } } diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EthCall.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EthCall.cs index 8118fe4d8e7..afbce2b63e2 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EthCall.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EthCall.cs @@ -1,8 +1,11 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Text.Json; using System.Threading.Tasks; using FluentAssertions; +using FluentAssertions.Execution; +using FluentAssertions.Json; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; @@ -14,6 +17,7 @@ using Nethermind.Specs.Test; using Nethermind.Specs.Test.ChainSpecStyle; using Nethermind.State; +using Newtonsoft.Json.Linq; using NUnit.Framework; namespace Nethermind.JsonRpc.Test.Modules.Eth; @@ -295,4 +299,75 @@ public async Task Eth_call_with_revert() Assert.That( serialized, Is.EqualTo("{\"jsonrpc\":\"2.0\",\"error\":{\"code\":-32015,\"message\":\"VM execution error.\",\"data\":\"revert\"},\"id\":67}")); } + + [TestCase( + "Nonce override doesn't cause failure", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000"}""", + """{"0x7f554713be84160fdf0178cc8df86f5aabd33397":{"nonce":"0x123"}}""", + """{"jsonrpc":"2.0","result":"0x","id":67}""" + )] + [TestCase( + "Uses account balance from state override", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000","value":"0x100"}""", + """{"0x7f554713be84160fdf0178cc8df86f5aabd33397":{"balance":"0x100"}}""", + """{"jsonrpc":"2.0","result":"0x","id":67}""" + )] + [TestCase( + "Executes code from state override", + """{"from":"0xb7705ae4c6f81b66cdb323c65f4e8133690fc099","to":"0xc200000000000000000000000000000000000000","input":"0xf8b2cb4f000000000000000000000000b7705ae4c6f81b66cdb323c65f4e8133690fc099"}""", + """{"0xc200000000000000000000000000000000000000":{"code":"0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063f8b2cb4f14610030575b600080fd5b61004a600480360381019061004591906100e4565b610060565b604051610057919061012a565b60405180910390f35b60008173ffffffffffffffffffffffffffffffffffffffff16319050919050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b60006100b182610086565b9050919050565b6100c1816100a6565b81146100cc57600080fd5b50565b6000813590506100de816100b8565b92915050565b6000602082840312156100fa576100f9610081565b5b6000610108848285016100cf565b91505092915050565b6000819050919050565b61012481610111565b82525050565b600060208201905061013f600083018461011b565b9291505056fea2646970667358221220172c443a163d8a43e018c339d1b749c312c94b6de22835953d960985daf228c764736f6c63430008120033"}}""", + """{"jsonrpc":"2.0","result":"0x00000000000000000000000000000000000000000000003635c9adc5de9f09e5","id":67}""" + )] + [TestCase( + "Executes precompile using overriden address", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000","input":"0xB6E16D27AC5AB427A7F68900AC5559CE272DC6C37C82B3E052246C82244C50E4000000000000000000000000000000000000000000000000000000000000001C7B8B1991EB44757BC688016D27940DF8FB971D7C87F77A6BC4E938E3202C44037E9267B0AEAA82FA765361918F2D8ABD9CDD86E64AA6F2B81D3C4E0B69A7B055"}""", + """{"0x0000000000000000000000000000000000000001":{"movePrecompileToAddress":"0xc200000000000000000000000000000000000000", "code": "0x"}}""", + """{"jsonrpc":"2.0","result":"0x000000000000000000000000b7705ae4c6f81b66cdb323c65f4e8133690fc099","id":67}""" + )] + public async Task Eth_call_with_state_override(string name, string transactionJson, string stateOverrideJson, string expectedResult) + { + var transaction = JsonSerializer.Deserialize(transactionJson); + var stateOverride = JsonSerializer.Deserialize(stateOverrideJson); + + using Context ctx = await Context.Create(); + + string serialized = await ctx.Test.TestEthRpc("eth_call", transaction, "latest", stateOverride); + + JToken.Parse(serialized).Should().BeEquivalentTo(expectedResult); + } + + [TestCase( + "When balance and nonce is overriden", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xbe5c953dd0ddb0ce033a98f36c981f1b74d3b33f","value":"0x1"}""", + """{"0x7f554713be84160fdf0178cc8df86f5aabd33397":{"balance":"0x123", "nonce": "0x123"}}""" + )] + [TestCase( + "When address code is overriden", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000","input":"0xf8b2cb4f000000000000000000000000b7705ae4c6f81b66cdb323c65f4e8133690fc099"}""", + """{"0xc200000000000000000000000000000000000000":{"code":"0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063f8b2cb4f14610030575b600080fd5b61004a600480360381019061004591906100e4565b610060565b604051610057919061012a565b60405180910390f35b60008173ffffffffffffffffffffffffffffffffffffffff16319050919050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b60006100b182610086565b9050919050565b6100c1816100a6565b81146100cc57600080fd5b50565b6000813590506100de816100b8565b92915050565b6000602082840312156100fa576100f9610081565b5b6000610108848285016100cf565b91505092915050565b6000819050919050565b61012481610111565b82525050565b600060208201905061013f600083018461011b565b9291505056fea2646970667358221220172c443a163d8a43e018c339d1b749c312c94b6de22835953d960985daf228c764736f6c63430008120033"}}""" + )] + [TestCase( + "When precompile address is changed", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000","input":"0xB6E16D27AC5AB427A7F68900AC5559CE272DC6C37C82B3E052246C82244C50E4000000000000000000000000000000000000000000000000000000000000001C7B8B1991EB44757BC688016D27940DF8FB971D7C87F77A6BC4E938E3202C44037E9267B0AEAA82FA765361918F2D8ABD9CDD86E64AA6F2B81D3C4E0B69A7B055"}""", + """{"0x0000000000000000000000000000000000000001":{"movePrecompileToAddress":"0xc200000000000000000000000000000000000000", "code": "0x"}}""" + )] + public async Task Eth_call_with_state_override_does_not_affect_other_calls(string name, string transactionJson, string stateOverrideJson) + { + var transaction = JsonSerializer.Deserialize(transactionJson); + var stateOverride = JsonSerializer.Deserialize(stateOverrideJson); + + using Context ctx = await Context.Create(); + + var resultOverrideBefore = await ctx.Test.TestEthRpc("eth_call", transaction, "latest", stateOverride); + + var resultNoOverride = await ctx.Test.TestEthRpc("eth_call", transaction, "latest"); + + var resultOverrideAfter = await ctx.Test.TestEthRpc("eth_call", transaction, "latest", stateOverride); + + using (new AssertionScope()) + { + JToken.Parse(resultOverrideBefore).Should().BeEquivalentTo(resultOverrideAfter); + JToken.Parse(resultNoOverride).Should().NotBeEquivalentTo(resultOverrideAfter); + } + } } diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/Simulate/EthSimulateTestsBlocksAndTransactions.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/Simulate/EthSimulateTestsBlocksAndTransactions.cs index bcfa68f448a..a7f8b54e2dc 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/Simulate/EthSimulateTestsBlocksAndTransactions.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/Simulate/EthSimulateTestsBlocksAndTransactions.cs @@ -12,8 +12,8 @@ using Nethermind.Core.Extensions; using Nethermind.Core.Test.Builders; using Nethermind.Crypto; +using Nethermind.Evm; using Nethermind.Facade.Eth.RpcTransaction; -using Nethermind.Facade.Proxy.Models; using Nethermind.Facade.Proxy.Models.Simulate; using Nethermind.Int256; using Nethermind.JsonRpc.Modules.Eth; diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TestRpcBlockchain.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TestRpcBlockchain.cs index 1a63fa846cd..5d81df7afa7 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TestRpcBlockchain.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TestRpcBlockchain.cs @@ -30,6 +30,7 @@ using Nethermind.Config; using Nethermind.Db; using Nethermind.Facade.Simulate; +using Nethermind.State; using Nethermind.Synchronization.ParallelSync; using NSubstitute; @@ -44,6 +45,7 @@ public class TestRpcBlockchain : TestBlockchain public ITxSender TxSender { get; private set; } = null!; public IReceiptFinder ReceiptFinder { get; private set; } = null!; public IGasPriceOracle GasPriceOracle { get; private set; } = null!; + public OverridableWorldStateManager OverridableWorldStateManager { get; private set; } = null!; public IKeyStore KeyStore { get; } = new MemKeyStore(TestItem.PrivateKeys, Path.Combine("testKeyStoreDir", Path.GetRandomFileName())); public IWallet TestWallet { get; } = @@ -145,8 +147,9 @@ protected override async Task Build( IFilterManager filterManager = new FilterManager(filterStore, BlockProcessor, TxPool, LimboLogs.Instance); var dbProvider = new ReadOnlyDbProvider(DbProvider, false); IReadOnlyBlockTree? roBlockTree = BlockTree!.AsReadOnly(); - ReadOnlyTxProcessingEnv processingEnv = new( - WorldStateManager, + OverridableWorldStateManager overridableWorldStateManager = new(DbProvider, WorldStateManager.TrieStore, LogManager); + OverridableTxProcessingEnv processingEnv = new( + overridableWorldStateManager, roBlockTree, SpecProvider, LimboLogs.Instance); @@ -169,6 +172,7 @@ protected override async Task Build( GasPriceOracle ??= new GasPriceOracle(BlockFinder, SpecProvider, LogManager); FeeHistoryOracle ??= new FeeHistoryOracle(BlockTree, ReceiptStorage, SpecProvider); EthRpcModule = _ethRpcModuleBuilder(this); + OverridableWorldStateManager = overridableWorldStateManager; return this; } diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Trace/ParityStyleTracerTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Trace/ParityStyleTracerTests.cs index 1c4023f547f..f4c12aeaaa6 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Trace/ParityStyleTracerTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Trace/ParityStyleTracerTests.cs @@ -41,6 +41,7 @@ public class ParityStyleTracerTests private Tracer? _tracer; private IPoSSwitcher? _poSSwitcher; private IStateReader _stateReader; + private TraceRpcModule _traceRpcModule; private readonly IJsonRpcConfig _jsonRpcConfig = new JsonRpcConfig(); [SetUp] @@ -84,7 +85,9 @@ public void Setup() _blockTree.SuggestBlock(genesis); _processor.Process(genesis, ProcessingOptions.None, NullBlockTracer.Instance); + IOverridableTxProcessorSource txProcessingSource = Substitute.For(); _tracer = new Tracer(stateProvider, _processor, _processor); + _traceRpcModule = new(NullReceiptStorage.Instance, _tracer, _blockTree, _jsonRpcConfig, _stateReader, txProcessingSource); } [TearDown] @@ -93,16 +96,14 @@ public void Setup() [Test] public void Can_trace_raw_parity_style() { - TraceRpcModule traceRpcModule = new(NullReceiptStorage.Instance, _tracer, _blockTree, _jsonRpcConfig, _stateReader); - ResultWrapper result = traceRpcModule.trace_rawTransaction(Bytes.FromHexString("f889808609184e72a00082271094000000000000000000000000000000000000000080a47f74657374320000000000000000000000000000000000000000000000000000006000571ca08a8bbf888cfa37bbf0bb965423625641fc956967b81d12e23709cead01446075a01ce999b56a8a88504be365442ea61239198e23d1fce7d00fcfc5cd3b44b7215f"), new[] { "trace" }); + ResultWrapper result = _traceRpcModule.trace_rawTransaction(Bytes.FromHexString("f889808609184e72a00082271094000000000000000000000000000000000000000080a47f74657374320000000000000000000000000000000000000000000000000000006000571ca08a8bbf888cfa37bbf0bb965423625641fc956967b81d12e23709cead01446075a01ce999b56a8a88504be365442ea61239198e23d1fce7d00fcfc5cd3b44b7215f"), new[] { "trace" }); Assert.That(result.Data, Is.Not.Null); } [Test] public void Can_trace_raw_parity_style_berlin_tx() { - TraceRpcModule traceRpcModule = new(NullReceiptStorage.Instance, _tracer, _blockTree, _jsonRpcConfig, _stateReader); - ResultWrapper result = traceRpcModule.trace_rawTransaction(Bytes.FromHexString("01f85b821e8e8204d7847735940083030d408080853a60005500c080a0f43e70c79190701347517e283ef63753f6143a5225cbb500b14d98eadfb7616ba070893923d8a1fc97499f426524f9e82f8e0322dfac7c3d7e8a9eee515f0bcdc4"), new[] { "trace" }); + ResultWrapper result = _traceRpcModule.trace_rawTransaction(Bytes.FromHexString("01f85b821e8e8204d7847735940083030d408080853a60005500c080a0f43e70c79190701347517e283ef63753f6143a5225cbb500b14d98eadfb7616ba070893923d8a1fc97499f426524f9e82f8e0322dfac7c3d7e8a9eee515f0bcdc4"), new[] { "trace" }); Assert.That(result.Data, Is.Not.Null); } @@ -114,8 +115,7 @@ public void Should_return_correct_block_reward(bool isPostMerge) _blockTree!.SuggestBlock(block).Should().Be(AddBlockResult.Added); _poSSwitcher!.IsPostMerge(Arg.Any()).Returns(isPostMerge); - TraceRpcModule traceRpcModule = new(NullReceiptStorage.Instance, _tracer, _blockTree, _jsonRpcConfig, _stateReader); - ParityTxTraceFromStore[] result = traceRpcModule.trace_block(new BlockParameter(block.Number)).Data.ToArray(); + ParityTxTraceFromStore[] result = _traceRpcModule.trace_block(new BlockParameter(block.Number)).Data.ToArray(); if (isPostMerge) { result.Length.Should().Be(1); diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TraceRpcModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TraceRpcModuleTests.cs index 37712eabbcd..2938cc32807 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TraceRpcModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TraceRpcModuleTests.cs @@ -3,10 +3,11 @@ using System.Collections.Generic; using System.Linq; +using System.Text.Json; using System.Threading.Tasks; using FluentAssertions; -using Nethermind.Blockchain; -using Nethermind.Blockchain.Receipts; +using FluentAssertions.Execution; +using FluentAssertions.Json; using Nethermind.Core; using Nethermind.Core.Extensions; using Nethermind.Core.Specs; @@ -16,18 +17,18 @@ using Nethermind.JsonRpc.Modules.Trace; using NUnit.Framework; using Nethermind.Blockchain.Find; -using Nethermind.Consensus.Processing; using Nethermind.Consensus.Rewards; -using Nethermind.Consensus.Tracing; -using Nethermind.Consensus.Validators; using Nethermind.Core.Crypto; +using Nethermind.Crypto; using Nethermind.Evm; -using Nethermind.Evm.TransactionProcessing; +using Nethermind.Evm.Tracing.ParityStyle; using Nethermind.Facade.Eth.RpcTransaction; using Nethermind.Serialization.Json; using Nethermind.Specs.Forks; using Nethermind.Specs.Test; using Nethermind.JsonRpc.Data; +using Nethermind.Serialization.Rlp; +using Newtonsoft.Json.Linq; using Nethermind.JsonRpc.Modules; using Nethermind.Specs.Test.ChainSpecStyle; @@ -41,59 +42,47 @@ public async Task Build(ISpecProvider? specProvider = null, bool isAura = false) { JsonRpcConfig = new JsonRpcConfig(); Blockchain = await TestRpcBlockchain.ForTest(isAura ? SealEngineType.AuRa : SealEngineType.NethDev).Build(specProvider); + await Blockchain.AddFunds(TestItem.AddressA, 1000.Ether()); await Blockchain.AddFunds(TestItem.AddressB, 1000.Ether()); await Blockchain.AddFunds(TestItem.AddressC, 1000.Ether()); - ReceiptsRecovery receiptsRecovery = - new(Blockchain.EthereumEcdsa, Blockchain.SpecProvider); - IReceiptFinder receiptFinder = new FullInfoReceiptFinder(Blockchain.ReceiptStorage, receiptsRecovery, Blockchain.BlockFinder); - ReadOnlyTxProcessingEnv txProcessingEnv = - new(Blockchain.WorldStateManager, Blockchain.BlockTree.AsReadOnly(), Blockchain.SpecProvider, Blockchain.LogManager); - IReadOnlyTxProcessingScope scope = txProcessingEnv.Build(Keccak.EmptyTreeHash); - - RewardCalculator rewardCalculatorSource = new(Blockchain.SpecProvider); - - IRewardCalculator rewardCalculator = rewardCalculatorSource.Get(scope.TransactionProcessor); - - RpcBlockTransactionsExecutor rpcBlockTransactionsExecutor = new(scope.TransactionProcessor, scope.WorldState); - BlockProcessor.BlockValidationTransactionsExecutor executeBlockTransactionsExecutor = new(scope.TransactionProcessor, - scope.WorldState); - - ReadOnlyChainProcessingEnv CreateChainProcessingEnv(IBlockProcessor.IBlockTransactionsExecutor transactionsExecutor) => new( - scope, - Always.Valid, - Blockchain.BlockPreprocessorStep, - rewardCalculator, - Blockchain.ReceiptStorage, - Blockchain.SpecProvider, - Blockchain.BlockTree, - Blockchain.StateReader, - Blockchain.LogManager, - transactionsExecutor); - - ReadOnlyChainProcessingEnv traceProcessingEnv = CreateChainProcessingEnv(rpcBlockTransactionsExecutor); - ReadOnlyChainProcessingEnv executeProcessingEnv = CreateChainProcessingEnv(executeBlockTransactionsExecutor); - - Tracer tracer = new(scope.WorldState, traceProcessingEnv.ChainProcessor, executeProcessingEnv.ChainProcessor); - TraceRpcModule = new TraceRpcModule(receiptFinder, tracer, Blockchain.BlockFinder, JsonRpcConfig, txProcessingEnv.StateReader); for (int i = 1; i < 10; i++) { List transactions = new(); for (int j = 0; j < i; j++) { - transactions.Add(Core.Test.Builders.Build.A.Transaction.WithNonce(Blockchain.State.GetNonce(TestItem.AddressB) + (UInt256)j) + transactions.Add(Core.Test.Builders.Build.A.Transaction + .WithTo(Address.Zero) + .WithNonce(Blockchain.State.GetNonce(TestItem.AddressB) + (UInt256)j) .SignedAndResolved(Blockchain.EthereumEcdsa, TestItem.PrivateKeyB).TestObject); } await Blockchain.AddBlock(transactions.ToArray()); } + + Factory = new( + Blockchain.OverridableWorldStateManager.TrieStore, + Blockchain.DbProvider, + Blockchain.BlockTree, + JsonRpcConfig, + Blockchain.BlockPreprocessorStep, + new RewardCalculator(Blockchain.SpecProvider), + Blockchain.ReceiptStorage, + Blockchain.SpecProvider, + Blockchain.PoSSwitcher, + Blockchain.LogManager + ); + + TraceRpcModule = Factory.Create(); } public ITraceRpcModule TraceRpcModule { get; private set; } = null!; + public TraceModuleFactory Factory { get; private set; } = null!; public IJsonRpcConfig JsonRpcConfig { get; private set; } = null!; public TestRpcBlockchain Blockchain { get; set; } = null!; } + [Test] public async Task Tx_positions_are_fine() { @@ -676,6 +665,93 @@ public async Task Trace_call_without_blockParameter_provided_test() Assert.That(traces.Data.Action.To, Is.EqualTo(TestItem.AddressC)); } + [Test] + public async Task Trace_call_runs_on_top_of_specified_block() + { + Context context = new(); + await context.Build(); + TestRpcBlockchain blockchain = context.Blockchain; + + PrivateKey addressKey = Build.A.PrivateKey.TestObject; + Address address = addressKey.Address; + UInt256 balance = 100.Ether(), send = balance / 2; + + await blockchain.AddFunds(address, balance); + Hash256 lastBlockHash = blockchain.BlockTree.Head!.Hash!; + + string[] traceTypes = ["stateDiff"]; + Transaction transaction = Build.A.Transaction + .SignedAndResolved(addressKey) + .WithTo(TestItem.AddressC) + .WithValue(send) + .TestObject; + + ResultWrapper traces = context.TraceRpcModule.trace_call( + TransactionForRpc.FromTransaction(transaction), traceTypes, new(lastBlockHash) + ); + + ParityAccountStateChange? stateChanges = traces.Data.StateChanges?.GetValueOrDefault(address); + stateChanges?.Balance?.Should().BeEquivalentTo(new ParityStateChange(balance, balance - send)); + } + + [Test] + public async Task Trace_callMany_runs_on_top_of_specified_block() + { + Context context = new(); + await context.Build(); + TestRpcBlockchain blockchain = context.Blockchain; + + PrivateKey addressKey = Build.A.PrivateKey.TestObject; + Address address = addressKey.Address; + UInt256 balance = 100.Ether(), send = balance / 2; + + await blockchain.AddFunds(address, balance); + Hash256 lastBlockHash = blockchain.BlockTree.Head!.Hash!; + + string[] traceTypes = ["stateDiff"]; + Transaction transaction = Build.A.Transaction + .SignedAndResolved(addressKey) + .WithTo(TestItem.AddressC) + .WithValue(send) + .TestObject; + + ResultWrapper> traces = context.TraceRpcModule.trace_callMany( + [new() { Transaction = TransactionForRpc.FromTransaction(transaction), TraceTypes = traceTypes }], + new(lastBlockHash) + ); + + ParityAccountStateChange? stateChanges = traces.Data.Single().StateChanges?.GetValueOrDefault(address); + stateChanges?.Balance?.Should().BeEquivalentTo(new ParityStateChange(balance, balance - send)); + } + + [Test] + public async Task Trace_rawTransaction_runs_on_top_of_specified_block() + { + Context context = new(); + await context.Build(); + TestRpcBlockchain blockchain = context.Blockchain; + + PrivateKey addressKey = Build.A.PrivateKey.TestObject; + Address address = addressKey.Address; + UInt256 balance = 100.Ether(), send = balance / 2; + + await blockchain.AddFunds(address, balance); + + string[] traceTypes = ["stateDiff"]; + Transaction transaction = Build.A.Transaction + .WithTo(TestItem.AddressC) + .WithValue(send) + .SignedAndResolved(addressKey) + .TestObject; + + ResultWrapper traces = context.TraceRpcModule.trace_rawTransaction( + TxDecoder.Instance.Encode(transaction).Bytes, traceTypes + ); + + ParityAccountStateChange? stateChanges = traces.Data.StateChanges?.GetValueOrDefault(address); + stateChanges?.Balance?.Should().BeEquivalentTo(new ParityStateChange(balance, balance - send)); + } + [Test] public async Task Trace_call_simple_tx_test() { @@ -775,7 +851,7 @@ public async Task Trace_callMany_accumulates_state_changes() context.TraceRpcModule, "trace_callMany", calls); - Assert.That(serialized, Is.EqualTo("{\"jsonrpc\":\"2.0\",\"result\":[{\"output\":null,\"stateDiff\":{\"0x0000000000000000000000000000000000000000\":{\"balance\":{\"*\":{\"from\":\"0x24\",\"to\":\"0x25\"}},\"code\":\"=\",\"nonce\":\"=\",\"storage\":{}},\"0xb7705ae4c6f81b66cdb323c65f4e8133690fc099\":{\"balance\":{\"*\":{\"from\":\"0x3635c9adc5de9f09e5\",\"to\":\"0x3635c9adc5de9f09e4\"}},\"code\":\"=\",\"nonce\":{\"*\":{\"from\":\"0x3\",\"to\":\"0x4\"}},\"storage\":{}}},\"trace\":[],\"vmTrace\":null},{\"output\":null,\"stateDiff\":{\"0x0000000000000000000000000000000000000000\":{\"balance\":{\"*\":{\"from\":\"0x25\",\"to\":\"0x26\"}},\"code\":\"=\",\"nonce\":\"=\",\"storage\":{}},\"0xb7705ae4c6f81b66cdb323c65f4e8133690fc099\":{\"balance\":{\"*\":{\"from\":\"0x3635c9adc5de9f09e4\",\"to\":\"0x3635c9adc5de9f09e3\"}},\"code\":\"=\",\"nonce\":{\"*\":{\"from\":\"0x4\",\"to\":\"0x5\"}},\"storage\":{}}},\"trace\":[],\"vmTrace\":null}],\"id\":67}"), serialized.Replace("\"", "\\\"")); + Assert.That(serialized, Is.EqualTo("{\"jsonrpc\":\"2.0\",\"result\":[{\"output\":null,\"stateDiff\":{\"0x0000000000000000000000000000000000000000\":{\"balance\":{\"*\":{\"from\":\"0x2d\",\"to\":\"0x2e\"}},\"code\":\"=\",\"nonce\":\"=\",\"storage\":{}},\"0xb7705ae4c6f81b66cdb323c65f4e8133690fc099\":{\"balance\":{\"*\":{\"from\":\"0x3635c9adc5de9f09e5\",\"to\":\"0x3635c9adc5de9f09e4\"}},\"code\":\"=\",\"nonce\":{\"*\":{\"from\":\"0x3\",\"to\":\"0x4\"}},\"storage\":{}}},\"trace\":[],\"vmTrace\":null},{\"output\":null,\"stateDiff\":{\"0x0000000000000000000000000000000000000000\":{\"balance\":{\"*\":{\"from\":\"0x2e\",\"to\":\"0x2f\"}},\"code\":\"=\",\"nonce\":\"=\",\"storage\":{}},\"0xb7705ae4c6f81b66cdb323c65f4e8133690fc099\":{\"balance\":{\"*\":{\"from\":\"0x3635c9adc5de9f09e4\",\"to\":\"0x3635c9adc5de9f09e3\"}},\"code\":\"=\",\"nonce\":{\"*\":{\"from\":\"0x4\",\"to\":\"0x5\"}},\"storage\":{}}},\"trace\":[],\"vmTrace\":null}],\"id\":67}"), serialized.Replace("\"", "\\\"")); } [Test] @@ -853,4 +929,89 @@ public async Task Trace_replayBlockTransactions_stateDiff() state[TestItem.AddressA].Balance!.After.Should().Be(accountA.Balance - 21000 * tx.GasPrice - tx.Value); state[TestItem.AddressF].Balance!.After.Should().Be(accountF.Balance + tx.Value); } + + [TestCase( + "Nonce increments from state override", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000"}""", + "stateDiff", + """{"0x7f554713be84160fdf0178cc8df86f5aabd33397":{"nonce":"0x123"}}""", + """{"jsonrpc":"2.0","result":{"output":null,"stateDiff":{"0x7f554713be84160fdf0178cc8df86f5aabd33397":{"balance":"=","code":"=","nonce":{"*":{"from":"0x123","to":"0x124"}},"storage":{}}},"trace":[],"vmTrace":null},"id":67}""" + )] + [TestCase( + "Uses account balance from state override", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xbe5c953dd0ddb0ce033a98f36c981f1b74d3b33f","value":"0x100"}""", + "stateDiff", + """{"0x7f554713be84160fdf0178cc8df86f5aabd33397":{"balance":"0x100"}}""", + """{"jsonrpc":"2.0","result":{"output":null,"stateDiff":{"0x7f554713be84160fdf0178cc8df86f5aabd33397":{"balance":{"*":{"from":"0x100","to":"0x0"}},"code":"=","nonce":{"*":{"from":"0x0","to":"0x1"}},"storage":{}},"0xbe5c953dd0ddb0ce033a98f36c981f1b74d3b33f":{"balance":{"\u002B":"0x100"},"code":"=","nonce":{"\u002B":"0x0"},"storage":{}}},"trace":[],"vmTrace":null},"id":67}""" + )] + [TestCase( + "Executes code from state override", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000","input":"0x60fe47b1112233445566778899001122334455667788990011223344556677889900112233445566778899001122"}""", + "stateDiff", + """{"0xc200000000000000000000000000000000000000":{"code":"0x6080604052348015600e575f80fd5b50600436106030575f3560e01c80632a1afcd914603457806360fe47b114604d575b5f80fd5b603b5f5481565b60405190815260200160405180910390f35b605c6058366004605e565b5f55565b005b5f60208284031215606d575f80fd5b503591905056fea2646970667358221220fd4e5f3894be8e57fc7460afebb5c90d96c3486d79bf47b00c2ed666ab2f82b364736f6c634300081a0033"}}""", + """{"jsonrpc":"2.0","result":{"output":null,"stateDiff":{"0x7f554713be84160fdf0178cc8df86f5aabd33397":{"balance":{"\u002B":"0x0"},"code":"=","nonce":{"\u002B":"0x1"},"storage":{}},"0xc200000000000000000000000000000000000000":{"balance":"=","code":"=","nonce":"=","storage":{"0x0000000000000000000000000000000000000000000000000000000000000000":{"*":{"from":"0x0000000000000000000000000000000000000000000000000000000000000000","to":"0x1122334455667788990011223344556677889900112233445566778899001122"}}}}},"trace":[],"vmTrace":null},"id":67}""" + )] + [TestCase( + "Uses storage from state override", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0xc200000000000000000000000000000000000000","input":"0x60fe47b1112233445566778899001122334455667788990011223344556677889900112233445566778899001122"}""", + "stateDiff", + """{"0xc200000000000000000000000000000000000000":{"state": {"0x0000000000000000000000000000000000000000000000000000000000000000": "0x0000000000000000000000000000000000000000000000000000000000123456"}, "code":"0x6080604052348015600e575f80fd5b50600436106030575f3560e01c80632a1afcd914603457806360fe47b114604d575b5f80fd5b603b5f5481565b60405190815260200160405180910390f35b605c6058366004605e565b5f55565b005b5f60208284031215606d575f80fd5b503591905056fea2646970667358221220fd4e5f3894be8e57fc7460afebb5c90d96c3486d79bf47b00c2ed666ab2f82b364736f6c634300081a0033"}}""", + """{"jsonrpc":"2.0","result":{"output":null,"stateDiff":{"0x7f554713be84160fdf0178cc8df86f5aabd33397":{"balance":{"\u002B":"0x0"},"code":"=","nonce":{"\u002B":"0x1"},"storage":{}},"0xc200000000000000000000000000000000000000":{"balance":"=","code":"=","nonce":"=","storage":{"0x0000000000000000000000000000000000000000000000000000000000000000":{"*":{"from":"0x0000000000000000000000000000000000000000000000000000000000123456","to":"0x1122334455667788990011223344556677889900112233445566778899001122"}}}}},"trace":[],"vmTrace":null},"id":67}""" + )] + [TestCase( + "Executes precompile using overriden address", + """{"from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","to":"0x0000000000000000000000000000000000123456","input":"0xB6E16D27AC5AB427A7F68900AC5559CE272DC6C37C82B3E052246C82244C50E4000000000000000000000000000000000000000000000000000000000000001C7B8B1991EB44757BC688016D27940DF8FB971D7C87F77A6BC4E938E3202C44037E9267B0AEAA82FA765361918F2D8ABD9CDD86E64AA6F2B81D3C4E0B69A7B055"}""", + "trace", + """{"0x0000000000000000000000000000000000000001":{"movePrecompileToAddress":"0x0000000000000000000000000000000000123456", "code": "0x"}}""", + """{"jsonrpc":"2.0","result":{"output":"0x000000000000000000000000b7705ae4c6f81b66cdb323c65f4e8133690fc099","stateDiff":null,"trace":[{"action":{"callType":"call","from":"0x7f554713be84160fdf0178cc8df86f5aabd33397","gas":"0x5f58878","input":"0xb6e16d27ac5ab427a7f68900ac5559ce272dc6c37c82b3e052246c82244c50e4000000000000000000000000000000000000000000000000000000000000001c7b8b1991eb44757bc688016d27940df8fb971d7c87f77a6bc4e938e3202c44037e9267b0aeaa82fa765361918f2d8abd9cdd86e64aa6f2b81d3c4e0b69a7b055","to":"0x0000000000000000000000000000000000123456","value":"0x0"},"result":{"gasUsed":"0xbb8","output":"0x000000000000000000000000b7705ae4c6f81b66cdb323c65f4e8133690fc099"},"subtraces":0,"traceAddress":[],"type":"call"}],"vmTrace":null},"id":67}""" + )] + public async Task Trace_call_with_state_override(string name, string transactionJson, string traceType, string stateOverrideJson, string expectedResult) + { + var transaction = JsonSerializer.Deserialize(transactionJson); + var stateOverride = JsonSerializer.Deserialize(stateOverrideJson); + + Context context = new(); + await context.Build(new TestSpecProvider(Prague.Instance)); + string serialized = await RpcTest.TestSerializedRequest( + context.TraceRpcModule, + "trace_call", transaction, new[] { traceType }, "latest", stateOverride); + + JToken.Parse(serialized).Should().BeEquivalentTo(expectedResult); + } + + [TestCase( + """{"from":"0xb7705ae4c6f81b66cdb323c65f4e8133690fc099","to":"0xc200000000000000000000000000000000000000"}""", + "stateDiff", + """{"0xb7705ae4c6f81b66cdb323c65f4e8133690fc099":{"balance":"0x123", "nonce": "0x123"}}""" + )] + [TestCase( + """{"from":"0xb7705ae4c6f81b66cdb323c65f4e8133690fc099","to":"0xc200000000000000000000000000000000000000","input":"0xf8b2cb4f000000000000000000000000b7705ae4c6f81b66cdb323c65f4e8133690fc099"}""", + "trace", + """{"0xc200000000000000000000000000000000000000":{"code":"0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063f8b2cb4f14610030575b600080fd5b61004a600480360381019061004591906100e4565b610060565b604051610057919061012a565b60405180910390f35b60008173ffffffffffffffffffffffffffffffffffffffff16319050919050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b60006100b182610086565b9050919050565b6100c1816100a6565b81146100cc57600080fd5b50565b6000813590506100de816100b8565b92915050565b6000602082840312156100fa576100f9610081565b5b6000610108848285016100cf565b91505092915050565b6000819050919050565b61012481610111565b82525050565b600060208201905061013f600083018461011b565b9291505056fea2646970667358221220172c443a163d8a43e018c339d1b749c312c94b6de22835953d960985daf228c764736f6c63430008120033"}}""" + )] + public async Task Trace_call_with_state_override_does_not_affect_other_calls(string transactionJson, string traceType, string stateOverrideJson) + { + var transaction = JsonSerializer.Deserialize(transactionJson); + var stateOverride = JsonSerializer.Deserialize(stateOverrideJson); + + Context context = new(); + await context.Build(); + + var traceTypes = new[] { traceType }; + + var resultOverrideBefore = await RpcTest.TestSerializedRequest(context.TraceRpcModule, "trace_call", + transaction, traceTypes, null, stateOverride); + + var resultNoOverride = await RpcTest.TestSerializedRequest(context.TraceRpcModule, "trace_call", + transaction, traceTypes, null); + + var resultOverrideAfter = await RpcTest.TestSerializedRequest(context.TraceRpcModule, "trace_call", + transaction, traceTypes, null, stateOverride); + + using (new AssertionScope()) + { + JToken.Parse(resultOverrideBefore).Should().BeEquivalentTo(resultOverrideAfter); + JToken.Parse(resultNoOverride).Should().NotBeEquivalentTo(resultOverrideAfter); + } + } } diff --git a/src/Nethermind/Nethermind.JsonRpc.TraceStore/TraceStoreRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc.TraceStore/TraceStoreRpcModule.cs index 12b38afe90a..f73ed60d07f 100644 --- a/src/Nethermind/Nethermind.JsonRpc.TraceStore/TraceStoreRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc.TraceStore/TraceStoreRpcModule.cs @@ -6,6 +6,7 @@ using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Db; +using Nethermind.Evm; using Nethermind.Evm.Tracing.ParityStyle; using Nethermind.Facade.Eth.RpcTransaction; using Nethermind.JsonRpc.Data; @@ -52,8 +53,9 @@ public TraceStoreRpcModule(ITraceRpcModule traceModule, _logger = logManager.GetClassLogger(); } - public ResultWrapper trace_call(TransactionForRpc call, string[] traceTypes, BlockParameter? blockParameter = null) => - _traceModule.trace_call(call, traceTypes, blockParameter); + public ResultWrapper trace_call(TransactionForRpc call, string[] traceTypes, BlockParameter? blockParameter = null, + Dictionary? stateOverride = null) => + _traceModule.trace_call(call, traceTypes, blockParameter, stateOverride); public ResultWrapper> trace_callMany(TransactionForRpcWithTraceTypes[] calls, BlockParameter? blockParameter = null) => _traceModule.trace_callMany(calls, blockParameter); diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugModuleFactory.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugModuleFactory.cs index 9856232921d..842adf7fa3a 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugModuleFactory.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugModuleFactory.cs @@ -18,12 +18,13 @@ using Nethermind.Logging; using Nethermind.State; using Nethermind.Synchronization.ParallelSync; +using Nethermind.Trie.Pruning; namespace Nethermind.JsonRpc.Modules.DebugModule; public class DebugModuleFactory : ModuleFactoryBase { - private readonly IWorldStateManager _worldStateManager; + private readonly IReadOnlyTrieStore _trieStore; private readonly IJsonRpcConfig _jsonRpcConfig; private readonly IBlockValidator _blockValidator; private readonly IRewardCalculatorSource _rewardCalculatorSource; @@ -38,10 +39,9 @@ public class DebugModuleFactory : ModuleFactoryBase private readonly ISyncModeSelector _syncModeSelector; private readonly IBadBlockStore _badBlockStore; private readonly IFileSystem _fileSystem; - private readonly ILogger _logger; public DebugModuleFactory( - IWorldStateManager worldStateManager, + IReadOnlyTrieStore trieStore, IDbProvider dbProvider, IBlockTree blockTree, IJsonRpcConfig jsonRpcConfig, @@ -57,7 +57,7 @@ public DebugModuleFactory( IFileSystem fileSystem, ILogManager logManager) { - _worldStateManager = worldStateManager; + _trieStore = trieStore; _dbProvider = dbProvider.AsReadOnly(false); _blockTree = blockTree.AsReadOnly(); _jsonRpcConfig = jsonRpcConfig ?? throw new ArgumentNullException(nameof(jsonRpcConfig)); @@ -72,16 +72,12 @@ public DebugModuleFactory( _syncModeSelector = syncModeSelector ?? throw new ArgumentNullException(nameof(syncModeSelector)); _badBlockStore = badBlockStore; _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); - _logger = logManager.GetClassLogger(); } public override IDebugRpcModule Create() { - ReadOnlyTxProcessingEnv txEnv = new( - _worldStateManager, - _blockTree, - _specProvider, - _logManager); + OverridableWorldStateManager worldStateManager = new(_dbProvider, _trieStore, _logManager); + OverridableTxProcessingEnv txEnv = new(worldStateManager, _blockTree, _specProvider, _logManager); IReadOnlyTxProcessingScope scope = txEnv.Build(Keccak.EmptyTreeHash); @@ -95,7 +91,7 @@ public override IDebugRpcModule Create() _receiptStorage, _specProvider, _blockTree, - _worldStateManager.GlobalStateReader, + worldStateManager.GlobalStateReader, _logManager, transactionsExecutor); @@ -107,7 +103,8 @@ public override IDebugRpcModule Create() _badBlockStore, _specProvider, transactionProcessorAdapter, - _fileSystem); + _fileSystem, + txEnv); DebugBridge debugBridge = new( _configProvider, diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.TransactionExecutor.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.TransactionExecutor.cs index 146df3f42a2..5f573a07c3f 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.TransactionExecutor.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.TransactionExecutor.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Collections.Generic; using System.Threading; using Nethermind.Blockchain.Find; using Nethermind.Core; @@ -30,7 +31,7 @@ protected override Transaction Prepare(TransactionForRpc call) return tx; } - protected override ResultWrapper Execute(BlockHeader header, Transaction tx, CancellationToken token) + protected override ResultWrapper Execute(BlockHeader header, Transaction tx, Dictionary? stateOverride, CancellationToken token) { BlockHeader clonedHeader = header.Clone(); if (NoBaseFee) @@ -41,35 +42,36 @@ protected override ResultWrapper Execute(BlockHeader header, Transactio { return ResultWrapper.Fail("Contract creation without any data provided.", ErrorCodes.InvalidInput); } - return ExecuteTx(clonedHeader, tx, token); + return ExecuteTx(clonedHeader, tx, stateOverride, token); } public override ResultWrapper Execute( TransactionForRpc transactionCall, - BlockParameter? blockParameter) + BlockParameter? blockParameter, + Dictionary? stateOverride = null) { NoBaseFee = !transactionCall.ShouldSetBaseFee(); transactionCall.EnsureDefaults(_rpcConfig.GasCap); - return base.Execute(transactionCall, blockParameter); + return base.Execute(transactionCall, blockParameter, stateOverride); } - public ResultWrapper ExecuteTx(TransactionForRpc transactionCall, BlockParameter? blockParameter) => Execute(transactionCall, blockParameter); + public ResultWrapper ExecuteTx(TransactionForRpc transactionCall, BlockParameter? blockParameter, Dictionary? stateOverride = null) + => Execute(transactionCall, blockParameter, stateOverride); - protected abstract ResultWrapper ExecuteTx(BlockHeader header, Transaction tx, CancellationToken token); + protected abstract ResultWrapper ExecuteTx(BlockHeader header, Transaction tx, Dictionary? stateOverride, CancellationToken token); } private class CallTxExecutor(IBlockchainBridge blockchainBridge, IBlockFinder blockFinder, IJsonRpcConfig rpcConfig) : TxExecutor(blockchainBridge, blockFinder, rpcConfig) { - protected override ResultWrapper ExecuteTx(BlockHeader header, Transaction tx, CancellationToken token) + protected override ResultWrapper ExecuteTx(BlockHeader header, Transaction tx, Dictionary? stateOverride, CancellationToken token) { - CallOutput result = _blockchainBridge.Call(header, tx, token); + CallOutput result = _blockchainBridge.Call(header, tx, stateOverride, token); return result.Error is null ? ResultWrapper.Success(result.OutputData.ToHexString(true)) : TryGetInputError(result) ?? ResultWrapper.Fail("VM execution error.", ErrorCodes.ExecutionError, result.Error); } - } private class EstimateGasTxExecutor(IBlockchainBridge blockchainBridge, IBlockFinder blockFinder, IJsonRpcConfig rpcConfig) @@ -77,9 +79,9 @@ private class EstimateGasTxExecutor(IBlockchainBridge blockchainBridge, IBlockFi { private readonly int _errorMargin = rpcConfig.EstimateErrorMargin; - protected override ResultWrapper ExecuteTx(BlockHeader header, Transaction tx, CancellationToken token) + protected override ResultWrapper ExecuteTx(BlockHeader header, Transaction tx, Dictionary stateOverride, CancellationToken token) { - CallOutput result = _blockchainBridge.EstimateGas(header, tx, _errorMargin, token); + CallOutput result = _blockchainBridge.EstimateGas(header, tx, _errorMargin, stateOverride, token); return result switch { @@ -93,7 +95,7 @@ private class EstimateGasTxExecutor(IBlockchainBridge blockchainBridge, IBlockFi private class CreateAccessListTxExecutor(IBlockchainBridge blockchainBridge, IBlockFinder blockFinder, IJsonRpcConfig rpcConfig, bool optimize) : TxExecutor(blockchainBridge, blockFinder, rpcConfig) { - protected override ResultWrapper ExecuteTx(BlockHeader header, Transaction tx, CancellationToken token) + protected override ResultWrapper ExecuteTx(BlockHeader header, Transaction tx, Dictionary stateOverride, CancellationToken token) { CallOutput result = _blockchainBridge.CreateAccessList(header, tx, token, optimize); diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.cs index 97246ca48de..70f06eeb6b0 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.cs @@ -22,6 +22,7 @@ using Nethermind.Facade.Eth; using Nethermind.Facade.Eth.RpcTransaction; using Nethermind.Facade.Filters; +using Nethermind.Facade.Proxy.Models; using Nethermind.Facade.Proxy.Models.Simulate; using Nethermind.Int256; using Nethermind.JsonRpc.Data; @@ -327,17 +328,17 @@ private async Task> SendTx(Transaction tx, } } - public ResultWrapper eth_call(TransactionForRpc transactionCall, BlockParameter? blockParameter = null) => + public ResultWrapper eth_call(TransactionForRpc transactionCall, BlockParameter? blockParameter = null, Dictionary? stateOverride = null) => new CallTxExecutor(_blockchainBridge, _blockFinder, _rpcConfig) - .ExecuteTx(transactionCall, blockParameter); + .ExecuteTx(transactionCall, blockParameter, stateOverride); public ResultWrapper> eth_simulateV1(SimulatePayload payload, BlockParameter? blockParameter = null) => new SimulateTxExecutor(_blockchainBridge, _blockFinder, _rpcConfig, _secondsPerSlot) .Execute(payload, blockParameter); - public ResultWrapper eth_estimateGas(TransactionForRpc transactionCall, BlockParameter? blockParameter) => + public ResultWrapper eth_estimateGas(TransactionForRpc transactionCall, BlockParameter? blockParameter, Dictionary? stateOverride = null) => new EstimateGasTxExecutor(_blockchainBridge, _blockFinder, _rpcConfig) - .ExecuteTx(transactionCall, blockParameter); + .ExecuteTx(transactionCall, blockParameter, stateOverride); public ResultWrapper eth_createAccessList(TransactionForRpc transactionCall, BlockParameter? blockParameter = null, bool optimize = true) => new CreateAccessListTxExecutor(_blockchainBridge, _blockFinder, _rpcConfig, optimize) diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/ExecutorBase.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/ExecutorBase.cs index 2e04d10b455..1721a52d804 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/ExecutorBase.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/ExecutorBase.cs @@ -1,9 +1,11 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Collections.Generic; using System.Threading; using Nethermind.Blockchain.Find; using Nethermind.Core; +using Nethermind.Evm; using Nethermind.Facade; namespace Nethermind.JsonRpc.Modules.Eth; @@ -23,7 +25,8 @@ protected ExecutorBase(IBlockchainBridge blockchainBridge, IBlockFinder blockFin public virtual ResultWrapper Execute( TRequest call, - BlockParameter? blockParameter) + BlockParameter? blockParameter, + Dictionary? stateOverride = null) { SearchResult searchResult = _blockFinder.SearchForHeader(blockParameter); if (searchResult.IsError) return ResultWrapper.Fail(searchResult); @@ -35,12 +38,12 @@ public virtual ResultWrapper Execute( using CancellationTokenSource cancellationTokenSource = new(_rpcConfig.Timeout); TProcessing? toProcess = Prepare(call); - return Execute(header.Clone(), toProcess, cancellationTokenSource.Token); + return Execute(header.Clone(), toProcess, stateOverride, cancellationTokenSource.Token); } protected abstract TProcessing Prepare(TRequest call); - protected abstract ResultWrapper Execute(BlockHeader header, TProcessing tx, CancellationToken token); + protected abstract ResultWrapper Execute(BlockHeader header, TProcessing tx, Dictionary? stateOverride, CancellationToken token); protected ResultWrapper? TryGetInputError(CallOutput result) { diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/IEthRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/IEthRpcModule.cs index 3ded7a3a957..4e7f22a9665 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/IEthRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/IEthRpcModule.cs @@ -6,6 +6,7 @@ using Nethermind.Blockchain.Find; using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Evm; using Nethermind.Facade.Eth; using Nethermind.Facade.Eth.RpcTransaction; using Nethermind.Facade.Filters; @@ -150,7 +151,7 @@ public interface IEthRpcModule : IRpcModule Description = "Executes a tx call (does not create a transaction)", IsSharable = false, ExampleResponse = "0x")] - ResultWrapper eth_call([JsonRpcParameter(ExampleValue = "[{\"from\":\"0x0001020304050607080910111213141516171819\",\"gasPrice\":\"0x100000\", \"data\": \"0x70a082310000000000000000000000006c1f09f6271fbe133db38db9c9280307f5d22160\", \"to\": \"0x0d8775f648430679a709e98d2b0cb6250d2887ef\"}]")] TransactionForRpc transactionCall, BlockParameter? blockParameter = null); + ResultWrapper eth_call([JsonRpcParameter(ExampleValue = "[{\"from\":\"0x0001020304050607080910111213141516171819\",\"gasPrice\":\"0x100000\", \"data\": \"0x70a082310000000000000000000000006c1f09f6271fbe133db38db9c9280307f5d22160\", \"to\": \"0x0d8775f648430679a709e98d2b0cb6250d2887ef\"}]")] TransactionForRpc transactionCall, BlockParameter? blockParameter = null, Dictionary? stateOverride = null); [JsonRpcMethod(IsImplemented = true, Description = "Executes a simulation across multiple blocks (does not create a transaction or block)", @@ -163,7 +164,7 @@ ResultWrapper> eth_simulateV1([JsonRpcParamet Description = "Executes a tx call and returns gas used (does not create a transaction)", IsSharable = false, ExampleResponse = "0x")] - ResultWrapper eth_estimateGas([JsonRpcParameter(ExampleValue = "[\"{\"from\": \"0x0001020304050607080910111213141516171819\", \"gasPrice\": \"1048576\", \"to\": \"0x0d8775f648430679a709e98d2b0cb6250d2887ef\"}\"]")] TransactionForRpc transactionCall, BlockParameter? blockParameter = null); + ResultWrapper eth_estimateGas([JsonRpcParameter(ExampleValue = "[\"{\"from\": \"0x0001020304050607080910111213141516171819\", \"gasPrice\": \"1048576\", \"to\": \"0x0d8775f648430679a709e98d2b0cb6250d2887ef\"}\"]")] TransactionForRpc transactionCall, BlockParameter? blockParameter = null, Dictionary? stateOverride = null); [JsonRpcMethod(IsImplemented = true, Description = "Creates an [EIP2930](https://eips.ethereum.org/EIPS/eip-2930) type AccessList for the given transaction", diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/SimulateTxExecutor.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/SimulateTxExecutor.cs index d87ae39bd1f..48db6e094e4 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/SimulateTxExecutor.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/SimulateTxExecutor.cs @@ -8,6 +8,7 @@ using Nethermind.Blockchain.Find; using Nethermind.Config; using Nethermind.Core; +using Nethermind.Evm; using Nethermind.Facade; using Nethermind.Facade.Eth.RpcTransaction; using Nethermind.Facade.Proxy.Models.Simulate; @@ -93,7 +94,8 @@ private static TransactionForRpc UpdateTxType(TransactionForRpc rpcTransaction) public override ResultWrapper> Execute( SimulatePayload call, - BlockParameter? blockParameter) + BlockParameter? blockParameter, + Dictionary? stateOverride = null) { if (call.BlockStateCalls is null) return ResultWrapper>.Fail("Must contain BlockStateCalls", ErrorCodes.InvalidParams); @@ -207,11 +209,11 @@ .. call.BlockStateCalls.Select(b => (long)(b.BlockOverrides?.Number ?? ulong.Min using CancellationTokenSource cancellationTokenSource = new(_rpcConfig.Timeout); //TODO remove! SimulatePayload toProcess = Prepare(call); - return Execute(header.Clone(), toProcess, cancellationTokenSource.Token); + return Execute(header.Clone(), toProcess, stateOverride, cancellationTokenSource.Token); } protected override ResultWrapper> Execute(BlockHeader header, - SimulatePayload tx, CancellationToken token) + SimulatePayload tx, Dictionary? stateOverride, CancellationToken token) { SimulateOutput results = _blockchainBridge.Simulate(header, tx, token); diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/ITraceRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/ITraceRpcModule.cs index d9a78740241..ef339ab2612 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/ITraceRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/ITraceRpcModule.cs @@ -3,7 +3,9 @@ using System.Collections.Generic; using Nethermind.Blockchain.Find; +using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Evm; using Nethermind.Facade.Eth.RpcTransaction; using Nethermind.JsonRpc.Data; @@ -13,7 +15,9 @@ namespace Nethermind.JsonRpc.Modules.Trace public interface ITraceRpcModule : IRpcModule { [JsonRpcMethod(Description = "", IsImplemented = true, IsSharable = false)] - ResultWrapper trace_call(TransactionForRpc call, [JsonRpcParameter(Description = "Possible values : [\"VmTrace\", \"StateDiff\", \"Trace\", \"Rewards\", \"All\"]")] string[] traceTypes, BlockParameter? blockParameter = null); + ResultWrapper trace_call(TransactionForRpc call, + [JsonRpcParameter(Description = "Possible values : [\"VmTrace\", \"StateDiff\", \"Trace\", \"Rewards\", \"All\"]")] string[] traceTypes, + BlockParameter? blockParameter = null, Dictionary? stateOverride = null); [JsonRpcMethod(Description = "Performs multiple traces on top of a block", IsImplemented = true, diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceModuleFactory.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceModuleFactory.cs index fce0c4f0e1d..11a2ce1ba68 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceModuleFactory.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceModuleFactory.cs @@ -11,14 +11,17 @@ using Nethermind.Consensus.Validators; using Nethermind.Core.Crypto; using Nethermind.Core.Specs; +using Nethermind.Db; using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; using Nethermind.State; +using Nethermind.Trie.Pruning; namespace Nethermind.JsonRpc.Modules.Trace; public class TraceModuleFactory( - IWorldStateManager worldStateManager, + IReadOnlyTrieStore trieStore, + IDbProvider dbProvider, IBlockTree blockTree, IJsonRpcConfig jsonRpcConfig, IBlockPreprocessorStep recoveryStep, @@ -28,7 +31,7 @@ public class TraceModuleFactory( IPoSSwitcher poSSwitcher, ILogManager logManager) : ModuleFactoryBase { - protected readonly IWorldStateManager _worldStateManager = worldStateManager; + protected readonly IReadOnlyTrieStore _trieStore = trieStore; protected readonly IReadOnlyBlockTree _blockTree = blockTree.AsReadOnly(); protected readonly IJsonRpcConfig _jsonRpcConfig = jsonRpcConfig ?? throw new ArgumentNullException(nameof(jsonRpcConfig)); protected readonly IReceiptStorage _receiptStorage = receiptFinder ?? throw new ArgumentNullException(nameof(receiptFinder)); @@ -38,9 +41,9 @@ public class TraceModuleFactory( protected readonly IRewardCalculatorSource _rewardCalculatorSource = rewardCalculatorSource ?? throw new ArgumentNullException(nameof(rewardCalculatorSource)); protected readonly IPoSSwitcher _poSSwitcher = poSSwitcher ?? throw new ArgumentNullException(nameof(poSSwitcher)); - protected virtual ReadOnlyTxProcessingEnv CreateTxProcessingEnv() => new(_worldStateManager, _blockTree, _specProvider, _logManager); + protected virtual OverridableTxProcessingEnv CreateTxProcessingEnv(OverridableWorldStateManager worldStateManager) => new(worldStateManager, _blockTree, _specProvider, _logManager); - protected virtual ReadOnlyChainProcessingEnv CreateChainProcessingEnv(IBlockProcessor.IBlockTransactionsExecutor transactionsExecutor, IReadOnlyTxProcessingScope scope, IRewardCalculator rewardCalculator) => new( + protected virtual ReadOnlyChainProcessingEnv CreateChainProcessingEnv(OverridableWorldStateManager worldStateManager, IBlockProcessor.IBlockTransactionsExecutor transactionsExecutor, IReadOnlyTxProcessingScope scope, IRewardCalculator rewardCalculator) => new( scope, Always.Valid, _recoveryStep, @@ -48,13 +51,14 @@ public class TraceModuleFactory( _receiptStorage, _specProvider, _blockTree, - _worldStateManager.GlobalStateReader, + worldStateManager.GlobalStateReader, _logManager, transactionsExecutor); public override ITraceRpcModule Create() { - ReadOnlyTxProcessingEnv txProcessingEnv = CreateTxProcessingEnv(); + OverridableWorldStateManager worldStateManager = new(dbProvider, _trieStore, logManager); + OverridableTxProcessingEnv txProcessingEnv = CreateTxProcessingEnv(worldStateManager); IReadOnlyTxProcessingScope scope = txProcessingEnv.Build(Keccak.EmptyTreeHash); IRewardCalculator rewardCalculator = @@ -64,12 +68,12 @@ public override ITraceRpcModule Create() RpcBlockTransactionsExecutor rpcBlockTransactionsExecutor = new(scope.TransactionProcessor, scope.WorldState); BlockProcessor.BlockValidationTransactionsExecutor executeBlockTransactionsExecutor = new(scope.TransactionProcessor, scope.WorldState); - ReadOnlyChainProcessingEnv traceProcessingEnv = CreateChainProcessingEnv(rpcBlockTransactionsExecutor, scope, rewardCalculator); - ReadOnlyChainProcessingEnv executeProcessingEnv = CreateChainProcessingEnv(executeBlockTransactionsExecutor, scope, rewardCalculator); + ReadOnlyChainProcessingEnv traceProcessingEnv = CreateChainProcessingEnv(worldStateManager, rpcBlockTransactionsExecutor, scope, rewardCalculator); + ReadOnlyChainProcessingEnv executeProcessingEnv = CreateChainProcessingEnv(worldStateManager, executeBlockTransactionsExecutor, scope, rewardCalculator); - Tracer tracer = new(scope.WorldState, traceProcessingEnv.ChainProcessor, executeProcessingEnv.ChainProcessor); + Tracer tracer = new(scope.WorldState, traceProcessingEnv.ChainProcessor, executeProcessingEnv.ChainProcessor, + traceOptions: ProcessingOptions.TraceTransactions); - return new TraceRpcModule(_receiptStorage, tracer, _blockTree, _jsonRpcConfig, txProcessingEnv.StateReader); + return new TraceRpcModule(_receiptStorage, tracer, _blockTree, _jsonRpcConfig, txProcessingEnv); } - } diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceRpcModule.cs index 622ba612700..05df3a6ff8d 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceRpcModule.cs @@ -8,12 +8,14 @@ using FastEnumUtility; using Nethermind.Blockchain.Find; using Nethermind.Blockchain.Receipts; +using Nethermind.Consensus.Processing; using Nethermind.Consensus.Tracing; using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Evm; using Nethermind.Evm.Tracing; using Nethermind.Evm.Tracing.ParityStyle; -using Nethermind.Facade; +using Nethermind.Evm.TransactionProcessing; using Nethermind.Facade.Eth.RpcTransaction; using Nethermind.Int256; using Nethermind.JsonRpc.Data; @@ -39,31 +41,36 @@ public class TraceRpcModule : ITraceRpcModule private readonly IJsonRpcConfig _jsonRpcConfig; private readonly TimeSpan _cancellationTokenTimeout; private readonly IStateReader _stateReader; + private readonly IOverridableTxProcessorSource _env; - public TraceRpcModule(IReceiptFinder? receiptFinder, ITracer? tracer, IBlockFinder? blockFinder, IJsonRpcConfig? jsonRpcConfig, IStateReader stateReader) + public TraceRpcModule(IReceiptFinder? receiptFinder, ITracer? tracer, IBlockFinder? blockFinder, IJsonRpcConfig? jsonRpcConfig, IStateReader? stateReader, IOverridableTxProcessorSource? env) { _receiptFinder = receiptFinder ?? throw new ArgumentNullException(nameof(receiptFinder)); _tracer = tracer ?? throw new ArgumentNullException(nameof(tracer)); _blockFinder = blockFinder ?? throw new ArgumentNullException(nameof(blockFinder)); _jsonRpcConfig = jsonRpcConfig ?? throw new ArgumentNullException(nameof(jsonRpcConfig)); _stateReader = stateReader ?? throw new ArgumentNullException(nameof(stateReader)); + _env = env ?? throw new ArgumentNullException(nameof(env)); _cancellationTokenTimeout = TimeSpan.FromMilliseconds(_jsonRpcConfig.Timeout); } + public TraceRpcModule(IReceiptFinder? receiptFinder, ITracer? tracer, IBlockFinder? blockFinder, IJsonRpcConfig? jsonRpcConfig, OverridableTxProcessingEnv? env) + : this(receiptFinder, tracer, blockFinder, jsonRpcConfig, env?.StateReader, env) { } + public static ParityTraceTypes GetParityTypes(string[] types) => types.Select(s => FastEnum.Parse(s, true)).Aggregate((t1, t2) => t1 | t2); /// /// Traces one transaction. Doesn't charge fees. /// - public ResultWrapper trace_call(TransactionForRpc call, string[] traceTypes, BlockParameter? blockParameter = null) + public ResultWrapper trace_call(TransactionForRpc call, string[] traceTypes, BlockParameter? blockParameter = null, Dictionary? stateOverride = null) { blockParameter ??= BlockParameter.Latest; call.EnsureDefaults(_jsonRpcConfig.GasCap); Transaction tx = call.ToTransaction(); - return TraceTx(tx, traceTypes, blockParameter); + return TraceTx(tx, traceTypes, blockParameter, stateOverride); } /// @@ -73,15 +80,16 @@ public ResultWrapper> trace_callMany(Transa { blockParameter ??= BlockParameter.Latest; - SearchResult headerSearch = SearchBlockHeaderForTraceCall(blockParameter); + SearchResult headerSearch = _blockFinder.SearchForHeader(blockParameter); if (headerSearch.IsError) { return ResultWrapper>.Fail(headerSearch); } - if (!_stateReader.HasStateForBlock(headerSearch.Object)) + BlockHeader header = headerSearch.Object!; + if (!_stateReader.HasStateForBlock(header)) { - return GetStateFailureResult>(headerSearch.Object); + return GetStateFailureResult>(header); } Dictionary traceTypeByTransaction = new(calls.Length); @@ -96,8 +104,8 @@ public ResultWrapper> trace_callMany(Transa traceTypeByTransaction.Add(tx.Hash, traceTypes); } - Block block = new(headerSearch.Object!, txs, Enumerable.Empty()); - IReadOnlyCollection? traces = TraceBlock(block, new ParityLikeBlockTracer(traceTypeByTransaction)); + Block block = new(header, txs, Enumerable.Empty()); + IReadOnlyCollection? traces = TraceBlock(block, new(traceTypeByTransaction)); return ResultWrapper>.Success(traces.Select(t => new ParityTxTraceFromReplay(t))); } @@ -110,46 +118,19 @@ public ResultWrapper trace_rawTransaction(byte[] data, return TraceTx(tx, traceTypes, BlockParameter.Latest); } - private SearchResult SearchBlockHeaderForTraceCall(BlockParameter blockParameter) + private ResultWrapper TraceTx(Transaction tx, string[] traceTypes, BlockParameter blockParameter, + Dictionary? stateOverride = null) { SearchResult headerSearch = _blockFinder.SearchForHeader(blockParameter); if (headerSearch.IsError) - { - return headerSearch; - } - - BlockHeader header = headerSearch.Object; - if (header!.IsGenesis) - { - UInt256 baseFee = header.BaseFeePerGas; - header = new BlockHeader( - header.Hash!, - Keccak.OfAnEmptySequenceRlp, - Address.Zero, - header.Difficulty, - header.Number + 1, - header.GasLimit, - header.Timestamp + 1, - header.ExtraData, - header.BlobGasUsed, - header.ExcessBlobGas); - - header.TotalDifficulty = 2 * header.Difficulty; - header.BaseFeePerGas = baseFee; - } - - return new SearchResult(header); - } - - private ResultWrapper TraceTx(Transaction tx, string[] traceTypes, BlockParameter blockParameter) - { - SearchResult headerSearch = SearchBlockHeaderForTraceCall(blockParameter); - if (headerSearch.IsError) { return ResultWrapper.Fail(headerSearch); } - Block block = new(headerSearch.Object!, new[] { tx }, Enumerable.Empty()); + BlockHeader header = headerSearch.Object!.Clone(); + Block block = new(header, [tx], []); + + using IOverridableTxProcessingScope? scope = stateOverride != null ? _env.BuildAndOverride(header, stateOverride) : null; ParityTraceTypes traceTypes1 = GetParityTypes(traceTypes); IReadOnlyCollection result = TraceBlock(block, new(traceTypes1)); diff --git a/src/Nethermind/Nethermind.Logging/PathUtils.cs b/src/Nethermind/Nethermind.Logging/PathUtils.cs index 62a8cd6b29b..9a16cedbfca 100644 --- a/src/Nethermind/Nethermind.Logging/PathUtils.cs +++ b/src/Nethermind/Nethermind.Logging/PathUtils.cs @@ -7,57 +7,56 @@ using System.Linq; using System.Reflection; -namespace Nethermind.Logging +namespace Nethermind.Logging; + +public static class PathUtils { - public static class PathUtils + public static string ExecutingDirectory { get; } + + static PathUtils() { - public static string ExecutingDirectory { get; } + Process process = Process.GetCurrentProcess(); + if (process.ProcessName.StartsWith("dotnet", StringComparison.InvariantCultureIgnoreCase) + || process.ProcessName.Equals("ReSharperTestRunner", StringComparison.InvariantCultureIgnoreCase)) + { + ExecutingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + } + else + { + ExecutingDirectory = Path.GetDirectoryName(process.MainModule.FileName); + //Console.WriteLine($"Resolved executing directory as {ExecutingDirectory}."); + } + } - static PathUtils() + public static string GetApplicationResourcePath(this string resourcePath, string overridePrefixPath = null) + { + if (string.IsNullOrWhiteSpace(resourcePath)) { - Process process = Process.GetCurrentProcess(); - if (process.ProcessName.StartsWith("dotnet", StringComparison.InvariantCultureIgnoreCase) - || process.ProcessName.Equals("ReSharperTestRunner", StringComparison.InvariantCultureIgnoreCase)) - { - ExecutingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); - } - else - { - ExecutingDirectory = Path.GetDirectoryName(process.MainModule.FileName); - Console.WriteLine($"Resolved executing directory as {ExecutingDirectory}."); - } + resourcePath = string.Empty; } - public static string GetApplicationResourcePath(this string resourcePath, string overridePrefixPath = null) + if (Path.IsPathRooted(resourcePath) || IsExplicitlyRelative(resourcePath)) { - if (string.IsNullOrWhiteSpace(resourcePath)) - { - resourcePath = string.Empty; - } - - if (Path.IsPathRooted(resourcePath) || IsExplicitlyRelative(resourcePath)) - { - return resourcePath; - } - - if (string.IsNullOrEmpty(overridePrefixPath)) - { - return Path.Combine(ExecutingDirectory, resourcePath); - } - - return Path.IsPathRooted(overridePrefixPath) || IsExplicitlyRelative(overridePrefixPath) - ? Path.Combine(overridePrefixPath, resourcePath) - : Path.Combine(ExecutingDirectory, overridePrefixPath, resourcePath); + return resourcePath; } - static readonly string[] RelativePrefixes = new[] + if (string.IsNullOrEmpty(overridePrefixPath)) { - "." + Path.DirectorySeparatorChar, - "." + Path.AltDirectorySeparatorChar, - ".." + Path.DirectorySeparatorChar, - ".." + Path.AltDirectorySeparatorChar, - }.Distinct().ToArray(); + return Path.Combine(ExecutingDirectory, resourcePath); + } - public static bool IsExplicitlyRelative(string resourcePath) => RelativePrefixes.Any(resourcePath.StartsWith); + return Path.IsPathRooted(overridePrefixPath) || IsExplicitlyRelative(overridePrefixPath) + ? Path.Combine(overridePrefixPath, resourcePath) + : Path.Combine(ExecutingDirectory, overridePrefixPath, resourcePath); } + + static readonly string[] RelativePrefixes = new[] + { + "." + Path.DirectorySeparatorChar, + "." + Path.AltDirectorySeparatorChar, + ".." + Path.DirectorySeparatorChar, + ".." + Path.AltDirectorySeparatorChar, + }.Distinct().ToArray(); + + public static bool IsExplicitlyRelative(string resourcePath) => RelativePrefixes.Any(resourcePath.StartsWith); } diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/MergePluginTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/MergePluginTests.cs index 2487dc5c71a..9810fb3c89f 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/MergePluginTests.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/MergePluginTests.cs @@ -74,7 +74,7 @@ public void Setup() public void SlotPerSeconds_has_different_value_in_mergeConfig_and_blocksConfig() { - JsonConfigSource? jsonSource = new("MisconfiguredConfig.cfg"); + JsonConfigSource? jsonSource = new("MisconfiguredConfig.json"); ConfigProvider? configProvider = new(); configProvider.AddSource(jsonSource); configProvider.Initialize(); diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/MisconfiguredConfig.cfg b/src/Nethermind/Nethermind.Merge.Plugin.Test/MisconfiguredConfig.json similarity index 100% rename from src/Nethermind/Nethermind.Merge.Plugin.Test/MisconfiguredConfig.cfg rename to src/Nethermind/Nethermind.Merge.Plugin.Test/MisconfiguredConfig.json diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/Nethermind.Merge.Plugin.Test.csproj b/src/Nethermind/Nethermind.Merge.Plugin.Test/Nethermind.Merge.Plugin.Test.csproj index b89fce1e281..91f60f830a6 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/Nethermind.Merge.Plugin.Test.csproj +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/Nethermind.Merge.Plugin.Test.csproj @@ -28,7 +28,7 @@ - + PreserveNewest diff --git a/src/Nethermind/Nethermind.Optimism/OptimismOverridableTxProcessingEnv.cs b/src/Nethermind/Nethermind.Optimism/OptimismOverridableTxProcessingEnv.cs new file mode 100644 index 00000000000..9e06be1ddcd --- /dev/null +++ b/src/Nethermind/Nethermind.Optimism/OptimismOverridableTxProcessingEnv.cs @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Blockchain; +using Nethermind.Consensus.Processing; +using Nethermind.Core.Specs; +using Nethermind.Evm; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Logging; +using Nethermind.State; + +namespace Nethermind.Optimism; + +public class OptimismOverridableTxProcessingEnv( + OverridableWorldStateManager worldStateManager, + IReadOnlyBlockTree readOnlyBlockTree, + ISpecProvider specProvider, + ILogManager logManager, + IL1CostHelper l1CostHelper, + IOptimismSpecHelper opSpecHelper, + IWorldState? worldStateToWarmUp = null) + : OverridableTxProcessingEnv(worldStateManager, readOnlyBlockTree, specProvider, logManager, worldStateToWarmUp) +{ + protected override ITransactionProcessor CreateTransactionProcessor() + { + ArgumentNullException.ThrowIfNull(LogManager); + + BlockhashProvider blockhashProvider = new(BlockTree, SpecProvider, StateProvider, LogManager); + VirtualMachine virtualMachine = new(blockhashProvider, SpecProvider, CodeInfoRepository, LogManager); + return new OptimismTransactionProcessor(SpecProvider, StateProvider, virtualMachine, LogManager, l1CostHelper, opSpecHelper, CodeInfoRepository); + } +} diff --git a/src/Nethermind/Nethermind.Optimism/Rpc/OptimismTraceModuleFactory.cs b/src/Nethermind/Nethermind.Optimism/Rpc/OptimismTraceModuleFactory.cs index 8a1b5da4d21..bf5b837c883 100644 --- a/src/Nethermind/Nethermind.Optimism/Rpc/OptimismTraceModuleFactory.cs +++ b/src/Nethermind/Nethermind.Optimism/Rpc/OptimismTraceModuleFactory.cs @@ -9,16 +9,19 @@ using Nethermind.Consensus.Validators; using Nethermind.Consensus.Withdrawals; using Nethermind.Core.Specs; +using Nethermind.Db; using Nethermind.Evm.TransactionProcessing; using Nethermind.JsonRpc; using Nethermind.JsonRpc.Modules.Trace; using Nethermind.Logging; using Nethermind.State; +using Nethermind.Trie.Pruning; namespace Nethermind.Optimism.Rpc; public class OptimismTraceModuleFactory( - IWorldStateManager worldStateManager, + IReadOnlyTrieStore trieStore, + IDbProvider dbProvider, IBlockTree blockTree, IJsonRpcConfig jsonRpcConfig, IBlockPreprocessorStep recoveryStep, @@ -31,7 +34,8 @@ public class OptimismTraceModuleFactory( IOptimismSpecHelper opSpecHelper, Create2DeployerContractRewriter contractRewriter, IWithdrawalProcessor withdrawalProcessor) : TraceModuleFactory( - worldStateManager, + trieStore, + dbProvider, blockTree, jsonRpcConfig, recoveryStep, @@ -41,10 +45,10 @@ public class OptimismTraceModuleFactory( poSSwitcher, logManager) { - protected override ReadOnlyTxProcessingEnv CreateTxProcessingEnv() => - new OptimismReadOnlyTxProcessingEnv(_worldStateManager, _blockTree, _specProvider, _logManager, l1CostHelper, opSpecHelper); + protected override OverridableTxProcessingEnv CreateTxProcessingEnv(OverridableWorldStateManager worldStateManager) => + new OptimismOverridableTxProcessingEnv(worldStateManager, _blockTree, _specProvider, _logManager, l1CostHelper, opSpecHelper); - protected override ReadOnlyChainProcessingEnv CreateChainProcessingEnv(IBlockProcessor.IBlockTransactionsExecutor transactionsExecutor, IReadOnlyTxProcessingScope scope, IRewardCalculator rewardCalculator) => new OptimismReadOnlyChainProcessingEnv( + protected override ReadOnlyChainProcessingEnv CreateChainProcessingEnv(OverridableWorldStateManager worldStateManager, IBlockProcessor.IBlockTransactionsExecutor transactionsExecutor, IReadOnlyTxProcessingScope scope, IRewardCalculator rewardCalculator) => new OptimismReadOnlyChainProcessingEnv( scope, Always.Valid, _recoveryStep, @@ -52,7 +56,7 @@ protected override ReadOnlyTxProcessingEnv CreateTxProcessingEnv() => _receiptStorage, _specProvider, _blockTree, - _worldStateManager.GlobalStateReader, + worldStateManager.GlobalStateReader, _logManager, opSpecHelper, contractRewriter, diff --git a/src/Nethermind/Nethermind.Optimism/Rpc/RegisterOptimismRpcModules.cs b/src/Nethermind/Nethermind.Optimism/Rpc/RegisterOptimismRpcModules.cs index a53257f1ec3..3f3072485f4 100644 --- a/src/Nethermind/Nethermind.Optimism/Rpc/RegisterOptimismRpcModules.cs +++ b/src/Nethermind/Nethermind.Optimism/Rpc/RegisterOptimismRpcModules.cs @@ -93,6 +93,7 @@ protected override void RegisterEthRpcModule(IRpcModuleProvider rpcModuleProvide protected override void RegisterTraceRpcModule(IRpcModuleProvider rpcModuleProvider) { StepDependencyException.ThrowIfNull(_api.WorldStateManager); + StepDependencyException.ThrowIfNull(_api.DbProvider); StepDependencyException.ThrowIfNull(_api.BlockTree); StepDependencyException.ThrowIfNull(_api.ReceiptStorage); StepDependencyException.ThrowIfNull(_api.RewardCalculatorSource); @@ -102,7 +103,8 @@ protected override void RegisterTraceRpcModule(IRpcModuleProvider rpcModuleProvi StepDependencyException.ThrowIfNull(_api.SpecHelper); OptimismTraceModuleFactory traceModuleFactory = new( - _api.WorldStateManager, + _api.WorldStateManager.TrieStore, + _api.DbProvider, _api.BlockTree, _jsonRpcConfig, _api.BlockPreprocessor, diff --git a/src/Nethermind/Nethermind.Overseer.Test/Framework/TestBuilder.cs b/src/Nethermind/Nethermind.Overseer.Test/Framework/TestBuilder.cs index 72dd3fac275..17ee00acea8 100644 --- a/src/Nethermind/Nethermind.Overseer.Test/Framework/TestBuilder.cs +++ b/src/Nethermind/Nethermind.Overseer.Test/Framework/TestBuilder.cs @@ -10,292 +10,291 @@ using Nethermind.Overseer.Test.Framework.Steps; using NUnit.Framework; -namespace Nethermind.Overseer.Test.Framework +namespace Nethermind.Overseer.Test.Framework; + +/// +/// https://stackoverflow.com/questions/32112418/how-to-make-a-fluent-async-inferface-in-c-sharp +/// +public class TestBuilder { - /// - /// https://stackoverflow.com/questions/32112418/how-to-make-a-fluent-async-inferface-in-c-sharp - /// - public class TestBuilder + [TearDown] + public void TearDown() { - [TearDown] - public void TearDown() - { - var passedCount = _results.Count(r => r.Passed); - var failedCount = _results.Count - passedCount; + var passedCount = _results.Count(r => r.Passed); + var failedCount = _results.Count - passedCount; - TestContext.Out.WriteLine("=========================== TESTS RESULTS ==========================="); - TestContext.Out.WriteLine($"TESTS PASSED: {passedCount}, FAILED: {failedCount}"); - foreach (var testResult in _results) - { - string message = $"{testResult.Order}. {testResult.Name} has " + - $"{(testResult.Passed ? "passed [+]" : "failed [-]")}"; - TestContext.Out.WriteLine(message); - } + TestContext.Out.WriteLine("=========================== TESTS RESULTS ==========================="); + TestContext.Out.WriteLine($"TESTS PASSED: {passedCount}, FAILED: {failedCount}"); + foreach (var testResult in _results) + { + string message = $"{testResult.Order}. {testResult.Name} has " + + $"{(testResult.Passed ? "passed [+]" : "failed [-]")}"; + TestContext.Out.WriteLine(message); } + } #pragma warning disable NUnit1032 - /// - /// Gets the task representing the fluent work. - /// - /// - /// The task. - /// - public Task ScenarioCompletion { get; private set; } + /// + /// Gets the task representing the fluent work. + /// + /// + /// The task. + /// + public Task ScenarioCompletion { get; private set; } #pragma warning restore NUnit1032 - /// - /// Queues up asynchronous work. - /// - /// The work to be queued. - public void QueueWork(Action work) + /// + /// Queues up asynchronous work. + /// + /// The work to be queued. + public void QueueWork(Action work) + { + // queue up the work + ScenarioCompletion = ScenarioCompletion.ContinueWith(task => { - // queue up the work - ScenarioCompletion = ScenarioCompletion.ContinueWith(task => + try { - try - { - work(); - } - catch (Exception e) - { - TestContext.Out.WriteLine(e.ToString()); - throw; - } - - return this; - }, TaskContinuationOptions.OnlyOnRanToCompletion); - } - - /// - /// Queues up asynchronous work. - /// - /// The work to be queued. - public void QueueWork(Func work) - { - // queue up the work - ScenarioCompletion = ScenarioCompletion.ContinueWith(async task => + work(); + } + catch (Exception e) { - try - { - await work(); - } - catch (Exception e) - { - TestContext.Out.WriteLine(e.ToString()); - throw; - } - - return this; - }, TaskContinuationOptions.OnlyOnRanToCompletion); - } + TestContext.Out.WriteLine(e.ToString()); + throw; + } - public void QueueWork(TestStepBase step) + return this; + }, TaskContinuationOptions.OnlyOnRanToCompletion); + } + + /// + /// Queues up asynchronous work. + /// + /// The work to be queued. + public void QueueWork(Func work) + { + // queue up the work + ScenarioCompletion = ScenarioCompletion.ContinueWith(async task => { - // queue up the work - ScenarioCompletion = ScenarioCompletion.ContinueWith(async task => + try { - TestContext.Out.WriteLine($"Awaiting step {step.Name}"); - try - { - _results.Add(await step.ExecuteAsync()); - } - catch (Exception e) - { - TestContext.Out.WriteLine($"Step {step.Name} failed with error: {e}"); - throw; - } - - TestContext.Out.WriteLine($"Step {step.Name} complete"); - }, TaskContinuationOptions.OnlyOnRanToCompletion).Unwrap(); - } - - private readonly ProcessBuilder _processBuilder; + await work(); + } + catch (Exception e) + { + TestContext.Out.WriteLine(e.ToString()); + throw; + } - private static readonly string _runnerDir; - private static readonly string _dbsDir; - private static readonly string _configsDir; + return this; + }, TaskContinuationOptions.OnlyOnRanToCompletion); + } - static TestBuilder() + public void QueueWork(TestStepBase step) + { + // queue up the work + ScenarioCompletion = ScenarioCompletion.ContinueWith(async task => { - string testContextDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "context"); - _runnerDir = Path.Combine(testContextDir, "runner"); - _configsDir = Path.Combine(testContextDir, "configs"); - _dbsDir = Path.Combine(testContextDir, "dbs"); - - if (Directory.Exists(testContextDir)) + TestContext.Out.WriteLine($"Awaiting step {step.Name}"); + try { - Directory.Delete(testContextDir, true); + _results.Add(await step.ExecuteAsync()); + } + catch (Exception e) + { + TestContext.Out.WriteLine($"Step {step.Name} failed with error: {e}"); + throw; } - Directory.CreateDirectory(_dbsDir); - Directory.CreateDirectory(_configsDir); - } + TestContext.Out.WriteLine($"Step {step.Name} complete"); + }, TaskContinuationOptions.OnlyOnRanToCompletion).Unwrap(); + } - public TestBuilder() - { - _processBuilder = new ProcessBuilder(); + private readonly ProcessBuilder _processBuilder; - if (!Directory.Exists(_runnerDir)) - { - Directory.CreateDirectory(_runnerDir); - CopyRunnerFiles(_runnerDir); - } + private static readonly string _runnerDir; + private static readonly string _dbsDir; + private static readonly string _configsDir; - // The entry point for the async work. - // Spin up a completed task to start with - // so that we dont have to do null checks - this.ScenarioCompletion = Task.FromResult(0); + static TestBuilder() + { + string testContextDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "context"); + _runnerDir = Path.Combine(testContextDir, "runner"); + _configsDir = Path.Combine(testContextDir, "configs"); + _dbsDir = Path.Combine(testContextDir, "dbs"); + + if (Directory.Exists(testContextDir)) + { + Directory.Delete(testContextDir, true); } - public T SetContext(T newContext) where T : ITestContext + Directory.CreateDirectory(_dbsDir); + Directory.CreateDirectory(_configsDir); + } + + public TestBuilder() + { + _processBuilder = new ProcessBuilder(); + + if (!Directory.Exists(_runnerDir)) { - newContext.SetBuilder(this); - return newContext; + Directory.CreateDirectory(_runnerDir); + CopyRunnerFiles(_runnerDir); } - private const int _startHttpPort = 8600; - private const int _startPort = 30200; + // The entry point for the async work. + // Spin up a completed task to start with + // so that we dont have to do null checks + this.ScenarioCompletion = Task.FromResult(0); + } - private byte _nodeCounter; + public T SetContext(T newContext) where T : ITestContext + { + newContext.SetBuilder(this); + return newContext; + } - public NethermindProcessWrapper CurrentNode { get; private set; } + private const int _startHttpPort = 8600; + private const int _startPort = 30200; - public List _results = new List(); + private byte _nodeCounter; - public TestBuilder SwitchNode(string node) - { - CurrentNode = Nodes[node]; - return this; - } + public NethermindProcessWrapper CurrentNode { get; private set; } - public TestBuilder Wait(int delay = 5000, string name = "Wait") - { - QueueWork(async () => await Task.Delay(delay)); - return this; - } + public List _results = new List(); - public TestBuilder StartCliqueNode(string name) - { - return StartNode(name, "configs/cliqueNode.cfg"); - } + public TestBuilder SwitchNode(string node) + { + CurrentNode = Nodes[node]; + return this; + } - public TestBuilder StartCliqueMiner(string name) - { - return StartNode(name, "configs/cliqueMiner.cfg"); - } + public TestBuilder Wait(int delay = 5000, string name = "Wait") + { + QueueWork(async () => await Task.Delay(delay)); + return this; + } - public TestBuilder StartAuRaMiner(string name, string key) - { - return StartNode(name, "configs/auRaMiner.cfg", key); - } + public TestBuilder StartCliqueNode(string name) + { + return StartNode(name, "configs/cliqueNode.json"); + } - public TestBuilder StartNode(string name, string baseConfigFile, string key = null) - { - CurrentNode = GetOrCreateNode(name, baseConfigFile, key); - var step = new StartProcessTestStep($"Start {name}", CurrentNode); - QueueWork(step); - return this; - } + public TestBuilder StartCliqueMiner(string name) + { + return StartNode(name, "configs/cliqueMiner.json"); + } - private NethermindProcessWrapper GetOrCreateNode(string name, string baseConfigFile, string key) - { - if (!Nodes.TryGetValue(name, out NethermindProcessWrapper value)) - { - string bootnodes = string.Empty; - foreach ((_, NethermindProcessWrapper process) in Nodes) - { - bootnodes += $",{process.Enode}"; - } - - bootnodes = bootnodes.TrimStart(','); - - var nodeKey = GetNodeKey(key); - - string dbDir = Path.Combine(_dbsDir, name); - string configPath = Path.Combine(_configsDir, $"{name}.cfg"); - File.Copy(baseConfigFile, configPath); - int p2pPort = _startPort + _nodeCounter; - int httpPort = _startHttpPort + _nodeCounter; - TestContext.Out.WriteLine($"Creating {name} at {p2pPort}, http://localhost:{httpPort}"); - value = _processBuilder.Create(name, _runnerDir, configPath, dbDir, httpPort, p2pPort, nodeKey, bootnodes); - Nodes[name] = value; - _nodeCounter++; - } + public TestBuilder StartAuRaMiner(string name, string key) + { + return StartNode(name, "configs/auRaMiner.json", key); + } - return value; - } + public TestBuilder StartNode(string name, string baseConfigFile, string key = null) + { + CurrentNode = GetOrCreateNode(name, baseConfigFile, key); + var step = new StartProcessTestStep($"Start {name}", CurrentNode); + QueueWork(step); + return this; + } - private string GetNodeKey(string key) + private NethermindProcessWrapper GetOrCreateNode(string name, string baseConfigFile, string key) + { + if (!Nodes.TryGetValue(name, out NethermindProcessWrapper value)) { - if (key is null) + string bootnodes = string.Empty; + foreach ((_, NethermindProcessWrapper process) in Nodes) { - byte[] keyArray = new byte[32]; - keyArray[0] = 1; - keyArray[31] = _nodeCounter; - key = keyArray.ToHexString(); + bootnodes += $",{process.Enode}"; } - return key; + bootnodes = bootnodes.TrimStart(','); + + var nodeKey = GetNodeKey(key); + + string dbDir = Path.Combine(_dbsDir, name); + string configPath = Path.Combine(_configsDir, $"{name}.json"); + File.Copy(baseConfigFile, configPath); + int p2pPort = _startPort + _nodeCounter; + int httpPort = _startHttpPort + _nodeCounter; + TestContext.Out.WriteLine($"Creating {name} at {p2pPort}, http://localhost:{httpPort}"); + value = _processBuilder.Create(name, _runnerDir, configPath, dbDir, httpPort, p2pPort, nodeKey, bootnodes); + Nodes[name] = value; + _nodeCounter++; } - public Dictionary Nodes { get; } = new Dictionary(); + return value; + } - public TestBuilder Kill() + private string GetNodeKey(string key) + { + if (key is null) { - return Kill(CurrentNode.Name); + byte[] keyArray = new byte[32]; + keyArray[0] = 1; + keyArray[31] = _nodeCounter; + key = keyArray.ToHexString(); } - public TestBuilder Kill(string name) + return key; + } + + public Dictionary Nodes { get; } = new Dictionary(); + + public TestBuilder Kill() + { + return Kill(CurrentNode.Name); + } + + public TestBuilder Kill(string name) + { + var step = new KillProcessTestStep($"Kill {name}", Nodes[name]); + QueueWork(step); + return this; + } + + public TestBuilder KillAll() + { + foreach (KeyValuePair keyValuePair in Nodes) { - var step = new KillProcessTestStep($"Kill {name}", Nodes[name]); + var step = new KillProcessTestStep($"Kill {keyValuePair.Key}", Nodes[keyValuePair.Key]); QueueWork(step); - return this; } - public TestBuilder KillAll() - { - foreach (KeyValuePair keyValuePair in Nodes) - { - var step = new KillProcessTestStep($"Kill {keyValuePair.Key}", Nodes[keyValuePair.Key]); - QueueWork(step); - } - - return this; - } + return this; + } #if DEBUG - const string buildConfiguration = "Debug"; + const string buildConfiguration = "Debug"; #else - const string buildConfiguration = "Release"; + const string buildConfiguration = "Release"; #endif - private void CopyRunnerFiles(string targetDirectory) + private void CopyRunnerFiles(string targetDirectory) + { + string sourceDirectory = Path.Combine(Directory.GetCurrentDirectory(), $"../../../../artifacts/bin/Nethermind.Runner/{buildConfiguration}/"); + if (!Directory.Exists(sourceDirectory)) { - string sourceDirectory = Path.Combine(Directory.GetCurrentDirectory(), $"../../../../artifacts/bin/Nethermind.Runner/{buildConfiguration}/"); - if (!Directory.Exists(sourceDirectory)) - { - throw new IOException($"Runner not found at {sourceDirectory}"); - } - - TestContext.Out.WriteLine($"Copying runner files from {sourceDirectory} to {targetDirectory}"); - CopyDir(sourceDirectory, targetDirectory); - string chainsDir = Path.Combine(Directory.GetCurrentDirectory(), "chainspec"); - CopyDir(chainsDir, Path.Combine(targetDirectory, "chainspec")); + throw new IOException($"Runner not found at {sourceDirectory}"); } - private void CopyDir(string sourceDirectory, string targetDirectory) + TestContext.Out.WriteLine($"Copying runner files from {sourceDirectory} to {targetDirectory}"); + CopyDir(sourceDirectory, targetDirectory); + string chainsDir = Path.Combine(Directory.GetCurrentDirectory(), "chainspec"); + CopyDir(chainsDir, Path.Combine(targetDirectory, "chainspec")); + } + + private void CopyDir(string sourceDirectory, string targetDirectory) + { + foreach (string file in Directory.GetFiles(sourceDirectory)) { - foreach (string file in Directory.GetFiles(sourceDirectory)) - { - File.Copy(file, Path.Combine(targetDirectory, Path.GetFileName(file)), true); - } + File.Copy(file, Path.Combine(targetDirectory, Path.GetFileName(file)), true); + } - foreach (string directory in Directory.GetDirectories(sourceDirectory)) - { - string targetSubDir = Path.Combine(targetDirectory, Path.GetFileName(directory)); - Directory.CreateDirectory(targetSubDir); - CopyDir(directory, targetSubDir); - } + foreach (string directory in Directory.GetDirectories(sourceDirectory)) + { + string targetSubDir = Path.Combine(targetDirectory, Path.GetFileName(directory)); + Directory.CreateDirectory(targetSubDir); + CopyDir(directory, targetSubDir); } } } diff --git a/src/Nethermind/Nethermind.Overseer.Test/Nethermind.Overseer.Test.csproj b/src/Nethermind/Nethermind.Overseer.Test/Nethermind.Overseer.Test.csproj index b88dcc1ca88..f7f3b45b2f5 100644 --- a/src/Nethermind/Nethermind.Overseer.Test/Nethermind.Overseer.Test.csproj +++ b/src/Nethermind/Nethermind.Overseer.Test/Nethermind.Overseer.Test.csproj @@ -26,13 +26,13 @@ PreserveNewest - + PreserveNewest - + PreserveNewest - + PreserveNewest diff --git a/src/Nethermind/Nethermind.Overseer.Test/configs/auRaMiner.cfg b/src/Nethermind/Nethermind.Overseer.Test/configs/auRaMiner.json similarity index 100% rename from src/Nethermind/Nethermind.Overseer.Test/configs/auRaMiner.cfg rename to src/Nethermind/Nethermind.Overseer.Test/configs/auRaMiner.json diff --git a/src/Nethermind/Nethermind.Overseer.Test/configs/cliqueMiner.cfg b/src/Nethermind/Nethermind.Overseer.Test/configs/cliqueMiner.json similarity index 100% rename from src/Nethermind/Nethermind.Overseer.Test/configs/cliqueMiner.cfg rename to src/Nethermind/Nethermind.Overseer.Test/configs/cliqueMiner.json diff --git a/src/Nethermind/Nethermind.Overseer.Test/configs/cliqueNode.cfg b/src/Nethermind/Nethermind.Overseer.Test/configs/cliqueNode.json similarity index 100% rename from src/Nethermind/Nethermind.Overseer.Test/configs/cliqueNode.cfg rename to src/Nethermind/Nethermind.Overseer.Test/configs/cliqueNode.json diff --git a/src/Nethermind/Nethermind.Runner.Test/ConfigFilesTests.cs b/src/Nethermind/Nethermind.Runner.Test/ConfigFilesTests.cs index 49a9cbf7d50..e5f1956dde8 100644 --- a/src/Nethermind/Nethermind.Runner.Test/ConfigFilesTests.cs +++ b/src/Nethermind/Nethermind.Runner.Test/ConfigFilesTests.cs @@ -42,7 +42,7 @@ public void Required_config_files_exist(string configWildcard) // maybe leave in test since deprecation has not fully happened? [TestCase("validators", true)] - [TestCase("poacore_validator.cfg", true)] + [TestCase("poacore_validator.json", true)] [TestCase("spaceneth", false)] [TestCase("archive", false)] [TestCase("fast", true)] @@ -149,8 +149,8 @@ public void Cache_state_index(string configWildcard, bool expectedValue) [TestCase("gnosis ^archive", 768000000)] [TestCase("poacore archive", 1024000000)] [TestCase("poacore ^archive", 768000000)] - [TestCase("spaceneth.cfg", 64000000)] - [TestCase("spaceneth_persistent.cfg", 128000000)] + [TestCase("spaceneth.json", 64000000)] + [TestCase("spaceneth_persistent.json", 128000000)] public void Memory_hint_values_are_correct(string configWildcard, long expectedValue) { Test(configWildcard, c => c.MemoryHint, expectedValue); @@ -160,7 +160,7 @@ public void Memory_hint_values_are_correct(string configWildcard, long expectedV public void Metrics_disabled_by_default(string configWildcard) { Test(configWildcard, c => c.Enabled, false); - Test(configWildcard, c => c.NodeName.ToUpperInvariant(), (cf, p) => cf.Replace("_", " ").Replace(".cfg", "").ToUpperInvariant().Replace("POACORE", "POA CORE")); + Test(configWildcard, c => c.NodeName.ToUpperInvariant(), (cf, p) => cf.Replace("_", " ").Replace(".json", "").ToUpperInvariant().Replace("POACORE", "POA CORE")); Test(configWildcard, c => c.IntervalSeconds, 5); Test(configWildcard, c => c.PushGatewayUrl, ""); } @@ -229,12 +229,12 @@ public void Fast_sync_settings_as_expected(string configWildcard, bool downloadB } [TestCase("archive", false)] - [TestCase("mainnet.cfg", true)] - [TestCase("sepolia.cfg", true)] - [TestCase("gnosis.cfg", true)] - [TestCase("chiado.cfg", true)] - [TestCase("energyweb.cfg", false)] - [TestCase("volta.cfg", false)] + [TestCase("mainnet.json", true)] + [TestCase("sepolia.json", true)] + [TestCase("gnosis.json", true)] + [TestCase("chiado.json", true)] + [TestCase("energyweb.json", false)] + [TestCase("volta.json", false)] public void Snap_sync_settings_as_expected(string configWildcard, bool enabled) { Test(configWildcard, c => c.SnapSync, enabled); @@ -251,7 +251,7 @@ public void Stays_on_full_sync(string configWildcard, bool stickToFullSyncAfterF Test(configWildcard, c => c.FastSyncCatchUpHeightDelta, stickToFullSyncAfterFastSync ? 10_000_000_000 : 8192); } - [TestCase("^spaceneth.cfg")] + [TestCase("^spaceneth.json")] public void Diagnostics_mode_is_not_enabled_by_default(string configWildcard) { Test(configWildcard, c => c.DiagnosticMode, DiagnosticMode.None); @@ -294,8 +294,8 @@ public void Stores_receipts(string configWildcard, bool storeReceipts) Test(configWildcard, c => c.StoreReceipts, storeReceipts); } - [TestCase("mainnet_archive.cfg", true)] - [TestCase("mainnet.cfg", true)] + [TestCase("mainnet_archive.json", true)] + [TestCase("mainnet.json", true)] [TestCase("poacore", true)] [TestCase("gnosis", true)] [TestCase("volta", false)] @@ -312,7 +312,7 @@ public void Basic_configs_are_as_expected(string configWildcard, bool isProducti Test(configWildcard, c => c.EnableUnsecuredDevWallet, false); } - Test(configWildcard, c => c.LogFileName, (cf, p) => p.Should().Be(cf.Replace("cfg", "logs.txt"), cf)); + Test(configWildcard, c => c.LogFileName, (cf, p) => p.Should().Be(cf.Replace("json", "logs.txt"), cf)); } [TestCase("*")] @@ -334,11 +334,11 @@ public void Blob_txs_support_is_correct(string configWildcard, BlobsSupportMode [TestCase("mainnet")] - [TestCase("poacore.cfg", new[] { 16, 16, 16, 16 })] - [TestCase("poacore_archive.cfg", new[] { 16, 16, 16, 16 })] - [TestCase("poacore_validator.cfg", null, false)] - [TestCase("gnosis.cfg", new[] { 16, 16, 16 })] - [TestCase("gnosis_archive.cfg", new[] { 16, 16, 16 })] + [TestCase("poacore.json", new[] { 16, 16, 16, 16 })] + [TestCase("poacore_archive.json", new[] { 16, 16, 16, 16 })] + [TestCase("poacore_validator.json", null, false)] + [TestCase("gnosis.json", new[] { 16, 16, 16 })] + [TestCase("gnosis_archive.json", new[] { 16, 16, 16 })] [TestCase("volta")] public void Bloom_configs_are_as_expected(string configWildcard, int[] levels = null, bool index = true) { @@ -416,24 +416,24 @@ public void Memory_hint_is_enough(string configWildcard) protected override IEnumerable Configs { get; } = new HashSet { - "holesky.cfg", - "holesky_archive.cfg", - "mainnet_archive.cfg", - "mainnet.cfg", - "poacore.cfg", - "poacore_archive.cfg", - "gnosis.cfg", - "gnosis_archive.cfg", - "spaceneth.cfg", - "spaceneth_persistent.cfg", - "volta.cfg", - "volta_archive.cfg", - "energyweb.cfg", - "energyweb_archive.cfg", - "sepolia.cfg", - "sepolia_archive.cfg", - "chiado.cfg", - "chiado_archive.cfg", + "holesky.json", + "holesky_archive.json", + "mainnet_archive.json", + "mainnet.json", + "poacore.json", + "poacore_archive.json", + "gnosis.json", + "gnosis_archive.json", + "spaceneth.json", + "spaceneth_persistent.json", + "volta.json", + "volta_archive.json", + "energyweb.json", + "energyweb_archive.json", + "sepolia.json", + "sepolia_archive.json", + "chiado.json", + "chiado_archive.json", }; public IEnumerable AllIndexesOf(string str, string searchString) diff --git a/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/Migrations/ReceiptMigrationTests.cs b/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/Migrations/ReceiptMigrationTests.cs index 575fa52694b..d7cb8645bc5 100644 --- a/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/Migrations/ReceiptMigrationTests.cs +++ b/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/Migrations/ReceiptMigrationTests.cs @@ -130,14 +130,14 @@ public TestReceiptStorage(IReceiptStorage inStorage, IReceiptStorage outStorage) public Hash256 FindBlockHash(Hash256 txHash) => _inStorage.FindBlockHash(txHash); - public TxReceipt[] Get(Block block, bool recover = true) => _inStorage.Get(block, recover); + public TxReceipt[] Get(Block block, bool recover = true, bool recoverSender = true) => _inStorage.Get(block, recover, recoverSender); public TxReceipt[] Get(Hash256 blockHash, bool recover = true) => _inStorage.Get(blockHash, recover); public bool CanGetReceiptsByHash(long blockNumber) => _inStorage.CanGetReceiptsByHash(blockNumber); public bool TryGetReceiptsIterator(long blockNumber, Hash256 blockHash, out ReceiptsIterator iterator) => _inStorage.TryGetReceiptsIterator(blockNumber, blockHash, out iterator); - public void Insert(Block block, TxReceipt[] txReceipts, bool ensureCanonical) => _outStorage.Insert(block, txReceipts); + public void Insert(Block block, TxReceipt[] txReceipts, bool ensureCanonical, WriteFlags writeFlags) => _outStorage.Insert(block, txReceipts, ensureCanonical, writeFlags); public long? LowestInsertedReceiptBlockNumber { diff --git a/src/Nethermind/Nethermind.Runner.Test/EthereumRunnerTests.cs b/src/Nethermind/Nethermind.Runner.Test/EthereumRunnerTests.cs index 1f9b0af6f43..65453770a02 100644 --- a/src/Nethermind/Nethermind.Runner.Test/EthereumRunnerTests.cs +++ b/src/Nethermind/Nethermind.Runner.Test/EthereumRunnerTests.cs @@ -11,24 +11,18 @@ using System.Threading; using System.Threading.Tasks; using Nethermind.Api; -using Nethermind.Blockchain.Synchronization; using Nethermind.Config; using Nethermind.Consensus.Clique; -using Nethermind.Consensus.Ethash; using Nethermind.Core.Test.IO; -using Nethermind.Db.Rocks.Config; -using Nethermind.EthStats; +using Nethermind.Hive; using Nethermind.JsonRpc; using Nethermind.JsonRpc.Modules; -using Nethermind.KeyStore.Config; using Nethermind.Logging; using Nethermind.Network.Config; using Nethermind.Runner.Ethereum; -using Nethermind.Db.Blooms; using Nethermind.Optimism; using Nethermind.Runner.Ethereum.Api; using Nethermind.Taiko; -using Nethermind.TxPool; using NUnit.Framework; namespace Nethermind.Runner.Test; @@ -106,21 +100,8 @@ public async Task Smoke_cancel((string file, ConfigProvider configProvider) test private static async Task SmokeTest(ConfigProvider configProvider, int testIndex, int basePort, bool cancel = false) { - Type type1 = typeof(ITxPoolConfig); - Type type2 = typeof(INetworkConfig); - Type type3 = typeof(IKeyStoreConfig); - Type type4 = typeof(IDbConfig); - Type type7 = typeof(IEthStatsConfig); - Type type8 = typeof(ISyncConfig); - Type type9 = typeof(IBloomConfig); - - Console.WriteLine(type1.Name); - Console.WriteLine(type2.Name); - Console.WriteLine(type3.Name); - Console.WriteLine(type4.Name); - Console.WriteLine(type7.Name); - Console.WriteLine(type8.Name); - Console.WriteLine(type9.Name); + // An ugly hack to keep unused types + Console.WriteLine(typeof(IHiveConfig)); var tempPath = TempPath.GetTempDirectory(); Directory.CreateDirectory(tempPath.Path); diff --git a/src/Nethermind/Nethermind.Runner.Test/Nethermind.Runner.Test.csproj b/src/Nethermind/Nethermind.Runner.Test/Nethermind.Runner.Test.csproj index 27a88a759bb..3b363b262d2 100644 --- a/src/Nethermind/Nethermind.Runner.Test/Nethermind.Runner.Test.csproj +++ b/src/Nethermind/Nethermind.Runner.Test/Nethermind.Runner.Test.csproj @@ -54,7 +54,7 @@ Chains\gnosis.json - + PreserveNewest diff --git a/src/Nethermind/Nethermind.Runner/Ethereum/Api/ApiBuilder.cs b/src/Nethermind/Nethermind.Runner/Ethereum/Api/ApiBuilder.cs index 9d17ac28787..04c7827288a 100644 --- a/src/Nethermind/Nethermind.Runner/Ethereum/Api/ApiBuilder.cs +++ b/src/Nethermind/Nethermind.Runner/Ethereum/Api/ApiBuilder.cs @@ -11,84 +11,83 @@ using Nethermind.Config; using Nethermind.Consensus; using Nethermind.Core; -using Nethermind.Facade.Eth.RpcTransaction; +using Nethermind.Hive; using Nethermind.Logging; using Nethermind.Serialization.Json; using Nethermind.Specs.ChainSpecStyle; -namespace Nethermind.Runner.Ethereum.Api +namespace Nethermind.Runner.Ethereum.Api; + +public class ApiBuilder { - public class ApiBuilder + private readonly IConfigProvider _configProvider; + private readonly IJsonSerializer _jsonSerializer; + private readonly ILogManager _logManager; + private readonly Nethermind.Logging.ILogger _logger; + private readonly IInitConfig _initConfig; + + public ApiBuilder(IConfigProvider configProvider, ILogManager logManager) { - private readonly IConfigProvider _configProvider; - private readonly IJsonSerializer _jsonSerializer; - private readonly ILogManager _logManager; - private readonly Nethermind.Logging.ILogger _logger; - private readonly IInitConfig _initConfig; + _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); + _logger = _logManager.GetClassLogger(); + _configProvider = configProvider ?? throw new ArgumentNullException(nameof(configProvider)); + _initConfig = configProvider.GetConfig(); + _jsonSerializer = new EthereumJsonSerializer(); + } - public ApiBuilder(IConfigProvider configProvider, ILogManager logManager) + public INethermindApi Create(params IConsensusPlugin[] consensusPlugins) => + Create((IEnumerable)consensusPlugins); + + public INethermindApi Create(IEnumerable consensusPlugins) + { + ChainSpec chainSpec = LoadChainSpec(_jsonSerializer); + bool wasCreated = Interlocked.CompareExchange(ref _apiCreated, 1, 0) == 1; + if (wasCreated) { - _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); - _logger = _logManager.GetClassLogger(); - _configProvider = configProvider ?? throw new ArgumentNullException(nameof(configProvider)); - _initConfig = configProvider.GetConfig(); - _jsonSerializer = new EthereumJsonSerializer(); + throw new NotSupportedException("Creation of multiple APIs not supported."); } - public INethermindApi Create(params IConsensusPlugin[] consensusPlugins) => - Create((IEnumerable)consensusPlugins); + string engine = chainSpec.SealEngineType; + IConsensusPlugin? enginePlugin = consensusPlugins.FirstOrDefault(p => p.SealEngineType == engine); - public INethermindApi Create(IEnumerable consensusPlugins) - { - ChainSpec chainSpec = LoadChainSpec(_jsonSerializer); - bool wasCreated = Interlocked.CompareExchange(ref _apiCreated, 1, 0) == 1; - if (wasCreated) - { - throw new NotSupportedException("Creation of multiple APIs not supported."); - } - - string engine = chainSpec.SealEngineType; - IConsensusPlugin? enginePlugin = consensusPlugins.FirstOrDefault(p => p.SealEngineType == engine); - - INethermindApi nethermindApi = - enginePlugin?.CreateApi(_configProvider, _jsonSerializer, _logManager, chainSpec) ?? - new NethermindApi(_configProvider, _jsonSerializer, _logManager, chainSpec); - nethermindApi.SealEngineType = engine; - nethermindApi.SpecProvider = new ChainSpecBasedSpecProvider(chainSpec, _logManager); - nethermindApi.GasLimitCalculator = new FollowOtherMiners(nethermindApi.SpecProvider); - - SetLoggerVariables(chainSpec); - - return nethermindApi; - } + INethermindApi nethermindApi = + enginePlugin?.CreateApi(_configProvider, _jsonSerializer, _logManager, chainSpec) ?? + new NethermindApi(_configProvider, _jsonSerializer, _logManager, chainSpec); + nethermindApi.SealEngineType = engine; + nethermindApi.SpecProvider = new ChainSpecBasedSpecProvider(chainSpec, _logManager); + nethermindApi.GasLimitCalculator = new FollowOtherMiners(nethermindApi.SpecProvider); - private int _apiCreated; + SetLoggerVariables(chainSpec); - private ChainSpec LoadChainSpec(IJsonSerializer ethereumJsonSerializer) - { - bool hiveEnabled = Environment.GetEnvironmentVariable("NETHERMIND_HIVE_ENABLED")?.ToLowerInvariant() == "true"; - bool hiveChainSpecExists = File.Exists(_initConfig.HiveChainSpecPath); + return nethermindApi; + } - string chainSpecFile; - if (hiveEnabled && hiveChainSpecExists) - chainSpecFile = _initConfig.HiveChainSpecPath; - else - chainSpecFile = _initConfig.ChainSpecPath; + private int _apiCreated; - if (_logger.IsDebug) _logger.Debug($"Loading chain spec from {chainSpecFile}"); + private ChainSpec LoadChainSpec(IJsonSerializer ethereumJsonSerializer) + { + IHiveConfig hiveConfig = _configProvider.GetConfig(); + bool hiveChainSpecExists = File.Exists(_initConfig.HiveChainSpecPath); - ThisNodeInfo.AddInfo("Chainspec :", $"{chainSpecFile}"); + string chainSpecFile; + if (hiveConfig.Enabled && hiveChainSpecExists) + chainSpecFile = _initConfig.HiveChainSpecPath; + else + chainSpecFile = _initConfig.ChainSpecPath; - IChainSpecLoader loader = new ChainSpecLoader(ethereumJsonSerializer); - ChainSpec chainSpec = loader.LoadEmbeddedOrFromFile(chainSpecFile, _logger); - return chainSpec; - } + if (_logger.IsDebug) _logger.Debug($"Loading chain spec from {chainSpecFile}"); - private void SetLoggerVariables(ChainSpec chainSpec) - { - _logManager.SetGlobalVariable("chain", chainSpec.Name); - _logManager.SetGlobalVariable("chainId", chainSpec.ChainId); - _logManager.SetGlobalVariable("engine", chainSpec.SealEngineType); - } + ThisNodeInfo.AddInfo("Chainspec :", $"{chainSpecFile}"); + + IChainSpecLoader loader = new ChainSpecLoader(ethereumJsonSerializer); + ChainSpec chainSpec = loader.LoadEmbeddedOrFromFile(chainSpecFile, _logger); + return chainSpec; + } + + private void SetLoggerVariables(ChainSpec chainSpec) + { + _logManager.SetGlobalVariable("chain", chainSpec.Name); + _logManager.SetGlobalVariable("chainId", chainSpec.ChainId); + _logManager.SetGlobalVariable("engine", chainSpec.SealEngineType); } } diff --git a/src/Nethermind/Nethermind.Runner/Ethereum/EthereumRunner.cs b/src/Nethermind/Nethermind.Runner/Ethereum/EthereumRunner.cs index 5e79b03e1cb..7e896261fee 100644 --- a/src/Nethermind/Nethermind.Runner/Ethereum/EthereumRunner.cs +++ b/src/Nethermind/Nethermind.Runner/Ethereum/EthereumRunner.cs @@ -13,102 +13,100 @@ using Nethermind.Init.Steps; using Nethermind.Logging; -namespace Nethermind.Runner.Ethereum +namespace Nethermind.Runner.Ethereum; + +public class EthereumRunner(INethermindApi api) { - public class EthereumRunner + private readonly INethermindApi _api = api; + private readonly ILogger _logger = api.LogManager.GetClassLogger(); + + public async Task Start(CancellationToken cancellationToken) + { + if (_logger.IsDebug) _logger.Debug("Starting Ethereum runner"); + + EthereumStepsLoader stepsLoader = new(GetStepsAssemblies(_api)); + EthereumStepsManager stepsManager = new(stepsLoader, _api, _api.LogManager); + + await stepsManager.InitializeAll(cancellationToken); + + string infoScreen = ThisNodeInfo.BuildNodeInfoScreen(); + + if (_logger.IsInfo) _logger.Info(infoScreen); + } + + private IEnumerable GetStepsAssemblies(INethermindApi api) { - private readonly INethermindApi _api; + yield return typeof(IStep).Assembly; + yield return GetType().Assembly; - private readonly ILogger _logger; + IEnumerable enabledInitializationPlugins = + _api.Plugins.OfType().Where(p => p.ShouldRunSteps(api)); - public EthereumRunner(INethermindApi api) + foreach (IInitializationPlugin initializationPlugin in enabledInitializationPlugins) { - _api = api; - _logger = api.LogManager.GetClassLogger(); + yield return initializationPlugin.GetType().Assembly; } + } - public async Task Start(CancellationToken cancellationToken) + public async Task StopAsync() + { + Stop(() => _api.SessionMonitor?.Stop(), "Stopping session monitor"); + Stop(() => _api.SyncModeSelector?.Stop(), "Stopping session sync mode selector"); + Task discoveryStopTask = Stop(() => _api.DiscoveryApp?.StopAsync(), "Stopping discovery app"); + Task blockProducerTask = Stop(() => _api.BlockProducerRunner?.StopAsync(), "Stopping block producer"); + Task syncPeerPoolTask = Stop(() => _api.SyncPeerPool?.StopAsync(), "Stopping sync peer pool"); + Task peerPoolTask = Stop(() => _api.PeerPool?.StopAsync(), "Stopping peer pool"); + Task peerManagerTask = Stop(() => _api.PeerManager?.StopAsync(), "Stopping peer manager"); + Task synchronizerTask = Stop(() => _api.Synchronizer?.StopAsync(), "Stopping synchronizer"); + Task blockchainProcessorTask = Stop(() => _api.BlockchainProcessor?.StopAsync(), "Stopping blockchain processor"); + Task rlpxPeerTask = Stop(() => _api.RlpxPeer?.Shutdown(), "Stopping RLPx peer"); + await Task.WhenAll(discoveryStopTask, rlpxPeerTask, peerManagerTask, synchronizerTask, syncPeerPoolTask, peerPoolTask, blockchainProcessorTask, blockProducerTask); + + foreach (INethermindPlugin plugin in _api.Plugins) { - if (_logger.IsDebug) _logger.Debug("Initializing Ethereum"); - - EthereumStepsLoader stepsLoader = new EthereumStepsLoader(GetStepsAssemblies(_api)); - EthereumStepsManager stepsManager = new EthereumStepsManager(stepsLoader, _api, _api.LogManager); - await stepsManager.InitializeAll(cancellationToken); - - string infoScreen = ThisNodeInfo.BuildNodeInfoScreen(); - if (_logger.IsInfo) _logger.Info(infoScreen); + await Stop(async () => await plugin.DisposeAsync(), $"Disposing plugin {plugin.Name}"); } - private IEnumerable GetStepsAssemblies(INethermindApi api) + while (_api.DisposeStack.Count != 0) { - yield return typeof(IStep).Assembly; - yield return GetType().Assembly; - IEnumerable enabledInitializationPlugins = - _api.Plugins.OfType().Where(p => p.ShouldRunSteps(api)); - - foreach (IInitializationPlugin initializationPlugin in enabledInitializationPlugins) - { - yield return initializationPlugin.GetType().Assembly; - } + IAsyncDisposable disposable = _api.DisposeStack.Pop(); + await Stop(async () => await disposable.DisposeAsync(), $"Disposing {disposable}"); } - public async Task StopAsync() + Stop(() => _api.DbProvider?.Dispose(), "Closing DBs"); + + if (_logger.IsInfo) { - Stop(() => _api.SessionMonitor?.Stop(), "Stopping session monitor"); - Stop(() => _api.SyncModeSelector?.Stop(), "Stopping session sync mode selector"); - Task discoveryStopTask = Stop(() => _api.DiscoveryApp?.StopAsync(), "Stopping discovery app"); - Task blockProducerTask = Stop(() => _api.BlockProducerRunner?.StopAsync(), "Stopping block producer"); - Task syncPeerPoolTask = Stop(() => _api.SyncPeerPool?.StopAsync(), "Stopping sync peer pool"); - Task peerPoolTask = Stop(() => _api.PeerPool?.StopAsync(), "Stopping peer pool"); - Task peerManagerTask = Stop(() => _api.PeerManager?.StopAsync(), "Stopping peer manager"); - Task synchronizerTask = Stop(() => _api.Synchronizer?.StopAsync(), "Stopping synchronizer"); - Task blockchainProcessorTask = Stop(() => _api.BlockchainProcessor?.StopAsync(), "Stopping blockchain processor"); - Task rlpxPeerTask = Stop(() => _api.RlpxPeer?.Shutdown(), "Stopping rlpx peer"); - await Task.WhenAll(discoveryStopTask, rlpxPeerTask, peerManagerTask, synchronizerTask, syncPeerPoolTask, peerPoolTask, blockchainProcessorTask, blockProducerTask); - - foreach (INethermindPlugin plugin in _api.Plugins) - { - await Stop(async () => await plugin.DisposeAsync(), $"Disposing plugin {plugin.Name}"); - } - - while (_api.DisposeStack.Count != 0) - { - IAsyncDisposable disposable = _api.DisposeStack.Pop(); - await Stop(async () => await disposable.DisposeAsync(), $"Disposing {disposable}"); - } - - Stop(() => _api.DbProvider?.Dispose(), "Closing DBs"); - - if (_logger.IsInfo) _logger.Info("All DBs closed."); - - if (_logger.IsInfo) _logger.Info("Ethereum shutdown complete... please wait for all components to close"); + _logger.Info("All DBs closed"); + _logger.Info("Ethereum runner stopped"); } + } - private void Stop(Action stopAction, string description) + private void Stop(Action stopAction, string description) + { + try { - try - { - if (_logger.IsInfo) _logger.Info($"{description}..."); - stopAction(); - } - catch (Exception e) - { - if (_logger.IsError) _logger.Error($"{description} shutdown error.", e); - } + if (_logger.IsInfo) _logger.Info(description); + + stopAction(); } + catch (Exception e) + { + if (_logger.IsError) _logger.Error($"{description} shutdown error.", e); + } + } - private Task Stop(Func stopAction, string description) + private Task Stop(Func stopAction, string description) + { + try + { + if (_logger.IsInfo) _logger.Info(description); + return stopAction() ?? Task.CompletedTask; + } + catch (Exception e) { - try - { - if (_logger.IsInfo) _logger.Info($"{description}..."); - return stopAction() ?? Task.CompletedTask; - } - catch (Exception e) - { - if (_logger.IsError) _logger.Error($"{description} shutdown error.", e); - return Task.CompletedTask; - } + if (_logger.IsError) _logger.Error($"{description} shutdown error.", e); + return Task.CompletedTask; } } } diff --git a/src/Nethermind/Nethermind.Runner/Logging/NLogConfigurator.cs b/src/Nethermind/Nethermind.Runner/Logging/NLogConfigurator.cs index 400118352e6..7a2ceca06a5 100644 --- a/src/Nethermind/Nethermind.Runner/Logging/NLogConfigurator.cs +++ b/src/Nethermind/Nethermind.Runner/Logging/NLogConfigurator.cs @@ -3,95 +3,92 @@ using System; using System.Linq; -using McMaster.Extensions.CommandLineUtils; using Nethermind.Core.Collections; using NLog; using NLog.Config; using NLog.Targets; using NLog.Targets.Seq; -namespace Nethermind.Runner.Logging +namespace Nethermind.Runner.Logging; + +public static class NLogConfigurator { - public static class NLogConfigurator + public static void ConfigureSeqBufferTarget( + string url = "http://localhost:5341", + string apiKey = "", + string minLevel = "Off") { - public static void ConfigureSeqBufferTarget( - string url = "http://localhost:5341", - string apiKey = "", - string minLevel = "Off") + LoggingConfiguration loggingConfiguration = LogManager.Configuration; + if (loggingConfiguration is not null) { - LoggingConfiguration loggingConfiguration = LogManager.Configuration; - if (loggingConfiguration is not null) + if (loggingConfiguration.AllTargets is not null) { - if (loggingConfiguration.AllTargets is not null) + foreach (SeqTarget target in loggingConfiguration.AllTargets.OfType()) { - foreach (SeqTarget target in loggingConfiguration.AllTargets.OfType()) + target.ApiKey = apiKey; + target.ServerUrl = url; + foreach (LoggingRule? rule in loggingConfiguration.LoggingRules) { - target.ApiKey = apiKey; - target.ServerUrl = url; - foreach (LoggingRule? rule in loggingConfiguration.LoggingRules) + foreach (Target? ruleTarget in rule.Targets) { - foreach (Target? ruleTarget in rule.Targets) + if (ruleTarget.Name == "seq" && rule.LoggerNamePattern == "*") { - if (ruleTarget.Name == "seq" && rule.LoggerNamePattern == "*") - { - rule.EnableLoggingForLevels(LogLevel.FromString(minLevel), LogLevel.Fatal); - } + rule.EnableLoggingForLevels(LogLevel.FromString(minLevel), LogLevel.Fatal); } } } } - - // // // re-initialize single target - loggingConfiguration.AllTargets?.OfType().ForEach(t => t.Dispose()); - LogManager.ReconfigExistingLoggers(); } - } - public static void ClearSeqTarget() - { - LoggingConfiguration loggingConfiguration = LogManager.Configuration; - loggingConfiguration?.RemoveTarget("seq"); + // // // re-initialize single target + loggingConfiguration.AllTargets?.OfType().ForEach(t => t.Dispose()); + LogManager.ReconfigExistingLoggers(); } + } - public static void ConfigureLogLevels(CommandOption logLevelOverride) + public static void ClearSeqTarget() + { + LoggingConfiguration loggingConfiguration = LogManager.Configuration; + loggingConfiguration?.RemoveTarget("seq"); + } + + public static void ConfigureLogLevels(string logLevel) + { + LogLevel nLogLevel = logLevel.ToUpperInvariant() switch { - string logLevel = logLevelOverride.Value(); - LogLevel nLogLevel = logLevel.ToUpperInvariant() switch - { - "OFF" => LogLevel.Off, - "ERROR" => LogLevel.Error, - "WARN" => LogLevel.Warn, - "INFO" => LogLevel.Info, - "DEBUG" => LogLevel.Debug, - "TRACE" => LogLevel.Trace, - _ => LogLevel.Info - }; + "OFF" => LogLevel.Off, + "ERROR" => LogLevel.Error, + "WARN" => LogLevel.Warn, + "INFO" => LogLevel.Info, + "DEBUG" => LogLevel.Debug, + "TRACE" => LogLevel.Trace, + _ => LogLevel.Info + }; - Console.WriteLine($"Enabling log level override: {logLevel.ToUpperInvariant()}"); + //Console.WriteLine($"Enabling log level override: {logLevel.ToUpperInvariant()}"); - // There are some rules for which we don't want to override the log level - // but instead preserve the original config defined in the 'NLog.config' file - string[] ignoredRuleNames = - { - "JsonWebAPI*", - "JsonWebAPI.Microsoft.Extensions.Diagnostics.HealthChecks.DefaultHealthCheckService", - }; - foreach (LoggingRule rule in LogManager.Configuration.LoggingRules) - { - if (ignoredRuleNames.Contains(rule.LoggerNamePattern)) { continue; } + // There are some rules for which we don't want to override the log level + // but instead preserve the original config defined in the 'NLog.config' file + string[] ignoredRuleNames = + { + "JsonWebAPI*", + "JsonWebAPI.Microsoft.Extensions.Diagnostics.HealthChecks.DefaultHealthCheckService", + }; + foreach (LoggingRule rule in LogManager.Configuration.LoggingRules) + { + if (ignoredRuleNames.Contains(rule.LoggerNamePattern)) { continue; } - foreach (var ruleTarget in rule.Targets) + foreach (var ruleTarget in rule.Targets) + { + if (ruleTarget.Name != "seq") { - if (ruleTarget.Name != "seq") - { - Console.WriteLine($"{ruleTarget.Name} TEST"); - rule.DisableLoggingForLevels(LogLevel.Trace, nLogLevel); - rule.EnableLoggingForLevels(nLogLevel, LogLevel.Off); - } + //Console.WriteLine($"{ruleTarget.Name} TEST"); + rule.DisableLoggingForLevels(LogLevel.Trace, nLogLevel); + rule.EnableLoggingForLevels(nLogLevel, LogLevel.Off); } } - - LogManager.ReconfigExistingLoggers(); } + + LogManager.ReconfigExistingLoggers(); } } diff --git a/src/Nethermind/Nethermind.Runner/NLog.config b/src/Nethermind/Nethermind.Runner/NLog.config index b9e3d620738..11993b49c13 100644 --- a/src/Nethermind/Nethermind.Runner/NLog.config +++ b/src/Nethermind/Nethermind.Runner/NLog.config @@ -53,10 +53,10 @@ - + - + diff --git a/src/Nethermind/Nethermind.Runner/Nethermind.Runner.csproj b/src/Nethermind/Nethermind.Runner/Nethermind.Runner.csproj index d439a869bd9..2cc6fa3b095 100644 --- a/src/Nethermind/Nethermind.Runner/Nethermind.Runner.csproj +++ b/src/Nethermind/Nethermind.Runner/Nethermind.Runner.csproj @@ -25,10 +25,10 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + @@ -72,7 +72,7 @@ PreserveNewest - + PreserveNewest PreserveNewest true @@ -83,7 +83,6 @@ true - Always diff --git a/src/Nethermind/Nethermind.Runner/Program.cs b/src/Nethermind/Nethermind.Runner/Program.cs index ed04e3ef6bf..ffc60b4971d 100644 --- a/src/Nethermind/Nethermind.Runner/Program.cs +++ b/src/Nethermind/Nethermind.Runner/Program.cs @@ -1,23 +1,23 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only using System; using System.Collections.Generic; -using System.Diagnostics; +using System.CommandLine; +using System.CommandLine.Help; +using System.CommandLine.Invocation; +using System.CommandLine.Parsing; using System.IO; using System.IO.Abstractions; using System.Linq; using System.Reflection; -using System.Runtime.InteropServices; -using System.Runtime.Loader; -using System.Text; +using System.Runtime; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; #if !DEBUG using DotNetty.Common; #endif -using McMaster.Extensions.CommandLineUtils; using Nethermind.Api; using Nethermind.Api.Extensions; using Nethermind.Config; @@ -32,6 +32,7 @@ using Nethermind.KeyStore.Config; using Nethermind.Logging; using Nethermind.Logging.NLog; +using Nethermind.Runner; using Nethermind.Runner.Ethereum; using Nethermind.Runner.Ethereum.Api; using Nethermind.Runner.Logging; @@ -41,557 +42,502 @@ using NLog; using NLog.Config; using ILogger = Nethermind.Logging.ILogger; +using NullLogger = Nethermind.Logging.NullLogger; -namespace Nethermind.Runner; +Console.Title = ProductInfo.Name; +// Increase regex cache size as more added in log coloring matches +Regex.CacheSize = 128; +#if !DEBUG +ResourceLeakDetector.Level = ResourceLeakDetector.DetectionLevel.Disabled; +#endif -public static partial class Program +ManualResetEventSlim exit = new(true); +ILogger logger = new(SimpleConsoleLogger.Instance); +ProcessExitSource? processExitSource = default; +var unhandledError = "A critical error has occurred"; + +AppDomain.CurrentDomain.UnhandledException += (sender, e) => { - private const string FailureString = "Failure"; - private const string DefaultConfigsDirectory = "configs"; - private const string DefaultConfigFile = "configs/mainnet.cfg"; + ILogger criticalLogger = GetCriticalLogger(); - private static ILogger _logger = new(SimpleConsoleLogger.Instance); + if (e.ExceptionObject is Exception ex) + criticalLogger.Error($"{unhandledError}.", ex); + else + criticalLogger.Error($"{unhandledError}: {e.ExceptionObject}"); +}; - private static readonly ProcessExitSource _processExitSource = new(); - private static readonly ManualResetEventSlim _appClosed = new(true); +try +{ + return await ConfigureAsync(args); +} +catch (Exception ex) +{ + ILogger criticalLogger = GetCriticalLogger(); - public static void Main(string[] args) - { - // Increase regex cache size as more added in log coloring matches - Regex.CacheSize = 128; -#if !DEBUG - ResourceLeakDetector.Level = ResourceLeakDetector.DetectionLevel.Disabled; -#endif - AppDomain.CurrentDomain.UnhandledException += (sender, eventArgs) => - { - ILogger logger = GetCriticalLogger(); - if (eventArgs.ExceptionObject is Exception e) - { - logger.Error(FailureString, e); - } - else - { - logger.Error(FailureString + eventArgs.ExceptionObject); - } - }; + ex = ex is AggregateException aex ? aex.InnerException : ex; - try - { - Run(args); - } - catch (AggregateException e) - { - ILogger logger = GetCriticalLogger(); - logger.Error(FailureString, e.InnerException); - } - catch (Exception e) - { - ILogger logger = GetCriticalLogger(); - logger.Error(FailureString, e); - } - finally - { - NLogManager.Shutdown(); - } - } + criticalLogger.Error($"{unhandledError}.", ex); + + return ex is IExceptionWithExitCode exc ? exc.ExitCode : ExitCodes.GeneralError; +} +finally +{ + NLogManager.Shutdown(); +} - private static ILogger GetCriticalLogger() +async Task ConfigureAsync(string[] args) +{ + CliConfiguration cli = ConfigureCli(); + ParseResult parseResult = cli.Parse(args); + // Suppress logs if run with `--help` or `--version` + bool silent = parseResult.CommandResult.Children + .Any(c => c is OptionResult { Option: HelpOption or VersionOption }); + + ConsoleHelpers.EnableConsoleColorOutput(); + + ConfigureLogger(parseResult); + + if (!silent) { - try - { - return new NLogManager("logs.txt").GetClassLogger(); - } - catch - { - if (_logger.IsWarn) _logger.Warn("Critical file logging could not be instantiated! Sticking to console logging till config is loaded."); - return _logger; - } + logger.Info("Nethermind is starting up"); + logger.Info($"Version: {ProductInfo.Version}"); } - private static void Run(string[] args) + AppDomain.CurrentDomain.ProcessExit += (_, _) => { - _logger.Info("Nethermind starting initialization."); - _logger.Info($"Client version: {ProductInfo.ClientId}"); - - AppDomain.CurrentDomain.ProcessExit += CurrentDomainOnProcessExit; - AssemblyLoadContext.Default.ResolvingUnmanagedDll += OnResolvingUnmanagedDll; - - GlobalDiagnosticsContext.Set("version", ProductInfo.Version); - CommandLineApplication app = new() { Name = "Nethermind.Runner" }; - _ = app.HelpOption("-?|-h|--help"); - _ = app.VersionOption("-v|--version", () => ProductInfo.Version, GetProductInfo); - - ConsoleHelpers.EnableConsoleColorOutput(); - - CommandOption dataDir = app.Option("-dd|--datadir ", "Data directory", CommandOptionType.SingleValue); - CommandOption configFile = app.Option("-c|--config ", "Config file path", CommandOptionType.SingleValue); - CommandOption dbBasePath = app.Option("-d|--baseDbPath ", "Base db path", CommandOptionType.SingleValue); - CommandOption logLevelOverride = app.Option("-l|--log ", "Log level override. Possible values: OFF|TRACE|DEBUG|INFO|WARN|ERROR", CommandOptionType.SingleValue); - CommandOption configsDirectory = app.Option("-cd|--configsDirectory ", "Configs directory", CommandOptionType.SingleValue); - CommandOption loggerConfigSource = app.Option("-lcs|--loggerConfigSource ", "Path to the NLog config file", CommandOptionType.SingleValue); - _ = app.Option("-pd|--pluginsDirectory ", "plugins directory", CommandOptionType.SingleValue); - - IFileSystem fileSystem = new FileSystem(); - - string pluginsDirectoryPath = LoadPluginsDirectory(args); - PluginLoader pluginLoader = new(pluginsDirectoryPath, fileSystem, - typeof(AuRaPlugin), - typeof(CliquePlugin), - typeof(EthashPlugin), - typeof(NethDevPlugin), - typeof(HivePlugin), - typeof(UPnPPlugin) - ); - - // leaving here as an example of adding Debug plugin - // IPluginLoader mevLoader = SinglePluginLoader.Instance; - // CompositePluginLoader pluginLoader = new (pluginLoader, mevLoader); - pluginLoader.Load(SimpleConsoleLogManager.Instance); - TypeDiscovery.Initialize(typeof(INethermindPlugin)); - - BuildOptionsFromConfigFiles(app); - - app.OnExecute(async () => - { - IConfigProvider configProvider = BuildConfigProvider(app, loggerConfigSource, logLevelOverride, configsDirectory, configFile); - IInitConfig initConfig = configProvider.GetConfig(); - IKeyStoreConfig keyStoreConfig = configProvider.GetConfig(); - ISnapshotConfig snapshotConfig = configProvider.GetConfig(); - IPluginConfig pluginConfig = configProvider.GetConfig(); + processExitSource?.Exit(ExitCodes.SigTerm); + exit.Wait(); + }; + GlobalDiagnosticsContext.Set("version", ProductInfo.Version); + + PluginLoader pluginLoader = new( + parseResult.GetValue(BasicOptions.PluginsDirectory) ?? "plugins", + new FileSystem(), + silent ? NullLogger.Instance : logger, + typeof(AuRaPlugin), + typeof(CliquePlugin), + typeof(EthashPlugin), + typeof(NethDevPlugin), + typeof(HivePlugin), + typeof(UPnPPlugin) + ); + pluginLoader.Load(); + + // leaving here as an example of adding Debug plugin + // IPluginLoader mevLoader = SinglePluginLoader.Instance; + // CompositePluginLoader pluginLoader = new (pluginLoader, mevLoader); + + TypeDiscovery.Initialize(typeof(INethermindPlugin)); + + AddConfigurationOptions(cli.RootCommand); + + cli.RootCommand.SetAction((result, token) => RunAsync(result, pluginLoader, token)); + + try + { + return await cli.InvokeAsync(args); + } + finally + { + exit.Wait(); + } +} - pluginLoader.OrderPlugins(pluginConfig); - Console.Title = initConfig.LogFileName; - Console.CancelKeyPress += ConsoleOnCancelKeyPress; +async Task RunAsync(ParseResult parseResult, PluginLoader pluginLoader, CancellationToken cancellationToken) +{ + processExitSource = new(cancellationToken); - SetFinalDataDirectory(dataDir.HasValue() ? dataDir.Value() : null, initConfig, keyStoreConfig, snapshotConfig); - NLogManager logManager = new(initConfig.LogFileName, initConfig.LogDirectory, initConfig.LogRules); + IConfigProvider configProvider = CreateConfigProvider(parseResult); + IInitConfig initConfig = configProvider.GetConfig(); + IKeyStoreConfig keyStoreConfig = configProvider.GetConfig(); + ISnapshotConfig snapshotConfig = configProvider.GetConfig(); + IPluginConfig pluginConfig = configProvider.GetConfig(); - _logger = logManager.GetClassLogger(); - ConfigureSeqLogger(configProvider); - SetFinalDbPath(dbBasePath.HasValue() ? dbBasePath.Value() : null, initConfig); - LogMemoryConfiguration(); + pluginLoader.OrderPlugins(pluginConfig); - EthereumJsonSerializer serializer = new(); - if (_logger.IsDebug) _logger.Debug($"Nethermind config:{Environment.NewLine}{serializer.Serialize(initConfig, true)}{Environment.NewLine}"); - if (_logger.IsInfo) _logger.Info($"RocksDb Version: {DbOnTheRocks.GetRocksDbVersion()}"); + ResolveDataDirectory(parseResult.GetValue(BasicOptions.DataDirectory), + initConfig, keyStoreConfig, snapshotConfig); - ApiBuilder apiBuilder = new(configProvider, logManager); + NLogManager logManager = new(initConfig.LogFileName, initConfig.LogDirectory, initConfig.LogRules); - IList plugins = new List(); - foreach (Type pluginType in pluginLoader.PluginTypes) - { - try - { - if (Activator.CreateInstance(pluginType) is INethermindPlugin plugin) - { - plugins.Add(plugin); - } - } - catch (Exception e) - { - if (_logger.IsError) _logger.Error($"Failed to create plugin {pluginType.FullName}", e); - } - } + logger = logManager.GetClassLogger(); - INethermindApi nethermindApi = apiBuilder.Create(plugins.OfType()); - ((List)nethermindApi.Plugins).AddRange(plugins); - nethermindApi.ProcessExit = _processExitSource; + ConfigureSeqLogger(configProvider); + ResolveDatabaseDirectory(parseResult.GetValue(BasicOptions.DatabasePath), initConfig); - _appClosed.Reset(); - EthereumRunner ethereumRunner = new(nethermindApi); - try - { - await ethereumRunner.Start(_processExitSource.Token); + logger.Info("Configuration complete"); - await _processExitSource.ExitTask; - } - catch (TaskCanceledException) - { - if (_logger.IsTrace) _logger.Trace("Runner Task was canceled"); - } - catch (OperationCanceledException) - { - if (_logger.IsTrace) _logger.Trace("Runner operation was canceled"); - } - catch (Exception e) - { - if (_logger.IsError) _logger.Error("Error during ethereum runner start", e); - _processExitSource.Exit(e is IExceptionWithExitCode withExit ? withExit.ExitCode : ExitCodes.GeneralError); - } + EthereumJsonSerializer serializer = new(); - _logger.Info("Closing, please wait until all functions are stopped properly..."); - await ethereumRunner.StopAsync(); - _logger.Info("All done, goodbye!"); - _appClosed.Set(); - }); + if (logger.IsDebug) + { + logger.Debug($"Nethermind configuration:\n{serializer.Serialize(initConfig, true)}"); - try - { - Environment.ExitCode = app.Execute(args); - } - catch (UnrecognizedCommandParsingException e) - { - string[] matches = e.NearestMatches.Take(3).ToArray(); - string suggestion = matches.Length switch - { - 0 => "", - 1 => $" Did you mean {matches[0]}", - _ => $" Did you mean one of: {string.Join(", ", matches)}" - }; - _logger.Error($"{e.Message}.{suggestion}"); - Environment.ExitCode = ExitCodes.UnrecognizedOption; - } - catch (CommandParsingException e) - { - Regex regex = GetUnexpectedConfigValueRegex(); - Match match = regex.Match(e.Message); - if (match.Success) - { - string option = match.Groups["Name"].Value; - CommandOption? optionInfo = app.GetOptions().FirstOrDefault(o => o.ShortName == option || o.LongName == option); - switch (optionInfo?.OptionType) - { - case CommandOptionType.SingleValue or CommandOptionType.SingleOrNoValue: - _logger.Error($"Duplicated option '{option}'"); - Environment.ExitCode = ExitCodes.DuplicatedOption; - return; - case CommandOptionType.NoValue: - _logger.Error($"Value {match.Groups["Value"].Value} passed for value-less option '{option}'"); - Environment.ExitCode = ExitCodes.ForbiddenOptionValue; - return; - } - } + logger.Debug($"Server GC: {GCSettings.IsServerGC}"); + logger.Debug($"GC latency mode: {GCSettings.LatencyMode}"); + logger.Debug($"LOH compaction mode: {GCSettings.LargeObjectHeapCompactionMode}"); + } - _logger.Error($"{e.Message}"); - } - catch (Exception e) + if (logger.IsInfo) logger.Info($"RocksDB: v{DbOnTheRocks.GetRocksDbVersion()}"); + + ApiBuilder apiBuilder = new(configProvider, logManager); + IList plugins = []; + + foreach (Type pluginType in pluginLoader.PluginTypes) + { + try { - if (e is IExceptionWithExitCode withExit) - { - Environment.ExitCode = withExit.ExitCode; - } - else - { - Environment.ExitCode = ExitCodes.GeneralError; - } - throw; + if (Activator.CreateInstance(pluginType) is INethermindPlugin plugin) + plugins.Add(plugin); } - finally + catch (Exception ex) { - _appClosed.Wait(); + if (logger.IsError) logger.Error($"Failed to create plugin {pluginType.FullName}", ex); } } - [GeneratedRegex("^Unexpected value '(?.+)' for option '(?.+)'", RegexOptions.Singleline)] - private static partial Regex GetUnexpectedConfigValueRegex(); + INethermindApi nethermindApi = apiBuilder.Create(plugins.OfType()); + ((List)nethermindApi.Plugins).AddRange(plugins); + nethermindApi.ProcessExit = processExitSource; - private static IntPtr OnResolvingUnmanagedDll(Assembly _, string nativeLibraryName) + EthereumRunner ethereumRunner = new(nethermindApi); + try { - var alternativePath = nativeLibraryName switch - { - "libdl" => "libdl.so.2", - _ => null - }; + await ethereumRunner.Start(processExitSource.Token); + await processExitSource.ExitTask; + } + catch (OperationCanceledException) + { + if (logger.IsTrace) logger.Trace("Nethermind operation was canceled."); + } + catch (Exception ex) + { + if (logger.IsError) logger.Error(unhandledError, ex); - return alternativePath is null ? IntPtr.Zero : NativeLibrary.Load(alternativePath); + processExitSource.Exit(ex is IExceptionWithExitCode withExit ? withExit.ExitCode : ExitCodes.GeneralError); } - private static void BuildOptionsFromConfigFiles(CommandLineApplication app) + logger.Info("Nethermind is shutting down... Please wait until all activities are stopped."); + + await ethereumRunner.StopAsync(); + + logger.Info("Nethermind is shut down"); + + exit.Set(); + + return processExitSource.ExitCode; +} + +void AddConfigurationOptions(CliCommand command) +{ + IEnumerable configTypes = TypeDiscovery + .FindNethermindBasedTypes(typeof(IConfig)) + .Where(ct => ct.IsInterface); + + foreach (Type configType in + configTypes.Where(ct => !ct.IsAssignableTo(typeof(INoCategoryConfig))).OrderBy(c => c.Name)) { - Type configurationType = typeof(IConfig); - IEnumerable configTypes = TypeDiscovery.FindNethermindBasedTypes(configurationType) - .Where(ct => ct.IsInterface); + if (configType is null) + continue; - foreach (Type configType in configTypes.Where(ct => !ct.IsAssignableTo(typeof(INoCategoryConfig))).OrderBy(c => c.Name)) - { - if (configType is null) - { - continue; - } + ConfigCategoryAttribute? typeLevel = configType.GetCustomAttribute(); - ConfigCategoryAttribute? typeLevel = configType.GetCustomAttribute(); + if (typeLevel is not null && (typeLevel.DisabledForCli || typeLevel.HiddenFromDocs)) + continue; - if (typeLevel is not null && (typeLevel?.DisabledForCli ?? true)) - { - continue; - } + foreach (PropertyInfo propertyInfo in + configType.GetProperties(BindingFlags.Public | BindingFlags.Instance).OrderBy(p => p.Name)) + { + ConfigItemAttribute? configItemAttribute = propertyInfo.GetCustomAttribute(); - foreach (PropertyInfo propertyInfo in configType - .GetProperties(BindingFlags.Public | BindingFlags.Instance) - .OrderBy(p => p.Name)) + if (configItemAttribute?.DisabledForCli == false || configItemAttribute?.HiddenFromDocs == false) { - ConfigItemAttribute? configItemAttribute = propertyInfo.GetCustomAttribute(); - if (!(configItemAttribute?.DisabledForCli ?? false)) - { - _ = app.Option($"--{ConfigExtensions.GetCategoryName(configType)}.{propertyInfo.Name}", $"{(configItemAttribute is null ? "" : configItemAttribute.Description + $" (DEFAULT: {configItemAttribute.DefaultValue})" ?? "")}", CommandOptionType.SingleValue); - } - if (configItemAttribute?.IsPortOption == true) + command.Add(new CliOption( + $"--{ConfigExtensions.GetCategoryName(configType)}.{propertyInfo.Name}", + $"--{ConfigExtensions.GetCategoryName(configType)}-{propertyInfo.Name}".ToLowerInvariant()) { - ConfigExtensions.AddPortOptionName(configType, propertyInfo.Name); - } + Description = configItemAttribute?.Description, + HelpName = "value" + }); } + + if (configItemAttribute?.IsPortOption == true) + ConfigExtensions.AddPortOptionName(configType, propertyInfo.Name); } + } +} - // Create Help Text for environment variables - Type noCategoryConfig = configTypes.FirstOrDefault(ct => ct.IsAssignableTo(typeof(INoCategoryConfig))); - if (noCategoryConfig is not null) +CliConfiguration ConfigureCli() +{ + CliRootCommand rootCommand = + [ + BasicOptions.Configuration, + BasicOptions.ConfigurationDirectory, + BasicOptions.DatabasePath, + BasicOptions.DataDirectory, + BasicOptions.LoggerConfigurationSource, + BasicOptions.LogLevel, + BasicOptions.PluginsDirectory + ]; + + var versionOption = (VersionOption)rootCommand.Children.SingleOrDefault(c => c is VersionOption); + + if (versionOption is not null) + { + versionOption.Action = new AnonymousCliAction(r => { - StringBuilder sb = new(); - sb.AppendLine(); - sb.AppendLine("Configurable Environment Variables:"); - foreach (PropertyInfo propertyInfo in noCategoryConfig.GetProperties(BindingFlags.Public | BindingFlags.Instance).OrderBy(p => p.Name)) - { - ConfigItemAttribute? configItemAttribute = propertyInfo.GetCustomAttribute(); - if (configItemAttribute is not null && !(string.IsNullOrEmpty(configItemAttribute?.EnvironmentVariable))) - { - sb.AppendLine($"{configItemAttribute.EnvironmentVariable} - {(string.IsNullOrEmpty(configItemAttribute.Description) ? "" : configItemAttribute.Description)} (DEFAULT: {configItemAttribute.DefaultValue})"); - } - } + Console.WriteLine($""" + Version: {ProductInfo.Version} + Commit: {ProductInfo.Commit} + Build date: {ProductInfo.BuildTimestamp:u} + Runtime: {ProductInfo.Runtime} + Platform: {ProductInfo.OS} {ProductInfo.OSArchitecture} + """); - app.ExtendedHelpText = sb.ToString(); - } + return ExitCodes.Ok; + }); } - private static string LoadPluginsDirectory(string[] args) + return new(rootCommand) { - string shortCommand = "-pd"; - string longCommand = "--pluginsDirectory"; + EnableDefaultExceptionHandler = false, + ProcessTerminationTimeout = Timeout.InfiniteTimeSpan + }; +} - string[] GetPluginArgs() - { - for (int i = 0; i < args.Length; i++) - { - string arg = args[i]; - if (arg == shortCommand || arg == longCommand) - { - return i == args.Length - 1 ? new[] { arg } : new[] { arg, args[i + 1] }; - } - } +void ConfigureLogger(ParseResult parseResult) +{ + string nLogConfig = Path.GetFullPath( + parseResult.GetValue(BasicOptions.LoggerConfigurationSource) + ?? "NLog.config".GetApplicationResourcePath()); - return Array.Empty(); - } + try + { + LogManager.Configuration = new XmlLoggingConfiguration(nLogConfig); + } + catch (Exception ex) + { + logger.Error($"Failed to load logging configuration file.", ex); + return; + } - CommandLineApplication pluginsApp = new() { Name = "Nethermind.Runner.Plugins" }; - CommandOption pluginsAppDirectory = pluginsApp.Option($"{shortCommand}|{longCommand} ", "plugins directory", CommandOptionType.SingleValue); - string pluginDirectory = "plugins"; - pluginsApp.OnExecute(() => - { - if (pluginsAppDirectory.HasValue()) - { - pluginDirectory = pluginsAppDirectory.Value(); - } + using NLogManager logManager = new("nethermind.log"); - return 0; - }); - pluginsApp.Execute(GetPluginArgs()); - return pluginDirectory; + logger = logManager.GetClassLogger(); + + string logLevel = parseResult.GetValue(BasicOptions.LogLevel) ?? "info"; + + // TODO: dynamically switch log levels from CLI + if (logLevel is not null) + NLogConfigurator.ConfigureLogLevels(logLevel); +} + +void ConfigureSeqLogger(IConfigProvider configProvider) +{ + ISeqConfig seqConfig = configProvider.GetConfig(); + + if (!seqConfig.MinLevel.Equals("Off", StringComparison.Ordinal)) + { + if (logger.IsInfo) + logger.Info($"Seq logging is enabled on {seqConfig.ServerUrl} with level of {seqConfig.MinLevel}"); + + NLogConfigurator.ConfigureSeqBufferTarget(seqConfig.ServerUrl, seqConfig.ApiKey, seqConfig.MinLevel); } + else + { + // Clear it up; otherwise, internally it will keep requesting localhost as `all` target includes this. + NLogConfigurator.ClearSeqTarget(); + } +} - private static IConfigProvider BuildConfigProvider( - CommandLineApplication app, - CommandOption loggerConfigSource, - CommandOption logLevelOverride, - CommandOption configsDirectory, - CommandOption configFile) +IConfigProvider CreateConfigProvider(ParseResult parseResult) +{ + ConfigProvider configProvider = new(); + Dictionary configArgs = []; + + foreach (SymbolResult child in parseResult.RootCommandResult.Children) { - if (loggerConfigSource.HasValue()) + if (child is OptionResult result) { - string nLogPath = loggerConfigSource.Value(); - _logger.Info($"Loading NLog configuration file from {nLogPath}."); + var value = result.GetValueOrDefault(); - try - { - LogManager.Configuration = new XmlLoggingConfiguration(nLogPath); - } - catch (Exception e) - { - _logger.Info($"Failed to load NLog configuration from {nLogPath}. {e}"); - } + if (value is not null) + configArgs.Add(result.Option.Name.TrimStart('-'), value); } - else - { - _logger.Info($"Loading standard NLog.config file from {"NLog.config".GetApplicationResourcePath()}."); - long startTime = Stopwatch.GetTimestamp(); - LogManager.Configuration = new XmlLoggingConfiguration("NLog.config".GetApplicationResourcePath()); + } - _logger.Info($"NLog.config loaded in {Stopwatch.GetElapsedTime(startTime).TotalMilliseconds:N0}ms."); - } + IConfigSource argsSource = new ArgsConfigSource(configArgs); + configProvider.AddSource(argsSource); + configProvider.AddSource(new EnvConfigSource()); - // TODO: dynamically switch log levels from CLI! - if (logLevelOverride.HasValue()) - { - NLogConfigurator.ConfigureLogLevels(logLevelOverride); - } + string configFile = parseResult.GetValue(BasicOptions.Configuration) + ?? Environment.GetEnvironmentVariable("NETHERMIND_CONFIG") + ?? "mainnet"; + + // If configFile is not a path, handle it + if (string.IsNullOrEmpty(Path.GetDirectoryName(configFile))) + { + string configsDir = parseResult.GetValue(BasicOptions.ConfigurationDirectory) + ?? "configs".GetApplicationResourcePath(); + + configFile = Path.Join(configsDir, configFile); - ConfigProvider configProvider = new(); - Dictionary configArgs = new(); - foreach (CommandOption commandOption in app.Options) + // If the configFile doesn't have an extension, try with supported file extensions + if (!Path.HasExtension(configFile)) { - if (commandOption.HasValue()) + string? fallback; + + foreach (var ext in new[] { ".json", ".cfg" }) { - configArgs.Add(commandOption.LongName, commandOption.Value()); + fallback = $"{configFile}{ext}"; + + if (File.Exists(fallback)) + { + configFile = fallback; + break; + } } } + } + else + { + configFile = configFile.GetApplicationResourcePath(); + } - IConfigSource argsSource = new ArgsConfigSource(configArgs); - configProvider.AddSource(argsSource); - configProvider.AddSource(new EnvConfigSource()); + // Resolve the full path for logging purposes + configFile = Path.GetFullPath(configFile); - string configDir = configsDirectory.HasValue() ? configsDirectory.Value() : DefaultConfigsDirectory; - string configFilePath = configFile.HasValue() ? configFile.Value() : DefaultConfigFile; - string? configPathVariable = Environment.GetEnvironmentVariable("NETHERMIND_CONFIG"); - if (!string.IsNullOrWhiteSpace(configPathVariable)) - { - configFilePath = configPathVariable; - } + if (!File.Exists(configFile)) + throw new FileNotFoundException("Configuration file not found.", configFile); - if (!PathUtils.IsExplicitlyRelative(configFilePath)) - { - configFilePath = configDir == DefaultConfigsDirectory - ? configFilePath.GetApplicationResourcePath() - : Path.Combine(configDir, string.Concat(configFilePath)); - } + logger.Info($"Loading configuration from {configFile}"); - if (!Path.HasExtension(configFilePath) && !configFilePath.Contains(Path.DirectorySeparatorChar)) - { - string redirectedConfigPath = Path.Combine(configDir, string.Concat(configFilePath, ".cfg")); - configFilePath = redirectedConfigPath; - if (!File.Exists(configFilePath)) - { - throw new InvalidOperationException($"Configuration: {configFilePath} was not found."); - } - } + configProvider.AddSource(new JsonConfigSource(configFile)); + configProvider.Initialize(); - if (!Path.HasExtension(configFilePath)) - { - configFilePath = string.Concat(configFilePath, ".cfg"); - } + var incorrectSettings = configProvider.FindIncorrectSettings(); - // Fallback to "{executingDirectory}/configs/{configFile}" if "configs" catalog was not specified. - if (!File.Exists(configFilePath)) - { - string configName = Path.GetFileName(configFilePath); - string? configDirectory = Path.GetDirectoryName(configFilePath); - string redirectedConfigPath = Path.Combine(configDirectory ?? string.Empty, configDir, configName); - configFilePath = redirectedConfigPath; - if (!File.Exists(configFilePath)) - { - throw new InvalidOperationException($"Configuration: {configFilePath} was not found."); - } - } + if (incorrectSettings.Errors.Any()) + logger.Warn($"Invalid configuration settings:\n{incorrectSettings.ErrorMsg}"); - _logger.Info($"Reading config file from {configFilePath}"); - configProvider.AddSource(new JsonConfigSource(configFilePath)); - configProvider.Initialize(); - var incorrectSettings = configProvider.FindIncorrectSettings(); - if (incorrectSettings.Errors.Count > 0) - { - _logger.Warn($"Incorrect config settings found:{Environment.NewLine}{incorrectSettings.ErrorMsg}"); - } + return configProvider; +} - _logger.Info("Configuration initialized."); - return configProvider; +ILogger GetCriticalLogger() +{ + try + { + return new NLogManager("nethermind.log").GetClassLogger(); } - - private static void CurrentDomainOnProcessExit(object? sender, EventArgs e) + catch { - _processExitSource.Exit(ExitCodes.SigTerm); - _appClosed.Wait(); + if (logger.IsWarn) logger.Warn("File logging failed. Using console logging."); + + return logger; } +} - private static void LogMemoryConfiguration() +void ResolveDatabaseDirectory(string? path, IInitConfig initConfig) +{ + if (string.IsNullOrWhiteSpace(path)) { - if (_logger.IsDebug) - _logger.Debug($"Server GC : {System.Runtime.GCSettings.IsServerGC}"); - if (_logger.IsDebug) - _logger.Debug($"GC latency mode : {System.Runtime.GCSettings.LatencyMode}"); - if (_logger.IsDebug) - _logger.Debug($"LOH compaction mode : {System.Runtime.GCSettings.LargeObjectHeapCompactionMode}"); + initConfig.BaseDbPath ??= string.Empty.GetApplicationResourcePath("db"); } + else + { + string dbPath = initConfig.BaseDbPath.GetApplicationResourcePath(path); - private static void SetFinalDbPath(string? baseDbPath, IInitConfig initConfig) + if (logger.IsDebug) logger.Debug($"{nameof(initConfig.BaseDbPath)}: {Path.GetFullPath(dbPath)}"); + + initConfig.BaseDbPath = dbPath; + } +} + +void ResolveDataDirectory(string? path, IInitConfig initConfig, IKeyStoreConfig keyStoreConfig, ISnapshotConfig snapshotConfig) +{ + if (string.IsNullOrWhiteSpace(path)) { - if (!string.IsNullOrWhiteSpace(baseDbPath)) - { - string newDbPath = initConfig.BaseDbPath.GetApplicationResourcePath(baseDbPath); - if (_logger.IsDebug) _logger.Debug($"Adding prefix to baseDbPath, new value: {newDbPath}, old value: {initConfig.BaseDbPath}"); - initConfig.BaseDbPath = newDbPath; - } - else + initConfig.BaseDbPath ??= string.Empty.GetApplicationResourcePath("db"); + keyStoreConfig.KeyStoreDirectory ??= string.Empty.GetApplicationResourcePath("keystore"); + initConfig.LogDirectory ??= string.Empty.GetApplicationResourcePath("logs"); + } + else + { + string newDbPath = initConfig.BaseDbPath.GetApplicationResourcePath(path); + string newKeyStorePath = keyStoreConfig.KeyStoreDirectory.GetApplicationResourcePath(path); + string newLogDirectory = initConfig.LogDirectory.GetApplicationResourcePath(path); + string newSnapshotPath = snapshotConfig.SnapshotDirectory.GetApplicationResourcePath(path); + + if (logger.IsInfo) { - initConfig.BaseDbPath ??= string.Empty.GetApplicationResourcePath("db"); + logger.Info($"{nameof(initConfig.BaseDbPath)}: {Path.GetFullPath(newDbPath)}"); + logger.Info($"{nameof(keyStoreConfig.KeyStoreDirectory)}: {Path.GetFullPath(newKeyStorePath)}"); + logger.Info($"{nameof(initConfig.LogDirectory)}: {Path.GetFullPath(newLogDirectory)}"); + + if (snapshotConfig.Enabled) + logger.Info($"{nameof(snapshotConfig.SnapshotDirectory)}: {Path.GetFullPath(newSnapshotPath)}"); } + + initConfig.BaseDbPath = newDbPath; + keyStoreConfig.KeyStoreDirectory = newKeyStorePath; + initConfig.LogDirectory = newLogDirectory; + snapshotConfig.SnapshotDirectory = newSnapshotPath; } +} - private static void SetFinalDataDirectory(string? dataDir, IInitConfig initConfig, IKeyStoreConfig keyStoreConfig, ISnapshotConfig snapshotConfig) - { - if (!string.IsNullOrWhiteSpace(dataDir)) +static class BasicOptions +{ + public static CliOption Configuration { get; } = + new("--config", "-c") { - string newDbPath = initConfig.BaseDbPath.GetApplicationResourcePath(dataDir); - string newKeyStorePath = keyStoreConfig.KeyStoreDirectory.GetApplicationResourcePath(dataDir); - string newLogDirectory = initConfig.LogDirectory.GetApplicationResourcePath(dataDir); - string newSnapshotPath = snapshotConfig.SnapshotDirectory.GetApplicationResourcePath(dataDir); - - if (_logger.IsInfo) - { - _logger.Info($"Setting BaseDbPath to: {newDbPath}, from: {initConfig.BaseDbPath}"); - _logger.Info($"Setting KeyStoreDirectory to: {newKeyStorePath}, from: {keyStoreConfig.KeyStoreDirectory}"); - _logger.Info($"Setting LogDirectory to: {newLogDirectory}, from: {initConfig.LogDirectory}"); - if (snapshotConfig.Enabled) - { - _logger.Info($"Setting SnapshotPath to: {newSnapshotPath}"); - } - } + Description = "The path to the configuration file or the file name (also without extension) of any of the configuration files in the configuration files directory.", + HelpName = "network or file name" + }; - initConfig.BaseDbPath = newDbPath; - keyStoreConfig.KeyStoreDirectory = newKeyStorePath; - initConfig.LogDirectory = newLogDirectory; - snapshotConfig.SnapshotDirectory = newSnapshotPath; - } - else + public static CliOption ConfigurationDirectory { get; } = + new("--configs-dir", "--configsDirectory", "-cd") { - initConfig.BaseDbPath ??= string.Empty.GetApplicationResourcePath("db"); - keyStoreConfig.KeyStoreDirectory ??= string.Empty.GetApplicationResourcePath("keystore"); - initConfig.LogDirectory ??= string.Empty.GetApplicationResourcePath("logs"); - } - } + Description = "The path to the configuration files directory.", + HelpName = "path" + }; - private static void ConsoleOnCancelKeyPress(object? sender, ConsoleCancelEventArgs e) + public static CliOption DatabasePath { get; } = new("--db-dir", "--baseDbPath", "-d") { - _processExitSource.Exit(ExitCodes.SigInt); - e.Cancel = true; - } + Description = "The path to the Nethermind database directory.", + HelpName = "path" + }; - private static void ConfigureSeqLogger(IConfigProvider configProvider) + public static CliOption DataDirectory { get; } = new("--data-dir", "--datadir", "-dd") { - ISeqConfig seqConfig = configProvider.GetConfig(); - if (seqConfig.MinLevel != "Off") - { - if (_logger.IsInfo) - _logger.Info($"Seq Logging enabled on host: {seqConfig.ServerUrl} with level: {seqConfig.MinLevel}"); - NLogConfigurator.ConfigureSeqBufferTarget(seqConfig.ServerUrl, seqConfig.ApiKey, seqConfig.MinLevel); - } - else + Description = "The path to the Nethermind data directory.", + HelpName = "path" + }; + + public static CliOption LoggerConfigurationSource { get; } = + new("--logger-config", "--loggerConfigSource", "-lcs") { - // Clear it up, otherwise internally it will keep requesting to localhost as `all` target include this. - NLogConfigurator.ClearSeqTarget(); - } - } + Description = "The path to the logging configuration file.", + HelpName = "path" + }; - private static string GetProductInfo() + public static CliOption LogLevel { get; } = new("--log", "-l") { - var info = new StringBuilder(); - - info - .Append("Version: ").AppendLine(ProductInfo.Version) - .Append("Commit: ").AppendLine(ProductInfo.Commit) - .Append("Build Date: ").AppendLine(ProductInfo.BuildTimestamp.ToString("u")) - .Append("OS: ") - .Append(ProductInfo.OS) - .Append(' ') - .AppendLine(ProductInfo.OSArchitecture) - .Append("Runtime: ").AppendLine(ProductInfo.Runtime); - - return info.ToString(); - } + Description = "Log level (severity). Allowed values: off, trace, debug, info, warn, error.", + HelpName = "level" + }; + + public static CliOption PluginsDirectory { get; } = + new("--plugins-dir", "--pluginsDirectory", "-pd") + { + Description = "The path to the Nethermind plugins directory.", + HelpName = "path" + }; +} + +class AnonymousCliAction(Func action) : SynchronousCliAction +{ + private readonly Func _action = action ?? throw new ArgumentNullException(nameof(action)); + + /// + public override int Invoke(ParseResult parseResult) => _action(parseResult); } diff --git a/src/Nethermind/Nethermind.Runner/Properties/launchSettings.json b/src/Nethermind/Nethermind.Runner/Properties/launchSettings.json index 4d6bd1e6c04..7c43a7b79a7 100644 --- a/src/Nethermind/Nethermind.Runner/Properties/launchSettings.json +++ b/src/Nethermind/Nethermind.Runner/Properties/launchSettings.json @@ -1,204 +1,120 @@ { "profiles": { - "Chiado": { + "Base Mainnet": { "commandName": "Project", - "commandLineArgs": "-c chiado -dd .data", + "commandLineArgs": "-c base-mainnet --data-dir .data", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, - "Chiado (archive)": { + "Base Sepolia": { "commandName": "Project", - "commandLineArgs": "-c chiado_archive -dd .data", + "commandLineArgs": "-c base-sepolia --data-dir .data", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, - "EIP-4844 local trace": { + "Chiado": { "commandName": "Project", - "commandLineArgs": "-c eip4844_local -dd .data --log TRACE", + "commandLineArgs": "-c chiado --data-dir .data", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, "Energy Web": { "commandName": "Project", - "commandLineArgs": "-c energyweb -dd .data --JsonRpc.Enabled true", + "commandLineArgs": "-c energyweb --data-dir .data --jsonrpc-enabled true", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, "Gnosis": { "commandName": "Project", - "commandLineArgs": "-c gnosis -dd .data", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "Gnosis (archive)": { - "commandName": "Project", - "commandLineArgs": "-c gnosis_archive -dd .data", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "Goerli": { - "commandName": "Project", - "commandLineArgs": "-c goerli -dd .data", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "Goerli (archive)": { - "commandName": "Project", - "commandLineArgs": "-c goerli_archive -dd .data", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "OP Mainnet": { - "commandName": "Project", - "commandLineArgs": "-c op-mainnet -dd %NETHERMIND_DATA_DIR%", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "OP Sepolia": { - "commandName": "Project", - "commandLineArgs": "-c op-sepolia -dd %NETHERMIND_DATA_DIR%", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "Base Mainnet": { - "commandName": "Project", - "commandLineArgs": "-c base-mainnet -dd %NETHERMIND_DATA_DIR%", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "Base Sepolia": { - "commandName": "Project", - "commandLineArgs": "-c base-sepolia -dd %NETHERMIND_DATA_DIR%", + "commandLineArgs": "-c gnosis --data-dir .data", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, "Hive": { "commandName": "Project", - "commandLineArgs": "-c hive -dd .data --Init.DiagnosticMode MemDb", + "commandLineArgs": "-c hive --data-dir .data --init-diagnosticmode MemDb", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, "Holesky": { "commandName": "Project", - "commandLineArgs": "-c holesky -dd .data", + "commandLineArgs": "-c holesky --data-dir .data", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, - "Holesky (archive)": { + "JOC Mainnet": { "commandName": "Project", - "commandLineArgs": "-c holesky_archive -dd .data", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "Mainnet": { - "commandName": "Project", - "commandLineArgs": "-c mainnet -dd %NETHERMIND_DATA_DIR%", + "commandLineArgs": "-c joc-mainnet -dd .data", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, - "Mainnet (archive)": { + "JOC Testnet": { "commandName": "Project", - "commandLineArgs": "-c mainnet_archive -dd .data", + "commandLineArgs": "-c joc-testnet --data-dir .data", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, - "POA Core": { + "Mainnet": { "commandName": "Project", - "commandLineArgs": "-c poacore -dd .data --JsonRpc.Enabled true", + "commandLineArgs": "-c mainnet --data-dir .data", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, - "Sepolia": { + "OP Mainnet": { "commandName": "Project", - "commandLineArgs": "-c sepolia -dd .data", + "commandLineArgs": "-c op-mainnet --data-dir .data", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, - "Sepolia (archive)": { + "OP Sepolia": { "commandName": "Project", - "commandLineArgs": "-c sepolia_archive -dd .data", + "commandLineArgs": "-c op-sepolia --data-dir .data", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, - "Spaceneth": { + "Sepolia": { "commandName": "Project", - "commandLineArgs": "-c spaceneth -dd .data", + "commandLineArgs": "-c sepolia --data-dir .data", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, "Taiko Hekla": { "commandName": "Project", - "commandLineArgs": "-c taiko-hekla -dd .data", + "commandLineArgs": "-c taiko-hekla --data-dir .data", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, "Taiko Mainnet": { "commandName": "Project", - "commandLineArgs": "-c taiko-mainnet -dd .data", + "commandLineArgs": "-c taiko-mainnet --data-dir .data", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, "Volta": { "commandName": "Project", - "commandLineArgs": "-c volta -dd .data --JsonRpc.Enabled true", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "Joc-Mainnet": { - "commandName": "Project", - "commandLineArgs": "-c joc-mainnet -dd .data", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "Joc-Testnet": { - "commandName": "Project", - "commandLineArgs": "-c joc-testnet -dd .data", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "Joc-Mainnet (archive)": { - "commandName": "Project", - "commandLineArgs": "-c joc-mainnet_archive -dd .data", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "Joc-Testnet (archive)": { - "commandName": "Project", - "commandLineArgs": "-c joc-testnet_archive -dd .data", + "commandLineArgs": "-c volta --data-dir .data --jsonrpc-enabled true", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, "Docker": { "commandName": "Docker", - "commandLineArgs": "-c sepolia -dd /data --JsonRpc.EngineHost 0.0.0.0 --JsonRpc.EnginePort 8551 --JsonRpc.Host 0.0.0.0" + "commandLineArgs": "-c holesky --data-dir .data /data --jsonrpc-enginehost 0.0.0.0 --jsonrpc-engineport 8551 --jsonrpc-host 0.0.0.0" } } } diff --git a/src/Nethermind/Nethermind.Runner/configs/AuraTest.cfg b/src/Nethermind/Nethermind.Runner/configs/AuraTest.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/AuraTest.cfg rename to src/Nethermind/Nethermind.Runner/configs/AuraTest.json diff --git a/src/Nethermind/Nethermind.Runner/configs/base-mainnet.cfg b/src/Nethermind/Nethermind.Runner/configs/base-mainnet.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/base-mainnet.cfg rename to src/Nethermind/Nethermind.Runner/configs/base-mainnet.json diff --git a/src/Nethermind/Nethermind.Runner/configs/base-mainnet_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/base-mainnet_archive.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/base-mainnet_archive.cfg rename to src/Nethermind/Nethermind.Runner/configs/base-mainnet_archive.json diff --git a/src/Nethermind/Nethermind.Runner/configs/base-sepolia.cfg b/src/Nethermind/Nethermind.Runner/configs/base-sepolia.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/base-sepolia.cfg rename to src/Nethermind/Nethermind.Runner/configs/base-sepolia.json diff --git a/src/Nethermind/Nethermind.Runner/configs/base-sepolia_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/base-sepolia_archive.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/base-sepolia_archive.cfg rename to src/Nethermind/Nethermind.Runner/configs/base-sepolia_archive.json diff --git a/src/Nethermind/Nethermind.Runner/configs/chiado.cfg b/src/Nethermind/Nethermind.Runner/configs/chiado.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/chiado.cfg rename to src/Nethermind/Nethermind.Runner/configs/chiado.json diff --git a/src/Nethermind/Nethermind.Runner/configs/chiado_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/chiado_archive.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/chiado_archive.cfg rename to src/Nethermind/Nethermind.Runner/configs/chiado_archive.json diff --git a/src/Nethermind/Nethermind.Runner/configs/energyweb.cfg b/src/Nethermind/Nethermind.Runner/configs/energyweb.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/energyweb.cfg rename to src/Nethermind/Nethermind.Runner/configs/energyweb.json diff --git a/src/Nethermind/Nethermind.Runner/configs/energyweb_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/energyweb_archive.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/energyweb_archive.cfg rename to src/Nethermind/Nethermind.Runner/configs/energyweb_archive.json diff --git a/src/Nethermind/Nethermind.Runner/configs/exosama.cfg b/src/Nethermind/Nethermind.Runner/configs/exosama.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/exosama.cfg rename to src/Nethermind/Nethermind.Runner/configs/exosama.json diff --git a/src/Nethermind/Nethermind.Runner/configs/exosama_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/exosama_archive.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/exosama_archive.cfg rename to src/Nethermind/Nethermind.Runner/configs/exosama_archive.json diff --git a/src/Nethermind/Nethermind.Runner/configs/gnosis.cfg b/src/Nethermind/Nethermind.Runner/configs/gnosis.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/gnosis.cfg rename to src/Nethermind/Nethermind.Runner/configs/gnosis.json diff --git a/src/Nethermind/Nethermind.Runner/configs/gnosis_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/gnosis_archive.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/gnosis_archive.cfg rename to src/Nethermind/Nethermind.Runner/configs/gnosis_archive.json diff --git a/src/Nethermind/Nethermind.Runner/configs/hive.cfg b/src/Nethermind/Nethermind.Runner/configs/hive.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/hive.cfg rename to src/Nethermind/Nethermind.Runner/configs/hive.json diff --git a/src/Nethermind/Nethermind.Runner/configs/holesky.cfg b/src/Nethermind/Nethermind.Runner/configs/holesky.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/holesky.cfg rename to src/Nethermind/Nethermind.Runner/configs/holesky.json diff --git a/src/Nethermind/Nethermind.Runner/configs/holesky_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/holesky_archive.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/holesky_archive.cfg rename to src/Nethermind/Nethermind.Runner/configs/holesky_archive.json diff --git a/src/Nethermind/Nethermind.Runner/configs/joc-mainnet.cfg b/src/Nethermind/Nethermind.Runner/configs/joc-mainnet.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/joc-mainnet.cfg rename to src/Nethermind/Nethermind.Runner/configs/joc-mainnet.json diff --git a/src/Nethermind/Nethermind.Runner/configs/joc-mainnet_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/joc-mainnet_archive.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/joc-mainnet_archive.cfg rename to src/Nethermind/Nethermind.Runner/configs/joc-mainnet_archive.json diff --git a/src/Nethermind/Nethermind.Runner/configs/joc-testnet.cfg b/src/Nethermind/Nethermind.Runner/configs/joc-testnet.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/joc-testnet.cfg rename to src/Nethermind/Nethermind.Runner/configs/joc-testnet.json diff --git a/src/Nethermind/Nethermind.Runner/configs/joc-testnet_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/joc-testnet_archive.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/joc-testnet_archive.cfg rename to src/Nethermind/Nethermind.Runner/configs/joc-testnet_archive.json diff --git a/src/Nethermind/Nethermind.Runner/configs/linea-mainnet.cfg b/src/Nethermind/Nethermind.Runner/configs/linea-mainnet.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/linea-mainnet.cfg rename to src/Nethermind/Nethermind.Runner/configs/linea-mainnet.json diff --git a/src/Nethermind/Nethermind.Runner/configs/linea-mainnet_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/linea-mainnet_archive.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/linea-mainnet_archive.cfg rename to src/Nethermind/Nethermind.Runner/configs/linea-mainnet_archive.json diff --git a/src/Nethermind/Nethermind.Runner/configs/linea-sepolia.cfg b/src/Nethermind/Nethermind.Runner/configs/linea-sepolia.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/linea-sepolia.cfg rename to src/Nethermind/Nethermind.Runner/configs/linea-sepolia.json diff --git a/src/Nethermind/Nethermind.Runner/configs/linea-sepolia_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/linea-sepolia_archive.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/linea-sepolia_archive.cfg rename to src/Nethermind/Nethermind.Runner/configs/linea-sepolia_archive.json diff --git a/src/Nethermind/Nethermind.Runner/configs/mainnet.cfg b/src/Nethermind/Nethermind.Runner/configs/mainnet.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/mainnet.cfg rename to src/Nethermind/Nethermind.Runner/configs/mainnet.json diff --git a/src/Nethermind/Nethermind.Runner/configs/mainnet_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/mainnet_archive.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/mainnet_archive.cfg rename to src/Nethermind/Nethermind.Runner/configs/mainnet_archive.json diff --git a/src/Nethermind/Nethermind.Runner/configs/none.cfg b/src/Nethermind/Nethermind.Runner/configs/none.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/none.cfg rename to src/Nethermind/Nethermind.Runner/configs/none.json diff --git a/src/Nethermind/Nethermind.Runner/configs/op-mainnet.cfg b/src/Nethermind/Nethermind.Runner/configs/op-mainnet.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/op-mainnet.cfg rename to src/Nethermind/Nethermind.Runner/configs/op-mainnet.json diff --git a/src/Nethermind/Nethermind.Runner/configs/op-mainnet_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/op-mainnet_archive.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/op-mainnet_archive.cfg rename to src/Nethermind/Nethermind.Runner/configs/op-mainnet_archive.json diff --git a/src/Nethermind/Nethermind.Runner/configs/op-sepolia.cfg b/src/Nethermind/Nethermind.Runner/configs/op-sepolia.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/op-sepolia.cfg rename to src/Nethermind/Nethermind.Runner/configs/op-sepolia.json diff --git a/src/Nethermind/Nethermind.Runner/configs/op-sepolia_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/op-sepolia_archive.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/op-sepolia_archive.cfg rename to src/Nethermind/Nethermind.Runner/configs/op-sepolia_archive.json diff --git a/src/Nethermind/Nethermind.Runner/configs/poacore.cfg b/src/Nethermind/Nethermind.Runner/configs/poacore.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/poacore.cfg rename to src/Nethermind/Nethermind.Runner/configs/poacore.json diff --git a/src/Nethermind/Nethermind.Runner/configs/poacore_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/poacore_archive.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/poacore_archive.cfg rename to src/Nethermind/Nethermind.Runner/configs/poacore_archive.json diff --git a/src/Nethermind/Nethermind.Runner/configs/poacore_validator.cfg b/src/Nethermind/Nethermind.Runner/configs/poacore_validator.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/poacore_validator.cfg rename to src/Nethermind/Nethermind.Runner/configs/poacore_validator.json diff --git a/src/Nethermind/Nethermind.Runner/configs/sepolia.cfg b/src/Nethermind/Nethermind.Runner/configs/sepolia.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/sepolia.cfg rename to src/Nethermind/Nethermind.Runner/configs/sepolia.json diff --git a/src/Nethermind/Nethermind.Runner/configs/sepolia_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/sepolia_archive.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/sepolia_archive.cfg rename to src/Nethermind/Nethermind.Runner/configs/sepolia_archive.json diff --git a/src/Nethermind/Nethermind.Runner/configs/spaceneth.cfg b/src/Nethermind/Nethermind.Runner/configs/spaceneth.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/spaceneth.cfg rename to src/Nethermind/Nethermind.Runner/configs/spaceneth.json diff --git a/src/Nethermind/Nethermind.Runner/configs/spaceneth_persistent.cfg b/src/Nethermind/Nethermind.Runner/configs/spaceneth_persistent.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/spaceneth_persistent.cfg rename to src/Nethermind/Nethermind.Runner/configs/spaceneth_persistent.json diff --git a/src/Nethermind/Nethermind.Runner/configs/taiko-hekla.cfg b/src/Nethermind/Nethermind.Runner/configs/taiko-hekla.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/taiko-hekla.cfg rename to src/Nethermind/Nethermind.Runner/configs/taiko-hekla.json diff --git a/src/Nethermind/Nethermind.Runner/configs/taiko-mainnet.cfg b/src/Nethermind/Nethermind.Runner/configs/taiko-mainnet.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/taiko-mainnet.cfg rename to src/Nethermind/Nethermind.Runner/configs/taiko-mainnet.json diff --git a/src/Nethermind/Nethermind.Runner/configs/volta.cfg b/src/Nethermind/Nethermind.Runner/configs/volta.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/volta.cfg rename to src/Nethermind/Nethermind.Runner/configs/volta.json diff --git a/src/Nethermind/Nethermind.Runner/configs/volta_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/volta_archive.json similarity index 100% rename from src/Nethermind/Nethermind.Runner/configs/volta_archive.cfg rename to src/Nethermind/Nethermind.Runner/configs/volta_archive.json diff --git a/src/Nethermind/Nethermind.Serialization.Json/EthereumJsonSerializer.cs b/src/Nethermind/Nethermind.Serialization.Json/EthereumJsonSerializer.cs index 571ec402637..6c8b9b74c20 100644 --- a/src/Nethermind/Nethermind.Serialization.Json/EthereumJsonSerializer.cs +++ b/src/Nethermind/Nethermind.Serialization.Json/EthereumJsonSerializer.cs @@ -6,8 +6,10 @@ using System.Collections.Generic; using System.IO; using System.IO.Pipelines; +using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; +using System.Text.Unicode; using System.Threading.Tasks; using Nethermind.Core.Collections; @@ -68,6 +70,7 @@ private static JsonSerializerOptions CreateOptions(bool indented, IEnumerable "OverridableReleaseSpec"; @@ -31,7 +32,7 @@ public OverridableReleaseSpec(IReleaseSpec spec) public long GasLimitBoundDivisor => _spec.GasLimitBoundDivisor; - public UInt256 BlockReward => _spec.BlockReward; + public UInt256 BlockReward { get; set; } public long DifficultyBombDelay => _spec.DifficultyBombDelay; diff --git a/src/Nethermind/Nethermind.State/OverlayWorldStateManager.cs b/src/Nethermind/Nethermind.State/OverlayWorldStateManager.cs index a4dda8fdd65..7b0fc9a6da7 100644 --- a/src/Nethermind/Nethermind.State/OverlayWorldStateManager.cs +++ b/src/Nethermind/Nethermind.State/OverlayWorldStateManager.cs @@ -29,17 +29,11 @@ public class OverlayWorldStateManager( public IWorldState CreateResettableWorldState(IWorldState? forWarmup = null) { - PreBlockCaches? preBlockCaches = (forWarmup as IPreBlockCaches)?.Caches; - return preBlockCaches is not null - ? new WorldState( - new PreCachedTrieStore(overlayTrieStore, preBlockCaches.RlpCache), - _codeDb, - logManager, - preBlockCaches) - : new WorldState( - overlayTrieStore, - _codeDb, - logManager); + ITrieStore trieStore = (forWarmup as IPreBlockCaches)?.Caches is { } preBlockCaches + ? new PreCachedTrieStore(overlayTrieStore, preBlockCaches.RlpCache) + : overlayTrieStore; + + return new WorldState(trieStore, _codeDb, logManager); } public event EventHandler? ReorgBoundaryReached diff --git a/src/Nethermind/Nethermind.State/OverridableWorldState.cs b/src/Nethermind/Nethermind.State/OverridableWorldState.cs new file mode 100644 index 00000000000..70a1bfc0afb --- /dev/null +++ b/src/Nethermind/Nethermind.State/OverridableWorldState.cs @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Db; +using Nethermind.Logging; +using Nethermind.Trie.Pruning; + +namespace Nethermind.State; + +public class OverridableWorldState( + OverlayTrieStore trieStore, + IReadOnlyDbProvider dbProvider, + ILogManager? logManager, + PreBlockCaches? preBlockCaches = null, + bool populatePreBlockCache = true) + : WorldState(trieStore, dbProvider.GetDb(DbNames.Code), logManager, preBlockCaches, populatePreBlockCache) +{ + + /// + /// Resets changes applied via + /// + public void ResetOverrides() + { + trieStore.ResetOverrides(); + dbProvider.ClearTempChanges(); + } +} diff --git a/src/Nethermind/Nethermind.State/OverridableWorldStateManager.cs b/src/Nethermind/Nethermind.State/OverridableWorldStateManager.cs new file mode 100644 index 00000000000..82eb253ae56 --- /dev/null +++ b/src/Nethermind/Nethermind.State/OverridableWorldStateManager.cs @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Db; +using Nethermind.Logging; +using Nethermind.Trie.Pruning; + +namespace Nethermind.State; + +public class OverridableWorldStateManager : IWorldStateManager +{ + private readonly ReadOnlyDbProvider _readOnlyDbProvider; + private readonly StateReader _reader; + private readonly WorldState _state; + + private readonly OverlayTrieStore _overlayTrieStore; + private readonly ILogManager? _logManager; + + public OverridableWorldStateManager(IDbProvider dbProvider, IReadOnlyTrieStore trieStore, ILogManager? logManager) + { + dbProvider = _readOnlyDbProvider = new(dbProvider, true); + OverlayTrieStore overlayTrieStore = new(dbProvider.StateDb, trieStore, logManager); + + _logManager = logManager; + _reader = new(overlayTrieStore, dbProvider.GetDb(DbNames.Code), logManager); + _state = new(overlayTrieStore, dbProvider.GetDb(DbNames.Code), logManager); + _overlayTrieStore = overlayTrieStore; + } + + public IWorldState GlobalWorldState => _state; + public IStateReader GlobalStateReader => _reader; + public IReadOnlyTrieStore TrieStore => _overlayTrieStore.AsReadOnly(); + + public IWorldState CreateResettableWorldState(IWorldState? forWarmup = null) + { + if (forWarmup is not null) + throw new NotSupportedException("Overridable world state with warm up is not supported."); + + return new OverridableWorldState(_overlayTrieStore, _readOnlyDbProvider, _logManager); + } + + public event EventHandler? ReorgBoundaryReached + { + add => _overlayTrieStore.ReorgBoundaryReached += value; + remove => _overlayTrieStore.ReorgBoundaryReached -= value; + } +} diff --git a/src/Nethermind/Nethermind.State/StateReaderExtensions.cs b/src/Nethermind/Nethermind.State/StateReaderExtensions.cs index ac63366349c..2fcb0179f24 100644 --- a/src/Nethermind/Nethermind.State/StateReaderExtensions.cs +++ b/src/Nethermind/Nethermind.State/StateReaderExtensions.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Threading; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; @@ -47,9 +48,9 @@ public static bool HasStateForBlock(this IStateReader stateReader, BlockHeader h return stateReader.HasStateForRoot(header.StateRoot!); } - public static TrieStats CollectStats(this IStateReader stateProvider, Hash256 root, IKeyValueStore codeStorage, ILogManager logManager) + public static TrieStats CollectStats(this IStateReader stateProvider, Hash256 root, IKeyValueStore codeStorage, ILogManager logManager, CancellationToken cancellationToken = default) { - TrieStatsCollector collector = new(codeStorage, logManager); + TrieStatsCollector collector = new(codeStorage, logManager, cancellationToken); stateProvider.RunTreeVisitor(collector, root, new VisitingOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs index 8e4e7a52d99..98bf09d31f8 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using FluentAssertions; using Nethermind.Blockchain; +using Nethermind.Blockchain.Blocks; using Nethermind.Blockchain.Synchronization; using Nethermind.Core; using Nethermind.Core.Extensions; @@ -116,6 +117,29 @@ async Task HandleAndPrepareNextRequest() req.Dispose(); } + [Test] + public async Task ShouldNotReDownloadExistingBlock() + { + _feed.InitializeFeed(); + + _syncingToBlockTree.Insert(_syncingFromBlockTree.FindBlock(_pivotBlock.Number - 2)!); + _syncingToBlockTree.Insert(_syncingFromBlockTree.FindBlock(_pivotBlock.Number - 4)!); + + using BodiesSyncBatch req = (await _feed.PrepareRequest())!; + req.Infos + .Where((bi) => bi is not null) + .Select((bi) => bi!.BlockNumber) + .Take(4) + .Should() + .BeEquivalentTo([ + _pivotBlock.Number, + _pivotBlock.Number - 1, + // Skipped + _pivotBlock.Number - 3, + // Skipped + _pivotBlock.Number - 5]); + } + [Test] public async Task ShouldRecoverOnInsertFailure() { diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/SyncStatusListTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/SyncStatusListTests.cs index 98a375ba26a..bbd4ea0774c 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/SyncStatusListTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/SyncStatusListTests.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using FluentAssertions; @@ -10,6 +11,7 @@ using Nethermind.Core.Test.Builders; using Nethermind.Synchronization.FastBlocks; using NSubstitute; +using NSubstitute.Core; using NUnit.Framework; namespace Nethermind.Synchronization.Test.FastBlocks; @@ -50,12 +52,44 @@ public void Will_not_go_below_ancient_barrier() blockTree.FindCanonicalBlockInfo(Arg.Any()).Returns(new BlockInfo(TestItem.KeccakA, 0)); SyncStatusList syncStatusList = new SyncStatusList(blockTree, 1000, null, 900); - BlockInfo?[] infos = new BlockInfo?[500]; - syncStatusList.GetInfosForBatch(infos); + BlockInfo?[] infos; + syncStatusList.TryGetInfosForBatch(500, (_) => false, out infos); infos.Count((it) => it is not null).Should().Be(101); } + [Test] + public void Will_skip_existing_keys() + { + IBlockTree blockTree = Substitute.For(); + blockTree.FindCanonicalBlockInfo(Arg.Any()) + .Returns((Func)((ci) => + { + long blockNumber = (long)ci[0]; + return new BlockInfo(TestItem.KeccakA, 0) + { + BlockNumber = blockNumber + }; + })); + + SyncStatusList syncStatusList = new SyncStatusList(blockTree, 100000, null, 1000); + + HashSet needToFetchBlocks = [99999, 99995, 99950, 99000, 99001, 99003, 85000]; + + List TryGetInfos() + { + BlockInfo?[] infos; + syncStatusList.TryGetInfosForBatch(50, (bi) => !needToFetchBlocks.Contains(bi.BlockNumber), out infos); + return infos.Where(bi => bi != null).Select((bi) => bi!.BlockNumber).ToList(); + } + + TryGetInfos().Should().BeEquivalentTo([99999, 99995]); // first two as it will try the first 50 only + TryGetInfos().Should().BeEquivalentTo([99950]); // Then the next 50 + TryGetInfos().Should().BeEquivalentTo([99000, 99001, 99003]); // If the next 50 failed, it will try looking far back. + TryGetInfos().Should().BeEmpty(); // If it look far back enough and still does not find anything it will just return so that progress can update. + TryGetInfos().Should().BeEquivalentTo([85000]); // But as the existing blocks was already marked as inserted, it should be able to make progress on later call. + } + [Test] public void Can_read_back_all_parallel_set_values() { diff --git a/src/Nethermind/Nethermind.Synchronization.Test/ReceiptSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/ReceiptSyncFeedTests.cs index d92204e234e..eef137e816f 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/ReceiptSyncFeedTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/ReceiptSyncFeedTests.cs @@ -9,6 +9,7 @@ using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Synchronization; using Nethermind.Core; +using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Core.Test.Builders; using Nethermind.Db; @@ -25,54 +26,72 @@ namespace Nethermind.Synchronization.Test; public class ReceiptSyncFeedTests { + private IBlockTree _syncingFromBlockTree = null!; + private IBlockTree _syncingToBlockTree = null!; + private ReceiptsSyncFeed _feed = null!; + private ISyncConfig _syncConfig = null!; + private Block _pivotBlock = null!; + private InMemoryReceiptStorage _syncingFromReceiptStore; + private IReceiptStorage _receiptStorage; - [Test] - public async Task ShouldRecoverOnInsertFailure() + [SetUp] + public void Setup() { - InMemoryReceiptStorage syncingFromReceiptStore = new InMemoryReceiptStorage(); - BlockTree syncingFromBlockTree = Build.A.BlockTree() - .WithTransactions(syncingFromReceiptStore) + _syncingFromReceiptStore = new InMemoryReceiptStorage(); + _syncingFromBlockTree = Build.A.BlockTree() + .WithTransactions(_syncingFromReceiptStore) .OfChainLength(100) .TestObject; - BlockTree syncingTooBlockTree = Build.A.BlockTree() + _receiptStorage = Substitute.For(); + _syncingToBlockTree = Build.A.BlockTree() .TestObject; for (int i = 1; i < 100; i++) { - Block block = syncingFromBlockTree.FindBlock(i, BlockTreeLookupOptions.None)!; - syncingTooBlockTree.Insert(block.Header); - syncingTooBlockTree.Insert(block); + Block block = _syncingFromBlockTree.FindBlock(i, BlockTreeLookupOptions.None)!; + _syncingToBlockTree.Insert(block.Header); + _syncingToBlockTree.Insert(block); } - Block pivot = syncingFromBlockTree.FindBlock(99, BlockTreeLookupOptions.None)!; + _pivotBlock = _syncingFromBlockTree.FindBlock(99, BlockTreeLookupOptions.None)!; - SyncConfig syncConfig = new() + _syncConfig = new SyncConfig() { FastSync = true, - PivotHash = pivot.Hash!.ToString(), - PivotNumber = pivot.Number.ToString(), + PivotHash = _pivotBlock.Hash!.ToString(), + PivotNumber = _pivotBlock.Number.ToString(), AncientBodiesBarrier = 0, DownloadBodiesInFastSync = true, }; - IReceiptStorage receiptStorage = Substitute.For(); - ReceiptsSyncFeed syncFeed = new ReceiptsSyncFeed( + _feed = new ReceiptsSyncFeed( MainnetSpecProvider.Instance, - syncingTooBlockTree, - receiptStorage, + _syncingToBlockTree, + _receiptStorage, Substitute.For(), - syncConfig, + _syncConfig, new NullSyncReport(), new MemDb(), LimboLogs.Instance ); - syncFeed.InitializeFeed(); + } + + [TearDown] + public void TearDown() + { + _feed.Dispose(); + } + + [Test] + public async Task ShouldRecoverOnInsertFailure() + { + _feed.InitializeFeed(); - using ReceiptsSyncBatch req = (await syncFeed.PrepareRequest())!; - req.Response = req.Infos.Take(8).Select(info => syncingFromReceiptStore.Get(info!.BlockHash)).ToPooledList(8)!; + using ReceiptsSyncBatch req = (await _feed.PrepareRequest())!; + req.Response = req.Infos.Take(8).Select(info => _syncingFromReceiptStore.Get(info!.BlockHash)).ToPooledList(8)!; - receiptStorage + _receiptStorage .When((it) => it.Insert(Arg.Any(), Arg.Any(), Arg.Any())) .Do((callInfo) => { @@ -80,9 +99,32 @@ public async Task ShouldRecoverOnInsertFailure() if (block.Number == 95) throw new Exception("test exception"); }); - Func act = () => syncFeed.HandleResponse(req); + Func act = () => _feed.HandleResponse(req); act.Should().Throw(); - using ReceiptsSyncBatch req2 = (await syncFeed.PrepareRequest())!; + using ReceiptsSyncBatch req2 = (await _feed.PrepareRequest())!; req2.Infos[0]!.BlockNumber.Should().Be(95); } + + [Test] + public async Task ShouldNotRedownloadExistingReceipts() + { + _feed.InitializeFeed(); + _receiptStorage.HasBlock(Arg.Is(_pivotBlock.Number - 2), Arg.Any()).Returns(true); + _receiptStorage.HasBlock(Arg.Is(_pivotBlock.Number - 4), Arg.Any()).Returns(true); + + using ReceiptsSyncBatch req = (await _feed.PrepareRequest())!; + + req.Infos + .Where((bi) => bi is not null) + .Select((bi) => bi!.BlockNumber) + .Take(4) + .Should() + .BeEquivalentTo([ + _pivotBlock.Number, + _pivotBlock.Number - 1, + // Skipped + _pivotBlock.Number - 3, + // Skipped + _pivotBlock.Number - 5]); + } } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerModuleTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerModuleTests.cs new file mode 100644 index 00000000000..d7ba3f27cad --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerModuleTests.cs @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading; +using System.Threading.Tasks; +using Autofac; +using FluentAssertions; +using Nethermind.Blockchain.Synchronization; +using Nethermind.Config; +using Nethermind.Consensus.Processing; +using Nethermind.Core; +using Nethermind.Core.Test.Builders; +using Nethermind.Db; +using Nethermind.Logging; +using Nethermind.State; +using Nethermind.Synchronization.FastSync; +using Nethermind.Trie; +using NSubstitute; +using NUnit.Framework; + +namespace Nethermind.Synchronization.Test; + +public class SynchronizerModuleTests +{ + public IContainer CreateTestContainer() + { + ITreeSync treeSync = Substitute.For(); + IStateReader stateReader = Substitute.For(); + IBlockProcessingQueue blockQueue = Substitute.For(); + + return new ContainerBuilder() + .AddModule(new SynchronizerModule(new SyncConfig() + { + FastSync = true, + VerifyTrieOnStateSyncFinished = true + })) + .AddKeyedSingleton(DbNames.Code, Substitute.For()) + .AddSingleton(stateReader) + .AddSingleton(treeSync) + .AddSingleton(blockQueue) + .AddSingleton(Substitute.For()) + .AddSingleton(LimboLogs.Instance) + .Build(); + } + + [Test] + public void TestOnTreeSyncFinish_CallVisit() + { + IContainer ctx = CreateTestContainer(); + ITreeSync treeSync = ctx.Resolve(); + IStateReader stateReader = ctx.Resolve(); + + treeSync.SyncCompleted += Raise.EventWith(null, new ITreeSync.SyncCompletedEventArgs(TestItem.KeccakA)); + + stateReader + .Received() + .RunTreeVisitor(Arg.Any(), Arg.Is(TestItem.KeccakA), Arg.Any()); + } + + [Test] + public async Task TestOnTreeSyncFinish_BlockProcessingQueue_UntilFinished() + { + IContainer ctx = CreateTestContainer(); + ITreeSync treeSync = ctx.Resolve(); + IStateReader stateReader = ctx.Resolve(); + IBlockProcessingQueue blockQueue = ctx.Resolve(); + + ManualResetEvent treeVisitorBlocker = new ManualResetEvent(false); + + stateReader + .When(sr => sr.RunTreeVisitor(Arg.Any(), Arg.Is(TestItem.KeccakA), Arg.Any())) + .Do((ci) => + { + treeVisitorBlocker.WaitOne(); + }); + + Task triggerTask = Task.Run(() => + { + treeSync.SyncCompleted += Raise.EventWith(null, new ITreeSync.SyncCompletedEventArgs(TestItem.KeccakA)); + }); + + await Task.Delay(100); + + Task blockQueueTask = Task.Run(() => + { + blockQueue.BlockRemoved += + Raise.EventWith(null, new BlockRemovedEventArgs(null!, ProcessingResult.Success)); + }); + + await Task.Delay(100); + + blockQueueTask.IsCompleted.Should().BeFalse(); + treeVisitorBlocker.Set(); + + await triggerTask; + await blockQueueTask; + blockQueue.BlockRemoved += Raise.EventWith(null, new BlockRemovedEventArgs(null!, ProcessingResult.Success)); + } +} diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs index 1152f416054..edf132f066c 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs @@ -7,6 +7,7 @@ using Autofac.Features.AttributeFilters; using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; +using Nethermind.Blockchain.Blocks; using Nethermind.Blockchain.Synchronization; using Nethermind.Consensus.Validators; using Nethermind.Core; @@ -130,11 +131,20 @@ private void PostFinishCleanUp() BodiesSyncBatch? batch = null; if (ShouldBuildANewBatch()) { - BlockInfo?[] infos = new BlockInfo[_requestSize]; - _syncStatusList.GetInfosForBatch(infos); + BlockInfo?[] infos = null; + while (!_syncStatusList.TryGetInfosForBatch(_requestSize, (info) => _blockTree.HasBlock(info.BlockNumber, info.BlockHash), out infos)) + { + token.ThrowIfCancellationRequested(); + + // Otherwise, the progress does not update correctly + _blockTree.LowestInsertedBodyNumber = _syncStatusList.LowestInsertWithoutGaps; + UpdateSyncReport(); + } + if (infos[0] is not null) { batch = new BodiesSyncBatch(infos); + // Used for peer allocation. It pick peer which have the at least this number batch.MinNumber = infos[0].BlockNumber; batch.Prioritized = true; } diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs index cbfce46215c..0869d2bcbbc 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs @@ -133,8 +133,14 @@ private void PostFinishCleanUp() ReceiptsSyncBatch? batch = null; if (ShouldBuildANewBatch()) { - BlockInfo?[] infos = new BlockInfo[_requestSize]; - _syncStatusList.GetInfosForBatch(infos); + BlockInfo?[] infos = null; + while (!_syncStatusList.TryGetInfosForBatch(_requestSize, (info) => _receiptStorage.HasBlock(info.BlockNumber, info.BlockHash), out infos)) + { + token.ThrowIfCancellationRequested(); + _receiptStorage.LowestInsertedReceiptBlockNumber = _syncStatusList.LowestInsertWithoutGaps; + UpdateSyncReport(); + } + if (infos[0] is not null) { batch = new ReceiptsSyncBatch(infos); @@ -280,11 +286,9 @@ private int InsertReceipts(ReceiptsSyncBatch batch) } } + UpdateSyncReport(); AdjustRequestSize(batch, validResponsesCount); LogPostProcessingBatchInfo(batch, validResponsesCount); - - _syncReport.FastBlocksReceipts.Update(_pivotNumber - _syncStatusList.LowestInsertWithoutGaps); - _syncReport.ReceiptsInQueue.Update(_syncStatusList.QueueSize); return validResponsesCount; } @@ -295,6 +299,12 @@ private void LogPostProcessingBatchInfo(ReceiptsSyncBatch batch, int validRespon $"{nameof(ReceiptsSyncBatch)} back from {batch.ResponseSourcePeer} with {validResponsesCount}/{batch.Infos.Length}"); } + private void UpdateSyncReport() + { + _syncReport.FastBlocksReceipts.Update(_pivotNumber - _syncStatusList.LowestInsertWithoutGaps); + _syncReport.ReceiptsInQueue.Update(_syncStatusList.QueueSize); + } + private void AdjustRequestSize(ReceiptsSyncBatch batch, int validResponsesCount) { int currentRequestSize = Volatile.Read(ref _requestSize); diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/SyncStatusList.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/SyncStatusList.cs index 373afedd1d2..208d355a844 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/SyncStatusList.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/SyncStatusList.cs @@ -3,14 +3,17 @@ using System; using System.Threading; +using System.Threading.Tasks; using Nethermind.Blockchain; using Nethermind.Core; using Nethermind.Core.Caching; +using Nethermind.Core.Collections; namespace Nethermind.Synchronization.FastBlocks { internal class SyncStatusList { + private const int ParallelExistCheckSize = 1024; private long _queueSize; private readonly IBlockTree _blockTree; private readonly FastBlockStatusList _statuses; @@ -35,7 +38,7 @@ public SyncStatusList(IBlockTree blockTree, long pivotNumber, long? lowestInsert _lowerBound = lowerBound; } - public void GetInfosForBatch(BlockInfo?[] blockInfos) + private void GetInfosForBatch(Span blockInfos) { int collected = 0; long currentNumber = Volatile.Read(ref _lowestInsertWithoutGaps); @@ -77,6 +80,90 @@ public void GetInfosForBatch(BlockInfo?[] blockInfos) } } + /// + /// Try get block infos of size `batchSize`. + /// + /// + /// + /// + /// + public bool TryGetInfosForBatch(int batchSize, Func blockExist, out BlockInfo?[] infos) + { + ArrayPoolList workingArray = new(batchSize, batchSize); + + // Need to be a max attempt to update sync progress + const int maxAttempt = 8; + for (int attempt = 0; attempt < maxAttempt; attempt++) + { + // Because the last clause of GetInfosForBatch increment the _lowestInsertWithoutGap need to be run + // sequentially, can't find an easy way to parallelize the checking for block exist part in the check + // So here we are... + GetInfosForBatch(workingArray.AsSpan()); + + (bool hasNonNull, bool hasInserted) = ClearExistingBlock(); + + if (hasNonNull || !hasInserted) + { + CompileOutput(out infos); + return true; + } + + // At this point, hasNonNull is false and hasInserted is true, meaning all entry in workingArray + // already exist. We switch to a bigger array to improve parallelization throughput + if (workingArray.Count < ParallelExistCheckSize) + { + workingArray = new ArrayPoolList(ParallelExistCheckSize, ParallelExistCheckSize); + } + } + + infos = []; + return false; + + (bool, bool) ClearExistingBlock() + { + bool hasNonNull = false; + bool hasInserted = false; + Parallel.For(0, workingArray.Count, (i) => + { + if (workingArray[i] is not null) + { + if (blockExist(workingArray[i])) + { + MarkInserted(workingArray[i].BlockNumber); + hasInserted = true; + workingArray[i] = null; + } + else + { + hasNonNull = true; + } + } + }); + return (hasNonNull, hasInserted); + } + + void CompileOutput(out BlockInfo?[] outputArray) + { + int slot = 0; + outputArray = new BlockInfo?[batchSize]; + for (int i = 0; i < workingArray.Count; i++) + { + if (workingArray[i] is null) continue; + + if (slot < outputArray.Length) + { + outputArray[slot] = workingArray[i]; + slot++; + } + else + { + // Not enough space in output we'll need to put back the block + MarkPending(workingArray[i]); + } + } + } + } + public void MarkInserted(long blockNumber) { if (_statuses.TrySet(blockNumber, FastBlockStatus.Inserted)) diff --git a/src/Nethermind/Nethermind.Synchronization/FastSync/ITreeSync.cs b/src/Nethermind/Nethermind.Synchronization/FastSync/ITreeSync.cs new file mode 100644 index 00000000000..3dc7ca4450e --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/FastSync/ITreeSync.cs @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Core.Crypto; + +namespace Nethermind.Synchronization.FastSync; + +public interface ITreeSync +{ + public event EventHandler SyncCompleted; + + public class SyncCompletedEventArgs(Hash256 root) : EventArgs + { + public Hash256 Root => root; + } +} diff --git a/src/Nethermind/Nethermind.Synchronization/FastSync/StateSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastSync/StateSyncFeed.cs index 875eb380899..23f4001a023 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastSync/StateSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastSync/StateSyncFeed.cs @@ -11,7 +11,7 @@ namespace Nethermind.Synchronization.FastSync { - public partial class StateSyncFeed : SyncFeed, IDisposable + public class StateSyncFeed : SyncFeed, IDisposable { private const StateSyncBatch EmptyBatch = null; diff --git a/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs b/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs index 12d2540e789..03e05a65660 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs @@ -26,7 +26,7 @@ namespace Nethermind.Synchronization.FastSync { - public class TreeSync + public class TreeSync : ITreeSync { public const int AlreadySavedCapacity = 1024 * 1024; public const int MaxRequestSize = 384; @@ -76,6 +76,8 @@ public class TreeSync private long _blockNumber; private readonly SyncMode _syncMode; + public event EventHandler? SyncCompleted; + public TreeSync([KeyFilter(DbNames.Code)] IDb codeDb, INodeStorage nodeStorage, IBlockTree blockTree, ILogManager logManager) : this(SyncMode.StateNodes, codeDb, nodeStorage, blockTree, logManager) { @@ -707,7 +709,6 @@ private void VerifyPostSyncCleanUp() } _dependencies = new Dictionary>(); - // _alreadySaved = new LruKeyCache(AlreadySavedCapacity, "saved nodes"); } if (_pendingItems.Count != 0) @@ -716,6 +717,8 @@ private void VerifyPostSyncCleanUp() } CleanupMemory(); + + SyncCompleted?.Invoke(this, new ITreeSync.SyncCompletedEventArgs(_rootNode)); } private void CleanupMemory() diff --git a/src/Nethermind/Nethermind.Synchronization/FastSync/VerifyStateOnStateSyncFinished.cs b/src/Nethermind/Nethermind.Synchronization/FastSync/VerifyStateOnStateSyncFinished.cs new file mode 100644 index 00000000000..1b194729446 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/FastSync/VerifyStateOnStateSyncFinished.cs @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading; +using Autofac; +using Autofac.Features.AttributeFilters; +using Nethermind.Config; +using Nethermind.Consensus.Processing; +using Nethermind.Core.Crypto; +using Nethermind.Db; +using Nethermind.Logging; +using Nethermind.State; +using Nethermind.Trie; + +namespace Nethermind.Synchronization.FastSync; + +public class VerifyStateOnStateSyncFinished( + IBlockProcessingQueue processingQueue, + ITreeSync treeSync, + IStateReader stateReader, + [KeyFilter(DbNames.Code)] IDb codeDb, + IProcessExitSource exitSource, + ILogManager logManager) : IStartable +{ + private readonly ILogger _logger = logManager.GetClassLogger(); + + public void Start() + { + treeSync.SyncCompleted += TreeSyncOnOnVerifyPostSyncCleanup; + } + + private void TreeSyncOnOnVerifyPostSyncCleanup(object? sender, ITreeSync.SyncCompletedEventArgs evt) + { + ManualResetEvent processingBlocker = new ManualResetEvent(false); + + processingQueue.BlockRemoved += ProcessingQueueOnBlockRemoved; + + try + { + Hash256 rootNode = evt.Root; + _logger!.Info("Collecting trie stats and verifying that no nodes are missing..."); + TrieStats stats = stateReader.CollectStats(rootNode, codeDb, logManager, exitSource.Token); + if (stats.MissingNodes > 0) + { + _logger.Error($"Missing node found!"); + } + _logger.Info($"Stats after finishing state \n" + stats); + } + catch (Exception e) + { + _logger.Error($"Error in verify trie", e); + } + finally + { + processingBlocker.Set(); + processingQueue.BlockRemoved -= ProcessingQueueOnBlockRemoved; + } + + return; + + void ProcessingQueueOnBlockRemoved(object? o, BlockRemovedEventArgs blockRemovedEventArgs) + { + processingBlocker.WaitOne(); + } + } +} diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index e6a3077b3cd..451ea36a47a 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -380,12 +380,20 @@ private void ConfigureBodiesSyncComponent(ContainerBuilder serviceCollection) private void ConfigureStateSyncComponent(ContainerBuilder serviceCollection) { serviceCollection - .AddSingleton(); + .AddSingleton(); ConfigureSingletonSyncFeed(serviceCollection); // Disable it by setting noop if (!syncConfig.FastSync) serviceCollection.AddSingleton, NoopSyncFeed>(); + + if (syncConfig.FastSync && syncConfig.VerifyTrieOnStateSyncFinished) + { + serviceCollection + .RegisterType() + .WithAttributeFiltering() + .As(); + } } private static void ConfigureSingletonSyncFeed(ContainerBuilder serviceCollection) where TFeed : class, ISyncFeed where TDownloader : class, ISyncDownloader where TAllocationStrategy : class, IPeerAllocationStrategyFactory diff --git a/src/Nethermind/Nethermind.Test.Runner/Nethermind.Test.Runner.csproj b/src/Nethermind/Nethermind.Test.Runner/Nethermind.Test.Runner.csproj index fd96da1e534..7a0db5f7549 100644 --- a/src/Nethermind/Nethermind.Test.Runner/Nethermind.Test.Runner.csproj +++ b/src/Nethermind/Nethermind.Test.Runner/Nethermind.Test.Runner.csproj @@ -13,7 +13,7 @@ - + diff --git a/src/Nethermind/Nethermind.Test.Runner/Program.cs b/src/Nethermind/Nethermind.Test.Runner/Program.cs index ccb60229cb5..6161803bcea 100644 --- a/src/Nethermind/Nethermind.Test.Runner/Program.cs +++ b/src/Nethermind/Nethermind.Test.Runner/Program.cs @@ -2,96 +2,118 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.CommandLine; using System.IO; +using System.Threading; using System.Threading.Tasks; -using CommandLine; using Ethereum.Test.Base; using Ethereum.Test.Base.Interfaces; -namespace Nethermind.Test.Runner +namespace Nethermind.Test.Runner; + +internal class Program { - internal class Program + public class Options { - public class Options - { - [Option('i', "input", Required = false, HelpText = "Set the state test input file or directory. Either 'input' or 'stdin' is required")] - public string Input { get; set; } + public static CliOption Input { get; } = + new("--input", "-i") { Description = "Set the state test input file or directory. Either 'input' or 'stdin' is required." }; - [Option('f', "filter", Required = false, HelpText = "Set the test name that you want to run. Could also be a regular expression")] - public string Filter { get; set; } + public static CliOption Filter { get; } = + new("--filter", "-f") { Description = "Set the test name that you want to run. Could also be a regular expression." }; - [Option('b', "blockTest", Required = false, HelpText = "Set test as blockTest. if not, it will be by default assumed a state test.")] - public bool BlockTest { get; set; } + public static CliOption BlockTest { get; } = + new("--blockTest", "-b") { Description = "Set test as blockTest. if not, it will be by default assumed a state test." }; - [Option('t', "trace", Required = false, HelpText = "Set to always trace (by default traces are only generated for failing tests). [Only for State Test]")] - public bool TraceAlways { get; set; } + public static CliOption TraceAlways { get; } = + new("--trace", "-t") { Description = "Set to always trace (by default traces are only generated for failing tests). [Only for State Test]" }; - [Option('n', "neverTrace", Required = false, HelpText = "Set to never trace (by default traces are only generated for failing tests). [Only for State Test]")] - public bool TraceNever { get; set; } + public static CliOption TraceNever { get; } = + new("--neverTrace", "-n") { Description = "Set to never trace (by default traces are only generated for failing tests). [Only for State Test]" }; - [Option('m', "memory", Required = false, HelpText = "Exclude memory trace. [Only for State Test]")] - public bool ExcludeMemory { get; set; } + public static CliOption ExcludeMemory { get; } = + new("--memory", "-m") { Description = "Exclude memory trace. [Only for State Test]" }; - [Option('s', "stack", Required = false, HelpText = "Exclude stack trace. [Only for State Test]")] - public bool ExcludeStack { get; set; } + public static CliOption ExcludeStack { get; } = + new("--stack", "-s") { Description = "Exclude stack trace. [Only for State Test]" }; - [Option('w', "wait", Required = false, HelpText = "Wait for input after the test run.")] - public bool Wait { get; set; } + public static CliOption Wait { get; } = + new("--wait", "-w") { Description = "Wait for input after the test run." }; - [Option('x', "stdin", Required = false, HelpText = "If stdin is used, the state runner will read inputs (filenames) from stdin, and continue executing until empty line is read.")] - public bool Stdin { get; set; } - } + public static CliOption Stdin { get; } = + new("--stdin", "-x") { Description = "If stdin is used, the state runner will read inputs (filenames) from stdin, and continue executing until empty line is read." }; + } - public static async Task Main(params string[] args) - { - ParserResult result = Parser.Default.ParseArguments(args); - if (result is Parsed options) - await Run(options.Value); - } + public static async Task Main(params string[] args) + { + CliRootCommand rootCommand = + [ + Options.Input, + Options.Filter, + Options.BlockTest, + Options.TraceAlways, + Options.TraceNever, + Options.ExcludeMemory, + Options.ExcludeStack, + Options.Wait, + Options.Stdin + ]; + rootCommand.SetAction(Run); + + CliConfiguration configuration = new(rootCommand); + + return await configuration.InvokeAsync(args); + } - private static async Task Run(Options options) - { - WhenTrace whenTrace = WhenTrace.WhenFailing; - if (options.TraceNever) - whenTrace = WhenTrace.Never; - - if (options.TraceAlways) - whenTrace = WhenTrace.Always; - - string input = options.Input; - if (options.Stdin) - input = Console.ReadLine(); - - while (!string.IsNullOrWhiteSpace(input)) - { - if (options.BlockTest) - await RunBlockTest(input, source => new BlockchainTestsRunner(source, options.Filter)); - else - RunStateTest(input, source => new StateTestsRunner(source, whenTrace, !options.ExcludeMemory, !options.ExcludeStack, options.Filter)); - if (!options.Stdin) - break; - - input = Console.ReadLine(); - } - - if (options.Wait) - Console.ReadLine(); - } + private static async Task Run(ParseResult parseResult, CancellationToken cancellationToken) + { + WhenTrace whenTrace = WhenTrace.WhenFailing; - private static async Task RunBlockTest(string path, Func testRunnerBuilder) - { - ITestSourceLoader source = Path.HasExtension(path) - ? new TestsSourceLoader(new LoadBlockchainTestFileStrategy(), path) - : new TestsSourceLoader(new LoadBlockchainTestsStrategy(), path); - await testRunnerBuilder(source).RunTestsAsync(); - } + if (parseResult.GetValue(Options.TraceNever)) + whenTrace = WhenTrace.Never; + + if (parseResult.GetValue(Options.TraceAlways)) + whenTrace = WhenTrace.Always; + + string input = parseResult.GetValue(Options.Input); - private static void RunStateTest(string path, Func testRunnerBuilder) + if (parseResult.GetValue(Options.Stdin)) + input = Console.ReadLine(); + + while (!string.IsNullOrWhiteSpace(input)) { - ITestSourceLoader source = Path.HasExtension(path) - ? new TestsSourceLoader(new LoadGeneralStateTestFileStrategy(), path) - : new TestsSourceLoader(new LoadGeneralStateTestsStrategy(), path); - testRunnerBuilder(source).RunTests(); + if (parseResult.GetValue(Options.BlockTest)) + await RunBlockTest(input, source => new BlockchainTestsRunner(source, parseResult.GetValue(Options.Filter))); + else + RunStateTest(input, source => new StateTestsRunner(source, whenTrace, + !parseResult.GetValue(Options.ExcludeMemory), + !parseResult.GetValue(Options.ExcludeStack), + parseResult.GetValue(Options.Filter))); + + if (!parseResult.GetValue(Options.Stdin)) + break; + + input = Console.ReadLine(); } + + if (parseResult.GetValue(Options.Wait)) + Console.ReadLine(); + + return 0; + } + + private static async Task RunBlockTest(string path, Func testRunnerBuilder) + { + ITestSourceLoader source = Path.HasExtension(path) + ? new TestsSourceLoader(new LoadBlockchainTestFileStrategy(), path) + : new TestsSourceLoader(new LoadBlockchainTestsStrategy(), path); + await testRunnerBuilder(source).RunTestsAsync(); + } + + private static void RunStateTest(string path, Func testRunnerBuilder) + { + ITestSourceLoader source = Path.HasExtension(path) + ? new TestsSourceLoader(new LoadGeneralStateTestFileStrategy(), path) + : new TestsSourceLoader(new LoadGeneralStateTestsStrategy(), path); + testRunnerBuilder(source).RunTests(); } } diff --git a/src/Nethermind/Nethermind.Trie/Pruning/OverlayTrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/OverlayTrieStore.cs index c84a6538539..6108165f1a2 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/OverlayTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/OverlayTrieStore.cs @@ -23,4 +23,11 @@ public override TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, public override byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => base.TryLoadRlp(address, in path, hash, flags) ?? store.TryLoadRlp(address, in path, hash, flags); + + protected override void VerifyNewCommitSet(long blockNumber) + { + // Skip checks, as override can be applied using the same block number or without a state root + } + + public void ResetOverrides() => ClearCache(); } diff --git a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs index a5fdf3407e7..f3781550d06 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs @@ -643,9 +643,6 @@ private void PruneCache(bool skipRecalculateMemory = false) if (_logger.IsDebug) _logger.Debug($"Finished pruning nodes in {(long)Stopwatch.GetElapsedTime(start).TotalMilliseconds}ms {MemoryUsedByDirtyCache / 1.MB()} MB, last persisted block: {LastPersistedBlockNumber} current: {LatestCommittedBlockNumber}."); } - /// - /// This method is here to support testing. - /// public void ClearCache() { foreach (TrieStoreDirtyNodesCache dirtyNode in _dirtyNodes) @@ -706,10 +703,8 @@ private static ConcurrentQueue CreateQueueAtomic(ref ConcurrentQ return prior ?? instance; } - private BlockCommitSet CreateCommitSet(long blockNumber) + protected virtual void VerifyNewCommitSet(long blockNumber) { - if (_logger.IsDebug) _logger.Debug($"Beginning new {nameof(BlockCommitSet)} - {blockNumber}"); - if (_lastCommitSet is not null) { Debug.Assert(_lastCommitSet.IsSealed, "Not sealed when beginning new block"); @@ -719,6 +714,13 @@ private BlockCommitSet CreateCommitSet(long blockNumber) if (_logger.IsInfo) _logger.Info($"Non consecutive block commit. This is likely a reorg. Last block commit: {_lastCommitSet.BlockNumber}. New block commit: {blockNumber}."); } } + } + + private BlockCommitSet CreateCommitSet(long blockNumber) + { + if (_logger.IsDebug) _logger.Debug($"Beginning new {nameof(BlockCommitSet)} - {blockNumber}"); + + VerifyNewCommitSet(blockNumber); BlockCommitSet commitSet = new(blockNumber); CommitSetQueue.Enqueue(commitSet); diff --git a/src/Nethermind/Nethermind.Trie/TrieNode.Visitor.cs b/src/Nethermind/Nethermind.Trie/TrieNode.Visitor.cs index 9b8d37c1d53..b0a970f8ca9 100644 --- a/src/Nethermind/Nethermind.Trie/TrieNode.Visitor.cs +++ b/src/Nethermind/Nethermind.Trie/TrieNode.Visitor.cs @@ -3,11 +3,14 @@ using System.Collections.Generic; using System.IO; +using System.Linq; using System.Runtime.CompilerServices; using System.Threading.Tasks; using Nethermind.Core; +using Nethermind.Core.Collections; using Nethermind.Core.Cpu; using Nethermind.Core.Crypto; +using Nethermind.Core.Threading; using Nethermind.Serialization.Rlp; using Nethermind.Trie.Pruning; @@ -187,31 +190,47 @@ void VisitSingleThread(ref TreePath parentPath, ITreeVisitor treeV } [MethodImpl(MethodImplOptions.AggressiveInlining)] - void VisitMultiThread(TreePath parentPath, ITreeVisitor treeVisitor, in TNodeContext nodeContext, ITrieNodeResolver trieNodeResolver, TrieVisitContext visitContext, TrieNode?[] children) + void VisitMultiThread(TreePath parentPath, ITreeVisitor treeVisitor, in TNodeContext nodeContext, ITrieNodeResolver trieNodeResolver, TrieVisitContext visitContext) { - var copy = nodeContext; + // we need to preallocate children + TNodeContext contextCopy = nodeContext; - // multithreaded route - Parallel.For(0, BranchesCount, RuntimeInformation.ParallelOptionsPhysicalCores, i => + ArrayPoolList? tasks = null; + for (int i = 0; i < BranchesCount; i++) { - visitContext.Semaphore.Wait(); - try + if (i < BranchesCount - 1 && visitContext.ConcurrencyController.TryTakeSlot(out ConcurrencyController.Slot returner)) { - TreePath closureParentPath = parentPath; - // we need to have separate context for each thread as context tracks level and branch child index - TrieVisitContext childContext = visitContext.Clone(); - VisitChild(ref closureParentPath, i, children[i], trieNodeResolver, treeVisitor, copy, childContext); + tasks ??= new ArrayPoolList(BranchesCount); + tasks.Add(SpawnChildVisit(parentPath, i, GetChild(nodeResolver, ref parentPath, i), returner)); } - finally + else { - visitContext.Semaphore.Release(); + VisitChild(ref parentPath, i, GetChild(nodeResolver, ref parentPath, i), trieNodeResolver, treeVisitor, contextCopy, visitContext); } - }); + } + + if (tasks is { Count: > 0 }) + { + Task.WaitAll(tasks.ToArray()); + tasks.Dispose(); + } + return; + + Task SpawnChildVisit(TreePath closureParentPath, int i, TrieNode? childNode, ConcurrencyController.Slot slotReturner) => + Task.Run(() => + { + using ConcurrencyController.Slot _ = slotReturner; + + // we need to have separate context for each thread as context tracks level and branch child index + TrieVisitContext childContext = visitContext.Clone(); + VisitChild(ref closureParentPath, i, childNode, trieNodeResolver, treeVisitor, + contextCopy, childContext); + }); } static void VisitAllSingleThread(TrieNode currentNode, ref TreePath path, ITreeVisitor visitor, TNodeContext nodeContext, ITrieNodeResolver nodeResolver, TrieVisitContext visitContext) { - TrieNode?[] output = new TrieNode?[16]; + TrieNode?[] output = new TrieNode?[BranchesCount]; currentNode.ResolveAllChildBranch(nodeResolver, ref path, output); path.AppendMut(0); for (int i = 0; i < 16; i++) @@ -234,23 +253,11 @@ static void VisitAllSingleThread(TrieNode currentNode, ref TreePath path, ITreeV trieVisitContext.AddVisited(); trieVisitContext.Level++; - if (trieVisitContext.MaxDegreeOfParallelism != 1 && trieVisitContext.Semaphore.CurrentCount > 1) + // Limiting the multithread path to top state tree and first level storage double the throughput on mainnet. + // Top level state split to 16^3 while storage is 16, which should be ok for large contract in most case. + if (trieVisitContext.MaxDegreeOfParallelism != 1 && (trieVisitContext.IsStorage ? path.Length == 0 : path.Length <= 2)) { - // we need to preallocate children - TrieNode?[] children = new TrieNode?[BranchesCount]; - for (int i = 0; i < BranchesCount; i++) - { - children[i] = GetChild(nodeResolver, ref path, i); - } - - if (trieVisitContext.Semaphore.CurrentCount > 1) - { - VisitMultiThread(path, visitor, nodeContext, nodeResolver, trieVisitContext, children); - } - else - { - VisitSingleThread(ref path, visitor, nodeContext, nodeResolver, trieVisitContext); - } + VisitMultiThread(path, visitor, nodeContext, nodeResolver, trieVisitContext); } else { diff --git a/src/Nethermind/Nethermind.Trie/TrieStats.cs b/src/Nethermind/Nethermind.Trie/TrieStats.cs index adaf65d57cf..ee17e7259fa 100644 --- a/src/Nethermind/Nethermind.Trie/TrieStats.cs +++ b/src/Nethermind/Nethermind.Trie/TrieStats.cs @@ -8,50 +8,50 @@ namespace Nethermind.Trie public class TrieStats { private const int Levels = 128; - internal int _stateBranchCount; - internal int _stateExtensionCount; - internal int _accountCount; - internal int _storageBranchCount; - internal int _storageExtensionCount; - internal int _storageLeafCount; - internal int _codeCount; - internal int _missingState; - internal int _missingCode; - internal int _missingStorage; + internal long _stateBranchCount; + internal long _stateExtensionCount; + internal long _accountCount; + internal long _storageBranchCount; + internal long _storageExtensionCount; + internal long _storageLeafCount; + internal long _codeCount; + internal long _missingState; + internal long _missingCode; + internal long _missingStorage; internal long _storageSize; internal long _codeSize; internal long _stateSize; - internal readonly int[] _stateLevels = new int[Levels]; - internal readonly int[] _storageLevels = new int[Levels]; - internal readonly int[] _codeLevels = new int[Levels]; + internal readonly long[] _stateLevels = new long[Levels]; + internal readonly long[] _storageLevels = new long[Levels]; + internal readonly long[] _codeLevels = new long[Levels]; - public int StateBranchCount => _stateBranchCount; + public long StateBranchCount => _stateBranchCount; - public int StateExtensionCount => _stateExtensionCount; + public long StateExtensionCount => _stateExtensionCount; - public int AccountCount => _accountCount; + public long AccountCount => _accountCount; - public int StorageBranchCount => _storageBranchCount; + public long StorageBranchCount => _storageBranchCount; - public int StorageExtensionCount => _storageExtensionCount; + public long StorageExtensionCount => _storageExtensionCount; - public int StorageLeafCount => _storageLeafCount; + public long StorageLeafCount => _storageLeafCount; - public int CodeCount => _codeCount; + public long CodeCount => _codeCount; - public int MissingState => _missingState; + public long MissingState => _missingState; - public int MissingCode => _missingCode; + public long MissingCode => _missingCode; - public int MissingStorage => _missingStorage; + public long MissingStorage => _missingStorage; - public int MissingNodes => MissingCode + MissingState + MissingStorage; + public long MissingNodes => MissingCode + MissingState + MissingStorage; - public int StorageCount => StorageLeafCount + StorageExtensionCount + StorageBranchCount; + public long StorageCount => StorageLeafCount + StorageExtensionCount + StorageBranchCount; - public int StateCount => AccountCount + StateExtensionCount + StateBranchCount; + public long StateCount => AccountCount + StateExtensionCount + StateBranchCount; - public int NodesCount => StorageCount + StateCount + CodeCount; + public long NodesCount => StorageCount + StateCount + CodeCount; public long StorageSize => _storageSize; @@ -63,14 +63,14 @@ public class TrieStats // public List MissingNodes { get; set; } = new List(); - public int[] StateLevels => _stateLevels; - public int[] StorageLevels => _storageLevels; - public int[] CodeLevels => _codeLevels; - public int[] AllLevels + public long[] StateLevels => _stateLevels; + public long[] StorageLevels => _storageLevels; + public long[] CodeLevels => _codeLevels; + public long[] AllLevels { get { - int[] result = new int[Levels]; + long[] result = new long[Levels]; for (int i = 0; i < result.Length; i++) { result[i] = _stateLevels[i] + _storageLevels[i] + _codeLevels[i]; diff --git a/src/Nethermind/Nethermind.Trie/TrieStatsCollector.cs b/src/Nethermind/Nethermind.Trie/TrieStatsCollector.cs index 259a1e96139..6fce7f826e7 100644 --- a/src/Nethermind/Nethermind.Trie/TrieStatsCollector.cs +++ b/src/Nethermind/Nethermind.Trie/TrieStatsCollector.cs @@ -4,6 +4,7 @@ using System; using System.Threading; using Nethermind.Core; +using Nethermind.Core.Caching; using Nethermind.Core.Crypto; using Nethermind.Logging; @@ -11,15 +12,18 @@ namespace Nethermind.Trie { public class TrieStatsCollector : ITreeVisitor { + private readonly ClockCache _existingCodeHash = new ClockCache(1024 * 8); private readonly IKeyValueStore _codeKeyValueStore; - private int _lastAccountNodeCount = 0; + private long _lastAccountNodeCount = 0; private readonly ILogger _logger; + private readonly CancellationToken _cancellationToken; - public TrieStatsCollector(IKeyValueStore codeKeyValueStore, ILogManager logManager) + public TrieStatsCollector(IKeyValueStore codeKeyValueStore, ILogManager logManager, CancellationToken cancellationToken = default) { _codeKeyValueStore = codeKeyValueStore ?? throw new ArgumentNullException(nameof(codeKeyValueStore)); _logger = logManager.GetClassLogger(); + _cancellationToken = cancellationToken; } public TrieStats Stats { get; } = new(); @@ -49,6 +53,8 @@ public void VisitMissingNode(Hash256 nodeHash, TrieVisitContext trieVisitContext public void VisitBranch(TrieNode node, TrieVisitContext trieVisitContext) { + _cancellationToken.ThrowIfCancellationRequested(); + if (trieVisitContext.IsStorage) { Interlocked.Add(ref Stats._storageSize, node.FullRlp.Length); @@ -81,9 +87,10 @@ public void VisitExtension(TrieNode node, TrieVisitContext trieVisitContext) public void VisitLeaf(TrieNode node, TrieVisitContext trieVisitContext, ReadOnlySpan value) { - if (Stats.NodesCount - _lastAccountNodeCount > 1_000_000) + long lastAccountNodeCount = _lastAccountNodeCount; + long currentNodeCount = Stats.NodesCount; + if (currentNodeCount - lastAccountNodeCount > 1_000_000 && Interlocked.CompareExchange(ref _lastAccountNodeCount, currentNodeCount, lastAccountNodeCount) == lastAccountNodeCount) { - _lastAccountNodeCount = Stats.NodesCount; _logger.Warn($"Collected info from {Stats.NodesCount} nodes. Missing CODE {Stats.MissingCode} STATE {Stats.MissingState} STORAGE {Stats.MissingStorage}"); } @@ -103,10 +110,22 @@ public void VisitLeaf(TrieNode node, TrieVisitContext trieVisitContext, ReadOnly public void VisitCode(Hash256 codeHash, TrieVisitContext trieVisitContext) { - byte[] code = _codeKeyValueStore[codeHash.Bytes]; - if (code is not null) + ValueHash256 key = new ValueHash256(codeHash.Bytes); + bool codeExist = _existingCodeHash.TryGet(key, out int codeLength); + if (!codeExist) + { + byte[] code = _codeKeyValueStore[codeHash.Bytes]; + codeExist = code is not null; + if (codeExist) + { + codeLength = code.Length; + _existingCodeHash.Set(key, codeLength); + } + } + + if (codeExist) { - Interlocked.Add(ref Stats._codeSize, code.Length); + Interlocked.Add(ref Stats._codeSize, codeLength); Interlocked.Increment(ref Stats._codeCount); } else @@ -119,11 +138,11 @@ public void VisitCode(Hash256 codeHash, TrieVisitContext trieVisitContext) private void IncrementLevel(TrieVisitContext trieVisitContext) { - int[] levels = trieVisitContext.IsStorage ? Stats._storageLevels : Stats._stateLevels; + long[] levels = trieVisitContext.IsStorage ? Stats._storageLevels : Stats._stateLevels; IncrementLevel(trieVisitContext, levels); } - private static void IncrementLevel(TrieVisitContext trieVisitContext, int[] levels) + private static void IncrementLevel(TrieVisitContext trieVisitContext, long[] levels) { Interlocked.Increment(ref levels[trieVisitContext.Level]); } diff --git a/src/Nethermind/Nethermind.Trie/VisitContext.cs b/src/Nethermind/Nethermind.Trie/VisitContext.cs index 430dfa4882c..41595c71d44 100644 --- a/src/Nethermind/Nethermind.Trie/VisitContext.cs +++ b/src/Nethermind/Nethermind.Trie/VisitContext.cs @@ -4,15 +4,18 @@ using System; using System.Runtime.InteropServices; using System.Threading; +using Nethermind.Core.Threading; namespace Nethermind.Trie { public class TrieVisitContext : IDisposable { - private SemaphoreSlim? _semaphore; private readonly int _maxDegreeOfParallelism = 1; private int _visitedNodes; + private ConcurrencyController? _threadLimiter = null; + public ConcurrencyController ConcurrencyController => _threadLimiter ??= new ConcurrencyController(MaxDegreeOfParallelism); + public int Level { get; internal set; } public bool IsStorage { get; set; } public int? BranchChildIndex { get; internal set; } @@ -22,20 +25,10 @@ public class TrieVisitContext : IDisposable public int MaxDegreeOfParallelism { get => _maxDegreeOfParallelism; - internal init => _maxDegreeOfParallelism = VisitingOptions.AdjustMaxDegreeOfParallelism(value); - } - - public SemaphoreSlim Semaphore - { - get + internal init { - if (_semaphore is null) - { - if (MaxDegreeOfParallelism == 1) throw new InvalidOperationException("Can not create semaphore for single threaded trie visitor."); - _semaphore = new SemaphoreSlim(MaxDegreeOfParallelism, MaxDegreeOfParallelism); - } - - return _semaphore; + _maxDegreeOfParallelism = VisitingOptions.AdjustMaxDegreeOfParallelism(value); + _threadLimiter = null; } } @@ -43,7 +36,6 @@ public SemaphoreSlim Semaphore public void Dispose() { - _semaphore?.Dispose(); } public void AddVisited() diff --git a/src/Nethermind/nuget.config b/src/Nethermind/nuget.config index 84c8ccf735e..401f058f2ab 100644 --- a/src/Nethermind/nuget.config +++ b/src/Nethermind/nuget.config @@ -3,14 +3,19 @@ + - + diff --git a/tools/DocGen/ConfigGenerator.cs b/tools/DocGen/ConfigGenerator.cs index 3edaabcd26a..aa39b045c3d 100644 --- a/tools/DocGen/ConfigGenerator.cs +++ b/tools/DocGen/ConfigGenerator.cs @@ -107,6 +107,7 @@ private static void WriteMarkdown(StreamWriter file, Type configType) ``` + --{{moduleName.ToLowerInvariant()}}-{{prop.Name.ToLowerInvariant()}} --{{moduleName}}.{{prop.Name}} ``` diff --git a/tools/DocGen/DocGen.csproj b/tools/DocGen/DocGen.csproj index 1890c043d16..bf97292bb15 100644 --- a/tools/DocGen/DocGen.csproj +++ b/tools/DocGen/DocGen.csproj @@ -8,7 +8,8 @@ - + + diff --git a/tools/DocGen/Program.cs b/tools/DocGen/Program.cs index 80a62ef36ca..d4d70e110ac 100644 --- a/tools/DocGen/Program.cs +++ b/tools/DocGen/Program.cs @@ -1,66 +1,65 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System.ComponentModel; +using System.CommandLine; using Nethermind.DocGen; using Spectre.Console; -using Spectre.Console.Cli; -var app = new CommandApp(); - -app.Run(args); - -public sealed class AppCommand : Command +CliOption configOption = new("--config") { Description = "Generate configuration options docs" }; +CliOption dbSizeOption = new("--dbsize") { Description = "Generate DB sizes" }; +CliOption dbSizeSourceOption = new("--dbsize-src") { - public override int Execute(CommandContext context, AppSettings settings) - { - if (settings.DocsPath is null) - { - AnsiConsole.MarkupLine("[red]The path to the docs is not specified[/]"); - return 1; - } - - if (!Directory.Exists(settings.DocsPath)) - { - AnsiConsole.MarkupLine("[red]No docs not found at the path specified[/]"); - return 1; - } - - if (settings.GenerateConfig) - ConfigGenerator.Generate(settings.DocsPath); - - if (settings.GenerateDBSize) - DBSizeGenerator.Generate(settings.DocsPath, settings.DBSizeSourcePath); - - if (settings.GenerateJsonRpc) - JsonRpcGenerator.Generate(settings.DocsPath); + Description = "The path to the directory with DB size files", + HelpName = "path" +}; +CliArgument docsDirArg = new("docs-dir") +{ + Description = "The path to the docs directory", + HelpName = "path" +}; +CliOption jsonRpcOption = new("--jsonrpc") { Description = "Generate JSON-RPC API docs" }; +CliOption metricsOption = new("--metrics") { Description = "Generate metrics options docs" }; - if (settings.GenerateMetrics) - MetricsGenerator.Generate(settings.DocsPath); +dbSizeOption.Validators.Add(optionResult => +{ + if (optionResult.Parent?.GetValue(dbSizeSourceOption) is null) + optionResult.AddError($"{dbSizeSourceOption.Name} must be specified when {dbSizeOption.Name} is set"); +}); + +CliRootCommand rootCommand = +[ + configOption, + dbSizeOption, + dbSizeSourceOption, + docsDirArg, + jsonRpcOption, + metricsOption +]; +rootCommand.SetAction(parseResult => +{ + var docsPath = parseResult.GetValue(docsDirArg)!; - return 0; + if (!Directory.Exists(docsPath)) + { + AnsiConsole.MarkupLine("[red]The specified docs directory not found[/]"); + return 1; } -} -public sealed class AppSettings : CommandSettings -{ - [Description("Path to the directory with DB size files")] - [CommandOption("--dbsize-src")] - public string? DBSizeSourcePath { get; init; } + if (parseResult.GetValue(configOption)) + ConfigGenerator.Generate(docsPath); + + if (parseResult.GetValue(dbSizeOption)) + DBSizeGenerator.Generate(docsPath, parseResult.GetValue(dbSizeSourceOption)); - [Description("Path to the docs")] - [CommandArgument(0, "[docspath]")] - public string? DocsPath { get; init; } + if (parseResult.GetValue(jsonRpcOption)) + JsonRpcGenerator.Generate(docsPath); - [CommandOption("--config")] - public bool GenerateConfig { get; init; } + if (parseResult.GetValue(metricsOption)) + MetricsGenerator.Generate(docsPath); - [CommandOption("--dbsize")] - public bool GenerateDBSize { get; init; } + return 0; +}); - [CommandOption("--jsonrpc")] - public bool GenerateJsonRpc { get; init; } +CliConfiguration cli = new(rootCommand); - [CommandOption("--metrics")] - public bool GenerateMetrics { get; init; } -} +return cli.Invoke(args); diff --git a/tools/DocGen/Properties/launchSettings.json b/tools/DocGen/Properties/launchSettings.json index 14e64bfc589..090d2b5ec38 100644 --- a/tools/DocGen/Properties/launchSettings.json +++ b/tools/DocGen/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "DocGen": { "commandName": "Project", - "commandLineArgs": "/path/to/docs --config --dbsize --jsonrpc --metrics" + "commandLineArgs": "/path/to/docs --config --jsonrpc --metrics" } } } diff --git a/tools/DocGen/nuget.config b/tools/DocGen/nuget.config deleted file mode 100644 index 765346e5343..00000000000 --- a/tools/DocGen/nuget.config +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - diff --git a/tools/Evm/Evm.sln b/tools/Evm/Evm.sln new file mode 100644 index 00000000000..74ba8e12abf --- /dev/null +++ b/tools/Evm/Evm.sln @@ -0,0 +1,16 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Evm", "Evm\Evm.csproj", "{9D450C5A-C4B3-457A-8398-4690DFF4C47C}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {9D450C5A-C4B3-457A-8398-4690DFF4C47C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9D450C5A-C4B3-457A-8398-4690DFF4C47C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9D450C5A-C4B3-457A-8398-4690DFF4C47C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9D450C5A-C4B3-457A-8398-4690DFF4C47C}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/tools/Evm/Evm/Evm.csproj b/tools/Evm/Evm/Evm.csproj new file mode 100644 index 00000000000..73617641c01 --- /dev/null +++ b/tools/Evm/Evm/Evm.csproj @@ -0,0 +1,22 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + + + + + + + + + diff --git a/tools/Evm/Evm/Program.cs b/tools/Evm/Evm/Program.cs new file mode 100644 index 00000000000..ea9c991fabf --- /dev/null +++ b/tools/Evm/Evm/Program.cs @@ -0,0 +1,16 @@ +using System.CommandLine; +using Evm.t8n; + +namespace Evm; + +public static class Program +{ + public static async Task Main(string[] args) + { + var rootCmd = new RootCommand { Name = "Evm" }; + + T8NCommand.Configure(ref rootCmd); + + await rootCmd.InvokeAsync(args); + } +} diff --git a/tools/Evm/Evm/t8n/Errors/T8NErrorCodes.cs b/tools/Evm/Evm/t8n/Errors/T8NErrorCodes.cs new file mode 100644 index 00000000000..cfd9e5750e1 --- /dev/null +++ b/tools/Evm/Evm/t8n/Errors/T8NErrorCodes.cs @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Evm.t8n.Errors; + +public class T8NErrorCodes +{ + public const int ErrorEVM = 2; // Other EVM error + public const int ErrorConfig = 3; // Failed configuration: when a non-supported or invalid fork was specified. + + public const int ErrorMissingBlockhash = 4; // Block history is not supplied, but needed for a BLOCKHASH operation. If BLOCKHASH is invoked targeting a block which history has not been provided for, the program will exit with code 4. + + public const int ErrorJson = 10; // Invalid input json: the supplied data could not be marshalled + public const int ErrorIO = 11; // IO problems: failure to load or save files + public const int ErrorRlp = 12; // Invalid Rlp +} diff --git a/tools/Evm/Evm/t8n/Errors/T8NException.cs b/tools/Evm/Evm/t8n/Errors/T8NException.cs new file mode 100644 index 00000000000..c37211d2c5c --- /dev/null +++ b/tools/Evm/Evm/t8n/Errors/T8NException.cs @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Exceptions; + +namespace Evm.t8n.Errors; + +public class T8NException : Exception, IExceptionWithExitCode +{ + public T8NException(Exception e, int exitCode) : base(e.Message, e) + { + ExitCode = exitCode; + } + + public T8NException(Exception e, string message, int exitCode) : base(message, e) + { + ExitCode = exitCode; + } + + public T8NException(string message, int exitCode) : base(message) + { + ExitCode = exitCode; + } + + public int ExitCode { get; } +} diff --git a/tools/Evm/Evm/t8n/JsonTypes/EnvJson.cs b/tools/Evm/Evm/t8n/JsonTypes/EnvJson.cs new file mode 100644 index 00000000000..f79636d645f --- /dev/null +++ b/tools/Evm/Evm/t8n/JsonTypes/EnvJson.cs @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Int256; + +namespace Evm.t8n.JsonTypes; + +public class EnvJson +{ + public Address? CurrentCoinbase { get; set; } + public long CurrentGasLimit { get; set; } + public long CurrentNumber { get; set; } + public ulong CurrentTimestamp { get; set; } + public Withdrawal[]? Withdrawals { get; set; } + + public UInt256? CurrentRandom { get; set; } + public ulong ParentTimestamp { get; set; } + public UInt256? ParentDifficulty { get; set; } + public UInt256? CurrentBaseFee { get; set; } + public UInt256? CurrentDifficulty { get; set; } + public Hash256? ParentUncleHash { get; set; } + public Hash256? ParentBeaconBlockRoot { get; set; } + public UInt256? ParentBaseFee { get; set; } + public long ParentGasUsed { get; set; } + public long ParentGasLimit { get; set; } + public ulong? ParentExcessBlobGas { get; set; } + public ulong? CurrentExcessBlobGas { get; set; } + public ulong? ParentBlobGasUsed { get; set; } + + public Dictionary BlockHashes { get; set; } = []; + public Ommer[] Ommers { get; set; } = []; + + public Hash256? GetCurrentRandomHash256() + { + if (CurrentRandom is null) return null; + + Span bytes = stackalloc byte[32]; + CurrentRandom?.ToBigEndian(bytes); + return new Hash256(bytes); + } +} diff --git a/tools/Evm/Evm/t8n/JsonTypes/InputData.cs b/tools/Evm/Evm/t8n/JsonTypes/InputData.cs new file mode 100644 index 00000000000..6cf58a894f0 --- /dev/null +++ b/tools/Evm/Evm/t8n/JsonTypes/InputData.cs @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Ethereum.Test.Base; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Crypto; +using Nethermind.Facade.Eth.RpcTransaction; +using Nethermind.Serialization.Rlp; + +namespace Evm.t8n.JsonTypes; + +public class InputData +{ + public Dictionary? Alloc { get; set; } + public EnvJson? Env { get; set; } + public TransactionForRpc[]? Txs { get; set; } + public TransactionMetaData[]? TransactionMetaDataList { get; set; } + public string? TxRlp { get; set; } + + public Transaction[] GetTransactions(TxDecoder decoder) + { + List transactions = []; + if (TxRlp is not null) + { + RlpStream rlp = new(Bytes.FromHexString(TxRlp)); + transactions = decoder.DecodeArray(rlp).ToList(); + } + else if (Txs is not null && TransactionMetaDataList is not null) + { + for (int i = 0; i < Txs.Length; i++) + { + var transaction = Txs[i].ToTransaction(); + transaction.SenderAddress = null; // t8n does not accept SenderAddress from input, so need to reset senderAddress + + SignTransaction(transaction, TransactionMetaDataList[i], (LegacyTransactionForRpc) Txs[i]); + + transaction.Hash = transaction.CalculateHash(); + transactions.Add(transaction); + } + } + + return transactions.ToArray(); + } + + private static void SignTransaction(Transaction transaction, TransactionMetaData transactionMetaData, LegacyTransactionForRpc txLegacy) + { + if (transactionMetaData.SecretKey is not null) + { + var privateKey = new PrivateKey(transactionMetaData.SecretKey); + transaction.SenderAddress = privateKey.Address; + + EthereumEcdsa ecdsa = new(transaction.ChainId ?? TestBlockchainIds.ChainId); + + ecdsa.Sign(privateKey, transaction, transactionMetaData.Protected ?? true); + } + else if (txLegacy.R.HasValue && txLegacy.S.HasValue && txLegacy.V.HasValue) + { + transaction.Signature = new Signature(txLegacy.R.Value, txLegacy.S.Value, txLegacy.V.Value.ToUInt64(null)); + } + } +} diff --git a/tools/Evm/Evm/t8n/JsonTypes/Ommer.cs b/tools/Evm/Evm/t8n/JsonTypes/Ommer.cs new file mode 100644 index 00000000000..1f89ca61ac3 --- /dev/null +++ b/tools/Evm/Evm/t8n/JsonTypes/Ommer.cs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; + +namespace Evm.t8n.JsonTypes; + +public class Ommer(int delta, Address address) +{ + public int Delta { get; set; } = delta; + public Address Address { get; set; } = address; +} diff --git a/tools/Evm/Evm/t8n/JsonTypes/T8NTest.cs b/tools/Evm/Evm/t8n/JsonTypes/T8NTest.cs new file mode 100644 index 00000000000..f50703d26c5 --- /dev/null +++ b/tools/Evm/Evm/t8n/JsonTypes/T8NTest.cs @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Ethereum.Test.Base; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using Nethermind.Evm.Tracing.GethStyle; +using Nethermind.Int256; +using Nethermind.Specs; + +namespace Evm.t8n.JsonTypes; + +public class T8NTest(IReleaseSpec spec, ISpecProvider specProvider) +{ + public IReleaseSpec Spec { get; set; } = spec; + public ISpecProvider SpecProvider { get; set; } = specProvider; + public Address? CurrentCoinbase { get; set; } + public UInt256? CurrentDifficulty { get; set; } + + public UInt256? CurrentBaseFee { get; set; } + public long CurrentGasLimit { get; set; } + public long CurrentNumber { get; set; } + public ulong CurrentTimestamp { get; set; } + public Hash256? PreviousHash { get; set; } + public Dictionary Alloc { get; set; } = []; + public Hash256? PostHash { get; set; } + public Transaction[] Transactions { get; set; } = []; + public Hash256? CurrentRandom { get; set; } + public Hash256? CurrentBeaconRoot { get; set; } + public Hash256? CurrentWithdrawalsRoot { get; set; } + public ulong? CurrentExcessBlobGas { get; set; } + public UInt256? ParentBlobGasUsed { get; set; } + public UInt256? ParentExcessBlobGas { get; set; } + + public Withdrawal[]? Withdrawals { get; set; } + public ulong ParentTimestamp { get; set; } + public UInt256? ParentDifficulty { get; set; } + public Hash256? ParentUncleHash { get; set; } + public Hash256? ParentBeaconBlockRoot { get; set; } + public UInt256? ParentBaseFee { get; set; } + public long ParentGasUsed { get; set; } + public long ParentGasLimit { get; set; } + public Dictionary BlockHashes { get; set; } = []; + public Ommer[] Ommers { get; set; } = []; + public ulong StateChainId { get; set; } = MainnetSpecProvider.Instance.ChainId; + public GethTraceOptions GethTraceOptions { get; set; } = GethTraceOptions.Default; + public bool IsTraceEnabled { get; set; } = false; +} diff --git a/tools/Evm/Evm/t8n/JsonTypes/TransactionMetaData.cs b/tools/Evm/Evm/t8n/JsonTypes/TransactionMetaData.cs new file mode 100644 index 00000000000..d4b66eae1fe --- /dev/null +++ b/tools/Evm/Evm/t8n/JsonTypes/TransactionMetaData.cs @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Evm.t8n.JsonTypes; + +public class TransactionMetaData +{ + public bool? Protected { get; set; } + public byte[]? SecretKey { get; set; } +} diff --git a/tools/Evm/Evm/t8n/T8NCommand.cs b/tools/Evm/Evm/t8n/T8NCommand.cs new file mode 100644 index 00000000000..75cf7f2f558 --- /dev/null +++ b/tools/Evm/Evm/t8n/T8NCommand.cs @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.CommandLine; + +namespace Evm.t8n; + +public static class T8NCommand +{ + public static void Configure(ref RootCommand rootCmd) + { + Command cmd = T8NCommandOptions.CreateCommand(); + rootCmd.Add(cmd); + + cmd.SetHandler( + context => + { + var arguments = T8NCommandArguments.FromParseResult(context.ParseResult); + T8NExecutor.Execute(arguments); + }); + } +} diff --git a/tools/Evm/Evm/t8n/T8NCommandArguments.cs b/tools/Evm/Evm/t8n/T8NCommandArguments.cs new file mode 100644 index 00000000000..32194860cd6 --- /dev/null +++ b/tools/Evm/Evm/t8n/T8NCommandArguments.cs @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.CommandLine.Parsing; +using Nethermind.Specs; + +namespace Evm.t8n; + +public class T8NCommandArguments +{ + public string InputAlloc { get; set; } = "alloc.json"; + public string InputEnv { get; set; } = "env.json"; + public string InputTxs { get; set; } = "txs.json"; + + public string OutputAlloc { get; set; } = "alloc.json"; + public string OutputResult { get; set; } = "result.json"; + public string? OutputBody { get; set; } + public string? OutputBaseDir { get; set; } + + public ulong StateChainId { get; set; } = MainnetSpecProvider.Instance.ChainId; + public string StateFork { get; set; } = "GrayGlacier"; + public string StateReward { get; set; } = "0"; + + public bool Trace { get; set; } + public bool TraceMemory { get; set; } + public bool TraceNoStack { get; set; } + public bool TraceReturnData { get; set; } + + public static T8NCommandArguments FromParseResult(ParseResult parseResult) + { + var arguments = new T8NCommandArguments + { + OutputBody = parseResult.GetValueForOption(T8NCommandOptions.OutputBodyOpt), + OutputBaseDir = parseResult.GetValueForOption(T8NCommandOptions.OutputBaseDirOpt), + Trace = parseResult.GetValueForOption(T8NCommandOptions.TraceOpt), + TraceMemory = parseResult.GetValueForOption(T8NCommandOptions.TraceMemoryOpt), + TraceNoStack = parseResult.GetValueForOption(T8NCommandOptions.TraceNoStackOpt), + TraceReturnData = parseResult.GetValueForOption(T8NCommandOptions.TraceReturnDataOpt) + }; + + var inputAlloc = parseResult.GetValueForOption(T8NCommandOptions.InputAllocOpt); + if (inputAlloc is not null) + { + arguments.InputAlloc = inputAlloc; + } + + var inputEnv = parseResult.GetValueForOption(T8NCommandOptions.InputEnvOpt); + if (inputEnv is not null) + { + arguments.InputEnv = inputEnv; + } + + var inputTxs = parseResult.GetValueForOption(T8NCommandOptions.InputTxsOpt); + if (inputTxs is not null) + { + arguments.InputTxs = inputTxs; + } + + var outputAlloc = parseResult.GetValueForOption(T8NCommandOptions.OutputAllocOpt); + if (outputAlloc is not null) + { + arguments.OutputAlloc = outputAlloc; + } + + var outputResult = parseResult.GetValueForOption(T8NCommandOptions.OutputResultOpt); + if (outputResult is not null) + { + arguments.OutputResult = outputResult; + } + + var stateFork = parseResult.GetValueForOption(T8NCommandOptions.StateForkOpt); + if (stateFork is not null) + { + arguments.StateFork = stateFork; + } + + var stateReward = parseResult.GetValueForOption(T8NCommandOptions.StateRewardOpt); + if (stateReward is not null) + { + arguments.StateReward = stateReward; + } + + var stateChainId = parseResult.GetValueForOption(T8NCommandOptions.StateChainIdOpt); + if (stateChainId.HasValue) + { + arguments.StateChainId = stateChainId.Value; + } + + return arguments; + } + +} diff --git a/tools/Evm/Evm/t8n/T8NCommandOptions.cs b/tools/Evm/Evm/t8n/T8NCommandOptions.cs new file mode 100644 index 00000000000..e29d2b8247f --- /dev/null +++ b/tools/Evm/Evm/t8n/T8NCommandOptions.cs @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Evm.t8n; + +using System.CommandLine; + +public static class T8NCommandOptions +{ + public static Option InputAllocOpt { get; } = new("--input.alloc", description: "Input allocations"); + public static Option InputEnvOpt { get; } = new("--input.env", description: "Input environment"); + public static Option InputTxsOpt { get; } = new("--input.txs", description: "Input transactions"); + + public static Option OutputAllocOpt { get; } = new("--output.alloc", description: "Output allocations"); + public static Option OutputResultOpt { get; } = new("--output.result", description: "Output result"); + public static Option OutputBodyOpt { get; } = new("--output.body", description: "Output body"); + public static Option OutputBaseDirOpt { get; } = new("--output.basedir", description: "Output base directory"); + + public static Option StateChainIdOpt { get; } = new("--state.chainid", description: "State chain id"); + public static Option StateForkOpt { get; } = new("--state.fork", description: "State fork"); + public static Option StateRewardOpt { get; } = new("--state.reward", description: "State reward"); + + public static Option TraceOpt { get; } = new("--trace", description: "Configures the use of the JSON opcode tracer. This tracer emits traces to files as trace--.json"); + public static Option TraceMemoryOpt { get; } = new("--trace.memory", description: "Trace memory"); + public static Option TraceNoStackOpt { get; } = new("--trace.nostack", description: "Trace no stack"); + public static Option TraceReturnDataOpt { get; } = new("--trace.returndata", description: "Trace return data"); + + public static Command CreateCommand() + { + var cmd = new Command("t8n", "EVM State Transition command") + { + InputAllocOpt, + InputEnvOpt, + InputTxsOpt, + OutputAllocOpt, + OutputBaseDirOpt, + OutputBodyOpt, + OutputResultOpt, + StateChainIdOpt, + StateForkOpt, + StateRewardOpt, + TraceOpt, + TraceMemoryOpt, + TraceNoStackOpt, + TraceReturnDataOpt, + }; + cmd.AddAlias("transition"); + return cmd; + } +} diff --git a/tools/Evm/Evm/t8n/T8NExecutor.cs b/tools/Evm/Evm/t8n/T8NExecutor.cs new file mode 100644 index 00000000000..570f3c0f0ea --- /dev/null +++ b/tools/Evm/Evm/t8n/T8NExecutor.cs @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Evm.t8n.JsonTypes; + +namespace Evm.t8n; + +public static class T8NExecutor +{ + public static void Execute(T8NCommandArguments arguments) + { + T8NTest t8nTest = T8NInputProcessor.ProcessInputAndConvertToT8NTest(arguments); + } +} diff --git a/tools/Evm/Evm/t8n/T8NInputProcessor.cs b/tools/Evm/Evm/t8n/T8NInputProcessor.cs new file mode 100644 index 00000000000..cb7e44db382 --- /dev/null +++ b/tools/Evm/Evm/t8n/T8NInputProcessor.cs @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Ethereum.Test.Base; +using Evm.t8n.Errors; +using Evm.t8n.JsonTypes; +using Nethermind.Core.Specs; +using Nethermind.Evm.Tracing.GethStyle; +using Nethermind.Int256; +using Nethermind.Serialization.Rlp; +using Nethermind.Specs; +using Nethermind.Specs.Forks; +using Nethermind.Specs.Test; + +namespace Evm.t8n; + +public static class T8NInputProcessor +{ + private static readonly TxDecoder TxDecoder = TxDecoder.Instance; + + public static T8NTest ProcessInputAndConvertToT8NTest(T8NCommandArguments arguments) + { + InputData inputData = T8NInputReader.ReadInputData(arguments); + + if (inputData.Env is null) + { + throw new T8NException("Env is not provided", T8NErrorCodes.ErrorIO); + } + + (ISpecProvider specProvider, IReleaseSpec spec) = GetSpec(arguments, inputData.Env); + + T8NValidator.ApplyChecks(inputData.Env, specProvider, spec); + + var gethTraceOptions = new GethTraceOptions + { + EnableMemory = arguments.TraceMemory, + DisableStack = arguments.TraceNoStack + }; + + T8NTest test = new(spec, specProvider) + { + Alloc = inputData.Alloc ?? [], + Transactions = inputData.GetTransactions(TxDecoder), + CurrentCoinbase = inputData.Env.CurrentCoinbase, + CurrentGasLimit = inputData.Env.CurrentGasLimit, + CurrentTimestamp = inputData.Env.CurrentTimestamp, + CurrentNumber = inputData.Env.CurrentNumber, + Withdrawals = inputData.Env.Withdrawals, + CurrentRandom = inputData.Env.GetCurrentRandomHash256(), + ParentTimestamp = inputData.Env.ParentTimestamp, + ParentDifficulty = inputData.Env.ParentDifficulty, + CurrentBaseFee = inputData.Env.CurrentBaseFee, + CurrentDifficulty = inputData.Env.CurrentDifficulty, + ParentUncleHash = inputData.Env.ParentUncleHash, + ParentBaseFee = inputData.Env.ParentBaseFee, + ParentBeaconBlockRoot = inputData.Env.ParentBeaconBlockRoot, + ParentGasUsed = inputData.Env.ParentGasUsed, + ParentGasLimit = inputData.Env.ParentGasLimit, + ParentExcessBlobGas = inputData.Env.ParentExcessBlobGas, + CurrentExcessBlobGas = inputData.Env.CurrentExcessBlobGas, + ParentBlobGasUsed = inputData.Env.ParentBlobGasUsed, + Ommers = inputData.Env.Ommers, + BlockHashes = inputData.Env.BlockHashes, + StateChainId = arguments.StateChainId, + GethTraceOptions = gethTraceOptions, + IsTraceEnabled = arguments.Trace, + }; + + return test; + } + + private static (ISpecProvider, IReleaseSpec) GetSpec(T8NCommandArguments arguments, EnvJson env) + { + IReleaseSpec spec; + try + { + spec = JsonToEthereumTest.ParseSpec(arguments.StateFork); + } + catch (NotSupportedException e) + { + throw new T8NException(e, $"unsupported fork {arguments.StateFork}", T8NErrorCodes.ErrorConfig); + } + OverridableReleaseSpec overridableReleaseSpec = new(spec); + + if (!string.IsNullOrEmpty(arguments.StateReward) && arguments.StateReward != "-1") // (-1 means rewards are disabled) + { + overridableReleaseSpec.BlockReward = UInt256.Parse(arguments.StateReward); + } + ISpecProvider specProvider = arguments.StateChainId == GnosisSpecProvider.Instance.ChainId + ? GnosisSpecProvider.Instance + : new CustomSpecProvider(((ForkActivation)0, Frontier.Instance), + ((ForkActivation)1, overridableReleaseSpec)); + + if (spec is Paris) + { + specProvider.UpdateMergeTransitionInfo(env.CurrentNumber, 0); + } + + return (specProvider, overridableReleaseSpec); + } +} diff --git a/tools/Evm/Evm/t8n/T8NInputReader.cs b/tools/Evm/Evm/t8n/T8NInputReader.cs new file mode 100644 index 00000000000..4cf86c29d2e --- /dev/null +++ b/tools/Evm/Evm/t8n/T8NInputReader.cs @@ -0,0 +1,86 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; +using Ethereum.Test.Base; +using Evm.t8n.Errors; +using Evm.t8n.JsonTypes; +using Nethermind.Core; +using Nethermind.Facade.Eth.RpcTransaction; +using Nethermind.Serialization.Json; + +namespace Evm.t8n; + +public static class T8NInputReader +{ + private static readonly EthereumJsonSerializer EthereumJsonSerializer = new(); + private const string Stdin = "stdin"; + + public static InputData ReadInputData(T8NCommandArguments arguments) + { + InputData inputData = new(); + + if (arguments.InputAlloc == Stdin || arguments.InputEnv == Stdin || arguments.InputTxs == Stdin) + { + inputData = ReadStdInput(); + } + + if (arguments.InputAlloc != Stdin) + { + inputData.Alloc = LoadDataFromFile>(arguments.InputAlloc, "alloc"); + } + + if (arguments.InputEnv != Stdin) + { + inputData.Env = LoadDataFromFile(arguments.InputEnv, "env"); + } + + if (arguments.InputTxs != Stdin) + { + switch (Path.GetExtension(arguments.InputTxs)) + { + case ".json": + inputData.Txs = LoadDataFromFile(arguments.InputTxs, "txs"); + inputData.TransactionMetaDataList = LoadDataFromFile(arguments.InputTxs, "txs"); + break; + case ".rlp": + inputData.TxRlp = File.ReadAllText(arguments.InputTxs).Replace("\"", "").Replace("\n", ""); + break; + default: + throw new T8NException("Transactions file support only rlp, json formats", T8NErrorCodes.ErrorIO); + } + } + + return inputData; + } + + private static T LoadDataFromFile(string filePath, string description) + { + try + { + var fileContent = File.ReadAllText(filePath); + return EthereumJsonSerializer.Deserialize(fileContent); + } + catch (FileNotFoundException e) + { + throw new T8NException(e, "failed reading {filePath} file: {description}", T8NErrorCodes.ErrorIO); + } + catch (JsonException e) + { + throw new T8NException(e, $"failed unmarshalling {filePath} file: {description}", T8NErrorCodes.ErrorJson); + } + } + + private static InputData ReadStdInput() + { + using var reader = new StreamReader(Console.OpenStandardInput()); + try + { + return EthereumJsonSerializer.Deserialize(reader.ReadToEnd()); + } + catch (Exception e) + { + throw new T8NException(e, T8NErrorCodes.ErrorJson); + } + } +} diff --git a/tools/Evm/Evm/t8n/T8NValidator.cs b/tools/Evm/Evm/t8n/T8NValidator.cs new file mode 100644 index 00000000000..00ae49b80d6 --- /dev/null +++ b/tools/Evm/Evm/t8n/T8NValidator.cs @@ -0,0 +1,105 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Evm.t8n.Errors; +using Evm.t8n.JsonTypes; +using Nethermind.Consensus.Ethash; +using Nethermind.Core; +using Nethermind.Core.Specs; +using Nethermind.Core.Test.Builders; +using Nethermind.Specs.Forks; + +namespace Evm.t8n; + +public static class T8NValidator +{ + public static void ApplyChecks(EnvJson env, ISpecProvider specProvider, IReleaseSpec spec) + { + ApplyLondonChecks(env, spec); + ApplyShanghaiChecks(env, spec); + ApplyCancunChecks(env, spec); + ApplyMergeChecks(env, specProvider); + } + + private static void ApplyLondonChecks(EnvJson env, IReleaseSpec spec) + { + if (spec is not London) return; + if (env.CurrentBaseFee is not null) return; + + if (!env.ParentBaseFee.HasValue || env.CurrentNumber == 0) + { + throw new T8NException("EIP-1559 config but missing 'parentBaseFee' in env section", + T8NErrorCodes.ErrorConfig); + } + + var parent = Build.A.BlockHeader.WithNumber(env.CurrentNumber - 1).WithBaseFee(env.ParentBaseFee.Value) + .WithGasUsed(env.ParentGasUsed).WithGasLimit(env.ParentGasLimit).TestObject; + env.CurrentBaseFee = BaseFeeCalculator.Calculate(parent, spec); + } + + private static void ApplyShanghaiChecks(EnvJson env, IReleaseSpec spec) + { + if (spec is not Shanghai) return; + if (env.Withdrawals is null) + { + throw new T8NException("Shanghai config but missing 'withdrawals' in env section", + T8NErrorCodes.ErrorConfig); + } + } + + private static void ApplyCancunChecks(EnvJson env, IReleaseSpec spec) + { + if (spec is not Cancun) + { + env.ParentBeaconBlockRoot = null; + return; + } + + if (env.ParentBeaconBlockRoot is null) + { + throw new T8NException("post-cancun env requires parentBeaconBlockRoot to be set", + T8NErrorCodes.ErrorConfig); + } + } + + private static void ApplyMergeChecks(EnvJson env, ISpecProvider specProvider) + { + if (specProvider.TerminalTotalDifficulty?.IsZero ?? false) + { + if (env.CurrentRandom is null) + throw new T8NException("post-merge requires currentRandom to be defined in env", + T8NErrorCodes.ErrorConfig); + if (env.CurrentDifficulty?.IsZero ?? false) + throw new T8NException("post-merge difficulty must be zero (or omitted) in env", + T8NErrorCodes.ErrorConfig); + return; + } + + if (env.CurrentDifficulty is not null) return; + if (!env.ParentDifficulty.HasValue) + { + throw new T8NException( + "currentDifficulty was not provided, and cannot be calculated due to missing parentDifficulty", + T8NErrorCodes.ErrorConfig); + } + + if (env.CurrentNumber == 0) + { + throw new T8NException("currentDifficulty needs to be provided for block number 0", + T8NErrorCodes.ErrorConfig); + } + + if (env.CurrentTimestamp <= env.ParentTimestamp) + { + throw new T8NException( + $"currentDifficulty cannot be calculated -- currentTime ({env.CurrentTimestamp}) needs to be after parent time ({env.ParentTimestamp})", + T8NErrorCodes.ErrorConfig); + } + + EthashDifficultyCalculator difficultyCalculator = new(specProvider); + + env.CurrentDifficulty = difficultyCalculator.Calculate(env.ParentDifficulty.Value, env.ParentTimestamp, + env.CurrentTimestamp, env.CurrentNumber, env.ParentUncleHash is not null); + } +} + diff --git a/tools/HiveCompare/HiveCompare/HiveCompare.csproj b/tools/HiveCompare/HiveCompare/HiveCompare.csproj index aa57c232794..61ac6a5d41c 100644 --- a/tools/HiveCompare/HiveCompare/HiveCompare.csproj +++ b/tools/HiveCompare/HiveCompare/HiveCompare.csproj @@ -8,7 +8,7 @@ - + diff --git a/tools/HiveCompare/HiveCompare/Program.cs b/tools/HiveCompare/HiveCompare/Program.cs index 2107db69bd1..5da4acd2f65 100644 --- a/tools/HiveCompare/HiveCompare/Program.cs +++ b/tools/HiveCompare/HiveCompare/Program.cs @@ -1,5 +1,5 @@ using HiveCompare.Models; -using McMaster.Extensions.CommandLineUtils; +using System.CommandLine; using System.Diagnostics.CodeAnalysis; using System.Text.Json; @@ -13,53 +13,40 @@ internal class Program private static void Main(string[] args) { - CommandLineApplication cli = CreateCommandLineInterface(); - try + CliOption firstFileOption = new("--first-file", "-f") { - cli.Execute(args); - } - catch (CommandParsingException) + Description = "The first file to be used for comparison", + Required = true, + HelpName = "path" + }; + CliOption secondFileOption = new("--second-file", "-s") { - cli.ShowHelp(); - } - } - - static CommandLineApplication CreateCommandLineInterface() - { - CommandLineApplication cli = new() { Name = "HiveCompare" }; - cli.HelpOption("-?|-h|--help"); - CommandOption firstFileOption = cli.Option("-f|--first-file", "first file to be used for comparison", CommandOptionType.SingleValue); - CommandOption secondFileOption = cli.Option("-s|--second-file", "second file to be used for comparison", CommandOptionType.SingleValue); + Description = "The second file to be used for comparison", + Required = true, + HelpName = "path" + }; + CliRootCommand rootCommand = [firstFileOption, secondFileOption]; - cli.OnExecute(() => + rootCommand.SetAction(parseResult => { - bool HasRequiredOption(CommandOption option) + static bool RequiredFileExists(string? filePath) { - if (option.HasValue() && !string.IsNullOrEmpty(option.Value())) return true; + if (File.Exists(filePath)) return true; - cli.ShowHelp(); + Console.WriteLine($"Could not find file '{filePath}'."); return false; } - bool RequiredFileExists(CommandOption option) - { - if (File.Exists(option.Value())) return true; - - Console.WriteLine($"Could not find file '{option.Value()}'."); - return false; - - } + string? firstFileValue = parseResult.GetValue(firstFileOption); + string? secondFileValue = parseResult.GetValue(secondFileOption); - return HasRequiredOption(firstFileOption) && HasRequiredOption(secondFileOption) - ? RequiredFileExists(firstFileOption) && RequiredFileExists(secondFileOption) - ? ParseTests(firstFileOption.Value()!, secondFileOption.Value()!) - ? 0 - : 4 - : 2 - : 1; + return RequiredFileExists(firstFileValue) && RequiredFileExists(secondFileValue) + ? ParseTests(firstFileValue!, secondFileValue!) ? 0 : 4 + : 2; }); - return cli; + CliConfiguration cli = new(rootCommand); + cli.Invoke(args); } private static bool ParseTests(string firstFile, string secondFile) diff --git a/tools/Nethermind.Tools.Kute/Config.cs b/tools/Nethermind.Tools.Kute/Config.cs index a6467c30312..63cba1a916b 100644 --- a/tools/Nethermind.Tools.Kute/Config.cs +++ b/tools/Nethermind.Tools.Kute/Config.cs @@ -1,135 +1,80 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using CommandLine; using Nethermind.Tools.Kute.MetricsConsumer; +using System.CommandLine; namespace Nethermind.Tools.Kute; -public class Config +public static class Config { - [Option( - shortName: 'i', - longName: "input", + public static CliOption MessagesFilePath { get; } = new("--input", "-i") + { + Description = "Path to a file or directory containing JSON RPC messages", + HelpName = "path", Required = true, - HelpText = "Path to a Folder or a File containing JSON RPC messages" - )] - public string MessagesFilePath { get; } - - [Option( - shortName: 'a', - longName: "address", - Required = false, - Default = "http://localhost:8551", - HelpText = "Address where to send JSON RPC requests" - )] - public string HostAddress { get; } + }; - [Option( - shortName: 's', - longName: "secret", - Required = true, - HelpText = "Path to File with hex encoded secret for JWT authentication" - )] - public string JwtSecretFilePath { get; } + public static CliOption HostAddress { get; } = new("--address", "-a") + { + DefaultValueFactory = r => "http://localhost:8551", + Description = "Address where to send JSON RPC requests", + HelpName = "URL" + }; - [Option( - shortName: 't', - longName: "ttl", - Required = false, - Default = 60, - HelpText = "Authentication time to live (ttl) in seconds" - )] - public int AuthTtl { get; } + public static CliOption JwtSecretFilePath { get; } = new("--secret", "-s") + { + Description = "Path to file with hex-encoded secret for JWT authentication", + HelpName = "value", + Required = true + }; - [Option( - shortName: 'd', - longName: "dry", - Required = false, - Default = false, - HelpText = "Only log into console" - )] - public bool DryRun { get; } + public static CliOption AuthTtl { get; } = new("--ttl", "-t") + { + DefaultValueFactory = r => 60, + Description = "Authentication time to live (TTL), in seconds", + HelpName = "value" + }; - [Option( - shortName: 'p', - longName: "progress", - Required = false, - Default = false, - HelpText = "Show progress" - )] - public bool ShowProgress { get; } + public static CliOption DryRun { get; } = new("--dry", "-d") + { + Description = "Only log into console" + }; - [Option( - shortName: 'o', - longName: "output", - Required = false, - Default = MetricsOutputFormatter.Report, - HelpText = "Strategy to report metrics" - )] - public MetricsOutputFormatter MetricsOutputFormatter { get; } + public static CliOption ShowProgress { get; } = new("--progress", "-p") + { + Description = "Show progress" + }; - [Option( - shortName: 'f', - longName: "filters", - Separator = ',', - Required = false, - Default = new string[] { }, - HelpText = "A comma separated List of regexes of methods to be executed with optional limits" - )] - public IEnumerable MethodFilters { get; } + public static CliOption MetricsOutputFormatter { get; } = new("--output", "-o") + { + DefaultValueFactory = r => MetricsConsumer.MetricsOutputFormatter.Report, + Description = "Strategy to report metrics", + HelpName = "value", + }; - [Option( - shortName: 'r', - longName: "responses", - Required = false, - Default = null, - HelpText = "Path to File to store JSON-RPC responses" - )] - public string? ResponsesTraceFile { get; } + public static CliOption> MethodFilters { get; } = new("--filters", "-f") + { + DefaultValueFactory = r => [], + CustomParser = r => r.Tokens.Count == 1 ? r.Tokens[0].Value.Split(',') : null, + Description = "A comma separated List of regexes of methods to be executed with optional limits", + HelpName = "value", + }; - [Option( - shortName: 'e', - longName: "rps", - Required = false, - Default = 0, - HelpText = "If set to higher than 0, then requests will be send in selected RPS (Requests per seconds) rate. If 0 (or lower) then requests will be sent sequentionally." - )] - public int RequestsPerSecond { get; } + public static CliOption ResponsesTraceFile { get; } = new("--responses", "-r") + { + Description = "Path to file to store JSON-RPC responses", + HelpName = "path" + }; - [Option( - shortName: 'u', - longName: "unwrapBatch", - Required = false, - Default = false, - HelpText = "If true then each batched request will be unwraped to single requests." - )] - public bool UnwrapBatch { get; } + public static CliOption RequestsPerSecond { get; } = new("--rps", "-e") + { + Description = "If set to higher than 0, then requests will be send in selected RPS (Requests per seconds) rate. If 0 (or lower) then requests will be sent sequentially", + HelpName = "value" + }; - public Config( - string messagesFilePath, - string hostAddress, - string jwtSecretFilePath, - int authTtl, - bool dryRun, - bool showProgress, - MetricsOutputFormatter metricsOutputFormatter, - IEnumerable methodFilters, - string? responsesTraceFile, - int requestsPerSecond, - bool unwrapBatch - ) + public static CliOption UnwrapBatch { get; } = new("--unwrapBatch", "-u") { - MessagesFilePath = messagesFilePath; - HostAddress = hostAddress; - JwtSecretFilePath = jwtSecretFilePath; - AuthTtl = authTtl; - DryRun = dryRun; - ShowProgress = showProgress; - MetricsOutputFormatter = metricsOutputFormatter; - MethodFilters = methodFilters; - ResponsesTraceFile = responsesTraceFile; - RequestsPerSecond = requestsPerSecond; - UnwrapBatch = unwrapBatch; - } + Description = "If true then each batched request will be unwraped to single requests" + }; } diff --git a/tools/Nethermind.Tools.Kute/Nethermind.Tools.Kute.csproj b/tools/Nethermind.Tools.Kute/Nethermind.Tools.Kute.csproj index 0e5870bdf32..32fb32af4b3 100644 --- a/tools/Nethermind.Tools.Kute/Nethermind.Tools.Kute.csproj +++ b/tools/Nethermind.Tools.Kute/Nethermind.Tools.Kute.csproj @@ -9,9 +9,9 @@ - + diff --git a/tools/Nethermind.Tools.Kute/Program.cs b/tools/Nethermind.Tools.Kute/Program.cs index 3e5157cfcc9..67b6837d0c6 100644 --- a/tools/Nethermind.Tools.Kute/Program.cs +++ b/tools/Nethermind.Tools.Kute/Program.cs @@ -1,7 +1,6 @@ using App.Metrics.Formatters; using App.Metrics.Formatters.Ascii; using App.Metrics.Formatters.Json; -using CommandLine; using Microsoft.Extensions.DependencyInjection; using Nethermind.Tools.Kute.Auth; using Nethermind.Tools.Kute.FlowManager; @@ -15,30 +14,52 @@ using Nethermind.Tools.Kute.ResponseTracer; using Nethermind.Tools.Kute.SecretProvider; using Nethermind.Tools.Kute.SystemClock; +using System.CommandLine; namespace Nethermind.Tools.Kute; static class Program { - public static async Task Main(string[] args) + public static async Task Main(string[] args) { - await Parser.Default.ParseArguments(args).WithParsedAsync(async config => + CliRootCommand rootCommand = + [ + Config.MessagesFilePath, + Config.HostAddress, + Config.JwtSecretFilePath, + Config.AuthTtl, + Config.DryRun, + Config.ShowProgress, + Config.MetricsOutputFormatter, + Config.MethodFilters, + Config.ResponsesTraceFile, + Config.RequestsPerSecond, + Config.UnwrapBatch + ]; + rootCommand.SetAction((parseResult, cancellationToken) => { - IServiceProvider serviceProvider = BuildServiceProvider(config); + IServiceProvider serviceProvider = BuildServiceProvider(parseResult); Application app = serviceProvider.GetService()!; - await app.Run(); + return app.Run(); }); + + CliConfiguration cli = new(rootCommand); + + return await cli.InvokeAsync(args); } - static IServiceProvider BuildServiceProvider(Config config) + private static IServiceProvider BuildServiceProvider(ParseResult parseResult) { + bool dryRun = parseResult.GetValue(Config.DryRun); + bool unwrapBatch = parseResult.GetValue(Config.UnwrapBatch); + string? responsesTraceFile = parseResult.GetValue(Config.ResponsesTraceFile); IServiceCollection collection = new ServiceCollection(); collection.AddSingleton(); collection.AddSingleton(); collection.AddSingleton(); - collection.AddSingleton(new FileSecretProvider(config.JwtSecretFilePath)); + collection.AddSingleton(new FileSecretProvider(parseResult.GetValue(Config.JwtSecretFilePath)!)); collection.AddSingleton(provider => new TtlAuth( new JwtAuth( @@ -46,42 +67,39 @@ static IServiceProvider BuildServiceProvider(Config config) provider.GetRequiredService() ), provider.GetRequiredService(), - config.AuthTtl + parseResult.GetValue(Config.AuthTtl) ) ); - collection.AddSingleton>(new FileMessageProvider(config.MessagesFilePath)); + collection.AddSingleton>(new FileMessageProvider(parseResult.GetValue(Config.MessagesFilePath)!)); collection.AddSingleton>(serviceProvider => { var messageProvider = serviceProvider.GetRequiredService>(); var jsonMessageProvider = new JsonRpcMessageProvider(messageProvider); - return config.UnwrapBatch - ? new UnwrapBatchJsonRpcMessageProvider(jsonMessageProvider) - : jsonMessageProvider; + return unwrapBatch ? new UnwrapBatchJsonRpcMessageProvider(jsonMessageProvider) : jsonMessageProvider; }); - collection.AddSingleton( - config.DryRun - ? new NullJsonRpcValidator() - : new ComposedJsonRpcValidator(new List - { - new NonErrorJsonRpcValidator(), new NewPayloadJsonRpcValidator(), - }) + collection.AddSingleton(dryRun + ? new NullJsonRpcValidator() + : new ComposedJsonRpcValidator(new List + { + new NonErrorJsonRpcValidator(), new NewPayloadJsonRpcValidator(), + }) ); collection.AddSingleton( new ComposedJsonRpcMethodFilter( - config.MethodFilters + parseResult.GetValue(Config.MethodFilters)! .Select(pattern => new PatternJsonRpcMethodFilter(pattern) as IJsonRpcMethodFilter) .ToList() ) ); collection.AddSingleton(provider => { - if (!config.DryRun) + if (!dryRun) { return new HttpJsonRpcSubmitter( provider.GetRequiredService(), provider.GetRequiredService(), - config.HostAddress + parseResult.GetValue(Config.HostAddress)! ); } @@ -92,13 +110,13 @@ static IServiceProvider BuildServiceProvider(Config config) return new NullJsonRpcSubmitter(); }); collection.AddSingleton( - config is { DryRun: false, ResponsesTraceFile: not null } - ? new FileResponseTracer(config.ResponsesTraceFile) + !dryRun && responsesTraceFile is not null + ? new FileResponseTracer(responsesTraceFile) : new NullResponseTracer() ); collection.AddSingleton(provider => { - if (config.ShowProgress) + if (parseResult.GetValue(Config.ShowProgress)) { // NOTE: // Terrible, terrible hack since it forces a double enumeration: @@ -107,7 +125,7 @@ static IServiceProvider BuildServiceProvider(Config config) // We can reduce the cost by not parsing each message on the first enumeration // only when we're not unwrapping batches. If we are, we need to parse. // This optimization relies on implementation details. - IMessageProvider messagesProvider = config.UnwrapBatch + IMessageProvider messagesProvider = unwrapBatch ? provider.GetRequiredService>() : provider.GetRequiredService>(); var totalMessages = messagesProvider.Messages.ToEnumerable().Count(); @@ -118,14 +136,15 @@ static IServiceProvider BuildServiceProvider(Config config) }); collection.AddSingleton(); collection.AddSingleton( - config.MetricsOutputFormatter switch + parseResult.GetValue(Config.MetricsOutputFormatter) switch { MetricsOutputFormatter.Report => new MetricsTextOutputFormatter(), MetricsOutputFormatter.Json => new MetricsJsonOutputFormatter(), _ => throw new ArgumentOutOfRangeException(), } ); - collection.AddSingleton(new JsonRpcFlowManager(config.RequestsPerSecond, config.UnwrapBatch)); + collection.AddSingleton(new JsonRpcFlowManager( + parseResult.GetValue(Config.RequestsPerSecond), unwrapBatch)); return collection.BuildServiceProvider(); } diff --git a/tools/SendBlobs/Program.cs b/tools/SendBlobs/Program.cs index 109df4e7247..08f18a197a4 100644 --- a/tools/SendBlobs/Program.cs +++ b/tools/SendBlobs/Program.cs @@ -1,40 +1,16 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using McMaster.Extensions.CommandLineUtils; -using Nethermind.Cli; -using Nethermind.Cli.Console; -using Nethermind.Consensus; -using Nethermind.Core; -using Nethermind.Core.Crypto; -using Nethermind.Crypto; -using Nethermind.Evm; -using Nethermind.Facade.Proxy.Models; -using Nethermind.Int256; -using Nethermind.Logging; -using Nethermind.Serialization.Json; -using Nethermind.Serialization.Rlp; -using Org.BouncyCastle.Utilities.Encoders; using SendBlobs; +using System.CommandLine; -CommandLineApplication app = new() { Name = "SendBlobs" }; - -SetupCli.SetupExecute(app); -SetupCli.SetupDistributeCommand(app); -SetupCli.SetupReclaimCommand(app); -SetupCli.SetupSendFileCommand(app); - -try -{ - app.Execute(args); -} -catch (CommandParsingException ex) -{ - Console.WriteLine(ex.Message); - app.ShowHelp(); -} - - +CliRootCommand rootCommand = []; +SetupCli.SetupExecute(rootCommand); +SetupCli.SetupDistributeCommand(rootCommand); +SetupCli.SetupReclaimCommand(rootCommand); +SetupCli.SetupSendFileCommand(rootCommand); +CliConfiguration cli = new(rootCommand); +return await cli.InvokeAsync(args); diff --git a/tools/SendBlobs/SendBlobs.csproj b/tools/SendBlobs/SendBlobs.csproj index 93c44067294..df746e10d83 100644 --- a/tools/SendBlobs/SendBlobs.csproj +++ b/tools/SendBlobs/SendBlobs.csproj @@ -12,7 +12,7 @@ - + diff --git a/tools/SendBlobs/SetupCli.cs b/tools/SendBlobs/SetupCli.cs index 43352f1a5ee..c64e0b09770 100644 --- a/tools/SendBlobs/SetupCli.cs +++ b/tools/SendBlobs/SetupCli.cs @@ -1,84 +1,107 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using McMaster.Extensions.CommandLineUtils; using Nethermind.Cli; using Nethermind.Cli.Console; using Nethermind.Consensus; -using Nethermind.Core; using Nethermind.Crypto; using Nethermind.Int256; using Nethermind.Logging; using Nethermind.Serialization.Json; +using System.CommandLine; namespace SendBlobs; internal static class SetupCli { - public static void SetupExecute(CommandLineApplication app) + public static void SetupExecute(CliRootCommand command) { - app.HelpOption("--help"); - - CommandOption rpcUrlOption = app.Option("--rpcurl ", "Url of the Json RPC.", CommandOptionType.SingleValue); - CommandOption blobTxOption = app.Option("--bloboptions ", "Options in format '10x1-2', '2x5-5' etc. for the blobs.", CommandOptionType.MultipleValue); - CommandOption privateKeyOption = app.Option("--privatekey ", "The key to use for sending blobs.", CommandOptionType.SingleValue); - CommandOption privateKeyFileOption = app.Option("--keyfile ", "File containing private keys that each blob tx will be send from.", CommandOptionType.SingleValue); - CommandOption receiverOption = app.Option("--receiveraddress ", "Receiver address of the blobs.", CommandOptionType.SingleValue); - CommandOption maxFeePerDataGasOptionObsolete = app.Option("--maxfeeperdatagas ", "(Optional) Set the maximum fee per blob data.", CommandOptionType.SingleValue); - CommandOption maxFeePerBlobGasOption = app.Option("--maxfeeperblobgas ", "(Optional) Set the maximum fee per blob data.", CommandOptionType.SingleValue); - CommandOption feeMultiplierOption = app.Option("--feemultiplier ", "(Optional) A multiplier to use for gas fees.", CommandOptionType.SingleValue); - CommandOption maxPriorityFeeGasOption = app.Option("--maxpriorityfee ", "(Optional) The maximum priority fee for each transaction.", CommandOptionType.SingleValue); - CommandOption waitOption = app.Option("--wait", "(Optional) Wait for tx inclusion.", CommandOptionType.NoValue); - - app.OnExecuteAsync(async cancellationToken => + CliOption rpcUrlOption = new("--rpcurl") + { + Description = "The URL of the JSON RPC server", + HelpName = "URL", + Required = true + }; + CliOption blobTxOption = new("--bloboptions") + { + Description = "Options in format '10x1-2', '2x5-5' etc. for the blobs", + HelpName = "options" + }; + CliOption privateKeyOption = new("--privatekey") + { + Description = "The key to use for sending blobs", + HelpName = "key" + }; + CliOption privateKeyFileOption = new("--keyfile") + { + Description = "File containing private keys that each blob tx will be send from", + HelpName = "path" + }; + CliOption receiverOption = new("--receiveraddress") + { + Description = "Receiver address of the blobs", + HelpName = "address", + Required = true + }; + CliOption maxFeePerDataGasOptionObsolete = new("--maxfeeperdatagas") + { + Description = "Set the maximum fee per blob data", + HelpName = "fee" + }; + CliOption maxFeePerBlobGasOption = new("--maxfeeperblobgas") + { + Description = "Set the maximum fee per blob data", + HelpName = "fee" + }; + CliOption feeMultiplierOption = new("--feemultiplier") + { + DefaultValueFactory = r => 1UL, + Description = "A multiplier to use for gas fees", + HelpName = "value" + }; + CliOption maxPriorityFeeGasOption = new("--maxpriorityfee") + { + Description = "The maximum priority fee for each transaction", + HelpName = "fee" + }; + CliOption waitOption = new("--wait") { Description = "Wait for tx inclusion" }; + + command.Add(rpcUrlOption); + command.Add(blobTxOption); + command.Add(privateKeyOption); + command.Add(privateKeyFileOption); + command.Add(receiverOption); + command.Add(maxFeePerDataGasOptionObsolete); + command.Add(maxFeePerBlobGasOption); + command.Add(feeMultiplierOption); + command.Add(maxPriorityFeeGasOption); + command.Add(waitOption); + command.SetAction((parseResult, cancellationToken) => { - string rpcUrl = rpcUrlOption.Value()!; - (int count, int blobCount, string @break)[] blobTxCounts = ParseTxOptions(blobTxOption.Value()); - PrivateKey[] privateKeys; - if (privateKeyFileOption.HasValue()) - privateKeys = File.ReadAllLines(privateKeyFileOption.Value()!).Select(k => new PrivateKey(k)).ToArray(); - else if (privateKeyOption.HasValue()) - privateKeys = [new PrivateKey(privateKeyOption.Value()!)]; + string? privateKeyFileValue = parseResult.GetValue(privateKeyFileOption); + string? privateKeyValue = parseResult.GetValue(privateKeyOption); + + if (privateKeyFileValue is not null) + privateKeys = File.ReadAllLines(privateKeyFileValue).Select(k => new PrivateKey(k)).ToArray(); + else if (privateKeyValue is not null) + privateKeys = [new PrivateKey(privateKeyValue)]; else { Console.WriteLine("Missing private key argument."); - app.ShowHelp(); - return; - } - - string receiver = receiverOption.Value()!; - - UInt256? maxFeePerBlobGas = null; - if (maxFeePerBlobGasOption.HasValue()) - { - ulong.TryParse(maxFeePerBlobGasOption.Value(), out ulong shortMaxFeePerBlobGas); - maxFeePerBlobGas = shortMaxFeePerBlobGas; - } - else if (maxFeePerDataGasOptionObsolete.HasValue()) - { - ulong.TryParse(maxFeePerDataGasOptionObsolete.Value(), out ulong shortMaxFeePerBlobGas); - maxFeePerBlobGas = shortMaxFeePerBlobGas; + return Task.CompletedTask; } - ulong feeMultiplier = 1; - if (feeMultiplierOption.HasValue()) - ulong.TryParse(feeMultiplierOption.Value(), out feeMultiplier); - - UInt256 maxPriorityFeeGasArgs = 0; - if (maxPriorityFeeGasOption.HasValue()) UInt256.TryParse(maxPriorityFeeGasOption.Value()!, out maxPriorityFeeGasArgs); - - bool wait = waitOption.HasValue(); + BlobSender sender = new(parseResult.GetValue(rpcUrlOption)!, SimpleConsoleLogManager.Instance); - BlobSender sender = new(rpcUrl, SimpleConsoleLogManager.Instance); - await sender.SendRandomBlobs( - blobTxCounts, + return sender.SendRandomBlobs( + ParseTxOptions(parseResult.GetValue(blobTxOption)), privateKeys, - receiver, - maxFeePerBlobGas, - feeMultiplier, - maxPriorityFeeGasArgs, - wait); + parseResult.GetValue(receiverOption)!, + parseResult.GetValue(maxFeePerBlobGasOption) ?? parseResult.GetValue(maxFeePerDataGasOptionObsolete), + parseResult.GetValue(feeMultiplierOption), + parseResult.GetValue(maxPriorityFeeGasOption), + parseResult.GetValue(waitOption)); }); } @@ -97,7 +120,6 @@ private static (int count, int blobCount, string @break)[] ParseTxOptions(string nextComma = SplitToNext(chars[offSet..], ','); ReadOnlySpan @break = SplitToNext(nextComma, '-', true); - ReadOnlySpan rest = nextComma[..(nextComma.Length - (@break.Length == 0 ? 0 : @break.Length + 1))]; ReadOnlySpan count = SplitToNext(rest, 'x'); ReadOnlySpan txCount = SplitToNext(rest, 'x', true); @@ -121,70 +143,127 @@ private static ReadOnlySpan SplitToNext(ReadOnlySpan line, char sepa return returnRemainder ? line[(i + 1)..] : line[..i]; } - public static void SetupDistributeCommand(CommandLineApplication app) + public static void SetupDistributeCommand(CliCommand root) { - app.Command("distribute", (command) => + CliCommand command = new("distribute") { - command.Description = "Distribute funds from an address to a number of new addresses."; - command.HelpOption("--help"); - - CommandOption rpcUrlOption = command.Option("--rpcurl ", "Url of the Json RPC.", CommandOptionType.SingleValue, c => c.IsRequired()); - CommandOption privateKeyOption = command.Option("--privatekey ", "The private key to distribute funds from.", CommandOptionType.SingleValue, c => c.IsRequired()); - CommandOption keyNumberOption = command.Option("--number ", "The number of new addresses/keys to make.", CommandOptionType.SingleValue); - CommandOption keyFileOption = command.Option("--keyfile ", "File where the newly generated keys are written.", CommandOptionType.SingleValue); - CommandOption maxPriorityFeeGasOption = command.Option("--maxpriorityfee ", "(Optional) The maximum priority fee for each transaction.", CommandOptionType.SingleValue); - CommandOption maxFeeOption = command.Option("--maxfee ", "(Optional) The maxFeePerGas fee paid for each transaction.", CommandOptionType.SingleValue); - - command.OnExecute(async () => - { - uint keysToMake = keyNumberOption.HasValue() ? uint.Parse(keyNumberOption.Value()!) : 0; - PrivateKey privateKey = new(privateKeyOption.Value()!); - - ILogger logger = SimpleConsoleLogManager.Instance.GetClassLogger(); - INodeManager nodeManager = InitNodeManager(rpcUrlOption.Value()!, logger); - - string? chainIdString = await nodeManager.Post("eth_chainId") ?? "1"; - ulong chainId = HexConvert.ToUInt64(chainIdString); - - Signer signer = new Signer(chainId, privateKey, SimpleConsoleLogManager.Instance); - UInt256 maxFee = maxFeeOption.HasValue() ? UInt256.Parse(maxFeeOption.Value()!) : 0; - UInt256 maxPriorityFee = maxPriorityFeeGasOption.HasValue() ? UInt256.Parse(maxPriorityFeeGasOption.Value()!) : 0; - - FundsDistributor distributor = new FundsDistributor(nodeManager, chainId, keyFileOption.Value(), SimpleConsoleLogManager.Instance); - IEnumerable hashes = await distributor.DitributeFunds(signer, keysToMake, maxFee, maxPriorityFee); - }); + Description = "Distribute funds from an address to a number of new addresses" + }; + CliOption rpcUrlOption = new("--rpcurl") + { + Description = "The URL of the JSON RPC server", + HelpName = "URL", + Required = true + }; + CliOption privateKeyOption = new("--privatekey") + { + Description = "The private key to distribute funds from", + HelpName = "key", + Required = true, + }; + CliOption keyNumberOption = new("--number") + { + Description = "The number of new addresses/keys to make", + HelpName = "value" + }; + CliOption keyFileOption = new("--keyfile") + { + Description = "File where the newly generated keys are written", + HelpName = "path" + }; + CliOption maxPriorityFeeGasOption = new("--maxpriorityfee") + { + Description = "The maximum priority fee for each transaction", + HelpName = "fee" + }; + CliOption maxFeeOption = new("--maxfee") + { + Description = "The maxFeePerGas fee paid for each transaction", + HelpName = "fee" + }; + + command.Add(rpcUrlOption); + command.Add(privateKeyOption); + command.Add(keyNumberOption); + command.Add(keyFileOption); + command.Add(maxPriorityFeeGasOption); + command.Add(maxFeeOption); + command.SetAction(async (parseResult, cancellationToken) => + { + INodeManager nodeManager = InitNodeManager( + parseResult.GetValue(rpcUrlOption)!, SimpleConsoleLogManager.Instance.GetClassLogger()); + + string? chainIdString = await nodeManager.Post("eth_chainId") ?? "1"; + ulong chainId = HexConvert.ToUInt64(chainIdString); + + Signer signer = new(chainId, new PrivateKey(parseResult.GetValue(privateKeyOption)!), + SimpleConsoleLogManager.Instance); + + FundsDistributor distributor = new FundsDistributor( + nodeManager, chainId, parseResult.GetValue(keyFileOption), SimpleConsoleLogManager.Instance); + IEnumerable hashes = await distributor.DitributeFunds( + signer, + parseResult.GetValue(keyNumberOption), + parseResult.GetValue(maxFeeOption), + parseResult.GetValue(maxPriorityFeeGasOption)); }); + + root.Add(command); } - public static void SetupReclaimCommand(CommandLineApplication app) + public static void SetupReclaimCommand(CliCommand root) { - app.Command("reclaim", (command) => + CliCommand command = new("reclaim") { - command.Description = "Reclaim funds distributed from the 'distribute' command."; - command.HelpOption("--help"); - - CommandOption rpcUrlOption = command.Option("--rpcurl ", "Url of the Json RPC.", CommandOptionType.SingleValue, c => c.IsRequired()); - CommandOption receiverOption = command.Option("--receiveraddress ", "The address to send the funds to.", CommandOptionType.SingleValue, c => c.IsRequired()); - CommandOption keyFileOption = command.Option("--keyfile ", "File of the private keys to reclaim from.", CommandOptionType.SingleValue); - CommandOption maxPriorityFeeGasOption = command.Option("--maxpriorityfee ", "(Optional) The maximum priority fee for each transaction.", CommandOptionType.SingleValue); - CommandOption maxFeeOption = command.Option("--maxfee ", "(Optional) The maxFeePerGas paid for each transaction.", CommandOptionType.SingleValue); - - command.OnExecute(async () => - { - INodeManager nodeManager = InitNodeManager(rpcUrlOption.Value()!, SimpleConsoleLogManager.Instance.GetClassLogger()); - - string? chainIdString = await nodeManager.Post("eth_chainId") ?? "1"; - ulong chainId = HexConvert.ToUInt64(chainIdString); - - Address beneficiary = new Address(receiverOption.Value()!); + Description = "Reclaim funds distributed from the 'distribute' command" + }; + CliOption rpcUrlOption = new("--rpcurl") + { + Description = "The URL of the JSON RPC server", + HelpName = "URL", + Required = true + }; + CliOption receiverOption = new("--receiveraddress") + { + Description = "The address to send the funds to", + HelpName = "address", + Required = true, + }; + CliOption keyFileOption = new("--keyfile") + { + Description = "File of the private keys to reclaim from", + HelpName = "path" + }; + CliOption maxPriorityFeeGasOption = new("--maxpriorityfee") + { + Description = "The maximum priority fee for each transaction", + HelpName = "fee" + }; + CliOption maxFeeOption = new("--maxfee") + { + Description = "The maxFeePerGas fee paid for each transaction", + HelpName = "fee" + }; + + command.Add(rpcUrlOption); + command.Add(keyFileOption); + command.Add(maxPriorityFeeGasOption); + command.Add(maxFeeOption); + command.SetAction(async (parseResult, cancellationToken) => + { + INodeManager nodeManager = InitNodeManager(parseResult.GetValue(rpcUrlOption)!, SimpleConsoleLogManager.Instance.GetClassLogger()); - UInt256 maxFee = maxFeeOption.HasValue() ? UInt256.Parse(maxFeeOption.Value()!) : 0; - UInt256 maxPriorityFee = maxPriorityFeeGasOption.HasValue() ? UInt256.Parse(maxPriorityFeeGasOption.Value()!) : 0; + string? chainIdString = await nodeManager.Post("eth_chainId") ?? "1"; + ulong chainId = HexConvert.ToUInt64(chainIdString); - FundsDistributor distributor = new FundsDistributor(nodeManager, chainId, keyFileOption.Value(), SimpleConsoleLogManager.Instance); - IEnumerable hashes = await distributor.ReclaimFunds(beneficiary, maxFee, maxPriorityFee); - }); + FundsDistributor distributor = new(nodeManager, chainId, parseResult.GetValue(keyFileOption), SimpleConsoleLogManager.Instance); + IEnumerable hashes = await distributor.ReclaimFunds( + new(parseResult.GetValue(receiverOption)!), + parseResult.GetValue(maxFeeOption), + parseResult.GetValue(maxPriorityFeeGasOption)); }); + + root.Add(command); } public static INodeManager InitNodeManager(string rpcUrl, ILogger logger) @@ -198,71 +277,79 @@ public static INodeManager InitNodeManager(string rpcUrl, ILogger logger) return nodeManager; } - public static void SetupSendFileCommand(CommandLineApplication app) + public static void SetupSendFileCommand(CliCommand root) { - app.Command("send", (command) => + CliCommand command = new("send") { - command.Description = "Sends a file"; - command.HelpOption("--help"); - - CommandOption fileOption = command.Option("--file ", "File to send as is.", CommandOptionType.SingleValue, c => c.IsRequired()); - CommandOption rpcUrlOption = command.Option("--rpcurl ", "Url of the Json RPC.", CommandOptionType.SingleValue, c => c.IsRequired()); - CommandOption privateKeyOption = command.Option("--privatekey ", "The key to use for sending blobs.", CommandOptionType.SingleValue); - CommandOption receiverOption = command.Option("--receiveraddress ", "Receiver address of the blobs.", CommandOptionType.SingleValue, c => c.IsRequired()); - CommandOption maxFeePerBlobGasOption = command.Option("--maxfeeperblobgas ", "(Optional) Set the maximum fee per blob data.", CommandOptionType.SingleValue); - CommandOption feeMultiplierOption = command.Option("--feemultiplier ", "(Optional) A multiplier to use for gas fees.", CommandOptionType.SingleValue); - CommandOption maxPriorityFeeGasOption = command.Option("--maxpriorityfee ", "(Optional) The maximum priority fee for each transaction.", CommandOptionType.SingleValue); - CommandOption waitOption = app.Option("--wait", "(Optional) Wait for tx inclusion.", CommandOptionType.NoValue); - - command.OnExecuteAsync(async cancellationToken => - { - string rpcUrl = rpcUrlOption.Value()!; - - PrivateKey privateKey; - - if (privateKeyOption.HasValue()) - privateKey = new PrivateKey(privateKeyOption.Value()!); - else - { - Console.WriteLine("Missing private key argument."); - app.ShowHelp(); - return; - } - - string receiver = receiverOption.Value()!; - - UInt256 maxFeePerBlobGas = 1000; - if (maxFeePerBlobGasOption.HasValue()) - { - ulong.TryParse(maxFeePerBlobGasOption.Value(), out ulong shortMaxFeePerBlobGas); - maxFeePerBlobGas = shortMaxFeePerBlobGas; - } - - ulong feeMultiplier = 1; - if (feeMultiplierOption.HasValue()) - ulong.TryParse(feeMultiplierOption.Value(), out feeMultiplier); - - UInt256? maxPriorityFeeGas = null; - if (maxPriorityFeeGasOption.HasValue() && UInt256.TryParse(maxPriorityFeeGasOption.Value()!, out UInt256 maxPriorityFeeGasParsed)) - { - maxPriorityFeeGas = maxPriorityFeeGasParsed; - } - - bool wait = waitOption.HasValue(); - - byte[] data = File.ReadAllBytes(fileOption.Value()!); - - BlobSender sender = new(rpcUrl, SimpleConsoleLogManager.Instance); - await sender.SendData( - data, - privateKey, - receiver, - maxFeePerBlobGas, - feeMultiplier, - maxPriorityFeeGas, - wait); - }); + Description = "Sends a file" + }; + CliOption fileOption = new("--file") + { + Description = "File to send as is", + HelpName = "path", + Required = true + }; + CliOption rpcUrlOption = new("--rpcurl") + { + Description = "The URL of the JSON RPC server", + HelpName = "URL", + Required = true + }; + CliOption privateKeyOption = new("--privatekey") + { + Description = "The key to use for sending blobs", + HelpName = "key", + Required = true, + }; + CliOption receiverOption = new("--receiveraddress") + { + Description = "Receiver address of the blobs", + HelpName = "address", + Required = true, + }; + CliOption maxFeePerBlobGasOption = new("--maxfeeperblobgas") + { + DefaultValueFactory = r => 1000, + Description = "Set the maximum fee per blob data", + HelpName = "fee" + }; + CliOption feeMultiplierOption = new("--feemultiplier") + { + DefaultValueFactory = r => 1UL, + Description = "A multiplier to use for gas fees", + HelpName = "value" + }; + CliOption maxPriorityFeeGasOption = new("--maxpriorityfee") + { + Description = "The maximum priority fee for each transaction", + HelpName = "fee" + }; + CliOption waitOption = new("--wait") { Description = "Wait for tx inclusion" }; + + command.Add(fileOption); + command.Add(rpcUrlOption); + command.Add(privateKeyOption); + command.Add(receiverOption); + command.Add(maxFeePerBlobGasOption); + command.Add(feeMultiplierOption); + command.Add(maxPriorityFeeGasOption); + command.Add(waitOption); + command.SetAction((parseResult, cancellationToken) => + { + PrivateKey privateKey = new(parseResult.GetValue(privateKeyOption)!); + byte[] data = File.ReadAllBytes(parseResult.GetValue(fileOption)!); + BlobSender sender = new(parseResult.GetValue(rpcUrlOption)!, SimpleConsoleLogManager.Instance); + + return sender.SendData( + data, + privateKey, + parseResult.GetValue(receiverOption)!, + parseResult.GetValue(maxFeePerBlobGasOption), + parseResult.GetValue(feeMultiplierOption), + parseResult.GetValue(maxPriorityFeeGasOption), + parseResult.GetValue(waitOption)); }); - } + root.Add(command); + } } diff --git a/tools/nuget.config b/tools/nuget.config new file mode 100644 index 00000000000..401f058f2ab --- /dev/null +++ b/tools/nuget.config @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + +