Merge branch 'master' into feature/system-commandline-2

src/Nethermind/Directory.Packages.props

@@ -45,7 +45,7 @@
     <PackageVersion Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.1" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
     <PackageVersion Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.0" />
-    <PackageVersion Include="Nethermind.Crypto.Bls" Version="1.0.3" />
+    <PackageVersion Include="Nethermind.Crypto.Bls" Version="1.0.4" />
     <PackageVersion Include="Nethermind.Crypto.Pairings" Version="1.1.1" />
     <PackageVersion Include="Nethermind.Crypto.SecP256k1" Version="1.2.2" />
     <PackageVersion Include="Nethermind.DotNetty.Buffers" Version="1.0.1" />

src/Nethermind/Ethereum.Test.Base/AccountState.cs

@@ -8,9 +8,9 @@ namespace Ethereum.Test.Base
 {
     public class AccountState
     {
-        public byte[] Code { get; set; }
+        public byte[] Code { get; set; } = [];
         public UInt256 Balance { get; set; }
         public UInt256 Nonce { get; set; }
-        public Dictionary<UInt256, byte[]> Storage { get; set; }
+        public Dictionary<UInt256, byte[]> Storage { get; set; } = new();
     }
 }

src/Nethermind/Ethereum.Test.Base/BlockchainTestJson.cs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System.Collections.Generic;
+using Nethermind.Core;
 using Nethermind.Core.Crypto;
 using Nethermind.Core.Specs;
 
@@ -24,8 +25,8 @@ public class BlockchainTestJson
         public TestBlockJson[]? Blocks { get; set; }
         public TestBlockHeaderJson? GenesisBlockHeader { get; set; }
 
-        public Dictionary<string, AccountStateJson>? Pre { get; set; }
-        public Dictionary<string, AccountStateJson>? PostState { get; set; }
+        public Dictionary<Address, AccountState>? Pre { get; set; }
+        public Dictionary<Address, AccountState>? PostState { get; set; }
 
         public Hash256? PostStateHash { get; set; }
 

src/Nethermind/Ethereum.Test.Base/GeneralStateTestJson.cs

@@ -3,6 +3,7 @@
 
 using System.Collections.Generic;
 using System.Text.Json.Serialization;
+using Nethermind.Core;
 
 namespace Ethereum.Test.Base
 {
@@ -12,7 +13,7 @@ public class GeneralStateTestJson
         public GeneralStateTestInfoJson? Info { get; set; }
         public GeneralStateTestEnvJson? Env { get; set; }
         public Dictionary<string, PostStateJson[]>? Post { get; set; }
-        public Dictionary<string, AccountStateJson>? Pre { get; set; }
+        public Dictionary<Address, AccountState>? Pre { get; set; }
         public string? SealEngine { get; set; }
         public string? LoadFailure { get; set; }
         public TransactionJson? Transaction { get; set; }

src/Nethermind/Ethereum.Test.Base/GeneralTestBase.cs

@@ -4,14 +4,11 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using Nethermind.Blockchain;
-using Nethermind.Consensus.Ethash;
 using Nethermind.Consensus.Validators;
 using Nethermind.Core;
 using Nethermind.Core.Crypto;
 using Nethermind.Core.Extensions;
 using Nethermind.Core.Specs;
-using Nethermind.Core.Test.Builders;
 using Nethermind.Crypto;
 using Nethermind.Db;
 using Nethermind.Int256;
@@ -89,7 +86,7 @@ protected EthereumTestResult RunTest(GeneralStateTest test, ITxTracer txTracer)
                 codeInfoRepository,
                 _logManager);
 
-            InitializeTestState(test, stateProvider, specProvider);
+            InitializeTestState(test.Pre, stateProvider, specProvider);
 
             BlockHeader header = new(
                 test.PreviousHash,
@@ -135,8 +132,6 @@ protected EthereumTestResult RunTest(GeneralStateTest test, ITxTracer txTracer)
                 header.ExcessBlobGas = BlobGasCalculator.CalculateExcessBlobGas(parent, spec);
             }
 
-            Block block = Build.A.Block.WithTransactions(test.Transaction).WithHeader(header).TestObject;
-
             ValidationResult txIsValid = _txValidator.IsWellFormed(test.Transaction, spec);
 
             if (txIsValid)
@@ -172,9 +167,9 @@ protected EthereumTestResult RunTest(GeneralStateTest test, ITxTracer txTracer)
             return testResult;
         }
 
-        private static void InitializeTestState(GeneralStateTest test, WorldState stateProvider, ISpecProvider specProvider)
+        private static void InitializeTestState(Dictionary<Address, AccountState> preState, WorldState stateProvider, ISpecProvider specProvider)
         {
-            foreach (KeyValuePair<Address, AccountState> accountState in test.Pre)
+            foreach (KeyValuePair<Address, AccountState> accountState in preState)
             {
                 foreach (KeyValuePair<UInt256, byte[]> storageItem in accountState.Value.Storage)
                 {
@@ -192,22 +187,6 @@ private static void InitializeTestState(GeneralStateTest test, WorldState stateP
             stateProvider.Reset();
         }
 
-        private bool IsValidBlock(Block block, ISpecProvider specProvider)
-        {
-            IBlockTree blockTree = Build.A.BlockTree()
-                .WithSpecProvider(specProvider)
-                .WithoutSettingHead
-                .TestObject;
-
-            var difficultyCalculator = new EthashDifficultyCalculator(specProvider);
-            var sealer = new EthashSealValidator(_logManager, difficultyCalculator, new CryptoRandom(), new Ethash(_logManager), Timestamper.Default);
-            IHeaderValidator headerValidator = new HeaderValidator(blockTree, sealer, specProvider, _logManager);
-            IUnclesValidator unclesValidator = new UnclesValidator(blockTree, headerValidator, _logManager);
-            IBlockValidator blockValidator = new BlockValidator(_txValidator, headerValidator, unclesValidator, specProvider, _logManager);
-
-            return blockValidator.ValidateOrphanedBlock(block, out _);
-        }
-
         private List<string> RunAssertions(GeneralStateTest test, IWorldState stateProvider)
         {
             List<string> differences = [];

src/Nethermind/Ethereum.Test.Base/JsonToEthereumTest.cs

@@ -21,13 +21,13 @@ namespace Ethereum.Test.Base
 {
     public static class JsonToEthereumTest
     {
-        private static IReleaseSpec ParseSpec(string network)
+        public static IReleaseSpec ParseSpec(string network)
         {
             network = network.Replace("EIP150", "TangerineWhistle");
             network = network.Replace("EIP158", "SpuriousDragon");
             network = network.Replace("DAO", "Dao");
-            network = network.Replace("Merged", "GrayGlacier");
-            network = network.Replace("Merge", "GrayGlacier");
+            network = network.Replace("Merged", "Paris");
+            network = network.Replace("Merge", "Paris");
             network = network.Replace("London+3540+3670", "Shanghai");
             network = network.Replace("GrayGlacier+3540+3670", "Shanghai");
             network = network.Replace("GrayGlacier+3860", "Shanghai");
@@ -55,6 +55,7 @@ private static IReleaseSpec ParseSpec(string network)
                 "Istanbul" => Istanbul.Instance,
                 "Berlin" => Berlin.Instance,
                 "London" => London.Instance,
+                "ArrowGlacier" => ArrowGlacier.Instance,
                 "GrayGlacier" => GrayGlacier.Instance,
                 "Shanghai" => Shanghai.Instance,
                 "Cancun" => Cancun.Instance,
@@ -112,17 +113,6 @@ public static BlockHeader Convert(TestBlockHeaderJson? headerJson)
             return header;
         }
 
-        public static Block Convert(PostStateJson postStateJson, TestBlockJson testBlockJson)
-        {
-            BlockHeader? header = Convert(testBlockJson.BlockHeader);
-            BlockHeader[] uncles = testBlockJson.UncleHeaders?.Select(Convert).ToArray()
-                                   ?? Array.Empty<BlockHeader>();
-            Transaction[] transactions = testBlockJson.Transactions?.Select(Convert).ToArray()
-                                         ?? Array.Empty<Transaction>();
-            Block block = new(header, transactions, uncles);
-            return block;
-        }
-
         public static Transaction Convert(PostStateJson postStateJson, TransactionJson transactionJson)
         {
             Transaction transaction = new();
@@ -216,7 +206,7 @@ public static Transaction Convert(PostStateJson postStateJson, TransactionJson t
             return transaction;
         }
 
-        private static void ProcessAccessList(AccessListItemJson[]? accessList, AccessList.Builder builder)
+        public static void ProcessAccessList(AccessListItemJson[]? accessList, AccessList.Builder builder)
         {
             foreach (AccessListItemJson accessListItemJson in accessList ?? Array.Empty<AccessListItemJson>())
             {
@@ -242,20 +232,6 @@ public static Transaction Convert(LegacyTransactionJson transactionJson)
             return transaction;
         }
 
-        private static AccountState Convert(AccountStateJson accountStateJson)
-        {
-            AccountState state = new();
-            state.Balance = accountStateJson.Balance is not null ? Bytes.FromHexString(accountStateJson.Balance).ToUInt256() : 0;
-            state.Code = accountStateJson.Code is not null ? Bytes.FromHexString(accountStateJson.Code) : Array.Empty<byte>();
-            state.Nonce = accountStateJson.Nonce is not null ? Bytes.FromHexString(accountStateJson.Nonce).ToUInt256() : 0;
-            state.Storage = accountStateJson.Storage is not null
-                ? accountStateJson.Storage.ToDictionary(
-                    p => Bytes.FromHexString(p.Key).ToUInt256(),
-                    p => Bytes.FromHexString(p.Value))
-                : new();
-            return state;
-        }
-
         public static IEnumerable<GeneralStateTest> Convert(string name, GeneralStateTestJson testJson)
         {
             if (testJson.LoadFailure is not null)
@@ -294,7 +270,7 @@ public static IEnumerable<GeneralStateTest> Convert(string name, GeneralStateTes
                     test.ParentExcessBlobGas = testJson.Env.ParentExcessBlobGas;
                     test.PostReceiptsRoot = stateJson.Logs;
                     test.PostHash = stateJson.Hash;
-                    test.Pre = testJson.Pre.ToDictionary(p => new Address(p.Key), p => Convert(p.Value));
+                    test.Pre = testJson.Pre.ToDictionary(p => p.Key, p => p.Value);
                     test.Transaction = Convert(stateJson, testJson.Transaction);
 
                     blockchainTests.Add(test);
@@ -321,7 +297,7 @@ public static BlockchainTest Convert(string name, BlockchainTestJson testJson)
             test.GenesisRlp = testJson.GenesisRlp is null ? null : new Rlp(Bytes.FromHexString(testJson.GenesisRlp));
             test.GenesisBlockHeader = testJson.GenesisBlockHeader;
             test.Blocks = testJson.Blocks;
-            test.Pre = testJson.Pre.ToDictionary(p => new Address(p.Key), p => Convert(p.Value));
+            test.Pre = testJson.Pre.ToDictionary(p => p.Key, p => p.Value);
 
             HalfBlockchainTestJson half = testJson as HalfBlockchainTestJson;
             if (half is not null)
@@ -330,7 +306,7 @@ public static BlockchainTest Convert(string name, BlockchainTestJson testJson)
             }
             else
             {
-                test.PostState = testJson.PostState?.ToDictionary(p => new Address(p.Key), p => Convert(p.Value));
+                test.PostState = testJson.PostState?.ToDictionary(p => p.Key, p => p.Value);
                 test.PostStateRoot = testJson.PostStateHash;
             }
 

src/Nethermind/Nethermind.Blockchain.Test/BeaconBlockRootHandlerTests.cs

@@ -124,7 +124,7 @@ public void Test_StoreBeaconRoot_AccessListIsNull()
         BlockHeader header = Build.A.BlockHeader.TestObject;
         Block block = Build.A.Block.WithHeader(header).TestObject;
 
-        _beaconBlockRootHandler.StoreBeaconRoot(block, Cancun.Instance);
+        _beaconBlockRootHandler.StoreBeaconRoot(block, Cancun.Instance, NullTxTracer.Instance);
 
         _transactionProcessor.DidNotReceive().Execute(Arg.Any<Transaction>(), Arg.Any<BlockExecutionContext>(), Arg.Any<ITxTracer>());
     }
@@ -136,7 +136,7 @@ public void Test_StoreBeaconRoot_AccessListNotNull()
         Block block = Build.A.Block.WithHeader(header).TestObject;
         _worldState.AccountExists(Arg.Any<Address>()).Returns(true);
 
-        _beaconBlockRootHandler.StoreBeaconRoot(block, Cancun.Instance);
+        _beaconBlockRootHandler.StoreBeaconRoot(block, Cancun.Instance, NullTxTracer.Instance);
 
         Transaction transaction = new()
         {

src/Nethermind/Nethermind.Blockchain.Test/Receipts/PersistentReceiptStorageTests.cs

@@ -7,6 +7,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using FluentAssertions;
+using FluentAssertions.Equivalency;
 using Nethermind.Blockchain.Blocks;
 using Nethermind.Blockchain.Receipts;
 using Nethermind.Core;
@@ -18,6 +19,7 @@
 using Nethermind.Db;
 using Nethermind.Serialization.Rlp;
 using Nethermind.Specs;
+using Nethermind.Specs.Forks;
 using NSubstitute;
 using NSubstitute.Core;
 using NUnit.Framework;
@@ -29,6 +31,7 @@ namespace Nethermind.Blockchain.Test.Receipts;
 [TestFixture(false)]
 public class PersistentReceiptStorageTests
 {
+    private TestSpecProvider _specProvider = new TestSpecProvider(Byzantium.Instance);
     private TestMemColumnsDb<ReceiptsColumns> _receiptsDb = null!;
     private ReceiptsRecovery _receiptsRecovery = null!;
     private IBlockTree _blockTree = null!;
@@ -46,10 +49,9 @@ public PersistentReceiptStorageTests(bool useCompactReceipts)
     [SetUp]
     public void SetUp()
     {
-        MainnetSpecProvider specProvider = MainnetSpecProvider.Instance;
-        EthereumEcdsa ethereumEcdsa = new(specProvider.ChainId);
+        EthereumEcdsa ethereumEcdsa = new(_specProvider.ChainId);
         _receiptConfig = new ReceiptConfig();
-        _receiptsRecovery = new(ethereumEcdsa, specProvider);
+        _receiptsRecovery = new(ethereumEcdsa, _specProvider);
         _receiptsDb = new TestMemColumnsDb<ReceiptsColumns>();
         _receiptsDb.GetColumnDb(ReceiptsColumns.Blocks).Set(Keccak.Zero, Array.Empty<byte>());
         _blockTree = Substitute.For<IBlockTree>();
@@ -68,7 +70,7 @@ private void CreateStorage()
         _decoder = new ReceiptArrayStorageDecoder(_useCompactReceipts);
         _storage = new PersistentReceiptStorage(
             _receiptsDb,
-            MainnetSpecProvider.Instance,
+            _specProvider,
             _receiptsRecovery,
             _blockTree,
             _blockStore,
@@ -111,9 +113,10 @@ public void Adds_and_retrieves_receipts_for_block()
     {
         var (block, receipts) = InsertBlock();
 
-        _storage.Get(block).Should().BeEquivalentTo(receipts);
+        _storage.ClearCache();
+        _storage.Get(block).Should().BeEquivalentTo(receipts, ReceiptCompareOpt);
         // second should be from cache
-        _storage.Get(block).Should().BeEquivalentTo(receipts);
+        _storage.Get(block).Should().BeEquivalentTo(receipts, ReceiptCompareOpt);
     }
 
     [Test]
@@ -128,6 +131,38 @@ public void Adds_should_prefix_key_with_blockNumber()
         _receiptsDb.GetColumnDb(ReceiptsColumns.Blocks)[blockNumPrefixed].Should().NotBeNull();
     }
 
+    [Test]
+    public void Adds_should_forward_write_flags()
+    {
+        (Block block, _) = InsertBlock(writeFlags: WriteFlags.DisableWAL);
+
+        Span<byte> blockNumPrefixed = stackalloc byte[40];
+        block.Number.ToBigEndianByteArray().CopyTo(blockNumPrefixed); // TODO: We don't need to create an array here...
+        block.Hash!.Bytes.CopyTo(blockNumPrefixed[8..]);
+
+        TestMemDb blockDb = (TestMemDb)_receiptsDb.GetColumnDb(ReceiptsColumns.Blocks);
+
+        blockDb.KeyWasWrittenWithFlags(blockNumPrefixed.ToArray(), WriteFlags.DisableWAL);
+    }
+
+    [Test, MaxTime(Timeout.MaxTestTime)]
+    public void Get_receipts_for_block_without_recovering_sender()
+    {
+        var (block, receipts) = InsertBlock();
+        foreach (Transaction tx in block.Transactions)
+        {
+            tx.SenderAddress = null;
+        }
+
+        _storage.ClearCache();
+        _storage.Get(block, recoverSender: false).Should().BeEquivalentTo(receipts, ReceiptCompareOpt);
+
+        foreach (Transaction tx in block.Transactions)
+        {
+            tx.SenderAddress.Should().BeNull();
+        }
+    }
+
     [Test]
     public void Adds_should_attempt_hash_key_first_if_inserted_with_hashkey()
     {
@@ -455,12 +490,18 @@ public void When_NewHeadBlock_ClearOldTxIndex()
         return (block, receipts);
     }
 
-    private (Block block, TxReceipt[] receipts) InsertBlock(Block? block = null, bool isFinalized = false, long? headNumber = null)
+    private (Block block, TxReceipt[] receipts) InsertBlock(Block? block = null, bool isFinalized = false, long? headNumber = null, WriteFlags writeFlags = WriteFlags.None)
     {
         (block, TxReceipt[] receipts) = PrepareBlock(block, isFinalized, headNumber);
-        _storage.Insert(block, receipts);
+        _storage.Insert(block, receipts, writeFlags: writeFlags);
         _receiptsRecovery.TryRecover(new ReceiptRecoveryBlock(block), receipts);
 
         return (block, receipts);
     }
+
+    private EquivalencyAssertionOptions<TxReceipt> ReceiptCompareOpt(EquivalencyAssertionOptions<TxReceipt> opts)
+    {
+        return opts
+            .Excluding(su => su.Error);
+    }
 }

src/Nethermind/Nethermind.Blockchain/BeaconBlockRoot/BeaconBlockRootHandler.cs

@@ -42,7 +42,7 @@ public class BeaconBlockRootHandler(ITransactionProcessor processor, IWorldState
         return (eip4788ContractAddress, builder.Build());
     }
 
-    public void StoreBeaconRoot(Block block, IReleaseSpec spec)
+    public void StoreBeaconRoot(Block block, IReleaseSpec spec, ITxTracer tracer)
     {
         (Address? toAddress, AccessList? accessList) = BeaconRootsAccessList(block, spec, includeStorageCells: false);
 
@@ -62,7 +62,7 @@ public void StoreBeaconRoot(Block block, IReleaseSpec spec)
 
             transaction.Hash = transaction.CalculateHash();
 
-            processor.Execute(transaction, header, NullTxTracer.Instance);
+            processor.Execute(transaction, header, tracer);
         }
     }
 }