Merge remote-tracking branch 'origin/master' into feature/rpc-state-o…

…verride # Conflicts: # src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs
NethermindEth · Aug 26, 2024 · 37eb5e3 · 37eb5e3
2 parents c8fa006 + e856de5
commit 37eb5e3
Show file tree

Hide file tree

Showing 101 changed files with 1,463 additions and 517 deletions.
diff --git a/.github/tests/assertor.yaml b/.github/tests/assertor.yaml
@@ -0,0 +1,16 @@
+participants_matrix:
+  el:
+   - el_type: nethermind
+     el_image: <DOCKER_IMAGE_PLACEHOLDER>
+   - el_type: geth
+  cl:
+   - cl_type: prysm
+   - cl_type: lighthouse
+   - cl_type: teku
+   - cl_type: lodestar
+   - cl_type: nimbus
+network_params:
+  preset: "minimal"
+additional_services:
+  - assertoor
+<ASSERTOOR_PARAMS_PLACEHOLDER>
diff --git a/.github/workflows/rpc-comparison.yml b/.github/workflows/rpc-comparison.yml
@@ -8,17 +8,6 @@ on:
         description: "A comma-separated list of ips allowed to connect to the node"
         default: ''
         required: false
-      network:
-        description: "Network on which test should be started"
-        default: "mainnet"
-        type: choice
-        options:
-          - mainnet
-          - gnosis
-          - sepolia
-          - goerli
-          - chiado
-          - holesky
       branch_to_compare:
         type: string
         description: "A branch to compare with. Example: \"release/1.25.1\""
@@ -31,7 +20,7 @@ on:
         required: false
       is_performance_check:
         type: boolean
-        description: "When checked, attempts to check RPC performance (because of Linode limitations it would work on single VM). When not checked equality mode will be started."
+        description: "Checked = Load Testing on identical spec. Not checked = RPC Equality Testing (no identical spec)."
         default: false
         required: false
       convert_to_paprika:
@@ -102,7 +91,7 @@ jobs:
       additional_nethermind_flags: JsonRpc.EnabledModules=[Eth,Subscribe,Trace,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Debug,Admin] JsonRpc.Timeout=3600000 log=INFO
       nethermind_repo_ref: ${{ github.ref }}
       custom_run_id: ${{ github.run_id }}
-      network: "${{ inputs.network || 'mainnet' }}"
+      network: "${{ 'mainnet' }}"
       convert_to_paprika: "${{ inputs.convert_to_paprika }}"
 
   create_compare_node:
@@ -125,7 +114,7 @@ jobs:
       additional_nethermind_flags: JsonRpc.EnabledModules=[Eth,Subscribe,Trace,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Debug,Admin] JsonRpc.Timeout=3600000 log=INFO
       nethermind_repo_ref: ${{ inputs.branch_to_compare }}
       custom_run_id: ${{ github.run_id }}
-      network: "${{ inputs.network || 'mainnet' }}"
+      network: "${{ 'mainnet' }}"
 
   aggregate_rpcs:
     name: Collect all RPC Urls and pass it further
@@ -411,8 +400,8 @@ jobs:
             
             for TEST in "${tests[@]}"; do
                 TEST_perf_result="${TEST}_perf_result"
-                echo "flood "$TEST" ${compare_to_other_branch_params} --rates 10 50 100 500 1000 --output "$TEST_perf_result" --duration 30 --deep-check | tee -a "$TEST_perf_result.txt""
-                flood "$TEST" ${compare_to_other_branch_params} --rates 10 50 100 500 1000 --output "$TEST_perf_result" --duration 30 --deep-check | tee -a "$TEST_perf_result.txt"
+                echo "flood "$TEST" ${compare_to_other_branch_params} --rates 10 10 10 100 100 100 500 --output "$TEST_perf_result" --duration 30 --deep-check | tee -a "$TEST_perf_result.txt""
+                flood "$TEST" ${compare_to_other_branch_params} --rates 10 10 10 100 100 100 500 --output "$TEST_perf_result" --duration 30 --deep-check | tee -a "$TEST_perf_result.txt"
             done
           else
             echo "flood all ${compare_to_other_branch_params} --equality | tee output_other_branch.txt"

diff --git a/.github/workflows/run-a-single-node-from-branch.yml b/.github/workflows/run-a-single-node-from-branch.yml
@@ -337,6 +337,12 @@ jobs:
       - name: Display machine specs content
         run: |
           FILE=$(ls downloaded-artifacts/machine-details | head -n 1)
+          echo "<details>" >> $GITHUB_STEP_SUMMARY
+          echo "<summary>Details of node from branch: ${{ env.CLEAN_REF }}</summary>" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          cat "downloaded-artifacts/machine-details/$FILE" >> $GITHUB_STEP_SUMMARY
+          echo "</details>" >> $GITHUB_STEP_SUMMARY
+          
           cat "downloaded-artifacts/machine-details/$FILE" | tee spec-output.txt
           rpc_url=$(grep -oP '\s{2,3}RPC URL: \K.+' spec-output.txt)
           echo "RPC URL extracted is: $rpc_url"

diff --git a/.github/workflows/sync-supported-chains.yml b/.github/workflows/sync-supported-chains.yml
@@ -226,6 +226,13 @@ jobs:
     if: success()
     runs-on: ubuntu-latest
     steps:
+      - name: Authenticate App
+        id: gh-app
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ vars.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+
       - name: Trigger notification action on test repo
         if: github.event.workflow_run.head_branch == 'master' || startsWith(github.event.workflow_run.head_branch, 'release/')
         continue-on-error: true

diff --git a/.github/workflows/test-assertoor.yml b/.github/workflows/test-assertoor.yml
@@ -0,0 +1,123 @@
+name: Assertoor tests
+
+on:
+  workflow_dispatch:
+  workflow_run:
+    workflows: ["Publish Docker image"]
+    branches: [master]
+    types:
+      - completed
+
+jobs:
+  get_tests:
+    name: "Run assertoor tests on Nethermind pairs"
+    runs-on: ubuntu-latest
+    steps:
+    - name: Authenticate App
+      id: gh-app
+      uses: actions/create-github-app-token@v1
+      with:
+        app-id: ${{ vars.APP_ID }}
+        private-key: ${{ secrets.APP_PRIVATE_KEY }}
+
+    - name: Checkout this repository
+      uses: actions/checkout@v4
+      with:
+        path: main-repo
+
+    - name: Checkout assertoor-test repository
+      uses: actions/checkout@v4
+      with:
+        repository: kamilchodola/assertoor-test
+        path: assertoor-test
+
+    - name: Set up Python
+      uses: actions/setup-python@v5.1.1
+      with:
+        python-version: '3.x'
+
+    - name: Install dependencies
+      run: pip install requests beautifulsoup4
+
+    - name: Run retriveAllTests.py script
+      run: |
+        python assertoor-test/scripts/retriveAllTests/retriveAllTests.py --exclude "pectra-dev" "verkle" "validator-lifecycle" "slashing" "mev" > /tmp/output.txt
+        cat /tmp/output.txt
+        chmod 777 /tmp/output.txt
+        echo "ASSERTOOR_PARAMS_FILE=/tmp/output.txt" >> $GITHUB_ENV
+
+    - name: Prepare docker tag
+      id: prepare_ref
+      run: |
+        REF_NAME=${{ inputs.nethermind_repo_ref || github.ref }}
+        CLEAN_REF=$(echo "${REF_NAME/refs\/heads\//}" | sed 's/[^a-zA-Z0-9._-]/-/g')
+        echo "CLEAN_REF=$CLEAN_REF" >> $GITHUB_ENV
+
+    - name: Set Repo and Org Variables
+      run: |
+        echo "ORG_NAME=${{ github.repository_owner }}" >> $GITHUB_ENV
+        echo "REPO_NAME=${{ github.event.repository.name }}" >> $GITHUB_ENV
+        
+    - name: Check if master branch and default additional_options
+      id: check_conditions
+      run: |
+        ref="${{ github.ref }}"
+        
+        if [[ $ref != refs/heads/* ]]; then
+          ref="refs/heads/$ref"
+        fi
+
+        if 
+        [[ "$ref" == "refs/heads/master" ]] && 
+        [[ "${{ steps.extract_dockerfile.outputs.dockerfile }}" == "Dockerfile" ]] && 
+        [[ "${{ steps.extract_dockerfile.outputs.build-config }}" == "release" ]]; then
+          echo "skip_docker_build=true" >> $GITHUB_OUTPUT
+        else
+          echo "skip_docker_build=false" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Trigger Docker Build Action with Cleaned Ref
+      if: steps.check_conditions.outputs.skip_docker_build != 'true'
+      uses: benc-uk/workflow-dispatch@v1
+      with:
+          workflow: publish-docker.yml
+          ref: "${{ github.ref }}"
+          token: "${{ steps.gh-app.outputs.token }}"
+          inputs: '{
+              "tag": "${{ env.CLEAN_REF }}",
+              "dockerfile": "${{ steps.extract_dockerfile.outputs.dockerfile }}",
+              "build-config": "${{ steps.extract_dockerfile.outputs.build-config }}"
+           }'
+
+    - name: Wait for Docker Build Action to complete
+      if: steps.check_conditions.outputs.skip_docker_build != 'true'
+      env:
+        GITHUB_TOKEN: ${{ steps.gh-app.outputs.token }}
+        WORKFLOW_ID: 'publish-docker.yml'
+        MAX_WAIT_MINUTES: "5"
+        INTERVAL: "5"
+        TIMEOUT: '20'
+        ORG_NAME: ${{ env.ORG_NAME }}
+        REPO_NAME: ${{ env.REPO_NAME }}
+        REF: ${{ github.ref }}
+      run: |
+        chmod +x main-repo/scripts/wait-for-workflow-completed.sh
+        ./main-repo/scripts/wait-for-workflow-completed.sh
+      working-directory: ${{ github.workspace }}
+
+    - name: Replace Placeholders in YAML
+      run: |
+        assertoor_params=$(cat ${{ env.ASSERTOOR_PARAMS_FILE}})
+        sed -i "s|<DOCKER_IMAGE_PLACEHOLDER>|nethermindeth/nethermind:${{ env.CLEAN_REF }}|g" main-repo/.github/tests/assertor.yaml
+        sed -i "/<ASSERTOOR_PARAMS_PLACEHOLDER>/ {
+          r /tmp/output.txt
+          d
+        }" main-repo/.github/tests/assertor.yaml
+        cat main-repo/.github/tests/assertor.yaml
+      
+    - name: Kurtosis Assertoor GitHub Action
+      uses: ethpandaops/kurtosis-assertoor-github-action@v1
+      with:
+        ethereum_package_url: "github.com/ethpandaops/ethereum-package"
+        ethereum_package_branch: ""
+        ethereum_package_args: main-repo/.github/tests/assertor.yaml
diff --git a/src/Nethermind/Nethermind.AuRa.Test/Contract/AuRaContractGasLimitOverrideTests.cs b/src/Nethermind/Nethermind.AuRa.Test/Contract/AuRaContractGasLimitOverrideTests.cs
@@ -2,7 +2,6 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System.Collections.Generic;
-using System.ComponentModel.DataAnnotations;
 using System.Linq;
 using System.Threading.Tasks;
 using FluentAssertions;
@@ -11,13 +10,12 @@
 using Nethermind.Consensus;
 using Nethermind.Consensus.AuRa;
 using Nethermind.Consensus.AuRa.Contracts;
-using Nethermind.Consensus.AuRa.Withdrawals;
 using Nethermind.Consensus.Processing;
 using Nethermind.Consensus.Rewards;
 using Nethermind.Consensus.Validators;
+using Nethermind.Consensus.Withdrawals;
 using Nethermind.Core;
 using Nethermind.Logging;
-using Nethermind.Trie.Pruning;
 using NUnit.Framework;
 
 namespace Nethermind.AuRa.Test.Contract;
@@ -103,7 +101,8 @@ protected override BlockProcessor CreateBlockProcessor()
                 NullWithdrawalProcessor.Instance,
                 null,
                 null,
-                GasLimitCalculator as AuRaContractGasLimitOverride);
+                GasLimitCalculator as AuRaContractGasLimitOverride,
+                preWarmer: CreateBlockCachePreWarmer());
         }
 
         protected override Task AddBlocksOnStart() => Task.CompletedTask;

diff --git a/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxCertifierFilterTests.cs b/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxCertifierFilterTests.cs
@@ -10,11 +10,11 @@
 using Nethermind.Consensus.AuRa;
 using Nethermind.Consensus.AuRa.Contracts;
 using Nethermind.Consensus.AuRa.Transactions;
-using Nethermind.Consensus.AuRa.Withdrawals;
 using Nethermind.Consensus.Processing;
 using Nethermind.Consensus.Rewards;
 using Nethermind.Consensus.Transactions;
 using Nethermind.Consensus.Validators;
+using Nethermind.Consensus.Withdrawals;
 using Nethermind.Core;
 using Nethermind.Core.Specs;
 using Nethermind.Core.Test.Builders;
@@ -157,8 +157,8 @@ protected override BlockProcessor CreateBlockProcessor()
                 LimboLogs.Instance,
                 BlockTree,
                 NullWithdrawalProcessor.Instance,
-                null
-                );
+                null,
+                preWarmer: CreateBlockCachePreWarmer());
         }
 
         protected override Task AddBlocksOnStart() => Task.CompletedTask;

diff --git a/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxPermissionFilterTest.cs b/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxPermissionFilterTest.cs
@@ -12,10 +12,10 @@
 using Nethermind.Consensus.AuRa;
 using Nethermind.Consensus.AuRa.Contracts;
 using Nethermind.Consensus.AuRa.Transactions;
-using Nethermind.Consensus.AuRa.Withdrawals;
 using Nethermind.Consensus.Processing;
 using Nethermind.Consensus.Rewards;
 using Nethermind.Consensus.Validators;
+using Nethermind.Consensus.Withdrawals;
 using Nethermind.Core;
 using Nethermind.Core.Caching;
 using Nethermind.Core.Crypto;
@@ -299,7 +299,10 @@ protected override BlockProcessor CreateBlockProcessor()
                 BlockTree,
                 NullWithdrawalProcessor.Instance,
                 null,
-                PermissionBasedTxFilter);
+                PermissionBasedTxFilter,
+                null,
+                null,
+                CreateBlockCachePreWarmer());
         }
 
         protected override async Task AddBlocksOnStart()

diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Consensus/ClefSignerTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Consensus/ClefSignerTests.cs
@@ -8,6 +8,7 @@
 using Nethermind.Core.Crypto;
 using Nethermind.Core.Extensions;
 using Nethermind.Core.Test.Builders;
+using Nethermind.ExternalSigner.Plugin;
 using Nethermind.JsonRpc;
 using Nethermind.JsonRpc.Client;
 using NSubstitute;

diff --git a/src/Nethermind/Nethermind.Config/ConfigExtensions.cs b/src/Nethermind/Nethermind.Config/ConfigExtensions.cs
@@ -2,12 +2,35 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
+using System.Collections.Concurrent;
 using System.ComponentModel;
+using System.Linq;
 using System.Reflection;
 
 namespace Nethermind.Config;
 public static class ConfigExtensions
 {
+    private static readonly ConcurrentDictionary<string, bool> PortOptions = new();
+
+    public static string GetCategoryName(Type type)
+    {
+        if (type.IsAssignableTo(typeof(INoCategoryConfig)))
+            return null;
+
+        string categoryName = type.Name.RemoveEnd("Config");
+        if (type.IsInterface) categoryName = categoryName.RemoveStart('I');
+        return categoryName;
+    }
+
+    public static void AddPortOptionName(Type categoryType, string optionName) =>
+        PortOptions.TryAdd(
+            GetCategoryName(categoryType) is { } categoryName ? $"{categoryName}.{optionName}" : optionName,
+            true
+        );
+
+    public static string[] GetPortOptionNames() =>
+        PortOptions.Keys.OrderByDescending(x => x).ToArray();
+
     public static T GetDefaultValue<T>(this IConfig config, string propertyName)
     {
         Type type = config.GetType();

diff --git a/src/Nethermind/Nethermind.Config/ConfigItemAttribute.cs b/src/Nethermind/Nethermind.Config/ConfigItemAttribute.cs
@@ -16,5 +16,7 @@ public class ConfigItemAttribute : Attribute
         public bool DisabledForCli { get; set; }
 
         public string EnvironmentVariable { get; set; }
+
+        public bool IsPortOption { get; set; }
     }
 }
diff --git a/src/Nethermind/Nethermind.Config/ExitCodes.cs b/src/Nethermind/Nethermind.Config/ExitCodes.cs
@@ -16,6 +16,7 @@ public static class ExitCodes
     public const int ConflictingConfigurations = 103;
     public const int LowDiskSpace = 104;
     public const int DuplicatedArguments = 105;
+    public const int UnrecognizedArgument = 106;
 
     // Posix exit code
     // https://tldp.org/LDP/abs/html/exitcodes.html

diff --git a/src/Nethermind/Nethermind.Config/StringExtensions.cs b/src/Nethermind/Nethermind.Config/StringExtensions.cs
@@ -15,6 +15,9 @@ public static string RemoveStart(this string thisString, char removeChar) =>
         public static string RemoveEnd(this string thisString, char removeChar) =>
             thisString.EndsWith(removeChar) ? thisString[..^1] : thisString;
 
+        public static string RemoveEnd(this string thisString, string removeString) =>
+            thisString.EndsWith(removeString) ? thisString[..^removeString.Length] : thisString;
+
         public static bool Contains(this IEnumerable<string> collection, string value, StringComparison comparison) =>
             collection.Any(i => string.Equals(i, value, comparison));
     }

diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/InitializeBlockchainAuRa.cs b/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/InitializeBlockchainAuRa.cs
@@ -17,11 +17,11 @@
 using Nethermind.Consensus.AuRa.Services;
 using Nethermind.Consensus.AuRa.Transactions;
 using Nethermind.Consensus.AuRa.Validators;
-using Nethermind.Consensus.AuRa.Withdrawals;
 using Nethermind.Consensus.Comparers;
 using Nethermind.Consensus.Processing;
 using Nethermind.Consensus.Transactions;
 using Nethermind.Consensus.Validators;
+using Nethermind.Consensus.Withdrawals;
 using Nethermind.Core;
 using Nethermind.Init.Steps;
 using Nethermind.Logging;

diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/StartBlockProducerAuRa.cs b/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/StartBlockProducerAuRa.cs
@@ -15,10 +15,10 @@
 using Nethermind.Consensus.AuRa.Contracts.DataStore;
 using Nethermind.Consensus.AuRa.Transactions;
 using Nethermind.Consensus.AuRa.Validators;
-using Nethermind.Consensus.AuRa.Withdrawals;
 using Nethermind.Consensus.Processing;
 using Nethermind.Consensus.Producers;
 using Nethermind.Consensus.Transactions;
+using Nethermind.Consensus.Withdrawals;
 using Nethermind.Core;
 using Nethermind.Core.Crypto;
 using Nethermind.Crypto;