Merge branch 'master' into feature/snap-sync-parallel-storage

.github/workflows/build-tools.yml

@@ -18,6 +18,7 @@ jobs:
         config: [release]
         project:
           - DocGen/DocGen.sln
+          - Evm/Evm.sln
           - HiveCompare/HiveCompare.sln
           - HiveConsensusWorkflowGenerator/HiveConsensusWorkflowGenerator.csproj
           - Nethermind.Tools.Kute/Nethermind.Tools.Kute.csproj

.github/workflows/dependency-review.yml

@@ -0,0 +1,18 @@
+name: Dependency review
+
+on: [pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-review:
+    name: Dependency review
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+      - name: Dependency review
+        uses: actions/dependency-review-action@v4
+        with:
+          fail-on-severity: high

.github/workflows/sync-supported-chains.yml

@@ -101,6 +101,11 @@ jobs:
           check-latest: true
           cache: true
 
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.X"
+
       - name: Install Sedge environment
         run: |
           echo "Downloading sedge sources..."
@@ -160,6 +165,7 @@ jobs:
             --network $stripped_network \
             --consensus-url $CONSENSUS_URL \
             --execution-api-url $EXECUTION_URL \
+            --el-op-extra-flag Sync.VerifyTrieOnStateSyncFinished=true \
             $extra_param
           else
             ./build/sedge generate \
@@ -176,6 +182,7 @@ jobs:
             --el-extra-flag Sync.DownloadBodiesInFastSync=false \
             --el-extra-flag Sync.DownloadReceiptsInFastSync=false \
             --el-extra-flag JsonRpc.EnabledModules=[Eth,Subscribe,Trace,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Debug] \
+            --el-extra-flag Sync.VerifyTrieOnStateSyncFinished=true \
             --el-extra-flag Sync.SnapSync=true \
             --checkpoint-sync-url=${{ matrix.config.checkpoint-sync-url }}
           fi          
@@ -185,79 +192,10 @@ jobs:
 
       - name: Wait for ${{ matrix.config.network }} to sync
         id: wait
+        env:
+          NETWORK: ${{ matrix.config.network }}
         run: |
-          declare -A bad_logs
-          declare -A good_logs
-          declare -A required_count
-          
-          bad_logs["Corrupt"]=1
-          bad_logs["Exception"]=1
-
-          good_logs["Processed"]=0
-
-          required_count["Processed"]=20
-
-          network="${{ matrix.config.network }}"
-          if [[ "$network" != "joc-mainnet" && "$network" != "joc-testnet" && "$network" != "linea-mainnet" && "$network" != "linea-sepolia" ]]; then
-            good_logs["Synced Chain Head"]=0
-            required_count["Synced Chain Head"]=20
-          fi
-            
-          counter=0
-          found_bad_log=false
-
-          if [[ "$network" == base-* || "$network" == op-* ]]; then
-            container_name="sedge-execution-op-l2-client"
-          else
-            container_name="sedge-execution-client"
-          fi
-          
-          docker logs -f "$container_name" | while read -r line; do
-            echo "$line"
-          
-            if [[ "$line" == *"All done"* ]]; then
-              echo "Unexpected termination detected: $line"
-              exit 1
-            fi
-      
-            if [ "$found_bad_log" = true ]; then
-              counter=$((counter + 1))
-              if [ $counter -ge 100 ]; then
-                echo "Exiting after capturing extra logs due to error."
-                exit 1
-              else
-                continue
-              fi
-            fi
-
-            for bad_log in "${!bad_logs[@]}"; do
-              if [[ "$line" == *"$bad_log"* ]]; then
-                echo "Error: $bad_log found in Docker logs."
-                found_bad_log=true
-                break
-              fi
-            done
-
-            for good_log in "${!good_logs[@]}"; do
-              if [[ "$line" == *"$good_log"* ]]; then
-                good_logs["$good_log"]=$((good_logs["$good_log"]+1))
-              fi
-            done
-
-            # Check if all good logs have reached the required count
-            all_reached_required_count=true
-            for good_log in "${!good_logs[@]}"; do
-              if [[ ${good_logs[$good_log]} -lt ${required_count[$good_log]} ]]; then
-                all_reached_required_count=false
-                break
-              fi
-            done
-
-            if $all_reached_required_count; then
-              echo "All required logs found."
-              exit 0
-            fi
-          done
+          python scripts/waitForSync.py
           
       - name: Get Consensus Logs
         if: always() && matrix.config.network != 'joc-mainnet' && matrix.config.network != 'joc-testnet' && matrix.config.network != 'linea-mainnet' && matrix.config.network != 'linea-sepolia'
@@ -296,9 +234,9 @@ jobs:
         config: ${{fromJson(needs.setup-matrix.outputs.matrix)}}
     runs-on: ubuntu-latest
     steps:
-      - name: Destroy VM (if initialization failed)
+      - name: Destroy VM (make sure is removed)
         uses: kamilchodola/linode-github-runner/.github/actions/linode-machine-manager@main
-        if: ${{ failure() && needs.create_a_runner.result == 'failure' }}
+        continue-on-error: true
         with:
           linode_token: ${{ secrets.LINODE_TOKEN }}
           github_token: "${{ secrets.REPOSITORY_DISPATCH_TOKEN }}"

scripts/dev-setup.sh

@@ -34,11 +34,11 @@ echo =======================================================
 
 echo -e "\033[32m";
 read -e -p "Which configuration/s (space separated) you wish to run? " -i "mainnet" config
-for cfg in $config; do cp nethermind/configs/$cfg.cfg ~; done
+for cfg in $config; do cp nethermind/configs/$cfg.json ~; done
 echo -e "\033[00m";
 
 echo =======================================================
 echo To run the node type: 
 echo 1. screen -S node
-echo 2. ./infra.sh config-name.cfg
+echo 2. ./infra.sh config-name.json
 echo =======================================================

scripts/infra.sh

@@ -17,5 +17,5 @@ mkdir ~/nethermind_$CONFIG/keystore
 cp ~/$CONFIG.key ~/nethermind_$CONFIG/keystore/node.key.plain
 DB_PATH="/root/db/$CONFIG"
 echo "DB PATH: " $DB_PATH
-cat ~/$CONFIG.cfg | jq '.Init.BaseDbPath = "'$DB_PATH'"' | sponge ~/$CONFIG.cfg
-dotnet nethermind.dll -c ../$CONFIG.cfg
+cat ~/$CONFIG.json | jq '.Init.BaseDbPath = "'$DB_PATH'"' | sponge ~/$CONFIG.json
+dotnet nethermind.dll -c ../$CONFIG.json

scripts/private-networking/clique-validators.sh

@@ -127,7 +127,7 @@ docker-compose up
 #END of main
 
 function writeNethermindConfig() {
-cat <<EOF > node_$1/configs/config.cfg
+cat <<EOF > node_$1/configs/config.json
 {
     "Init": {
         "WebSocketsEnabled": false,
@@ -196,7 +196,7 @@ cat <<EOF >> docker-compose.yml
         command: --config config
         volumes:
             - ./genesis:/config/genesis
-            - ./node_$1/configs/config.cfg:/nethermind/configs/config.cfg
+            - ./node_$1/configs/config.json:/nethermind/configs/config.json
             - ./static-nodes.json:/nethermind/Data/static-nodes.json
             - ./node_$1/db/clique:/nethermind/nethermind_db/clique
             - ./node_$1/keystore:/nethermind/keystore
@@ -233,4 +233,4 @@ function clearDbs() {
     done
 }
 
-main
+main

scripts/syncSettings.py

@@ -121,12 +121,12 @@ def fastBlocksSettings(configuration, apiUrl, blockReduced, multiplierRequiremen
     print(configuration + 'PivotHash: ' + str(pivotHash))
     print(configuration + 'PivotTotalDifficulty: ' + str(pivotTotalDifficulty))
     data = {}
-    with open(f'{configsPath}/{configuration}.cfg', 'r') as mainnetCfg:
+    with open(f'{configsPath}/{configuration}.json', 'r') as mainnetCfg:
         data = json.load(mainnetCfg)
         data['Sync']['PivotNumber'] = baseBlock
         data['Sync']['PivotHash'] = pivotHash
         data['Sync']['PivotTotalDifficulty'] = str(pivotTotalDifficulty)
-        with open(f'{configsPath}/{configuration}.cfg', 'w') as mainnetCfgChanged:
+        with open(f'{configsPath}/{configuration}.json', 'w') as mainnetCfgChanged:
             json.dump(data, mainnetCfgChanged, indent=2)
 
 for config, value in configs.items():

scripts/waitForSync.py

@@ -0,0 +1,63 @@
+import os
+import subprocess
+import sys
+
+network = os.getenv("NETWORK")
+
+bad_logs = {"Exception": 1, "Missing node found!": 1}
+good_logs = {"Processed": 0, "Stats after finishing state": 0}
+required_count = {"Processed": 20, "Stats after finishing state": 1}
+
+if network not in {"joc-mainnet", "joc-testnet", "linea-mainnet", "linea-sepolia"}:
+    good_logs["Synced Chain Head"] = 0
+    required_count["Synced Chain Head"] = 20
+
+container_mapping = {
+    "base-": "sedge-execution-op-l2-client",
+    "op-": "sedge-execution-op-l2-client",
+}
+default_container_name = "sedge-execution-client"
+
+container_name = next(
+    (name for prefix, name in container_mapping.items() if network.startswith(prefix)),
+    default_container_name
+)
+
+process = subprocess.Popen(["docker", "logs", "-f", container_name], stdout=subprocess.PIPE, text=True)
+
+found_bad_log = False
+counter = 0
+
+try:
+    for line in process.stdout:
+        print(line.strip())
+
+        if found_bad_log:
+            counter += 1
+            if counter >= 100:
+                print("Exiting after capturing extra logs due to error.")
+                sys.exit(1)
+            continue
+
+        if any(bad_log in line for bad_log in bad_logs):
+            print(f"Error: Found bad log in line: {line.strip()}")
+            found_bad_log = True
+            continue
+
+        for good_log in good_logs:
+            if good_log in line:
+                good_logs[good_log] += 1
+
+        if all(good_logs[log] >= required_count[log] for log in required_count):
+            print("All required logs found.")
+            sys.exit(0)
+
+except Exception as e:
+    print(f"An error occurred: {e}")
+    sys.exit(1)
+finally:
+    process.terminate()
+
+# Final exit if we did not reach required lines after an error
+print("Unhandled termination. Probably critical issue in client. Stopping...")
+sys.exit(1)