Merge branch 'master' into fix/7594

NethermindEth · Oct 20, 2024 · f6c8001 · f6c8001
2 parents ad34ceb + 07def8d
commit f6c8001
Show file tree

Hide file tree

Showing 186 changed files with 4,417 additions and 2,839 deletions.
diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml
@@ -26,7 +26,7 @@ jobs:
           echo "Import GPG owner trust"
           echo ${{ secrets.GPG_OWNERTRUST }} | base64 --decode | gpg --import-ownertrust
       - name: Install PPA dependencies
-        run: sudo apt-get update && sudo apt-get install debhelper devscripts -y
+        run: sudo apt-get update && sudo apt-get install build-essential debhelper devscripts -y
       - name: Submit package
         env:
           PPA_GPG_KEYID: ${{ secrets.PPA_GPG_KEYID }}
@@ -65,6 +65,8 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install debhelper devscripts ubuntu-dev-tools -y
+      - name: Set up Python
+        uses: actions/setup-python@v5
       - name: Install launchpadlib
         run: pip install launchpadlib --upgrade
       - name: Copy to other series

diff --git a/.github/workflows/sync-supported-chains.yml b/.github/workflows/sync-supported-chains.yml
@@ -35,6 +35,7 @@ jobs:
     needs: [setup-matrix]
     strategy:
       fail-fast: false
+      max-parallel: 5 
       matrix:
         config: ${{fromJson(needs.setup-matrix.outputs.matrix)}}
     runs-on: ubuntu-latest
@@ -52,13 +53,14 @@ jobs:
           linode_token: ${{ secrets.LINODE_TOKEN }}
           github_token: "${{ secrets.REPOSITORY_DISPATCH_TOKEN }}"
           action: "create"
-          runner_label: ${{ matrix.config.network }}-${{ matrix.config.cl }}-${{ github.run_id }}
+          runner_label: t-${{ github.run_id }}-${{ matrix.config.network }}
           root_password: ${{ secrets.LINODE_ROOT_PASSWORD }}
           machine_type: "${{ matrix.config.agent }}"
           image: "linode/ubuntu24.04"
           tags: "core, self-hosted, dynamic"
           organization: "NethermindEth"
           repo_name: "nethermind"
+          blocked_ports: "8545,8546,8551,8552"
 
   sync-chain:
     needs: [setup-matrix, create_a_runner]
@@ -68,7 +70,7 @@ jobs:
         config: ${{fromJson(needs.setup-matrix.outputs.matrix)}}
 
     name: "Run sync of ${{ matrix.config.network }} chain"
-    runs-on: ${{ matrix.config.network }}-${{ matrix.config.cl }}-${{ github.run_id }}
+    runs-on: t-${{ github.run_id }}-${{ matrix.config.network }}
     timeout-minutes: ${{ matrix.config.timeout }}
 
     steps:
@@ -102,6 +104,7 @@ jobs:
         working-directory: sedge
         run: |
           docker_image=""
+          network="${{ matrix.config.network }}"
           
           if [ -z "${{ inputs.nethermind_image }}" ]; then
             REF_NAME=${{ github.ref }}
@@ -117,37 +120,89 @@ jobs:
           
           echo 'Generating sedge docker...'
           ./build/sedge deps install
-          ./build/sedge generate --logging none -p $GITHUB_WORKSPACE/sedge \
-          full-node --map-all --no-mev-boost --no-validator --network ${{ matrix.config.network }} \
-          -c ${{ matrix.config.cl }}:${{ matrix.config.cl_image }} -e nethermind:$docker_image \
-          --el-extra-flag Sync.NonValidatorNode=true --el-extra-flag Sync.DownloadBodiesInFastSync=false \
-          --el-extra-flag Sync.DownloadReceiptsInFastSync=false \
-          --el-extra-flag JsonRpc.EnabledModules=[Eth,Subscribe,Trace,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Debug] \
-          --el-extra-flag Sync.SnapSync=true \
-          --checkpoint-sync-url=${{ matrix.config.checkpoint-sync-url }}
+
+          if [[ "$network" == base-* || "$network" == op-* ]]; then
+            if [[ "$network" == *mainnet* ]]; then
+              CONSENSUS_URL="${{ secrets.MAINNET_CONSENSUS_URL }}"
+              EXECUTION_URL="${{ secrets.MAINNET_EXECUTION_URL }}"
+            elif [[ "$network" == *sepolia* ]]; then
+              CONSENSUS_URL="${{ secrets.SEPOLIA_CONSENSUS_URL }}"
+              EXECUTION_URL="${{ secrets.SEPOLIA_EXECUTION_URL }}"
+            else
+              echo "Unknown network"
+              exit 1
+            fi
+
+            if [[ "$network" == base-* ]]; then
+              extra_param="--base"
+            fi
+
+            stripped_network="${network#base-}"
+            stripped_network="${stripped_network#op-}"
+            
+            ./build/sedge generate \
+            --logging none \
+            -p $GITHUB_WORKSPACE/sedge \
+            op-full-node \
+            --op-execution opnethermind:$docker_image \
+            --op-image op-node:us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:latest \
+            --map-all \
+            --network $stripped_network \
+            --consensus-url $CONSENSUS_URL \
+            --execution-api-url $EXECUTION_URL \
+            $extra_param
+          else
+            ./build/sedge generate \
+            --logging none \
+            -p $GITHUB_WORKSPACE/sedge \
+            full-node \
+            -c ${{ matrix.config.cl }}:${{ matrix.config.cl_image }} \
+            -e nethermind:$docker_image \
+            --map-all \
+            --no-mev-boost \
+            --no-validator \
+            --network ${{ matrix.config.network }} \
+            --el-extra-flag Sync.NonValidatorNode=true \
+            --el-extra-flag Sync.DownloadBodiesInFastSync=false \
+            --el-extra-flag Sync.DownloadReceiptsInFastSync=false \
+            --el-extra-flag JsonRpc.EnabledModules=[Eth,Subscribe,Trace,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Debug] \
+            --el-extra-flag Sync.SnapSync=true \
+            --checkpoint-sync-url=${{ matrix.config.checkpoint-sync-url }}
+          fi          
+          
           echo 'Running sedge...'
           ./build/sedge run -p $GITHUB_WORKSPACE/sedge
 
       - name: Wait for ${{ matrix.config.network }} to sync
         id: wait
-        timeout-minutes: 180
         run: |
           declare -A bad_logs
+          declare -A good_logs
+          declare -A required_count
+          
           bad_logs["Corrupt"]=1
           bad_logs["Exception"]=1
-      
-          declare -A good_logs
-          good_logs["Synced Chain Head"]=0
+
           good_logs["Processed"]=0
-
-          declare -A required_count
-          required_count["Synced Chain Head"]=20
-          required_count["Processed"]=20      
+
+          required_count["Processed"]=20
+
+          network="${{ matrix.config.network }}"
+          if [[ "$network" != "joc-mainnet" && "$network" != "joc-testnet" ]]; then
+            good_logs["Synced Chain Head"]=0
+            required_count["Synced Chain Head"]=20
+          fi
             
           counter=0
           found_bad_log=false
-
-          docker logs -f sedge-execution-client | while read -r line; do
+
+          if [[ "$network" == base-* || "$network" == op-* ]]; then
+            container_name="sedge-execution-op-l2-client"
+          else
+            container_name="sedge-execution-client"
+          fi
+          
+          docker logs -f "$container_name" | while read -r line; do
             echo "$line"
           
             if [[ "$line" == *"All done"* ]]; then
@@ -195,9 +250,32 @@ jobs:
           done
 
       - name: Get Consensus Logs
-        if: always()
+        if: always() && matrix.config.network != 'joc-mainnet' && matrix.config.network != 'joc-testnet'
         run: |
-          docker logs sedge-consensus-client      
+          network="${{ matrix.config.network }}"
+          if [[ "$network" == base-* || "$network" == op-* ]]; then
+            docker logs sedge-consensus-op-l2-client
+          else
+            docker logs sedge-consensus-client      
+          fi
+          
+      - name: Check size of DB 
+        run: |
+          du -h $GITHUB_WORKSPACE/sedge
+
+      - name: Destroy VM
+        if: always()
+        id: run-linode-action
+        uses: kamilchodola/linode-github-runner/.github/actions/linode-machine-manager@main
+        with:
+          linode_token: ${{ secrets.LINODE_TOKEN }}
+          github_token: "${{ secrets.REPOSITORY_DISPATCH_TOKEN }}"
+          action: "destroy-machine"
+          runner_label: t-${{ github.run_id }}-${{ matrix.config.network }}
+          search_phrase: t-${{ github.run_id }}-${{ matrix.config.network }}
+          root_password: ${{ secrets.LINODE_ROOT_PASSWORD }}
+          organization: "NethermindEth"
+          repo_name: "nethermind"
 
   destroy_runner:
     needs: [setup-matrix, create_a_runner, sync-chain]
@@ -208,19 +286,30 @@ jobs:
         config: ${{fromJson(needs.setup-matrix.outputs.matrix)}}
     runs-on: ubuntu-latest
     steps:    
-      - name: Destroy a Runner
-        id: run-linode-action
+      - name: Destroy VM (make sure is removed if by any unexpected reason it did not removed it on )
+        continue-on-error: true
+        uses: kamilchodola/linode-github-runner/.github/actions/linode-machine-manager@main
+        with:
+          linode_token: ${{ secrets.LINODE_TOKEN }}
+          github_token: "${{ secrets.REPOSITORY_DISPATCH_TOKEN }}"
+          action: "destroy-machine"
+          runner_label: t-${{ github.run_id }}-${{ matrix.config.network }}
+          search_phrase: t-${{ github.run_id }}-${{ matrix.config.network }}
+          root_password: ${{ secrets.LINODE_ROOT_PASSWORD }}
+          organization: "NethermindEth"
+          repo_name: "nethermind"  
+
+      - name: Destroy Runner
         uses: kamilchodola/linode-github-runner/.github/actions/linode-machine-manager@main
         with:
           linode_token: ${{ secrets.LINODE_TOKEN }}
           github_token: "${{ secrets.REPOSITORY_DISPATCH_TOKEN }}"
-          action: "destroy"
-          runner_label: ${{ matrix.config.network }}-${{ matrix.config.cl }}-${{ github.run_id }}
-          search_phrase: ${{ matrix.config.network }}-${{ matrix.config.cl }}-${{ github.run_id }}
+          action: "destroy-runner"
+          runner_label: t-${{ github.run_id }}-${{ matrix.config.network }}
+          search_phrase: t-${{ github.run_id }}-${{ matrix.config.network }}
           root_password: ${{ secrets.LINODE_ROOT_PASSWORD }}
           organization: "NethermindEth"
           repo_name: "nethermind"
-
   trigger_tests:
     needs: [setup-matrix, create_a_runner, sync-chain, destroy_runner]
     if: success()

diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml
@@ -0,0 +1,46 @@
+name: Trivy scanner
+
+on:
+  pull_request:
+    branches: [master]
+  push:
+    branches: [master]
+  schedule:
+    - cron: '29 19 * * 4'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      security-events: write
+      actions: read
+    env:
+      IMAGE_TAG: nethermind:${{ github.sha }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 #v4.2.1
+
+      - name: Build Docker image
+        run: docker build -t $IMAGE_TAG .
+
+      - name: Scan
+        uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 #v0.28.0
+        with:
+          image-ref: ${{ env.IMAGE_TAG }}
+          format: template
+          template: '@/contrib/sarif.tpl'
+          output: trivy-results.sarif
+          severity: CRITICAL,HIGH
+        env:
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db
+
+      - name: Upload scan results
+        uses: github/codeql-action/upload-sarif@cf5b0a9041d3c1d336516f1944c96d96598193cc #v2.19.1
+        with:
+          sarif_file: trivy-results.sarif
diff --git a/.github/workflows/update-fast-sync.yml b/.github/workflows/update-fast-sync.yml
@@ -16,10 +16,12 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
       - name: Install dependencies
         run: |
-          pip3 install setuptools
-          pip3 install emoji
+          pip install setuptools
+          pip install emoji
       - name: Update config files
         run: python3 scripts/syncSettings.py ${{ secrets.ETHERSCAN_API_KEY }}
       - name: Create GitHub app token

diff --git a/scripts/workflow_config/sync_testnets_matrix.json b/scripts/workflow_config/sync_testnets_matrix.json
@@ -19,7 +19,7 @@
     "network": "sepolia",
     "cl": "prysm",
     "cl_image": "prysmaticlabs/prysm-beacon-chain:stable",
-    "checkpoint-sync-url": "https://beaconstate-sepolia.chainsafe.io",
+    "checkpoint-sync-url": "https://checkpoint-sync.sepolia.ethpandaops.io/",
     "timeout": 180,
     "agent": "g6-standard-6"
   },
@@ -35,8 +35,56 @@
     "network": "mainnet",
     "cl": "lodestar",
     "cl_image": "chainsafe/lodestar:latest",
-    "checkpoint-sync-url": "https://mainnet.checkpoint.sigp.io",
+    "checkpoint-sync-url": "https://mainnet-checkpoint-sync.attestant.io/",
     "timeout": 180,
     "agent": "g6-standard-16"
+  },
+  {
+    "network": "op-sepolia",
+    "cl": "",
+    "cl_image": "",
+    "checkpoint-sync-url": "",
+    "timeout": 180,
+    "agent": "g6-standard-8"
+  },
+  {
+    "network": "base-sepolia",
+    "cl": "",
+    "cl_image": "",
+    "checkpoint-sync-url": "",
+    "timeout": 180,
+    "agent": "g6-standard-8"
+  },
+  {
+    "network": "op-mainnet",
+    "cl": "",
+    "cl_image": "",
+    "checkpoint-sync-url": "",
+    "timeout": 600,
+    "agent": "g6-standard-16"
+  },
+  {
+    "network": "base-mainnet",
+    "cl": "",
+    "cl_image": "",
+    "checkpoint-sync-url": "",
+    "timeout": 600,
+    "agent": "g6-standard-16"
+  },
+  {
+    "network": "joc-mainnet",
+    "cl": "",
+    "cl_image": "",
+    "checkpoint-sync-url": "",
+    "timeout": 180,
+    "agent": "g6-standard-6"
+  },
+  {
+    "network": "joc-testnet",
+    "cl": "",
+    "cl_image": "",
+    "checkpoint-sync-url": "",
+    "timeout": 180,
+    "agent": "g6-standard-6"
   }
 ]
diff --git a/src/Nethermind/Chains/op-mainnet.json b/src/Nethermind/Chains/op-mainnet.json
@@ -73,7 +73,7 @@
     "rip7212TransitionTimestamp": "0x668eb001",
     "opGraniteTransitionTimestamp": "0x66e1be81",
 
-    "terminalTotalDifficulty": "0"
+    "terminalTotalDifficulty": "210470125"
   },
   "genesis": {
     "seal": {
@@ -93,6 +93,11 @@
     "stateRoot": "0xeddb4c1786789419153a27c4c80ff44a2226b6eda04f7e22ce5bae892ea568eb"
   },
   "nodes": [
+    "enode://87a32fd13bd596b2ffca97020e31aef4ddcc1bbd4b95bb633d16c1329f654f34049ed240a36b449fda5e5225d70fe40bc667f53c304b71f8e68fc9d448690b51@3.231.138.188:30301",
+    "enode://ca21ea8f176adb2e229ce2d700830c844af0ea941a1d8152a9513b966fe525e809c3a6c73a2c18a12b74ed6ec4380edf91662778fe0b79f6a591236e49e176f9@184.72.129.189:30301",
+    "enode://acf4507a211ba7c1e52cdf4eef62cdc3c32e7c9c47998954f7ba024026f9a6b2150cd3f0b734d9c78e507ab70d59ba61dfe5c45e1078c7ad0775fb251d7735a2@3.220.145.177:30301",
+    "enode://8a5a5006159bf079d06a04e5eceab2a1ce6e0f721875b2a9c96905336219dbe14203d38f70f3754686a6324f786c2f9852d8c0dd3adac2d080f4db35efc678c5@3.231.11.52:30301",
+    "enode://cdadbe835308ad3557f9a1de8db411da1a260a98f8421d62da90e71da66e55e98aaa8e90aa7ce01b408a54e4bd2253d701218081ded3dbe5efbbc7b41d7cef79@54.198.153.150:30301",
     "enode://ca2774c3c401325850b2477fd7d0f27911efbf79b1e8b335066516e2bd8c4c9e0ba9696a94b1cb030a88eac582305ff55e905e64fb77fe0edcd70a4e5296d3ec@34.65.175.185:30305",
     "enode://dd751a9ef8912be1bfa7a5e34e2c3785cc5253110bd929f385e07ba7ac19929fb0e0c5d93f77827291f4da02b2232240fbc47ea7ce04c46e333e452f8656b667@34.65.107.0:30305",
     "enode://c5d289b56a77b6a2342ca29956dfd07aadf45364dde8ab20d1dc4efd4d1bc6b4655d902501daea308f4d8950737a4e93a4dfedd17b49cd5760ffd127837ca965@34.65.202.239:30305",