Merge branch 'main' into badcursor

.github/workflows/ci-cd-main-branch-docker-images.yml

@@ -9,6 +9,7 @@ env:
   CHECKOUT_REF: "main"
   DOCKERHUB_REPOSITORY: "erigontech/erigon"
   LABEL_DESCRIPTION: "[docker image built on a last commit id from the main branch] Erigon is an implementation of Ethereum (execution layer with embeddable consensus layer), on the efficiency frontier. Archive Node by default."
+  KEEP_IMAGES: 100
 
 on:
   push:
@@ -127,7 +128,7 @@ jobs:
           --push \
           --platform linux/amd64,linux/arm64 .
 
-      - name: export and print docker build tag
+      - name: export and print docker build tag, cleanup old docker images
         id: built_tag_export
         env:
           BUILD_VERSION: "main-${{ steps.getCommitId.outputs.short_commit_id }}"
@@ -136,11 +137,41 @@ jobs:
           echo The following docker images have been published:
           echo "${{ env.DOCKERHUB_REPOSITORY }}:main-${{ env.BUILD_VERSION }}"
           echo "${{ env.DOCKERHUB_REPOSITORY }}:main-latest"
+          echo
+          echo "Cleanup old docker images matching pattern tag ~= main-XXXXXXX"
+          curl_cmd="curl -s -H \"Authorization: JWT ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_TOKEN }}\" "
+          dockerhub_url='https://hub.docker.com/v2/namespaces/erigontech/repositories/erigon'
+          my_list () {
+            # First page:
+            next_page="$dockerhub_url/tags?page=1&page_size=100"
+            while [ "$next_page" != "null" ]
+              do
+              # Print tags and push dates for tags matching "main-":
+              $curl_cmd $next_page | jq -r '.results|.[]|.name + " " + .tag_last_pushed' | grep 'main-'
+              next_page=`$curl_cmd $next_page | jq '.next' | sed -e 's/^\"//' -e 's/\"$//'`
+            done
+            }
+
+            my_list | tail -n+${{ env.KEEP_IMAGES }} | while read line; do
+              echo -n "Removing docker image/published - $line "
+              current_image=$(echo $line | sed -e 's/^\(main-.\{7\}\) .*/\1/')
+              output_code=$(curl --write-out %{http_code} --output curl-output.log \
+                    -s -X DELETE -H "Accept: application/json" \
+                    -H "Authorization: JWT ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_TOKEN }}" \
+                    https://hub.docker.com/v2/repositories/erigontech/erigon/tags/${current_image} )
+              if [ $output_code -ne 204 ]; then
+                echo "ERROR: failed to remove docker image erigon:${current_image}"
+                echo "ERROR: API response: $(cat curl-output.log)."
+              else
+                echo -n " - removed. "
+              fi
+              echo "Done."
+            done
 
   run-kurtosis-assertoor:
     needs: [define_matrix, Build]
     uses: erigontech/erigon/.github/workflows/test-kurtosis-assertoor.yml@main
     with:
       checkout_ref: ${{ github.sha }}
       os: ${{ needs.define_matrix.outputs.os }}
-      docker_build_tag: ${{ needs.Build.outputs.docker_build_tag }}
+      docker_build_tag: ${{ needs.Build.outputs.docker_build_tag }}

.github/workflows/lint.yml

@@ -26,13 +26,13 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.22'
+          go-version: '1.23'
 
       - name: Install golangci-lint
         if: runner.os == 'Linux'
         uses: golangci/golangci-lint-action@v6
         with:
-          version: v1.59.1
+          version: v1.61.0
           args: --help
 
       - name: Lint

.github/workflows/qa-constrained-tip-tracking.yml

@@ -2,15 +2,8 @@ name: QA - Constrained Tip tracking
 
 on:
   schedule:
-    - cron: '0 0 * * 0'  # Run on Sunday at 00:00 AM UTC
+    - cron: '0 20 * * 0'  # Run on Sunday at 08:00 PM UTC
   workflow_dispatch:     # Run manually
-  pull_request:
-    branches:
-      - qa_tests_constrained_tip_tracking
-    types:
-      - opened
-      - synchronize
-      - ready_for_review
 
 jobs:
   constrained-tip-tracking-test:

.github/workflows/qa-rpc-integration-tests.yml

@@ -17,6 +17,7 @@ on:
 jobs:
   integration-test-suite:
     runs-on: [ self-hosted, Erigon3 ]
+    timeout-minutes: 15
     env:
       ERIGON_REFERENCE_DATA_DIR: /opt/erigon-versions/reference-version/datadir
       ERIGON_TESTBED_DATA_DIR: /opt/erigon-testbed/datadir
@@ -83,97 +84,10 @@ jobs:
 
           cd ${{ runner.workspace }}/rpc-tests/integration
           rm -rf ./mainnet/results/
-          
+                    
           # Run RPC integration test runner via http
-          python3 ./run_tests.py -p 8545 --continue -f --json-diff -x \
-          debug_accountRange,debug_getModifiedAccountsByHash,debug_getModifiedAccountsByNumber,debug_storageRangeAt,debug_traceBlockByHash,\
-          debug_traceCallMany/test_02.tar,debug_traceCallMany/test_04.tar,debug_traceCallMany/test_05.tar,debug_traceCallMany/test_06.tar,debug_traceCallMany/test_07.tar,debug_traceCallMany/test_09.json,debug_traceCallMany/test_10.tar,\
-          debug_traceBlockByNumber/test_05.tar,debug_traceBlockByNumber/test_08.tar,debug_traceBlockByNumber/test_09.tar,debug_traceBlockByNumber/test_10.tar,debug_traceBlockByNumber/test_11.tar,debug_traceBlockByNumber/test_12.tar,\
-          debug_traceTransaction,\
-          engine_exchangeCapabilities/test_1.json,\
-          engine_exchangeTransitionConfigurationV1/test_01.json,\
-          engine_getClientVersionV1/test_1.json,\
-          erigon_getLogsByHash/test_04.json,\
-          erigon_getHeaderByHash/test_02.json,\
-          erigon_getHeaderByHash/test_03.json,\
-          erigon_getHeaderByHash/test_04.json,\
-          erigon_getHeaderByHash/test_06.json,\
-          erigon_getHeaderByNumber/test_01.json,\
-          erigon_getHeaderByNumber/test_02.json,\
-          erigon_getHeaderByNumber/test_03.json,\
-          erigon_getHeaderByNumber/test_04.json,\
-          erigon_getHeaderByNumber/test_05.json,\
-          erigon_getHeaderByNumber/test_06.json,\
-          erigon_getHeaderByNumber/test_07.json,\
-          erigon_getHeaderByNumber/test_08.json,\
-          erigon_getLatestLogs/test_01.json,\
-          erigon_getLatestLogs/test_02.json,\
-          erigon_getLatestLogs/test_03.json,\
-          erigon_getLatestLogs/test_04.json,\
-          erigon_getLatestLogs/test_05.json,\
-          erigon_getLatestLogs/test_06.json,\
-          erigon_getLatestLogs/test_08.json,\
-          erigon_getLatestLogs/test_09.json,\
-          erigon_getLatestLogs/test_10.json,\
-          erigon_getLatestLogs/test_11.json,\
-          erigon_getLatestLogs/test_12.json,\
-          erigon_getBalanceChangesInBlock,\
-          eth_createAccessList/test_16.json,\
-          parity_getBlockReceipts/test_01.json,\
-          parity_getBlockReceipts/test_02.json,\
-          parity_getBlockReceipts/test_03.json,\
-          parity_getBlockReceipts/test_04.json,\
-          parity_getBlockReceipts/test_05.json,\
-          parity_getBlockReceipts/test_06.json,\
-          parity_getBlockReceipts/test_07.json,\
-          parity_getBlockReceipts/test_08.json,\
-          parity_getBlockReceipts/test_09.json,\
-          parity_getBlockReceipts/test_10.json,\
-          trace_filter/test_16.json,\
-          trace_rawTransaction/test_01.json,\
-          trace_rawTransaction/test_03.json,\
-          admin_nodeInfo/test_01.json,\
-          admin_peers/test_01.json,\
-          erigon_nodeInfo/test_1.json,\
-          eth_coinbase/test_01.json,\
-          eth_feeHistory/test_01.json,\
-          eth_feeHistory/test_02.json,\
-          eth_feeHistory/test_03.json,\
-          eth_feeHistory/test_04.json,\
-          eth_feeHistory/test_05.json,\
-          eth_feeHistory/test_06.json,\
-          eth_feeHistory/test_08.json,\
-          eth_feeHistory/test_09.json,\
-          eth_feeHistory/test_10.json,\
-          eth_feeHistory/test_11.json,\
-          eth_getBlockByHash/test_01.json,\
-          eth_getBlockByHash/test_02.json,\
-          eth_getBlockByHash/test_05.json,\
-          eth_getBlockByHash/test_06.json,\
-          eth_getBlockByHash/test_07.json,\
-          eth_getBlockByHash/test_08.json,\
-          eth_getBlockByNumber/test_01.json,\
-          eth_getBlockByNumber/test_02.json,\
-          eth_getBlockByNumber/test_04.json,\
-          eth_getBlockByNumber/test_05.json,\
-          eth_getBlockByNumber/test_06.json,\
-          eth_getBlockByNumber/test_07.json,\
-          eth_getBlockByNumber/test_08.json,\
-          eth_getBlockByNumber/test_12.json,\
-          eth_getBlockByNumber/test_13.json,\
-          eth_getTransactionByHash/test_02.json,\
-          eth_getWork/test_01.json,\
-          eth_mining/test_01.json,\
-          eth_protocolVersion/test_1.json,\
-          eth_submitHashrate/test_1.json,\
-          eth_submitWork/test_1.json,\
-          net_peerCount/test_1.json,\
-          net_version/test_1.json,\
-          txpool_content/test_01.json,\
-          txpool_status/test_1.json,\
-          web3_clientVersion/test_1.json,\
-          eth_estimateGas/test_14.json,\
-          trace_replayBlockTransactions/test_29.tar
+          chmod +x ${{ runner.workspace }}/erigon/.github/workflows/scripts/run_rpc_tests.sh
+          ${{ runner.workspace }}/erigon/.github/workflows/scripts/run_rpc_tests.sh
           
           # Capture test runner script exit status
           test_exit_status=$?

.github/workflows/qa-snap-download.yml

@@ -2,7 +2,7 @@ name: QA - Snapshot Download
 
 on:
   schedule:
-    - cron: '0 22 * * 1-6'  # Run every night at 22:00 (10:00 PM) UTC except Sunday
+    - cron: '0 20 * * 1-6'  # Run every night at 20:00 (08:00 PM) UTC except Sunday
   workflow_dispatch:     # Run manually
 
 jobs:

.github/workflows/qa-tip-tracking.yml

@@ -2,7 +2,7 @@ name: QA - Tip tracking
 
 on:
   schedule:
-    - cron: '0 0 * * 1-6'  # Run every night at 00:00 AM UTC except Sunday
+    - cron: '0 20 * * 1-6'  # Run every night at 08:00 PM UTC except Sunday
   workflow_dispatch:     # Run manually
 
 jobs:

.github/workflows/scripts/run_rpc_tests.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+set +e # Disable exit on error
+
+# Array of disabled tests
+disabled_tests=(
+    # Erigon2 and Erigon3 never supported this api methods
+    trace_rawTransaction
+    # false positives: Erigon return expected response. but rpc-test somehow doesn't see 1 field.
+    erigon_getHeaderByHash,erigon_getHeaderByNumber,eth_feeHistory
+    # total difficulty field was removed, then added back
+    eth_getBlockByHash,eth_getBlockByNumber
+    # Erigon bugs
+    debug_accountRange,debug_storageRangeAt
+    # need update rpc-test - because Erigon is correct (@AskAlexSharov will do after https://github.com/erigontech/erigon/pull/12634)
+    debug_getModifiedAccountsByHash,debug_getModifiedAccountsByNumber
+    # Erigon bug https://github.com/erigontech/erigon/issues/12603
+    erigon_getLatestLogs,erigon_getLogsByHash/test_04.json
+    # Erigon bug https://github.com/erigontech/erigon/issues/12637
+    debug_traceBlockByNumber/test_05.tar
+    debug_traceBlockByNumber/test_08.tar
+    debug_traceBlockByNumber/test_09.tar
+    debug_traceBlockByNumber/test_10.tar
+    debug_traceBlockByNumber/test_11.tar
+    debug_traceBlockByNumber/test_12.tar
+    # remove this line after https://github.com/erigontech/rpc-tests/pull/281
+    parity_getBlockReceipts
+    # to investigate
+    debug_traceBlockByHash
+    debug_traceCallMany/test_02.tar
+    debug_traceCallMany/test_04.tar
+    debug_traceCallMany/test_05.tar
+    debug_traceCallMany/test_06.tar
+    debug_traceCallMany/test_07.tar
+    debug_traceCallMany/test_09.json
+    debug_traceCallMany/test_10.tar
+    engine_exchangeCapabilities/test_1.json
+    engine_exchangeTransitionConfigurationV1/test_01.json
+    engine_getClientVersionV1/test_1.json
+    erigon_getBalanceChangesInBlock
+    eth_createAccessList/test_16.json
+    admin_nodeInfo/test_01.json
+    admin_peers/test_01.json
+    erigon_nodeInfo/test_1.json
+    eth_coinbase/test_01.json
+    eth_getTransactionByHash/test_02.json
+    eth_getWork/test_01.json
+    eth_mining/test_01.json
+    eth_protocolVersion/test_1.json
+    eth_submitHashrate/test_1.json
+    eth_submitWork/test_1.json
+    net_peerCount/test_1.json
+    net_version/test_1.json
+    txpool_content/test_01.json
+    txpool_status/test_1.json
+    web3_clientVersion/test_1.json
+    eth_estimateGas/test_14.json
+    trace_replayBlockTransactions/test_29.tar
+    # recently started to fail
+    debug_traceTransaction/test_20.json
+    debug_traceTransaction/test_21.json
+    debug_traceTransaction/test_22.json
+    debug_traceTransaction/test_25.json
+    debug_traceTransaction/test_30.tar
+    debug_traceTransaction/test_33.json
+    debug_traceTransaction/test_35.tar
+    debug_traceTransaction/test_36.json
+    debug_traceTransaction/test_37.tar
+    debug_traceTransaction/test_38.tar
+    debug_traceTransaction/test_43.json
+    debug_traceTransaction/test_44.json
+    debug_traceTransaction/test_62.json
+    debug_traceTransaction/test_64.json
+    debug_traceTransaction/test_74.tar
+    debug_traceTransaction/test_75.tar
+    debug_traceTransaction/test_77.json
+    debug_traceTransaction/test_78.tar
+    debug_traceTransaction/test_79.tar
+    debug_traceTransaction/test_80.tar
+    debug_traceTransaction/test_81.tar
+    debug_traceTransaction/test_82.tar
+    debug_traceTransaction/test_83.tar
+    debug_traceTransaction/test_84.tar
+    debug_traceTransaction/test_85.tar
+    debug_traceTransaction/test_87.json
+    debug_traceTransaction/test_90.tar
+    debug_traceTransaction/test_91.tar
+    debug_traceTransaction/test_92.tar
+    debug_traceTransaction/test_93.json
+    debug_traceTransaction/test_96.json
+    trace_filter/test_16.json)
+
+# Transform the array into a comma-separated string
+disabled_test_list=$(IFS=,; echo "${disabled_tests[*]}")
+
+python3 ./run_tests.py -p 8545 --continue -f --json-diff -x "$disabled_test_list"
+
+exit $?

README.md

@@ -203,18 +203,20 @@ du -hsc /erigon/snapshots/*
 
 ### Erigon3 changes from Erigon2
 
-- Initial sync does download LatestState and it's history - no re-exec from 0 anymore.
-- ExecutionStage included many E2 stages: stage_hash_state, stage_trie, log_index, history_index, trace_index
-- E3 can execute 1 historical transaction - without executing it's block - because history/indices have
-  transaction-granularity, instead of block-granularity.
-- E3 doesn't store Logs (aka Receipts) - it always re-executing historical txn (but it's cheaper then in E2 - see point
-  above).
-- Restart doesn't loose much partial progress: `--sync.loop.block.limit=5_000` enabled by default
-- `chaindata` is less than `15gb`. It's ok to `rm -rf chaindata`. To prevent it's grow: recommend `--batchSize <= 1G`
-- can symlink/mount latest state to fast drive and history to cheap drive
-- `--internalcl` is enabled by default. to disable use `--externalcl`
-- `--prune` flags changed: default `--prune.mode=archive`, FullNode: `--prune.mode=full`, MinimalNode (EIP-4444):
-  `--prune.mode=minimal`.
+- **Initial sync doesn't re-exec from 0:** downloading 99% LatestState and History
+- **Per-Transaction granularity of history** (Erigon2 had per-block). Means:
+    - Can execute 1 historical transaction - without executing it's block
+    - If account X change V1->V2->V1 within 1 block (different transactions): `debug_getModifiedAccountsByNumber` return
+      it
+    - Erigon3 doesn't store Logs (aka Receipts) - it always re-executing historical txn (but it's cheaper)
+- **Validator mode**: added. `--internalcl` is enabled by default. to disable use `--externalcl`.
+- **Store most of data in immutable files (segments/snapshots):**
+    - can symlink/mount latest state to fast drive and history to cheap drive
+    - `chaindata` is less than `15gb`. It's ok to `rm -rf chaindata`. (to prevent grow: recommend `--batchSize <= 1G`)
+- **`--prune` flags changed**: see `--prune.mode` (default: `archive`, full: `full`, EIP-4444: `minimal`)
+- **Other changes:**
+    - ExecutionStage included many E2 stages: stage_hash_state, stage_trie, log_index, history_index, trace_index
+    - Restart doesn't loose much partial progress: `--sync.loop.block.limit=5_000` enabled by default
 
 ### Logging