Reth update

.config/nextest.toml

@@ -4,4 +4,4 @@ slow-timeout = { period = "30s", terminate-after = 4 }
 
 [[profile.default.overrides]]
 filter = "test(general_state_tests)"
-slow-timeout = { period = "1m", terminate-after = 4 }
+slow-timeout = { period = "1m", terminate-after = 10 }

.github/ISSUE_TEMPLATE/bug.yml

@@ -55,7 +55,7 @@ body:
         - Windows (ARM)
     validations:
       required: true
-  - type: input
+  - type: textarea
     id: client-version
     attributes:
       label: What version/commit are you on?

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+    interval: "weekly"

.github/workflows/assertoor.yml

@@ -0,0 +1,230 @@
+name: Assertoor Tests
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  get_tests:
+    name: "Run assertoor tests on reth pairs"
+    runs-on: ubuntu-latest
+    outputs:
+      test_result: ${{ steps.test_result.outputs.test_result }}
+      test_status: ${{ steps.test_result.outputs.test_status }}
+      failed_test_status: ${{ steps.test_result.outputs.failed_test_status }}
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v4
+    - name: Setup Kurtosis
+      shell: bash
+      run: |
+        echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+        sudo apt update
+        sudo apt install kurtosis-cli
+        kurtosis analytics disable
+
+    - name: Run Kurtosis
+      shell: bash
+      id: services
+      run: |
+        export github_sha=${{ github.sha }}
+        export github_repository=${{ github.repository }} 
+
+        cat etc/assertoor/assertoor-template.yaml | envsubst > etc/assertoor/assertoor.yaml
+
+        kurtosis run github.com/kurtosis-tech/ethereum-package --enclave assertoor-${{ github.run_id }} --args-file etc/assertoor/assertoor.yaml
+
+        enclave_dump=$(kurtosis enclave inspect assertoor-${{ github.run_id }})
+
+        assertoor_url=$(echo "$enclave_dump" | grep assertoor | grep http | sed 's/.*\(http:\/\/[0-9.:]\+\).*/\1/')
+        echo "assertoor_url: ${assertoor_url}"
+        echo "assertoor_url=${assertoor_url}" >> $GITHUB_OUTPUT
+
+    - name: Await test completion
+      shell: bash
+      id: test_result
+      run: |
+        assertoor_url="${{ steps.services.outputs.assertoor_url }}"
+
+        YELLOW='\033[1;33m'
+        GRAY='\033[0;37m'
+        GREEN='\033[0;32m'
+        RED='\033[0;31m'
+        NC='\033[0m'
+
+        # print assertor logs
+        assertoor_container=$(docker container list | grep assertoor | sed 's/^\([^ ]\+\) .*$/\1/')
+        docker logs -f $assertoor_container &
+
+        # helper to fetch task status for specific test id
+        get_tasks_status() {
+          tasks=$(curl -s ${assertoor_url}/api/v1/test_run/$1 | jq -c ".data.tasks[] | {index, parent_index, name, title, status, result}")
+          declare -A task_graph_map
+          task_graph_map[0]=""
+
+          while read task; do
+            task_id=$(echo "$task" | jq -r ".index")
+            task_parent=$(echo "$task" | jq -r ".parent_index")
+            task_name=$(echo "$task" | jq -r ".name")
+            task_title=$(echo "$task" | jq -r ".title")
+            task_status=$(echo "$task" | jq -r ".status")
+            task_result=$(echo "$task" | jq -r ".result")
+
+            task_graph="${task_graph_map[$task_parent]}"
+            task_graph_map[$task_id]="$task_graph |"
+            if [ ! -z "$task_graph" ]; then
+              task_graph="${task_graph}- "
+            fi
+
+            if [ "$task_status" == "pending" ]; then
+              task_status="${GRAY}pending ${NC}"
+            elif [ "$task_status" == "running" ]; then
+              task_status="${YELLOW}running ${NC}"
+            elif [ "$task_status" == "complete" ]; then
+              task_status="${GREEN}complete${NC}"
+            fi
+
+            if [ "$task_result" == "none" ]; then
+              task_result="${GRAY}none   ${NC}"
+            elif [ "$task_result" == "success" ]; then
+              task_result="${GREEN}success${NC}"
+            elif [ "$task_result" == "failure" ]; then
+              task_result="${RED}failure${NC}"
+            fi
+
+            echo -e " $(printf '%-4s' "$task_id")\t$task_status\t$task_result\t$(printf '%-50s' "$task_graph$task_name") \t$task_title"
+          done <<< $(echo "$tasks")
+        }
+
+        # poll & check test status
+        final_test_result=""
+        failed_test_id=""
+        while true
+        do
+          pending_tests=0
+          failed_tests=0
+          total_tests=0
+          running_test=""
+
+          status_lines=()
+          task_lines=""
+          status_lines+=("$(date +'%Y-%m-%d %H:%M:%S')  Test Status:")
+
+          tests=$(curl -s ${assertoor_url}/api/v1/test_runs | jq -c ".data[] | {run_id, test_id, name, status}")
+          while read test; do
+            if [ -z "$test" ]; then
+              continue
+            fi
+            run_id=$(echo "$test" | jq -r ".run_id")
+            test_id=$(echo "$test" | jq -r ".test_id")
+            test_name=$(echo "$test" | jq -r ".name")
+            test_status=$(echo "$test" | jq -r ".status")
+
+            if [ "$test_status" == "pending" ]; then
+              pending_tests=$(expr $pending_tests + 1)
+              status_name="${GRAY}pending${NC}"
+            elif [ "$test_status" == "running" ]; then
+              pending_tests=$(expr $pending_tests + 1)
+              running_test="$run_id"
+              status_name="${YELLOW}running${NC}"
+
+            elif [ "$test_status" == "success" ]; then
+              status_name="${GREEN}success${NC}"
+            elif [ "$test_status" == "failure" ]; then
+              failed_tests=$(expr $failed_tests + 1)
+              failed_test_id="$run_id"
+              status_name="${RED}failure${NC}"
+            else
+              status_name="$test_status"
+            fi
+            status_lines+=("  $(printf '%-3s' "$test_id") $status_name \t$test_name")
+            total_tests=$(expr $total_tests + 1)
+          done <<< $(echo "$tests")
+
+          for status_line in "${status_lines[@]}"
+          do
+            echo -e "$status_line"
+          done
+
+          if ! [ -z "$running_test" ]; then
+            task_lines=$(get_tasks_status "$running_test")
+            echo "Active Test Task Status:"
+            echo "$task_lines"
+          fi
+
+          if [ $failed_tests -gt 0 ]; then 
+            final_test_result="failure"
+            break
+          fi
+          if [ $total_tests -gt 0 ] && [ $pending_tests -le 0 ]; then
+            final_test_result="success"
+            break
+          fi
+
+          sleep 60
+        done
+
+        # save test results & status to github output
+        echo "test_result=$(echo "$final_test_result")" >> $GITHUB_OUTPUT
+        echo "test_status<<EOF" >> $GITHUB_OUTPUT
+        for status_line in "${status_lines[@]}"
+        do
+          echo -e "$status_line" >> $GITHUB_OUTPUT
+        done
+        echo "EOF" >> $GITHUB_OUTPUT
+
+        if ! [ -z "$failed_test_id" ]; then
+          echo "failed_test_status<<EOF" >> $GITHUB_OUTPUT
+          get_tasks_status "$failed_test_id" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+        else
+          echo "failed_test_status=" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Generate dump and remove kurtosis enclave
+      shell: bash
+      run: |
+        mkdir -p ./temp/dump
+        cd ./temp/dump
+        cp ../../etc/assertoor/assertoor.yaml ./kurtosis-params.yaml
+
+        kurtosis enclave dump assertoor-${{ github.run_id }}
+        kurtosis enclave rm -f assertoor-${{ github.run_id }}
+
+    - name: Upload dump artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: "kurtosis-enclave-dump-${{ github.run_id }}"
+        path: ./temp/dump
+
+    - name: Return test result
+      shell: bash
+      run: |
+        test_result="${{ steps.test_result.outputs.test_result }}"
+        test_status=$(
+        cat <<"EOF"
+        ${{ steps.test_result.outputs.test_status }}
+        EOF
+        )
+        failed_test_status=$(
+        cat <<"EOF"
+        ${{ steps.test_result.outputs.failed_test_status }}
+        EOF
+        )
+
+        echo "Test Result:  $test_result"
+        echo "$test_status"
+
+        if ! [ "$test_result" == "success" ]; then
+          echo ""
+          echo "Failed Test Task Status:"
+          echo "$failed_test_status"
+
+          echo ""
+          echo "See 'Await test completion' task for detailed logs about this failure!"
+          echo ""
+
+          exit 1 # fail action
+        fi
+

.github/workflows/book.yml

@@ -105,7 +105,7 @@ jobs:
             .
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: github-pages
           path: ${{ runner.temp }}/artifact.tar
@@ -132,4 +132,4 @@ jobs:
     steps:
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v2
+        uses: actions/deploy-pages@v4

.github/workflows/dependencies.yml

@@ -52,7 +52,7 @@ jobs:
           echo "EOF" >> $GITHUB_OUTPUT
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@v6
         with:
           add-paths: ./Cargo.lock
           commit-message: ${{ steps.msg.outputs.commit_message }}

.github/workflows/docker.yml

@@ -36,5 +36,9 @@ jobs:
         run: |
           docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
           docker buildx create --use --name cross-builder
+      - name: Build and push image, tag as "latest"
+        if: ${{ contains(github.event.ref, 'beta') }}
+        run: make PROFILE=maxperf docker-build-push-latest
       - name: Build and push image
-        run: make PROFILE=maxperf docker-build-latest
+        if: ${{ ! contains(github.event.ref, 'beta') }}
+        run: make PROFILE=maxperf docker-build-push

.github/workflows/hive.yml

@@ -17,19 +17,22 @@ concurrency:
 
 jobs:
   prepare:
+    timeout-minutes: 45
     runs-on:
       group: Reth
     steps:
       - uses: actions/checkout@v4
       - run: mkdir artifacts
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Build and export reth image
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
           tags: ghcr.io/paradigmxyz/reth:latest
-          build-args: BUILD_PROFILE=hivetests
+          build-args: |
+            BUILD_PROFILE=hivetests
+            FEATURES=asm-keccak
           outputs: type=docker,dest=./artifacts/reth_image.tar
           cache-from: type=gha
           cache-to: type=gha,mode=max
@@ -41,7 +44,7 @@ jobs:
           ref: master
           path: hivetests
 
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
           go-version: "^1.13.1"
       - run: go version
@@ -52,12 +55,13 @@ jobs:
           mv ./hive ../artifacts/
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: artifacts
           path: ./artifacts
 
   test:
+    timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
@@ -162,10 +166,11 @@ jobs:
     name: run
     runs-on:
       group: Reth
-
+    permissions:
+      issues: write
     steps:
       - name: Download artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: artifacts
           path: /tmp
@@ -191,6 +196,28 @@ jobs:
           cd hivetests
           hive --sim "${{ matrix.sim }}$" --sim.limit "${{matrix.limit}}/${{join(matrix.include, '|')}}" --client reth
 
+      - name: Create github issue if sim failed
+        env:
+          GH_TOKEN: ${{ github.token }}
+        if: ${{ failure() }}
+        run: |
+          echo "Simulator failed, creating issue"
+          # Check if issue already exists
+          # get all issues with the label C-hivetest, loop over each page and check if the issue already exists
+
+          existing_issues=$(gh api /repos/paradigmxyz/reth/issues -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -F "labels=C-hivetest" --method GET | jq '.[].title')
+          if [[ $existing_issues == *"Hive Test Failure: ${{ matrix.sim }}"* ]]; then
+            echo "Issue already exists"
+            exit 0
+          fi
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            /repos/${{ github.repository }}/issues \
+            -f title='Hive Test Failure: ${{ matrix.sim }}' \
+            -f body="!!!!!!! This is an automated issue created by the hive test failure !!!!!!!<br /><br />The hive test for ${{ matrix.sim }} failed. Please investigate and fix the issue.<br /><br />[Link to the failed run](https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }})" \
+            -f "labels[]=C-hivetest"
       - name: Print simulator output
         if: ${{ failure() }}
         run: |