perf(levm): automate flamegraph comparison

.github/workflows/daily_reports.yaml

@@ -203,6 +203,9 @@ jobs:
     name: Post to Slack link to Flamegraphs Page
     runs-on: ubuntu-latest
     steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
       - name: Post message to slack
         env:
           SLACK_WEBHOOKS: >

.github/workflows/flamegraph_reporter.yaml

@@ -5,16 +5,161 @@ permissions:
   pages: write
   id-token: write
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 on:
   push:
-    branches: [ "main", "automate-perf-tests" ]
+    branches: ["main"]
   workflow_dispatch:
 
 env:
   RUST_VERSION: 1.84.0
   RUST_RETH_VERSION: 1.84.0
 
 jobs:
+  generate-flamegraph-evm:
+    name: Generate Flamegraph LEVM
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        name: ["levm", "revm"]
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Rustup toolchain install
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Cache Extra Binaries
+        id: cache-binaries
+        uses: actions/cache@v4
+        with:
+          path: |
+            ${{ env.HOME }}/.cargo/bin/addr2line
+            ${{ env.HOME }}/.cargo/bin/flamegraph
+            ${{ env.HOME }}/.cargo/bin/inferno-*
+            ${{ env.HOME }}/ethrex/ethrex/cmd/ef_tests/levm/vectors
+          key: ${{ runner.os }}-${{ env.RUST_VERSION }}-extra-binaries
+
+      - name: Change perf settings
+        run: |
+          sudo sysctl kernel.perf_event_paranoid=-1
+          sudo sysctl -w kernel.kptr_restrict=0
+          sudo chmod +r /proc/kallsyms
+          sudo perf list hw
+
+      - name: Check addr2line installation
+        id: check-addr2line
+        run: |
+          if [ -f "$HOME/.cargo/bin/addr2line" ]; then
+            echo "$HOME/.cargo/bin/addr2line found"
+            echo "addr2line_exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "$HOME/.cargo/bin/addr2line NOT found"
+            echo "addr2line_exists=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Checkout gimli addr2line
+        if: steps.check-addr2line.outputs.addr2line_exists != 'true'
+        uses: actions/checkout@v4
+        with:
+          repository: gimli-rs/addr2line
+          path: "addr2line"
+
+      - name: Build gimli addr2line
+        if: steps.check-addr2line.outputs.addr2line_exists != 'true'
+        working-directory: ./addr2line
+        run: |
+          # Little hack we need else it throws error building
+          echo "[workspace]" >> ./Cargo.toml
+          cargo install --force --features="bin" addr2line
+
+      - name: Install flamegraph tools
+        run: |
+          if [ ! -f "$HOME/.cargo/bin/flamegraph" ]; then
+            cargo install --force flamegraph
+          else
+            echo "$HOME/.cargo/bin/flamegraph already found"
+          fi
+          if [ ! -f "$HOME/.cargo/bin/inferno-collapse-perf" ]; then
+            cargo install --force inferno
+          else
+            echo "$HOME/.cargo/bin/inferno-collapse-perf already found"
+          fi
+
+      - name: Download EF Tests
+        run: |
+          if [ ! -f "$HOME/ethrex/ethrex/cmd/ef_tests/levm/vectors" ]; then
+            cd crates/vm/levm
+            make download-evm-ef-tests
+          fi
+
+      - name: Generate Flamegraph data for ${{ matrix.name }}
+        run: |
+          start_time=$(date +%s)
+
+          if [ "${{ matrix.name }}" == "levm" ]; then
+            rm -rf target/debug/ef_tests_levm
+            CARGO_PROFILE_RELEASE_DEBUG=true cargo flamegraph -c "record -o perf.data -F997 --call-graph dwarf,16384 -g" \
+            -p ef_tests-levm -- --tests stSolidityTest,stCallCodes
+          elif [ "${{ matrix.name }}" == "revm" ]; then
+            rm -rf target/debug/ef_tests_levm
+            CARGO_PROFILE_RELEASE_DEBUG=true cargo flamegraph -c "record -o perf.data -F997 --call-graph dwarf,16384 -g" \
+            -p ef_tests-levm -- --revm --tests stSolidityTest,stCallCodes
+          fi
+
+          end_time=$(date +%s)
+          elapsed_time=$((end_time - start_time))
+          echo "time_${{ matrix.name }}=${elapsed_time}" >> time_${{ matrix.name }}.txt
+
+      - name: Generate SVG
+        shell: bash
+        run: |
+          PATH=$HOME/.cargo/bin:$PATH which addr2line
+          PATH=$HOME/.cargo/bin:$PATH perf script -v -i /home/runner/work/ethrex/ethrex/perf.data --no-inline > stack.data
+          PATH=$HOME/.cargo/bin:$PATH inferno-collapse-perf -q < stack.data > collapsed.data
+
+          if [ "${{ matrix.name }}" == "levm" ]; then
+            PATH=$HOME/.cargo/bin:$PATH inferno-flamegraph --title "LEVM Flamegraph" < collapsed.data > flamegraph_levm.svg
+          elif [ "${{ matrix.name }}" == "revm" ]; then
+            PATH=$HOME/.cargo/bin:$PATH inferno-flamegraph --title "REVM Flamegraph" < collapsed.data > flamegraph_revm.svg
+          fi
+
+      - name: Upload artifact - time_${{ matrix.name }}.txt
+        uses: actions/upload-artifact@v4
+        with:
+          name: time_${{ matrix.name }}.txt
+          path: time_${{ matrix.name }}.txt
+
+      - name: Upload artifact - ${{ matrix.name }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: flamegraph_${{ matrix.name }}.svg
+          path: ./flamegraph_${{ matrix.name }}.svg
+
+  test-output:
+    name: Test Output
+    runs-on: ubuntu-latest
+    needs: [generate-flamegraph-evm]
+    steps:
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: time_levm.txt
+          path: time_levm.txt
+
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: time_revm.txt
+          path: time_revm.txt
+      - name: echo Output
+        run: |
+          echo "LevM Time: $(grep "time_levm" time_levm.txt/time_levm.txt | cut -d '=' -f2)"
+          echo "Revm Time: $(grep "time_revm" time_revm.txt/time_revm.txt | cut -d '=' -f2)"
+
   flamegraph-ethrex:
     name: Generate Flamegraph for Ethrex
     runs-on: ubuntu-latest
@@ -163,8 +308,8 @@ jobs:
       - name: Caching
         uses: Swatinem/rust-cache@v2
         with:
-          workspaces: '. -> target
-          ./reth -> ./reth/target'
+          workspaces: ". -> target \
+            ./reth -> ./reth/target"
 
       - name: Cache Extra Binaries
         id: cache-binaries
@@ -260,7 +405,6 @@ jobs:
           bash /home/runner/work/ethrex/ethrex/.github/scripts/flamegraph_watcher.sh &&
           echo "Load test finished"
 
-
       - name: Generate SVG
         shell: bash
         run: |
@@ -281,7 +425,7 @@ jobs:
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
-    needs: [ flamegraph-ethrex, flamegraph-reth ]
+    needs: [generate-flamegraph-evm, flamegraph-ethrex, flamegraph-reth]
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
@@ -298,23 +442,67 @@ jobs:
           name: flamegraph_reth.svg
           path: flamegraph_reth.svg
 
+      - name: Download levm flamegraph artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: flamegraph_levm.svg
+          path: flamegraph_levm.svg
+
+      - name: Download revm flamegraph artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: flamegraph_revm.svg
+          path: flamegraph_revm.svg
+
+      - name: Download levm time artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: time_levm.txt
+          path: time_levm.txt
+
+      - name: Download revm time artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: time_revm.txt
+          path: time_revm.txt
+
       - name: Update static page locally with new data
         shell: bash
         run: |
+          cp -r flamegraph_levm.svg pages/
+          cp -r flamegraph_revm.svg pages/
           cp -r flamegraph_ethrex.svg pages/
           cp -r flamegraph_reth.svg pages/
+
+          # ETHREX HTML
           sed -i "s/{{LAST_UPDATE}}/$(TZ='Etc/GMT+3' date +'%Y-%m-%dT%H:%M:%S')/g" pages/index.html
           sed -i "s/{{ETHREX_TIME}}/${{ needs.flamegraph-ethrex.outputs.time }}/g" pages/index.html
           sed -i "s/{{RETH_TIME}}/${{ needs.flamegraph-reth.outputs.time }}/g" pages/index.html
 
+          # LEVM HTML
+          sed -i "s/{{LAST_UPDATE}}/$(TZ='Etc/GMT+3' date +'%Y-%m-%dT%H:%M:%S')/g" pages/levm.html
+          time_levm=$(grep "time_levm" time_levm.txt/time_levm.txt | cut -d '=' -f2)
+
+          minutes=$(bc -l <<< "scale=0; $time_levm / 60")
+          seconds=$(bc -l <<< "scale=0; $time_levm % 60")
+
+          time_levm_str="${minutes} minutes ${seconds} seconds"
+          sed -i "s/{{LEVM_TIME}}/${time_levm_str}/g" pages/levm.html
+
+          time_revm=$(grep "time_revm" time_revm.txt/time_revm.txt | cut -d '=' -f2)
+
+          minutes=$(bc -l <<< "scale=0; $time_revm / 60")
+          seconds=$(bc -l <<< "scale=0; $time_revm % 60")
+
+          time_revm_str="${minutes} minutes ${seconds} seconds"
+          sed -i "s/{{REVM_TIME}}/${time_revm_str}/g" pages/levm.html
+
       - name: Setup Pages
         uses: actions/configure-pages@v5
-
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3
         with:
           path: "pages/"
-
       - name: Deploy to GitHub Pages
         id: deployment
         uses: actions/deploy-pages@v4

cmd/ef_tests/levm/Cargo.toml

@@ -35,6 +35,10 @@ hex = "0.4.3"
 [lib]
 path = "./ef_tests.rs"
 
+[[bin]]
+name = "ef_tests_levm"
+path = "tests/ef_tests_levm.rs"
+
 [[test]]
 name = "ef_tests_levm"
 harness = false

pages/index.html

@@ -1,79 +1,52 @@
 <!DOCTYPE html>
 <html lang="en">
+
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>Ethrex Flamegraph Comparison</title>
-    <link rel="stylesheet" type="text/css" href="style.css"/>
+    <link rel="stylesheet" type="text/css" href="style.css" />
 
     <!-- This shows in the Slack link preview -->
     <meta content="Ethrex flamegraph comparison" property="og:title">
     <meta content="Last updated {{LAST_UPDATE}}" property="og:description">
-    <meta property="og:image" content="flamegraph_ethrex.svg/flamegraph_ethrex.svg"/>
-    <meta name="twitter:label1" content="🕒 Ethrex"/>
-    <meta name="twitter:data1" content="{{ETHREX_TIME}}"/>
-    <meta name="twitter:label2" content="🕒 Reth"/>
-    <meta name="twitter:data2" content="{{RETH_TIME}}"/>
+    <meta property="og:image" content="flamegraph_ethrex.svg/flamegraph_ethrex.svg" />
+    <meta name="twitter:label1" content="🕒 Ethrex" />
+    <meta name="twitter:data1" content="{{ETHREX_TIME}}" />
+    <meta name="twitter:label2" content="🕒 Reth" />
+    <meta name="twitter:data2" content="{{RETH_TIME}}" />
 </head>
+
 <body>
-<header>
-    <h1>Ethrex Flamegraph Comparison</h1>
-    <p class="last-updated">Last update: {{LAST_UPDATE}}</p>
-</header>
-<main class="container">
-    <div class="column">
-        <div class="col-header">
-            <h2>Ethrex</h2>
-            <a href="flamegraph_ethrex.svg/flamegraph_ethrex.svg"
-               download="flamegraph_ethrex.svg/flamegraph_ethrex.svg">Download</a>
-        </div>
-        <p class="time-elapsed">Time elapsed: {{ETHREX_TIME}}</p>
-        <div class="object-container">
-            <object id="svg1" data="flamegraph_ethrex.svg/flamegraph_ethrex.svg" type="image/svg+xml"></object>
+    <header>
+        <h1>Ethrex Flamegraph Comparison</h1>
+        <p class="last-updated">Last update: {{LAST_UPDATE}}</p>
+        <a href="levm.html">LEVM Flamegraph Comparison</a>
+    </header>
+    <main class="container">
+        <div class="column">
+            <div class="col-header">
+                <h2>Ethrex</h2>
+                <a href="flamegraph_ethrex.svg/flamegraph_ethrex.svg" download="flamegraph_ethrex.svg">Download</a>
+            </div>
+            <p class="time-elapsed">Time elapsed: {{ETHREX_TIME}}</p>
+            <div class="object-container">
+                <object id="svg1" data="flamegraph_ethrex.svg/flamegraph_ethrex.svg" type="image/svg+xml"></object>
+            </div>
         </div>
-    </div>
-    <div class="column">
-        <div class="col-header">
-            <h2>Reth</h2>
-            <a href="flamegraph_reth.svg/flamegraph_reth.svg" download="flamegraph_reth.svg/flamegraph_reth.svg">
-                Download</a>
+        <div class="column">
+            <div class="col-header">
+                <h2>Reth</h2>
+                <a href="flamegraph_reth.svg/flamegraph_reth.svg" download="flamegraph_reth.svg">
+                    Download</a>
+            </div>
+            <p class="time-elapsed">Time elapsed: {{RETH_TIME}}</p>
+            <div class="object-container">
+                <object id="svg2" data="flamegraph_reth.svg/flamegraph_reth.svg" type="image/svg+xml"></object>
+            </div>
         </div>
-        <p class="time-elapsed">Time elapsed: {{RETH_TIME}}</p>
-        <div class="object-container">
-            <object id="svg2" data="flamegraph_reth.svg/flamegraph_reth.svg" type="image/svg+xml"></object>
-        </div>
-    </div>
-</main>
-
-<script>
-    // We need this so that the SVG can scale to the container.
-    function adjustViewBox(objectId) {
-        const object = document.getElementById(objectId);
-        object.addEventListener('load', () => {
-            const svgDocument = object.contentDocument;
-            const svgElement = svgDocument?.querySelector('svg');
-            if (svgElement) {
-                // Remove explicit width and height attributes to allow scaling
-                svgElement.removeAttribute('width');
-                svgElement.removeAttribute('height');
-                // Ensure the aspect ratio is preserved during scaling
-                svgElement.setAttribute('preserveAspectRatio', 'xMidYMid meet');
-
-                // Get the current viewBox or calculate a new one based on the bounding box
-                const bbox = svgElement.getBBox();
-                svgElement.setAttribute(
-                    'viewBox',
-                    `${bbox.x} ${bbox.y} ${bbox.width} ${bbox.height}`
-                );
-            } else {
-                console.warn(`SVG element not found in object with ID: ${objectId}`);
-            }
-        });
-    }
-
-
-    adjustViewBox('svg1');
-    adjustViewBox('svg2');
-</script>
+    </main>
+    <script src="script.js"></script>
 </body>
+
 </html>