pr_comments: include inputs, compare against main

.github/workflows/ci_levm.yaml

@@ -60,44 +60,51 @@ jobs:
 
   hive-report-creation:
     uses: ./.github/workflows/common/hive_reports.yaml
-  # Check we don't have a regression
+    with:
+      evm: "levm"
+      job_type: "trigger"
+
+  # Check we don't have a regression against main
   hive-test:
-    # "Integration Test" is a required check, don't change the name
     name: Hive Tests Check
     needs: hive-report-creation
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
 
-      - name: Download results (levm)
+      - name: Download results (pr)
         uses: actions/download-artifact@v4
         with:
-          name: results_levm.md
+          name: results_levm_trigger.md
 
       - name: Rename result (1)
         run: cp results.md results_levm.md
 
-      - name: Download results (revm)
+      - name: Download results (main)
         uses: actions/download-artifact@v4
         with:
-          name: results_revm.md
+          name: results_levm_daily.md
+        # If we don't have the file it will use the result(1) file
+        continue-on-error: true
 
       - name: Rename result (2)
-        run: cp results.md results_revm.md
+        run: cp results.md results_levm_daily.md
 
       - name: Create diff message
         run: |
-          bash .github/scripts/levm_revm_diff.sh results_revm.md results_levm.md >> diff.md
+          bash .github/scripts/hive_levm_revm_diff.sh results_levm_daily.md results_levm.md >> diff.md
           cat diff.md >> $GITHUB_STEP_SUMMARY
 
       - name: Check Regression
         run: |
           if grep -q "No differences found" diff.md; then
             echo "No regression found."
+          elif ! grep -q "regression" diff.md; then
+            echo "No regression found."
           else
-            echo "Regression found."
-            exit 1;
+            echo "Differences found."
+            exit 1
           fi
 
   # The purpose of this job is to add it as a required check in GitHub so that we don't have to add every individual job as a required check

.github/workflows/common/hive_reports.yaml

@@ -1,18 +1,28 @@
 name: Run Hive Tests
 
-on: workflow_call
+on:
+  workflow_call:
+    inputs:
+      # Must be 'levm' or 'revm'
+      evm:
+        required: true
+        type: string
+      # Must be 'daily' or 'trigger'
+      # If it's daily, it is assumed that the CI is checking out the main branch
+      job_type:
+        required: true
+        type: string
 
 env:
   CARGO_TERM_COLOR: always
   RUST_VERSION: 1.81.0
 
 jobs:
   run-hive:
-    name: Hive (${{ matrix.vm }}) - ${{ matrix.test.name }}
+    name: Hive (${{ inputs.evm }}) - ${{ matrix.test.name }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        vm: [levm, revm]
         test:
           - {
               name: "Rpc Compat tests",
@@ -29,7 +39,7 @@ jobs:
 
     steps:
       - name: Pull image
-        if: ${{ matrix.vm == 'revm' }}
+        if: ${{ inputs.evm == 'revm' }}
         run: |
           docker pull ghcr.io/lambdaclass/ethrex:latest
           docker tag ghcr.io/lambdaclass/ethrex:latest ethrex:latest
@@ -38,7 +48,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Build Image with LEVM
-        if: ${{ matrix.vm == 'levm' }}
+        if: ${{ inputs.evm == 'levm' }}
         run: cd crates/vm/levm && make build-image-levm
 
       - name: Setup Go
@@ -54,17 +64,14 @@ jobs:
       - name: Upload results
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.test.file_name }}_${{ matrix.vm }}_logs
+          name: ${{ matrix.test.file_name }}_${{ inputs.evm }}_${{ inputs.job_type }}_logs
           path: hive/workspace/logs/*-*.json
           if-no-files-found: error
 
   hive-report:
-    name: Generate and Save report (${{ matrix.vm }})
+    name: Generate and Save report (${{ inputs.evm }})
     needs: run-hive
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        vm: [levm, revm]
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
@@ -78,7 +85,7 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           path: hive/workspace/logs
-          pattern: "*_${{ matrix.vm }}_logs"
+          pattern: "*_${{ inputs.evm }}_${{ inputs.job_type }}_logs"
           merge-multiple: true
 
       - name: Caching
@@ -87,14 +94,15 @@ jobs:
       - name: Generate the hive report
         run: cargo run -p hive_report > results.md
 
-      - name: Upload ${{matrix.vm}} result
+      - name: Upload ${{inputs.evm}}_${{ inputs.job_type }} result
+        if: ${{ inputs.main_branch != 'true' }}
         uses: actions/upload-artifact@v4
         with:
-          name: results_${{matrix.vm}}.md
+          name: results_${{inputs.evm}}_${{ inputs.job_type }}.md
           path: results.md
           if-no-files-found: error
 
       - name: Post results in summary
         run: |
-          echo "# Hive coverage report (${{ matrix.vm }})" >> $GITHUB_STEP_SUMMARY
+          echo "# Hive coverage report (${{ inputs.evm }})" >> $GITHUB_STEP_SUMMARY
           cat results.md >> $GITHUB_STEP_SUMMARY

.github/workflows/daily_reports.yaml

@@ -10,12 +10,20 @@ env:
   RUST_VERSION: 1.81.0
 
 jobs:
-  hive-report-creation:
+  hive-report-creation-levm:
     uses: ./.github/workflows/common/hive_reports.yaml
+    with:
+      evm: "levm"
+      job_type: "daily"
+  hive-report-creation-revm:
+    uses: ./.github/workflows/common/hive_reports.yaml
+    with:
+      evm: "revm"
+      job_type: "daily"
 
   hive-diff-report:
     name: Post tests diff to levm slack
-    needs: hive-report-creation
+    needs: [hive-report-creation-levm, hive-report-creation-revm]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
@@ -24,22 +32,22 @@ jobs:
       - name: Download results (levm)
         uses: actions/download-artifact@v4
         with:
-          name: results_levm.md
+          name: results_levm_daily_main.md
 
       - name: Rename result (1)
         run: cp results.md results_levm.md
 
       - name: Download results (revm)
         uses: actions/download-artifact@v4
         with:
-          name: results_revm.md
+          name: results_revm_daily_main.md
 
       - name: Rename result (2)
         run: cp results.md results_revm.md
 
       - name: Create diff message
         run: |
-          bash .github/scripts/levm_revm_diff.sh results_revm.md results_levm.md >> diff.md
+          bash .github/scripts/hive_levm_revm_diff.sh results_revm.md results_levm.md >> diff.md
           cat diff.md >> $GITHUB_STEP_SUMMARY
 
       - name: Post results to Slack