From 2e91897da955d7203637a8c7134edba8b148adaf Mon Sep 17 00:00:00 2001 From: Tomas Fabrizio Orsi Date: Tue, 11 Feb 2025 15:01:58 -0300 Subject: [PATCH] ci(levm): add comparison between ef tests (#1894) **Motivation** Now that we have incorporated new tests, it would be nice if we could compare the EF-tests result of the current PR against main. **Description** Incorporate: - `compare-ef-tests` job that compares the results of the PR and the main branch. - This will post a comment in the PR that will only show the table when there's a difference - Update the Summary text (Previously it only showed until `Frontier` forks, now it will show all forks). --- .github/scripts/compare_ef_tests.sh | 41 ++++++ .github/scripts/parse_test_result.sh | 18 +++ .github/workflows/ci_bench_levm_in_pr.yaml | 4 +- .github/workflows/ci_levm.yaml | 152 ++++++++++++++++++++- 4 files changed, 209 insertions(+), 6 deletions(-) create mode 100644 .github/scripts/compare_ef_tests.sh create mode 100644 .github/scripts/parse_test_result.sh diff --git a/.github/scripts/compare_ef_tests.sh b/.github/scripts/compare_ef_tests.sh new file mode 100644 index 0000000000..8498fb277b --- /dev/null +++ b/.github/scripts/compare_ef_tests.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# $1 Main branch tests results +# $2 PR branch tests results +main_results=$(cat "$1") +IFS=$'\n' read -rd '' -a main_results <<<"${main_results}" + + +pr_results=$(cat "$2") +IFS=$'\n' read -rd '' -a pr_results <<<"${pr_results}" + + +echo "# EF Tests Comparison" +echo "|Test Name | MAIN | PR | DIFF | " +echo "|----------|----------|----|------|" + +num=0 +for i in "${main_results[@]}" +do + name_main=$(echo "$i" | awk -F " " '{print $1}') + result_main=$(echo "$i" | awk -F " " '{print $2}') + result_main=${result_main%(*} + + name_pr=$(echo "${pr_results[num]}" | awk -F " " '{print $1}') + result_pr=$(echo "${pr_results[num]}" | awk -F " " '{print $2}') + result_pr=${result_pr%(*} + + emoji="" + if (( $(echo "$result_main > $result_pr" |bc -l) )); then + emoji="⬇️️" + elif (( $(echo "$result_main < $result_pr" |bc -l) )); then + emoji="⬆️" + else + emoji="➖️" + fi + + echo "|$name_main|$result_main|$result_pr| $emoji |" + + num=$((num + 1)) + +done diff --git a/.github/scripts/parse_test_result.sh b/.github/scripts/parse_test_result.sh new file mode 100644 index 0000000000..808670c5ec --- /dev/null +++ b/.github/scripts/parse_test_result.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# To determine where the test summary ends, we use new lines. + +# Remove everything until the line that says "Summary: " +resulting_text=$(awk '/Summary: /, 0' $1) + +empty_lines=$(echo "${resulting_text}" | awk '/^$/{print NR}') +empty_lines=($empty_lines) + +resulting_text=$(echo "${resulting_text}" | sed -e "${empty_lines[0]}d") + +# We substract one because we deleted one before. This correction +# shouldn't be needed if all lines are deleted as once +empty_lines[1]=$((empty_lines[1] - 1)) + +resulting_text=$(echo "${resulting_text}" | sed -e "${empty_lines[1]},\$d") +echo "${resulting_text}" diff --git a/.github/workflows/ci_bench_levm_in_pr.yaml b/.github/workflows/ci_bench_levm_in_pr.yaml index 9bf43f7e15..bbf1dda4da 100644 --- a/.github/workflows/ci_bench_levm_in_pr.yaml +++ b/.github/workflows/ci_bench_levm_in_pr.yaml @@ -185,7 +185,7 @@ jobs: - name: Show test summary -- short run: | - cd crates/vm/levm && awk '/Summary: /,/Frontier/' test_result.txt; + bash ./.github/scripts/parse_test_result.sh crates/vm/levm/test_result.txt; - name: Check EF-TESTS status is 100% id: check_tests @@ -202,7 +202,7 @@ jobs: run: | { printf '```\n'; - cd crates/vm/levm && awk '/Summary: /,/Frontier/' test_result.txt; + bash ./.github/scripts/parse_test_result.sh crates/vm/levm/test_result.txt; printf '```\n'; } > test_summary.txt diff --git a/.github/workflows/ci_levm.yaml b/.github/workflows/ci_levm.yaml index 0030cd020f..6aef045897 100644 --- a/.github/workflows/ci_levm.yaml +++ b/.github/workflows/ci_levm.yaml @@ -41,24 +41,168 @@ jobs: - name: Run tests run: | cd crates/vm/levm - make run-evm-ef-tests-ci | tee test_result.txt + make run-evm-ef-tests-ci | tee test_result_pr.txt - name: Show test summary -- full run: | - cd crates/vm/levm && awk '/Summary: /,0' test_result.txt + cd crates/vm/levm && awk '/Summary: /,0' test_result_pr.txt - name: Show test summary -- short run: | - cd crates/vm/levm && awk '/Summary: /,/Frontier/' test_result.txt; + bash .github/scripts/parse_test_result.sh crates/vm/levm/test_result_pr.txt | tee crates/vm/levm/test_result_pr_short.txt + echo "PR's test results:" + cat crates/vm/levm/test_result_pr_short.txt + + - name: Upload PR branch EF-test results. + uses: actions/upload-artifact@v4 + with: + name: pr-ef-test-data + path: crates/vm/levm/test_result_pr_short.txt + + # This is only needed in the PR. As soon as this is merged, this should be deleted + - name: Upload comparison shell script + uses: actions/upload-artifact@v4 + with: + name: ef-test-shell-script + path: .github/scripts/parse_test_result.sh - name: Check EF-TESTS status is 100% run: | cd crates/vm/levm - if [ "$(awk '/Summary:/ {print $(NF)}' test_result.txt)" != "(100.00%)" ]; then + if [ "$(awk '/Summary:/ {print $(NF)}' test_result_pr_short.txt)" != "(100.00%)" ]; then echo "Percentage is not 100%." # exit 1 # uncomment when we expect 100% pass-rate fi + ef-test-main: + if: ${{ github.event_name != 'merge_group' }} + name: EF Tests Check main + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + with: + ref: main + + - name: Rustup toolchain install + uses: dtolnay/rust-toolchain@stable + with: + toolchain: ${{ env.RUST_VERSION }} + + - name: Caching + uses: Swatinem/rust-cache@v2 + + - name: Download EF Tests + run: | + cd crates/vm/levm + make download-evm-ef-tests + + - name: Run tests + run: | + cd crates/vm/levm + make run-evm-ef-tests-ci | tee test_result_main.txt + + - name: Show test summary -- full + run: | + cd crates/vm/levm && awk '/Summary: /,0' test_result_main.txt + + # This is only needed in the PR. As soon as this is merged, this should be deleted + - name: Download shell script + uses: actions/download-artifact@v4 + with: + name: ef-test-shell-script + path: .github/scripts/ + + - name: Show test summary -- short + run: | + bash .github/scripts/parse_test_result.sh crates/vm/levm/test_result_main.txt | tee crates/vm/levm/test_result_main_short.txt + echo "Main's test results:" + cat crates/vm/levm/test_result_main_short.txt + + - name: Upload main branch EF-test results. + uses: actions/upload-artifact@v4 + with: + name: main-ef-test-data + path: crates/vm/levm/test_result_main_short.txt + + - name: Check EF-TESTS status is 100% + run: | + cd crates/vm/levm + if [ "$(awk '/Summary:/ {print $(NF)}' test_result_main_short.txt)" != "(100.00%)" ]; then + echo "Percentage is not 100%." + # exit 1 # uncomment when we expect 100% pass-rate + fi + + compare-ef-tests: + if: ${{ github.event_name != 'merge_group' }} + name: EF Tests Compare + needs: [ef-test-main, ef-test] + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Download main branch ef tests + uses: actions/download-artifact@v4 + with: + name: main-ef-test-data + path: crates/vm/levm/ + + - name: Download PR branch ef tests + uses: actions/download-artifact@v4 + with: + name: pr-ef-test-data + path: crates/vm/levm/ + + # NOTE: diff will exit with a non 0 exit code when there are differences + - name: Compare files + id: branch_diffs + continue-on-error: true + run: | + cd crates/vm/levm + diff test_result_main_short.txt test_result_pr_short.txt + + - name: Compare results + if: ${{ steps.branch_diffs.outcome == 'failure' && github.event_name == 'pull_request' }} + run: | + bash .github/scripts/compare_ef_tests.sh crates/vm/levm/test_result_main_short.txt crates/vm/levm/test_result_pr_short.txt | tee crates/vm/levm/ef_tests_comparison.md + + - name: Find comment + continue-on-error: true + uses: peter-evans/find-comment@v3 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: "github-actions[bot]" + body-includes: "EF Tests Comparison" + + # If we have a failure, means that some ef-tests don't pass. + # If the condition is met, create or update the comment with the summary. + - name: Create comment + if: ${{ steps.branch_diffs.outcome == 'failure' && github.event_name == 'pull_request' }} + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + token: ${{ secrets.GITHUB_TOKEN }} + issue-number: ${{ github.event.pull_request.number }} + body-path: crates/vm/levm/ef_tests_comparison.md + edit-mode: replace + + # If we don't have a failure, means that all ef-tests pass. + # If comment-id != '', means that we've already created the comment. + # If both conditions are met, update the comment saying that all tests pass. + - name: Update comment + if: ${{ steps.branch_diffs.outcome != 'failure' && github.event_name == 'pull_request' && steps.fc.outputs.comment-id != '' }} + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + token: ${{ secrets.GITHUB_TOKEN }} + issue-number: ${{ github.event.pull_request.number }} + body: | + # EF Tests Comparison + Same results between main branch and the current PR. + edit-mode: replace + hive-report-creation: uses: ./.github/workflows/common_hive_reports.yaml with: