benchmark.yml

name: Benchmarks

on:
  pull_request:
    branches:
      - master
    paths-ignore:
      - 'docs/**'
      - 'devtools/**'
  workflow_dispatch:
    inputs:
      debug_enabled:
        type: boolean
        description: '(https://github.com/marketplace/actions/debugging-with-tmate)'
        required: false
        default: false

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  benchmark:
    runs-on: ubuntu-latest
    env:
      GH_TOKEN: ${{ github.token }}
    strategy:
      matrix:
        python-version: ['3.9']
        group: [1, 2]

    steps:
      # Enable tmate debugging of manually-triggered workflows if the input option was provided
      - name: Setup tmate session
        uses: mxschmitt/action-tmate@v3
        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Restore Python environment cache
        id: restore-env
        uses: actions/cache/restore@v4
        with:
          path: .venv-${{ matrix.python-version }}
          key: ${{ runner.os }}-venv-${{ matrix.python-version }}-${{ hashFiles('devtools/dev-requirements.txt', 'requirements.txt') }}

      - name: Set up virtual environment if not restored from cache
        if: steps.restore-env.outputs.cache-hit != 'true'
        run: |
          gh cache list
          python -m venv .venv-${{ matrix.python-version }}
          source .venv-${{ matrix.python-version }}/bin/activate
          python -m pip install --upgrade pip
          pip install -r devtools/dev-requirements.txt

      - name: Benchmark with pytest-benchmark (PR)
        run: |
          source .venv-${{ matrix.python-version }}/bin/activate
          pwd
          lscpu
          cd tests/benchmarks
          python -m pytest benchmark_cpu_small.py -vv \
            --benchmark-save='Latest_Commit' \
            --durations=0 \
            --benchmark-save-data \
            --splits 2 \
            --group ${{ matrix.group }} \
            --splitting-algorithm least_duration

      - name: Checkout current master
        uses: actions/checkout@v4
        with:
          ref: master
          clean: false

      - name: Checkout benchmarks from PR head
        run: git checkout ${{ github.event.pull_request.head.sha }} -- tests/benchmarks

      - name: Benchmark with pytest-benchmark (MASTER)
        run: |
          source .venv-${{ matrix.python-version }}/bin/activate
          pwd
          lscpu
          cd tests/benchmarks
          python -m pytest benchmark_cpu_small.py -vv \
            --benchmark-save='master' \
            --durations=0 \
            --benchmark-save-data \
            --splits 2 \
            --group ${{ matrix.group }} \
            --splitting-algorithm least_duration

      - name: Put benchmark results in same folder
        run: |
          source .venv-${{ matrix.python-version }}/bin/activate
          pwd
          cd tests/benchmarks
          find .benchmarks/ -type f -printf "%T@ %p\n" | sort -n | cut -d' ' -f 2- | tail -n 1 > temp1
          find .benchmarks/ -type f -printf "%T@ %p\n" | sort -n | cut -d' ' -f 2- | tail -n 2 | head -n 1 > temp2
          t1=$(cat temp1)
          t2=$(cat temp2)
          mkdir compare_results
          cp $t1 compare_results
          cp $t2 compare_results

      - name: Download artifact
        if: always()
        uses: actions/download-artifact@v4
        with:
          pattern: benchmark_artifact_*
          path: tests/benchmarks

      - name: Unzip artifacts if downloaded
        run: |
          cd tests/benchmarks
          ls
          if [ -f tests/benchmarks/benchmark_artifact_*.zip ]; then
            unzip tests/benchmarks/benchmark_artifact_*.zip -d tests/benchmarks
          else
            echo "No benchmark artifact file found."
          fi

      - name: Compare latest commit results to the master branch results
        run: |
          source .venv-${{ matrix.python-version }}/bin/activate
          cd tests/benchmarks
          pwd
          python compare_bench_results.py
          cat commit_msg.txt

      - name: Comment PR with the results
        uses: thollander/actions-comment-pull-request@v2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          filePath: tests/benchmarks/commit_msg.txt
          comment_tag: benchmark

      - name: Upload benchmark data
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: benchmark_artifact_${{ matrix.group }}
          path: tests/benchmarks/.benchmarks