benchmark.yml

name: Benchmark
on:
  push:
    branches: [master]
  pull_request:
    branches: [master]

permissions:
  contents: write
  deployments: write

jobs:
  benchmark:
    name: Run pytest-benchmark benchmark example
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install deps
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          uv pip install --system -r requirements/dev-requirements.txt
          uv pip install --system torch --index-url https://download.pytorch.org/whl/cpu
      - name: Install module
        run: |
          uv pip install --system .
      - name: Run benchmark
        run: |
          pytest tests/benchmark*.py --benchmark-json output.json
      - name: Store benchmark result
        if: github.ref == 'refs/heads/master'
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: Python Benchmark with pytest-benchmark
          tool: "pytest"
          output-file-path: output.json
          # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: true
          # Show alert with commit comment on detecting possible performance regression
          alert-threshold: "200%"
          comment-on-alert: false
          fail-on-alert: false
      - name: Check benchmark threshold
        if: github.ref != 'refs/heads/master'
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: Python Benchmark with pytest-benchmark
          tool: "pytest"
          output-file-path: output.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: false
          # Show alert with commit comment on detecting possible performance regression
          alert-threshold: "200%"
          comment-on-alert: true
          fail-on-alert: false