👷 [#663] Upload performance tests results to bencher

.github/workflows/ci.yml

@@ -76,34 +76,95 @@ jobs:
     name: Run the performance test suite
     runs-on: ubuntu-latest
 
+    services:
+      postgres:
+        image: postgis/postgis:17-3.5
+        env:
+          POSTGRES_HOST_AUTH_METHOD: trust
+        ports:
+          - 5432:5432
+        # needed because the postgres container does not provide a healthcheck
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v5
+      - name: Set up backend environment
+        uses: maykinmedia/setup-django-backend@v1.3
         with:
-          python-version: '3.12'
+          apt-packages: "libgdal-dev gdal-bin postgresql-client"
+          python-version: "3.12"
+          optimize-postgres: "no"
+          setup-node: "no"
 
-      - name: Bring up docker compose and load data
+      - name: Set up uWSGI and generate data
+        run: |
+          createdb -h localhost -U postgres objects
+          pip install uv
+          uv pip install --system -r requirements/ci.txt
+          src/manage.py migrate
+          SCRIPTPATH=bin UWSGI_PROCESSES=4 UWSGI_THREADS=4 bin/docker_start.sh &
+          src/manage.py shell < performance_test/create_data.py
+        env:
+          RUN_SETUP_CONFIG: false
+          DB_PASSWORD: ""
+          DB_USER: postgres
+          DB_NAME: objects
+          DB_HOST: localhost
+          SECRET_KEY: secret
+          DJANGO_SETTINGS_MODULE: objects.conf.production
+          SITE_DOMAIN: localhost:8000
+          ALLOWED_HOSTS: localhost,127.0.0.1
+          FUZZY_PAGINATION: true
+          DB_CONN_MAX_AGE: 60
+
+      - name: Install dependencies
         run: |
-          docker compose up -d --build || ( docker compose logs >&2 && exit 1; )
-          until docker compose logs web | grep -q "spawned uWSGI worker"; do
-            echo "uWSGI not running yet, waiting..."
-            sleep 3
-          done
-          docker compose exec --user root web pip install factory-boy
-          cat performance_test/create_data.py | docker compose exec -T web src/manage.py shell
+          curl --proto '=https' --tlsv1.2 -sSfL https://bencher.dev/download/install-cli.sh | sh
+          pip install uv
+          pkgs=$(cat requirements/performance-tests.in)
+          grep -E "^($(echo $pkgs | sed 's/ /|/g'))([=<>]|$)" requirements/ci.txt | xargs uv pip install --system
 
       - run: python -m pip install --upgrade pip
       - run: python -m pip install locust
 
       - name: Run Locust tests
         run: |
-          python -m locust --config performance_test/locust.conf
+          python -m locust --config performance_test/locust/locust.conf
 
-      - name: Run tests
+      - name: Run tests for master branch
+        if: github.ref == 'refs/heads/master'
+        run: |
+          bencher run --adapter python_pytest \
+                      --project objects-ed2ce35-z71n5gf8t4f40 \
+                      --token '${{ secrets.BENCHER_API_TOKEN }}' \
+                      --branch master \
+                      --testbed ubuntu-latest \
+                      --average median \
+                      --start-point-reset \
+                      --github-actions '${{ secrets.GITHUB_TOKEN }}' \
+                      --file output.json \
+                      "pytest performance_test/tests/ --benchmark-json output.json"
+
+      - name: Run tests for other branches
+        if: github.ref != 'refs/heads/master' && !startsWith(github.ref, 'refs/tags/')
         run: |
-          pip install -r requirements/ci.txt
-          pytest -v performance_test/test_objects_list.py --benchmark-json output.json
+          bencher run --adapter python_pytest \
+                      --project objects-ed2ce35-z71n5gf8t4f40 \
+                      --token '${{ secrets.BENCHER_API_TOKEN }}' \
+                      --branch "$GITHUB_HEAD_REF" \
+                      --start-point "$GITHUB_BASE_REF" \
+                      --start-point-reset \
+                      --start-point-max-versions 1 \
+                      --average median \
+                      --threshold-measure latency \
+                      --threshold-test percentage \
+                      --threshold-upper-boundary 0.05 \
+                      --thresholds-reset \
+                      --testbed ubuntu-latest \
+                      --github-actions '${{ secrets.GITHUB_TOKEN }}' \
+                      --file output.json \
+                      "pytest performance_test/tests/ --benchmark-json output.json"
 
   docs:
     runs-on: ubuntu-latest

bin/compile_dependencies.sh

@@ -32,12 +32,14 @@ uv pip compile \
     --output-file requirements/ci.txt \
     "$@" \
     requirements/test-tools.in \
-    requirements/docs.in
+    requirements/docs.in \
+    requirements/performance-tests.in
 
 # Dev depedencies - exact same set as CI + some extra tooling
 uv pip compile \
     --output-file requirements/dev.txt \
     "$@" \
-    requirements/dev.in
+    requirements/dev.in \
+    requirements/performance-tests.in
 
 cd "${cwd}"

performance_test/locust.conf → performance_test/locust/locust.conf

@@ -1,4 +1,4 @@
-locustfile = performance_test/test_locust.py
+locustfile = performance_test/locust/test_locust.py
 headless = true
 host = http://localhost:8000
 users = 100

requirements/base.txt

@@ -315,7 +315,7 @@ pyyaml==6.0.1
     #   pydantic-settings
 qrcode==6.1
     # via django-two-factor-auth
-redis==3.5.3
+redis==6.4.0
     # via django-redis
 requests==2.32.4
     # via

requirements/ci.txt

@@ -371,7 +371,9 @@ face==20.1.1
     #   -r requirements/base.txt
     #   glom
 factory-boy==3.2.0
-    # via -r requirements/test-tools.in
+    # via
+    #   -r requirements/performance-tests.in
+    #   -r requirements/test-tools.in
 faker==37.1.0
     # via factory-boy
 flower==2.0.1
@@ -384,6 +386,7 @@ furl==2.1.3
     # via
     #   -c requirements/base.txt
     #   -r requirements/base.txt
+    #   -r requirements/performance-tests.in
     #   ape-pie
 glom==23.5.0
     # via
@@ -544,6 +547,7 @@ pyjwt==2.10.1
     # via
     #   -c requirements/base.txt
     #   -r requirements/base.txt
+    #   -r requirements/performance-tests.in
     #   commonground-api-common
     #   zgw-consumers
 pyopenssl==24.3.0
@@ -562,10 +566,13 @@ pyrsistent==0.17.3
     #   jsonschema
 pytest==8.3.3
     # via
+    #   -r requirements/performance-tests.in
     #   -r requirements/test-tools.in
     #   pytest-benchmark
 pytest-benchmark==5.1.0
-    # via -r requirements/test-tools.in
+    # via
+    #   -r requirements/performance-tests.in
+    #   -r requirements/test-tools.in
 python-dateutil==2.9.0.post0
     # via
     #   -c requirements/base.txt
@@ -609,7 +616,7 @@ qrcode==6.1
     #   django-two-factor-auth
 recommonmark==0.7.1
     # via -r requirements/docs.in
-redis==3.5.3
+redis==6.4.0
     # via
     #   -c requirements/base.txt
     #   -r requirements/base.txt
@@ -618,6 +625,7 @@ requests==2.32.4
     # via
     #   -c requirements/base.txt
     #   -r requirements/base.txt
+    #   -r requirements/performance-tests.in
     #   ape-pie
     #   codecov
     #   commonground-api-common

requirements/dev.txt

@@ -422,6 +422,7 @@ factory-boy==3.2.0
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
+    #   -r requirements/performance-tests.in
 faker==37.1.0
     # via
     #   -c requirements/ci.txt
@@ -441,6 +442,7 @@ furl==2.1.3
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
+    #   -r requirements/performance-tests.in
     #   ape-pie
 glom==23.5.0
     # via
@@ -665,6 +667,7 @@ pyjwt==2.10.1
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
+    #   -r requirements/performance-tests.in
     #   commonground-api-common
     #   zgw-consumers
 pyopenssl==24.3.0
@@ -691,11 +694,13 @@ pytest==8.3.3
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
+    #   -r requirements/performance-tests.in
     #   pytest-benchmark
 pytest-benchmark==5.1.0
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
+    #   -r requirements/performance-tests.in
 python-dateutil==2.9.0.post0
     # via
     #   -c requirements/ci.txt
@@ -745,7 +750,7 @@ recommonmark==0.7.1
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   -r requirements/dev.in
-redis==3.5.3
+redis==6.4.0
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
@@ -754,6 +759,7 @@ requests==2.32.4
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
+    #   -r requirements/performance-tests.in
     #   ape-pie
     #   codecov
     #   commonground-api-common

requirements/performance-tests.in

@@ -0,0 +1,6 @@
+pytest
+pytest-benchmark
+furl
+requests
+pyjwt
+factory-boy