diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e8b9288..5576d065 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,34 +76,95 @@ jobs: name: Run the performance test suite runs-on: ubuntu-latest + services: + postgres: + image: postgis/postgis:17-3.5 + env: + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - name: Set up backend environment + uses: maykinmedia/setup-django-backend@v1.3 with: - python-version: '3.12' + apt-packages: "libgdal-dev gdal-bin postgresql-client" + python-version: "3.12" + optimize-postgres: "no" + setup-node: "no" - - name: Bring up docker compose and load data + - name: Set up uWSGI and generate data + run: | + createdb -h localhost -U postgres objects + pip install uv + uv pip install --system -r requirements/ci.txt + src/manage.py migrate + SCRIPTPATH=bin UWSGI_PROCESSES=4 UWSGI_THREADS=4 bin/docker_start.sh & + src/manage.py shell < performance_test/create_data.py + env: + RUN_SETUP_CONFIG: false + DB_PASSWORD: "" + DB_USER: postgres + DB_NAME: objects + DB_HOST: localhost + SECRET_KEY: secret + DJANGO_SETTINGS_MODULE: objects.conf.production + SITE_DOMAIN: localhost:8000 + ALLOWED_HOSTS: localhost,127.0.0.1 + FUZZY_PAGINATION: true + DB_CONN_MAX_AGE: 60 + + - name: Install dependencies run: | - docker compose up -d --build || ( docker compose logs >&2 && exit 1; ) - until docker compose logs web | grep -q "spawned uWSGI worker"; do - echo "uWSGI not running yet, waiting..." - sleep 3 - done - docker compose exec --user root web pip install factory-boy - cat performance_test/create_data.py | docker compose exec -T web src/manage.py shell + curl --proto '=https' --tlsv1.2 -sSfL https://bencher.dev/download/install-cli.sh | sh + pip install uv + pkgs=$(cat requirements/performance-tests.in) + grep -E "^($(echo $pkgs | sed 's/ /|/g'))([=<>]|$)" requirements/ci.txt | xargs uv pip install --system - run: python -m pip install --upgrade pip - run: python -m pip install locust - name: Run Locust tests run: | - python -m locust --config performance_test/locust.conf + python -m locust --config performance_test/locust/locust.conf - - name: Run tests + - name: Run tests for master branch + if: github.ref == 'refs/heads/master' + run: | + bencher run --adapter python_pytest \ + --project objects-ed2ce35-z71n5gf8t4f40 \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch master \ + --testbed ubuntu-latest \ + --average median \ + --start-point-reset \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --file output.json \ + "pytest performance_test/tests/ --benchmark-json output.json" + + - name: Run tests for other branches + if: github.ref != 'refs/heads/master' && !startsWith(github.ref, 'refs/tags/') run: | - pip install -r requirements/ci.txt - pytest -v performance_test/test_objects_list.py --benchmark-json output.json + bencher run --adapter python_pytest \ + --project objects-ed2ce35-z71n5gf8t4f40 \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch "$GITHUB_HEAD_REF" \ + --start-point "$GITHUB_BASE_REF" \ + --start-point-reset \ + --start-point-max-versions 1 \ + --average median \ + --threshold-measure latency \ + --threshold-test percentage \ + --threshold-upper-boundary 0.05 \ + --thresholds-reset \ + --testbed ubuntu-latest \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --file output.json \ + "pytest performance_test/tests/ --benchmark-json output.json" docs: runs-on: ubuntu-latest diff --git a/bin/compile_dependencies.sh b/bin/compile_dependencies.sh index 4c5fc48e..98c1bfc5 100755 --- a/bin/compile_dependencies.sh +++ b/bin/compile_dependencies.sh @@ -32,12 +32,14 @@ uv pip compile \ --output-file requirements/ci.txt \ "$@" \ requirements/test-tools.in \ - requirements/docs.in + requirements/docs.in \ + requirements/performance-tests.in # Dev depedencies - exact same set as CI + some extra tooling uv pip compile \ --output-file requirements/dev.txt \ "$@" \ - requirements/dev.in + requirements/dev.in \ + requirements/performance-tests.in cd "${cwd}" diff --git a/output.json b/output.json new file mode 100644 index 00000000..e69de29b diff --git a/performance_test/locust.conf b/performance_test/locust/locust.conf similarity index 63% rename from performance_test/locust.conf rename to performance_test/locust/locust.conf index b5d96f86..60bb71d2 100644 --- a/performance_test/locust.conf +++ b/performance_test/locust/locust.conf @@ -1,4 +1,4 @@ -locustfile = performance_test/test_locust.py +locustfile = performance_test/locust/test_locust.py headless = true host = http://localhost:8000 users = 100 diff --git a/performance_test/test_locust.py b/performance_test/locust/test_locust.py similarity index 100% rename from performance_test/test_locust.py rename to performance_test/locust/test_locust.py diff --git a/performance_test/test_objects_list.py b/performance_test/tests/test_objects_list.py similarity index 100% rename from performance_test/test_objects_list.py rename to performance_test/tests/test_objects_list.py diff --git a/requirements/base.txt b/requirements/base.txt index cb774c5e..906c2f13 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -315,7 +315,7 @@ pyyaml==6.0.1 # pydantic-settings qrcode==6.1 # via django-two-factor-auth -redis==3.5.3 +redis==6.4.0 # via django-redis requests==2.32.4 # via diff --git a/requirements/ci.txt b/requirements/ci.txt index a9e1c572..1ce800de 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -371,7 +371,9 @@ face==20.1.1 # -r requirements/base.txt # glom factory-boy==3.2.0 - # via -r requirements/test-tools.in + # via + # -r requirements/performance-tests.in + # -r requirements/test-tools.in faker==37.1.0 # via factory-boy flower==2.0.1 @@ -384,6 +386,7 @@ furl==2.1.3 # via # -c requirements/base.txt # -r requirements/base.txt + # -r requirements/performance-tests.in # ape-pie glom==23.5.0 # via @@ -544,6 +547,7 @@ pyjwt==2.10.1 # via # -c requirements/base.txt # -r requirements/base.txt + # -r requirements/performance-tests.in # commonground-api-common # zgw-consumers pyopenssl==24.3.0 @@ -562,10 +566,13 @@ pyrsistent==0.17.3 # jsonschema pytest==8.3.3 # via + # -r requirements/performance-tests.in # -r requirements/test-tools.in # pytest-benchmark pytest-benchmark==5.1.0 - # via -r requirements/test-tools.in + # via + # -r requirements/performance-tests.in + # -r requirements/test-tools.in python-dateutil==2.9.0.post0 # via # -c requirements/base.txt @@ -609,7 +616,7 @@ qrcode==6.1 # django-two-factor-auth recommonmark==0.7.1 # via -r requirements/docs.in -redis==3.5.3 +redis==6.4.0 # via # -c requirements/base.txt # -r requirements/base.txt @@ -618,6 +625,7 @@ requests==2.32.4 # via # -c requirements/base.txt # -r requirements/base.txt + # -r requirements/performance-tests.in # ape-pie # codecov # commonground-api-common diff --git a/requirements/dev.txt b/requirements/dev.txt index 221d0de7..b8aaab74 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -422,6 +422,7 @@ factory-boy==3.2.0 # via # -c requirements/ci.txt # -r requirements/ci.txt + # -r requirements/performance-tests.in faker==37.1.0 # via # -c requirements/ci.txt @@ -441,6 +442,7 @@ furl==2.1.3 # via # -c requirements/ci.txt # -r requirements/ci.txt + # -r requirements/performance-tests.in # ape-pie glom==23.5.0 # via @@ -665,6 +667,7 @@ pyjwt==2.10.1 # via # -c requirements/ci.txt # -r requirements/ci.txt + # -r requirements/performance-tests.in # commonground-api-common # zgw-consumers pyopenssl==24.3.0 @@ -691,11 +694,13 @@ pytest==8.3.3 # via # -c requirements/ci.txt # -r requirements/ci.txt + # -r requirements/performance-tests.in # pytest-benchmark pytest-benchmark==5.1.0 # via # -c requirements/ci.txt # -r requirements/ci.txt + # -r requirements/performance-tests.in python-dateutil==2.9.0.post0 # via # -c requirements/ci.txt @@ -745,7 +750,7 @@ recommonmark==0.7.1 # -c requirements/ci.txt # -r requirements/ci.txt # -r requirements/dev.in -redis==3.5.3 +redis==6.4.0 # via # -c requirements/ci.txt # -r requirements/ci.txt @@ -754,6 +759,7 @@ requests==2.32.4 # via # -c requirements/ci.txt # -r requirements/ci.txt + # -r requirements/performance-tests.in # ape-pie # codecov # commonground-api-common diff --git a/requirements/performance-tests.in b/requirements/performance-tests.in new file mode 100644 index 00000000..533607f0 --- /dev/null +++ b/requirements/performance-tests.in @@ -0,0 +1,6 @@ +pytest +pytest-benchmark +furl +requests +pyjwt +factory-boy