diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 371664e6..6f661705 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,8 +16,9 @@ jobs: tests: strategy: matrix: - postgres: ['13', '15', '16', '17'] + postgres: ['14', '15', '16', '17'] postgis: ['3.2', '3.5'] + use_pooling: [false] exclude: # postgis 3.2 is not compatible with recent postgres versions - postgres: '17' @@ -26,8 +27,13 @@ jobs: postgis: '3.2' - postgres: '15' postgis: '3.2' + include: + - postgres: '17' + postgis: '3.5' + use_pooling: true + + name: Run the Django test suite (PG ${{ matrix.postgres }}, postgis ${{ matrix.postgis }}) ${{matrix.use_pooling && 'with connection pooling' || ''}} - name: Run the Django test suite (PG ${{ matrix.postgres }}, postgis ${{ matrix.postgis }}) runs-on: ubuntu-latest @@ -59,6 +65,7 @@ jobs: SECRET_KEY: dummy DB_USER: postgres DB_PASSWORD: '' + DB_POOL_ENABLED: ${{ matrix.use_pooling }} - name: Publish coverage report uses: codecov/codecov-action@v4 diff --git a/INSTALL.rst b/INSTALL.rst index 7831998b..52b78dc9 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -20,7 +20,7 @@ You need the following libraries and/or programs: * `Python`_ 3.11 or above * Python `Virtualenv`_ and `Pip`_ -* `PostgreSQL`_ 13 or above with PostGIS extension +* `PostgreSQL`_ 14 or above with PostGIS extension * `Node.js`_ * `npm`_ diff --git a/docs/installation/config.rst b/docs/installation/config.rst index 7463f707..ec64fb3d 100644 --- a/docs/installation/config.rst +++ b/docs/installation/config.rst @@ -31,6 +31,16 @@ Database * ``DB_PASSWORD``: password of the database user. Defaults to: ``objects``. * ``DB_HOST``: hostname of the PostgreSQL database. Defaults to ``db`` for the docker environment, otherwise defaults to ``localhost``. * ``DB_PORT``: port number of the database. Defaults to: ``5432``. +* ``DB_CONN_MAX_AGE``: The lifetime of a database connection, as an integer of seconds. Use 0 to close database connections at the end of each request — Django’s historical behavior. This setting cannot be set in combination with connection pooling. Defaults to: ``0``. +* ``DB_POOL_ENABLED``: Whether to use connection pooling. Defaults to: ``False``. +* ``DB_POOL_MIN_SIZE``: The minimum number of connection the pool will hold. The pool will actively try to create new connections if some are lost (closed, broken) and will try to never go below min_size. Defaults to: ``4``. +* ``DB_POOL_MAX_SIZE``: The maximum number of connections the pool will hold. If None, or equal to min_size, the pool will not grow or shrink. If larger than min_size, the pool can grow if more than min_size connections are requested at the same time and will shrink back after the extra connections have been unused for more than max_idle seconds. Defaults to: ``None``. +* ``DB_POOL_TIMEOUT``: The default maximum time in seconds that a client can wait to receive a connection from the pool (using connection() or getconn()). Note that these methods allow to override the timeout default. Defaults to: ``30``. +* ``DB_POOL_MAX_WAITING``: Maximum number of requests that can be queued to the pool, after which new requests will fail, raising TooManyRequests. 0 means no queue limit. Defaults to: ``0``. +* ``DB_POOL_MAX_LIFETIME``: The maximum lifetime of a connection in the pool, in seconds. Connections used for longer get closed and replaced by a new one. The amount is reduced by a random 10% to avoid mass eviction. Defaults to: ``3600``. +* ``DB_POOL_MAX_IDLE``: Maximum time, in seconds, that a connection can stay unused in the pool before being closed, and the pool shrunk. This only happens to connections more than min_size, if max_size allowed the pool to grow. Defaults to: ``600``. +* ``DB_POOL_RECONNECT_TIMEOUT``: Maximum time, in seconds, the pool will try to create a connection. If a connection attempt fails, the pool will try to reconnect a few times, using an exponential backoff and some random factor to avoid mass attempts. If repeated attempts fail, after reconnect_timeout second the connection attempt is aborted and the reconnect_failed() callback invoked. Defaults to: ``300``. +* ``DB_POOL_NUM_WORKERS``: Number of background worker threads used to maintain the pool state. Background workers are used for example to create new connections and to clean up connections when they are returned to the pool. Defaults to: ``3``. Cross-Origin-Resource-Sharing diff --git a/publiccode.yaml b/publiccode.yaml index 3f39f559..61814a25 100644 --- a/publiccode.yaml +++ b/publiccode.yaml @@ -71,7 +71,7 @@ dependsOn: versionMin: '1.0' - name: PostgreSQL optional: false - versionMin: '13.0' + versionMin: '14.0' - name: PostGIS optional: false versionMin: '3.2' diff --git a/requirements/base.in b/requirements/base.in index 2943e35e..b87255f4 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -16,3 +16,5 @@ mozilla-django-oidc-db[setup-configuration] # TODO this should be moved to open-api-framework once it is verified that this fixes # maykinmedia/objects-api#541 kombu>=5.4.0 + +psycopg[pool] \ No newline at end of file diff --git a/requirements/base.txt b/requirements/base.txt index f39b4e35..b0d68d48 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -9,7 +9,7 @@ ape-pie==0.2.0 # commonground-api-common # notifications-api-common # zgw-consumers -asgiref==3.7.2 +asgiref==3.8.1 # via # django # django-axes @@ -70,7 +70,7 @@ cryptography==44.0.1 # mozilla-django-oidc # pyopenssl # webauthn -django==4.2.20 +django==5.2 # via # commonground-api-common # django-admin-index @@ -148,7 +148,7 @@ django-sendfile2==0.7.0 # via django-privates django-sessionprofile==3.0.0 # via open-api-framework -django-setup-configuration==0.7.1 +django-setup-configuration==0.7.2 # via # -r requirements/base.in # mozilla-django-oidc-db @@ -258,6 +258,10 @@ prometheus-client==0.20.0 # via flower prompt-toolkit==3.0.43 # via click-repl +psycopg==3.2.6 + # via -r requirements/base.in +psycopg-pool==3.2.6 + # via psycopg psycopg2==2.9.9 # via open-api-framework pycparser==2.20 @@ -336,6 +340,8 @@ tornado==6.4.2 typing-extensions==4.9.0 # via # mozilla-django-oidc-db + # psycopg + # psycopg-pool # pydantic # pydantic-core # zgw-consumers diff --git a/requirements/ci.txt b/requirements/ci.txt index f6067b59..ae5ca5fa 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -19,7 +19,7 @@ ape-pie==0.2.0 # commonground-api-common # notifications-api-common # zgw-consumers -asgiref==3.7.2 +asgiref==3.8.1 # via # -c requirements/base.txt # -r requirements/base.txt @@ -147,7 +147,7 @@ cryptography==44.0.1 # webauthn cssselect==1.1.0 # via pyquery -django==4.2.20 +django==5.2 # via # -c requirements/base.txt # -r requirements/base.txt @@ -282,7 +282,7 @@ django-sessionprofile==3.0.0 # -c requirements/base.txt # -r requirements/base.txt # open-api-framework -django-setup-configuration==0.7.1 +django-setup-configuration==0.7.2 # via # -c requirements/base.txt # -r requirements/base.txt @@ -306,7 +306,7 @@ django-two-factor-auth==1.17.0 # -c requirements/base.txt # -r requirements/base.txt # maykin-2fa -django-webtest==1.9.7 +django-webtest==1.9.13 # via -r requirements/test-tools.in djangorestframework==3.15.2 # via @@ -524,6 +524,14 @@ prompt-toolkit==3.0.43 # -c requirements/base.txt # -r requirements/base.txt # click-repl +psycopg==3.2.6 + # via + # -c requirements/base.txt + # -r requirements/base.txt +psycopg-pool==3.2.6 + # via + # -c requirements/base.txt + # -r requirements/base.txt psycopg2==2.9.9 # via # -c requirements/base.txt @@ -725,6 +733,8 @@ typing-extensions==4.9.0 # -c requirements/base.txt # -r requirements/base.txt # mozilla-django-oidc-db + # psycopg + # psycopg-pool # pydantic # pydantic-core # zgw-consumers diff --git a/requirements/dev.txt b/requirements/dev.txt index 86d03e33..25c87fc2 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -24,7 +24,7 @@ ape-pie==0.2.0 # commonground-api-common # notifications-api-common # zgw-consumers -asgiref==3.7.2 +asgiref==3.8.1 # via # -c requirements/ci.txt # -r requirements/ci.txt @@ -182,7 +182,7 @@ cssselect==1.1.0 # -c requirements/ci.txt # -r requirements/ci.txt # pyquery -django==4.2.20 +django==5.2 # via # -c requirements/ci.txt # -r requirements/ci.txt @@ -324,7 +324,7 @@ django-sessionprofile==3.0.0 # -c requirements/ci.txt # -r requirements/ci.txt # open-api-framework -django-setup-configuration==0.7.1 +django-setup-configuration==0.7.2 # via # -c requirements/ci.txt # -r requirements/ci.txt @@ -350,7 +350,7 @@ django-two-factor-auth==1.17.0 # -c requirements/ci.txt # -r requirements/ci.txt # maykin-2fa -django-webtest==1.9.7 +django-webtest==1.9.13 # via # -c requirements/ci.txt # -r requirements/ci.txt @@ -630,6 +630,14 @@ prompt-toolkit==3.0.43 # -r requirements/ci.txt # click-repl # questionary +psycopg==3.2.6 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt +psycopg-pool==3.2.6 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt psycopg2==2.9.9 # via # -c requirements/ci.txt @@ -910,6 +918,8 @@ typing-extensions==4.9.0 # -r requirements/ci.txt # anyio # mozilla-django-oidc-db + # psycopg + # psycopg-pool # pydantic # pydantic-core # rich-click diff --git a/src/objects/accounts/tests/test_oidc.py b/src/objects/accounts/tests/test_oidc.py index 9eaa0732..93ef4954 100644 --- a/src/objects/accounts/tests/test_oidc.py +++ b/src/objects/accounts/tests/test_oidc.py @@ -88,7 +88,7 @@ def test_duplicate_email_unique_constraint_violated(self): self.assertEqual( error_page.context["oidc_error"], 'duplicate key value violates unique constraint "filled_email_unique"\n' - "DETAIL: Key (email)=(admin@example.com) already exists.\n", + "DETAIL: Key (email)=(admin@example.com) already exists.", ) self.assertContains( error_page, "duplicate key value violates unique constraint" diff --git a/src/objects/conf/base.py b/src/objects/conf/base.py index a856db35..95264c15 100644 --- a/src/objects/conf/base.py +++ b/src/objects/conf/base.py @@ -4,7 +4,133 @@ from .api import * # noqa DATABASES["default"]["ENGINE"] = "django.contrib.gis.db.backends.postgis" +DATABASES["default"]["CONN_MAX_AGE"] = config( + "DB_CONN_MAX_AGE", + default=0, + help_text=( + "The lifetime of a database connection, as an integer of seconds. " + "Use 0 to close database connections at the end of each request — Django’s historical behavior. " + "This setting cannot be set in combination with connection pooling." + ), + group="Database", +) + + +# https://docs.djangoproject.com/en/5.2/ref/databases/#connection-pool +# https://www.psycopg.org/psycopg3/docs/api/pool.html#the-connectionpool-class + +DB_POOL_ENABLED = config( + "DB_POOL_ENABLED", + default=False, + help_text=("Whether to use connection pooling."), + group="Database", +) + +DB_POOL_MIN_SIZE = config( + "DB_POOL_MIN_SIZE", + default=4, + help_text=( + "The minimum number of connection the pool will hold. " + "The pool will actively try to create new connections if some are lost (closed, broken) " + "and will try to never go below min_size." + ), + group="Database", +) + +DB_POOL_MAX_SIZE = config( + "DB_POOL_MAX_SIZE", + default=None, + help_text=( + "The maximum number of connections the pool will hold. " + "If None, or equal to min_size, the pool will not grow or shrink. " + "If larger than min_size, the pool can grow if more than min_size connections " + "are requested at the same time and will shrink back after the extra connections " + "have been unused for more than max_idle seconds." + ), + group="Database", +) + +DB_POOL_TIMEOUT = config( + "DB_POOL_TIMEOUT", + default=30, + help_text=( + "The default maximum time in seconds that a client can wait " + "to receive a connection from the pool (using connection() or getconn()). " + "Note that these methods allow to override the timeout default." + ), + group="Database", +) + +DB_POOL_MAX_WAITING = config( + "DB_POOL_MAX_WAITING", + default=0, + help_text=( + "Maximum number of requests that can be queued to the pool, " + "after which new requests will fail, raising TooManyRequests. 0 means no queue limit." + ), + group="Database", +) + +DB_POOL_MAX_LIFETIME = config( + "DB_POOL_MAX_LIFETIME", + default=60 * 60, + help_text=( + "The maximum lifetime of a connection in the pool, in seconds. " + "Connections used for longer get closed and replaced by a new one. " + "The amount is reduced by a random 10% to avoid mass eviction" + ), + group="Database", +) + +DB_POOL_MAX_IDLE = config( + "DB_POOL_MAX_IDLE", + default=10 * 60, + help_text=( + "Maximum time, in seconds, that a connection can stay unused in the pool " + "before being closed, and the pool shrunk. This only happens to " + "connections more than min_size, if max_size allowed the pool to grow." + ), + group="Database", +) + +DB_POOL_RECONNECT_TIMEOUT = config( + "DB_POOL_RECONNECT_TIMEOUT", + default=5 * 60, + help_text=( + "Maximum time, in seconds, the pool will try to create a connection. " + "If a connection attempt fails, the pool will try to reconnect a few times, " + "using an exponential backoff and some random factor to avoid mass attempts. " + "If repeated attempts fail, after reconnect_timeout second the connection " + "attempt is aborted and the reconnect_failed() callback invoked." + ), + group="Database", +) + +DB_POOL_NUM_WORKERS = config( + "DB_POOL_NUM_WORKERS", + default=3, + help_text=( + "Number of background worker threads used to maintain the pool state. " + "Background workers are used for example to create new connections and " + "to clean up connections when they are returned to the pool." + ), + group="Database", +) + +if DB_POOL_ENABLED: + DATABASES["default"]["OPTIONS"] = { + "pool": { + "min_size": DB_POOL_MIN_SIZE, + "max_size": DB_POOL_MAX_SIZE, + "timeout": DB_POOL_TIMEOUT, + "max_waiting": DB_POOL_MAX_WAITING, + "max_lifetime": DB_POOL_MAX_LIFETIME, + "max_idle": DB_POOL_MAX_IDLE, + "reconnect_timeout": DB_POOL_RECONNECT_TIMEOUT, + "num_workers": DB_POOL_NUM_WORKERS, + } + } # Application definition diff --git a/src/objects/core/admin.py b/src/objects/core/admin.py index 61890128..a30faf96 100644 --- a/src/objects/core/admin.py +++ b/src/objects/core/admin.py @@ -1,8 +1,7 @@ import logging from django.contrib import admin -from django.contrib.gis import forms -from django.contrib.gis.db.models import GeometryField +from django.contrib.gis.admin.options import GeoModelAdminMixin from django.http import JsonResponse from django.urls import path @@ -47,7 +46,7 @@ def versions_view(self, request, objecttype_id): return JsonResponse(versions, safe=False) -class ObjectRecordInline(admin.TabularInline): +class ObjectRecordInline(GeoModelAdminMixin, admin.TabularInline): model = ObjectRecord extra = 1 readonly_fields = ("index", "registration_at", "end_at", "get_corrected_by") @@ -62,7 +61,6 @@ class ObjectRecordInline(admin.TabularInline): "get_corrected_by", "correct", ) - formfield_overrides = {GeometryField: {"widget": forms.OSMWidget}} def has_delete_permission(self, request, obj=None): return False diff --git a/src/objects/scss/admin/_admin_theme.scss b/src/objects/scss/admin/_admin_theme.scss index 50065125..07c96b9f 100644 --- a/src/objects/scss/admin/_admin_theme.scss +++ b/src/objects/scss/admin/_admin_theme.scss @@ -71,6 +71,17 @@ body { } } +#user-tools button { + border-bottom: none; + text-decoration: underline; + + &:focus, + &:hover { + color: var(--header-link-color); + margin-bottom: 0; + } +} + div.breadcrumbs { a { &:focus, diff --git a/src/objects/templates/admin/base_site.html b/src/objects/templates/admin/base_site.html index c1b8d723..6dc75745 100644 --- a/src/objects/templates/admin/base_site.html +++ b/src/objects/templates/admin/base_site.html @@ -31,7 +31,10 @@