From a861e360052b0657c70003f552563aaf99743ae9 Mon Sep 17 00:00:00 2001 From: Floris272 Date: Fri, 18 Apr 2025 16:36:52 +0200 Subject: [PATCH 01/14] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20[#568]=20update=20dj?= =?UTF-8?q?ango=20to=205.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements/base.in | 2 ++ requirements/base.txt | 12 +++++++++--- requirements/ci.txt | 16 +++++++++++++--- requirements/dev.txt | 16 +++++++++++++--- 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/requirements/base.in b/requirements/base.in index 2943e35e..b87255f4 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -16,3 +16,5 @@ mozilla-django-oidc-db[setup-configuration] # TODO this should be moved to open-api-framework once it is verified that this fixes # maykinmedia/objects-api#541 kombu>=5.4.0 + +psycopg[pool] \ No newline at end of file diff --git a/requirements/base.txt b/requirements/base.txt index f39b4e35..b0d68d48 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -9,7 +9,7 @@ ape-pie==0.2.0 # commonground-api-common # notifications-api-common # zgw-consumers -asgiref==3.7.2 +asgiref==3.8.1 # via # django # django-axes @@ -70,7 +70,7 @@ cryptography==44.0.1 # mozilla-django-oidc # pyopenssl # webauthn -django==4.2.20 +django==5.2 # via # commonground-api-common # django-admin-index @@ -148,7 +148,7 @@ django-sendfile2==0.7.0 # via django-privates django-sessionprofile==3.0.0 # via open-api-framework -django-setup-configuration==0.7.1 +django-setup-configuration==0.7.2 # via # -r requirements/base.in # mozilla-django-oidc-db @@ -258,6 +258,10 @@ prometheus-client==0.20.0 # via flower prompt-toolkit==3.0.43 # via click-repl +psycopg==3.2.6 + # via -r requirements/base.in +psycopg-pool==3.2.6 + # via psycopg psycopg2==2.9.9 # via open-api-framework pycparser==2.20 @@ -336,6 +340,8 @@ tornado==6.4.2 typing-extensions==4.9.0 # via # mozilla-django-oidc-db + # psycopg + # psycopg-pool # pydantic # pydantic-core # zgw-consumers diff --git a/requirements/ci.txt b/requirements/ci.txt index f6067b59..39f74423 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -19,7 +19,7 @@ ape-pie==0.2.0 # commonground-api-common # notifications-api-common # zgw-consumers -asgiref==3.7.2 +asgiref==3.8.1 # via # -c requirements/base.txt # -r requirements/base.txt @@ -147,7 +147,7 @@ cryptography==44.0.1 # webauthn cssselect==1.1.0 # via pyquery -django==4.2.20 +django==5.2 # via # -c requirements/base.txt # -r requirements/base.txt @@ -282,7 +282,7 @@ django-sessionprofile==3.0.0 # -c requirements/base.txt # -r requirements/base.txt # open-api-framework -django-setup-configuration==0.7.1 +django-setup-configuration==0.7.2 # via # -c requirements/base.txt # -r requirements/base.txt @@ -524,6 +524,14 @@ prompt-toolkit==3.0.43 # -c requirements/base.txt # -r requirements/base.txt # click-repl +psycopg==3.2.6 + # via + # -c requirements/base.txt + # -r requirements/base.txt +psycopg-pool==3.2.6 + # via + # -c requirements/base.txt + # -r requirements/base.txt psycopg2==2.9.9 # via # -c requirements/base.txt @@ -725,6 +733,8 @@ typing-extensions==4.9.0 # -c requirements/base.txt # -r requirements/base.txt # mozilla-django-oidc-db + # psycopg + # psycopg-pool # pydantic # pydantic-core # zgw-consumers diff --git a/requirements/dev.txt b/requirements/dev.txt index 86d03e33..05c1e61b 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -24,7 +24,7 @@ ape-pie==0.2.0 # commonground-api-common # notifications-api-common # zgw-consumers -asgiref==3.7.2 +asgiref==3.8.1 # via # -c requirements/ci.txt # -r requirements/ci.txt @@ -182,7 +182,7 @@ cssselect==1.1.0 # -c requirements/ci.txt # -r requirements/ci.txt # pyquery -django==4.2.20 +django==5.2 # via # -c requirements/ci.txt # -r requirements/ci.txt @@ -324,7 +324,7 @@ django-sessionprofile==3.0.0 # -c requirements/ci.txt # -r requirements/ci.txt # open-api-framework -django-setup-configuration==0.7.1 +django-setup-configuration==0.7.2 # via # -c requirements/ci.txt # -r requirements/ci.txt @@ -630,6 +630,14 @@ prompt-toolkit==3.0.43 # -r requirements/ci.txt # click-repl # questionary +psycopg==3.2.6 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt +psycopg-pool==3.2.6 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt psycopg2==2.9.9 # via # -c requirements/ci.txt @@ -910,6 +918,8 @@ typing-extensions==4.9.0 # -r requirements/ci.txt # anyio # mozilla-django-oidc-db + # psycopg + # psycopg-pool # pydantic # pydantic-core # rich-click From 7a5869a639e792568f260e7693994e091d101f0b Mon Sep 17 00:00:00 2001 From: Floris272 Date: Fri, 18 Apr 2025 16:37:41 +0200 Subject: [PATCH 02/14] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20[#568]=20use=20conne?= =?UTF-8?q?ction=20pooling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/objects/conf/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/objects/conf/base.py b/src/objects/conf/base.py index a856db35..4e5bb8f3 100644 --- a/src/objects/conf/base.py +++ b/src/objects/conf/base.py @@ -4,6 +4,7 @@ from .api import * # noqa DATABASES["default"]["ENGINE"] = "django.contrib.gis.db.backends.postgis" +DATABASES["default"]["OPTIONS"] = {"pool": True} # Application definition From 75631dc65672cde1d6492a9e4891a9646005d2c3 Mon Sep 17 00:00:00 2001 From: Floris272 Date: Tue, 22 Apr 2025 16:13:33 +0200 Subject: [PATCH 03/14] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20[#568]=20use=20GeoMo?= =?UTF-8?q?delAdminMixin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/objects/core/admin.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/objects/core/admin.py b/src/objects/core/admin.py index 61890128..a30faf96 100644 --- a/src/objects/core/admin.py +++ b/src/objects/core/admin.py @@ -1,8 +1,7 @@ import logging from django.contrib import admin -from django.contrib.gis import forms -from django.contrib.gis.db.models import GeometryField +from django.contrib.gis.admin.options import GeoModelAdminMixin from django.http import JsonResponse from django.urls import path @@ -47,7 +46,7 @@ def versions_view(self, request, objecttype_id): return JsonResponse(versions, safe=False) -class ObjectRecordInline(admin.TabularInline): +class ObjectRecordInline(GeoModelAdminMixin, admin.TabularInline): model = ObjectRecord extra = 1 readonly_fields = ("index", "registration_at", "end_at", "get_corrected_by") @@ -62,7 +61,6 @@ class ObjectRecordInline(admin.TabularInline): "get_corrected_by", "correct", ) - formfield_overrides = {GeometryField: {"widget": forms.OSMWidget}} def has_delete_permission(self, request, obj=None): return False From 4f81697dfd91893a688d66b238bb767a7ffce7cc Mon Sep 17 00:00:00 2001 From: Floris272 Date: Tue, 22 Apr 2025 16:57:12 +0200 Subject: [PATCH 04/14] =?UTF-8?q?=F0=9F=9A=A7=20[#568]=20add=20pool=20opti?= =?UTF-8?q?ons?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/objects/conf/base.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/objects/conf/base.py b/src/objects/conf/base.py index 4e5bb8f3..3f7b0f04 100644 --- a/src/objects/conf/base.py +++ b/src/objects/conf/base.py @@ -5,7 +5,17 @@ DATABASES["default"]["ENGINE"] = "django.contrib.gis.db.backends.postgis" DATABASES["default"]["OPTIONS"] = {"pool": True} - +# TODO ConnectionPool settings +# DATABASES["default"]["OPTIONS"] = {"pool": { +# "min_size": 4, +# "max_size": None, +# "timeout": 30, +# "max_waiting": 0, +# "max_lifetime": 60 * 60, +# "max_idle": 10 * 60, +# "reconnect_timeout": 5 * 60, +# "num_workers": 3 +# }} # Application definition From d9b1e1748068226c5942cbf0a3a6eab755be471d Mon Sep 17 00:00:00 2001 From: Floris272 Date: Thu, 24 Apr 2025 12:02:40 +0200 Subject: [PATCH 05/14] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20[#568]=20upgrade=20d?= =?UTF-8?q?jango-webtest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements/ci.txt | 2 +- requirements/dev.txt | 2 +- src/objects/accounts/tests/test_oidc.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index 39f74423..ae5ca5fa 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -306,7 +306,7 @@ django-two-factor-auth==1.17.0 # -c requirements/base.txt # -r requirements/base.txt # maykin-2fa -django-webtest==1.9.7 +django-webtest==1.9.13 # via -r requirements/test-tools.in djangorestframework==3.15.2 # via diff --git a/requirements/dev.txt b/requirements/dev.txt index 05c1e61b..25c87fc2 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -350,7 +350,7 @@ django-two-factor-auth==1.17.0 # -c requirements/ci.txt # -r requirements/ci.txt # maykin-2fa -django-webtest==1.9.7 +django-webtest==1.9.13 # via # -c requirements/ci.txt # -r requirements/ci.txt diff --git a/src/objects/accounts/tests/test_oidc.py b/src/objects/accounts/tests/test_oidc.py index 9eaa0732..93ef4954 100644 --- a/src/objects/accounts/tests/test_oidc.py +++ b/src/objects/accounts/tests/test_oidc.py @@ -88,7 +88,7 @@ def test_duplicate_email_unique_constraint_violated(self): self.assertEqual( error_page.context["oidc_error"], 'duplicate key value violates unique constraint "filled_email_unique"\n' - "DETAIL: Key (email)=(admin@example.com) already exists.\n", + "DETAIL: Key (email)=(admin@example.com) already exists.", ) self.assertContains( error_page, "duplicate key value violates unique constraint" From f48433b8c677acb7e2db09841af843e279e3a840 Mon Sep 17 00:00:00 2001 From: Floris272 Date: Thu, 24 Apr 2025 12:13:40 +0200 Subject: [PATCH 06/14] =?UTF-8?q?=F0=9F=92=9A=20[#568]=20bump=20unsupporte?= =?UTF-8?q?d=20PG=2013=20to=2014.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 2 +- publiccode.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 371664e6..ae277251 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: tests: strategy: matrix: - postgres: ['13', '15', '16', '17'] + postgres: ['14', '15', '16', '17'] postgis: ['3.2', '3.5'] exclude: # postgis 3.2 is not compatible with recent postgres versions diff --git a/publiccode.yaml b/publiccode.yaml index 3f39f559..61814a25 100644 --- a/publiccode.yaml +++ b/publiccode.yaml @@ -71,7 +71,7 @@ dependsOn: versionMin: '1.0' - name: PostgreSQL optional: false - versionMin: '13.0' + versionMin: '14.0' - name: PostGIS optional: false versionMin: '3.2' From 2316112b82c952214a450491843c42aef1052681 Mon Sep 17 00:00:00 2001 From: Floris272 Date: Thu, 24 Apr 2025 12:30:37 +0200 Subject: [PATCH 07/14] =?UTF-8?q?=F0=9F=90=9B=20[#568]=20change=20logout?= =?UTF-8?q?=20to=20post=20request?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/objects/scss/admin/_admin_theme.scss | 11 +++++++++++ src/objects/templates/admin/base_site.html | 5 ++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/objects/scss/admin/_admin_theme.scss b/src/objects/scss/admin/_admin_theme.scss index 50065125..07c96b9f 100644 --- a/src/objects/scss/admin/_admin_theme.scss +++ b/src/objects/scss/admin/_admin_theme.scss @@ -71,6 +71,17 @@ body { } } +#user-tools button { + border-bottom: none; + text-decoration: underline; + + &:focus, + &:hover { + color: var(--header-link-color); + margin-bottom: 0; + } +} + div.breadcrumbs { a { &:focus, diff --git a/src/objects/templates/admin/base_site.html b/src/objects/templates/admin/base_site.html index c1b8d723..6dc75745 100644 --- a/src/objects/templates/admin/base_site.html +++ b/src/objects/templates/admin/base_site.html @@ -31,7 +31,10 @@

{{ settings.PROJECT_NAME }} {% if user.has_usable_password %} {% trans 'Change password' %} / {% endif %} - {% trans 'Log out' %} +
+ {% csrf_token %} + +
{% endblock %} {% block nav-global %}{% endblock %} From 465714d6b1f5d62c4778cc2abb5ac5b48937c23f Mon Sep 17 00:00:00 2001 From: Floris272 Date: Thu, 24 Apr 2025 13:42:54 +0200 Subject: [PATCH 08/14] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20[#568]=20Add=20conne?= =?UTF-8?q?ction=20pool=20options?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/installation/config.rst | 14 ++++ src/objects/conf/base.py | 128 +++++++++++++++++++++++++++++++---- 2 files changed, 130 insertions(+), 12 deletions(-) diff --git a/docs/installation/config.rst b/docs/installation/config.rst index 7463f707..d20d1adc 100644 --- a/docs/installation/config.rst +++ b/docs/installation/config.rst @@ -70,6 +70,20 @@ Content Security Policy * ``CSP_OBJECT_SRC``: ``object-src`` urls. Defaults to: ``['"\'none\'"']``. +Connection Pooling +------------------ + +* ``DB_POOL_ENABLED``: Whether to use connection pooling. Defaults to: ``True``. +* ``DB_POOL_MIN_SIZE``: The minimum number of connection the pool will hold. The pool will actively try to create new connections if some are lost (closed, broken) and will try to never go below min_size. Defaults to: ``4``. +* ``DB_POOL_MAX_SIZE``: The maximum number of connections the pool will hold. If None, or equal to min_size, the pool will not grow or shrink. If larger than min_size, the pool can grow if more than min_size connections are requested at the same time and will shrink back after the extra connections have been unused for more than max_idle seconds. Defaults to: ``None``. +* ``DB_POOL_TIMEOUT``: The default maximum time in seconds that a client can wait to receive a connection from the pool (using connection() or getconn()). Note that these methods allow to override the timeout default. Defaults to: ``30``. +* ``DB_POOL_MAX_WAITING``: Maximum number of requests that can be queued to the pool, after which new requests will fail, raising TooManyRequests. 0 means no queue limit. Defaults to: ``0``. +* ``DB_POOL_MAX_LIFETIME``: The maximum lifetime of a connection in the pool, in seconds. Connections used for longer get closed and replaced by a new one. The amount is reduced by a random 10% to avoid mass eviction. Defaults to: ``3600``. +* ``DB_POOL_MAX_IDLE``: Maximum time, in seconds, that a connection can stay unused in the pool before being closed, and the pool shrunk. This only happens to connections more than min_size, if max_size allowed the pool to grow. Defaults to: ``600``. +* ``DB_POOL_RECONNECT_TIMEOUT``: Maximum time, in seconds, the pool will try to create a connection. If a connection attempt fails, the pool will try to reconnect a few times, using an exponential backoff and some random factor to avoid mass attempts. If repeated attempts fail, after reconnect_timeout second the connection attempt is aborted and the reconnect_failed() callback invoked. Defaults to: ``300``. +* ``DB_POOL_NUM_WORKERS``: Number of background worker threads used to maintain the pool state. Background workers are used for example to create new connections and to clean up connections when they are returned to the pool. Defaults to: ``3``. + + Cache ----- diff --git a/src/objects/conf/base.py b/src/objects/conf/base.py index 3f7b0f04..877c1a5e 100644 --- a/src/objects/conf/base.py +++ b/src/objects/conf/base.py @@ -4,18 +4,122 @@ from .api import * # noqa DATABASES["default"]["ENGINE"] = "django.contrib.gis.db.backends.postgis" -DATABASES["default"]["OPTIONS"] = {"pool": True} -# TODO ConnectionPool settings -# DATABASES["default"]["OPTIONS"] = {"pool": { -# "min_size": 4, -# "max_size": None, -# "timeout": 30, -# "max_waiting": 0, -# "max_lifetime": 60 * 60, -# "max_idle": 10 * 60, -# "reconnect_timeout": 5 * 60, -# "num_workers": 3 -# }} + +# https://docs.djangoproject.com/en/5.2/ref/databases/#connection-pool +# https://www.psycopg.org/psycopg3/docs/api/pool.html#the-connectionpool-class + +DB_POOL_ENABLED = config( + "DB_POOL_ENABLED", + default=True, + help_text=("Whether to use connection pooling."), + group="Connection Pooling", +) + +DB_POOL_MIN_SIZE = config( + "DB_POOL_MIN_SIZE", + default=4, + help_text=( + "The minimum number of connection the pool will hold. " + "The pool will actively try to create new connections if some are lost (closed, broken) " + "and will try to never go below min_size." + ), + group="Connection Pooling", +) + +DB_POOL_MAX_SIZE = config( + "DB_POOL_MAX_SIZE", + default=None, + help_text=( + "The maximum number of connections the pool will hold. " + "If None, or equal to min_size, the pool will not grow or shrink. " + "If larger than min_size, the pool can grow if more than min_size connections " + "are requested at the same time and will shrink back after the extra connections " + "have been unused for more than max_idle seconds." + ), + group="Connection Pooling", +) + +DB_POOL_TIMEOUT = config( + "DB_POOL_TIMEOUT", + default=30, + help_text=( + "The default maximum time in seconds that a client can wait " + "to receive a connection from the pool (using connection() or getconn()). " + "Note that these methods allow to override the timeout default." + ), + group="Connection Pooling", +) + +DB_POOL_MAX_WAITING = config( + "DB_POOL_MAX_WAITING", + default=0, + help_text=( + "Maximum number of requests that can be queued to the pool, " + "after which new requests will fail, raising TooManyRequests. 0 means no queue limit." + ), + group="Connection Pooling", +) + +DB_POOL_MAX_LIFETIME = config( + "DB_POOL_MAX_LIFETIME", + default=60 * 60, + help_text=( + "The maximum lifetime of a connection in the pool, in seconds. " + "Connections used for longer get closed and replaced by a new one. " + "The amount is reduced by a random 10% to avoid mass eviction" + ), + group="Connection Pooling", +) + +DB_POOL_MAX_IDLE = config( + "DB_POOL_MAX_IDLE", + default=10 * 60, + help_text=( + "Maximum time, in seconds, that a connection can stay unused in the pool " + "before being closed, and the pool shrunk. This only happens to " + "connections more than min_size, if max_size allowed the pool to grow." + ), + group="Connection Pooling", +) + +DB_POOL_RECONNECT_TIMEOUT = config( + "DB_POOL_RECONNECT_TIMEOUT", + default=5 * 60, + help_text=( + "Maximum time, in seconds, the pool will try to create a connection. " + "If a connection attempt fails, the pool will try to reconnect a few times, " + "using an exponential backoff and some random factor to avoid mass attempts. " + "If repeated attempts fail, after reconnect_timeout second the connection " + "attempt is aborted and the reconnect_failed() callback invoked." + ), + group="Connection Pooling", +) + +DB_POOL_NUM_WORKERS = config( + "DB_POOL_NUM_WORKERS", + default=3, + help_text=( + "Number of background worker threads used to maintain the pool state. " + "Background workers are used for example to create new connections and " + "to clean up connections when they are returned to the pool." + ), + group="Connection Pooling", +) + + +if DB_POOL_ENABLED: + DATABASES["default"]["OPTIONS"] = { + "pool": { + "min_size": DB_POOL_MIN_SIZE, + "max_size": DB_POOL_MAX_SIZE, + "timeout": DB_POOL_TIMEOUT, + "max_waiting": DB_POOL_MAX_WAITING, + "max_lifetime": DB_POOL_MAX_LIFETIME, + "max_idle": DB_POOL_MAX_IDLE, + "reconnect_timeout": DB_POOL_RECONNECT_TIMEOUT, + "num_workers": DB_POOL_NUM_WORKERS, + } + } # Application definition From 721cf8f0d15ba9626cd9bfcf8d35af461560a841 Mon Sep 17 00:00:00 2001 From: Floris272 Date: Fri, 25 Apr 2025 11:23:30 +0200 Subject: [PATCH 09/14] =?UTF-8?q?=E2=9C=A8=20[#566]=20add=20CONN=5FMAX=5FA?= =?UTF-8?q?GE=20setting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- INSTALL.rst | 2 +- docs/installation/config.rst | 3 ++- src/objects/conf/base.py | 14 +++++++++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/INSTALL.rst b/INSTALL.rst index 7831998b..52b78dc9 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -20,7 +20,7 @@ You need the following libraries and/or programs: * `Python`_ 3.11 or above * Python `Virtualenv`_ and `Pip`_ -* `PostgreSQL`_ 13 or above with PostGIS extension +* `PostgreSQL`_ 14 or above with PostGIS extension * `Node.js`_ * `npm`_ diff --git a/docs/installation/config.rst b/docs/installation/config.rst index d20d1adc..5e46e725 100644 --- a/docs/installation/config.rst +++ b/docs/installation/config.rst @@ -31,6 +31,7 @@ Database * ``DB_PASSWORD``: password of the database user. Defaults to: ``objects``. * ``DB_HOST``: hostname of the PostgreSQL database. Defaults to ``db`` for the docker environment, otherwise defaults to ``localhost``. * ``DB_PORT``: port number of the database. Defaults to: ``5432``. +* ``CONN_MAX_AGE``: The lifetime of a database connection, as an integer of seconds. Use 0 to close database connections at the end of each request — Django’s historical behavior — and None for unlimited persistent database connections. This setting cannot be set in combination with connection pooling. Defaults to: ``0``. Cross-Origin-Resource-Sharing @@ -73,7 +74,7 @@ Content Security Policy Connection Pooling ------------------ -* ``DB_POOL_ENABLED``: Whether to use connection pooling. Defaults to: ``True``. +* ``DB_POOL_ENABLED``: Whether to use connection pooling. Defaults to: ``False``. * ``DB_POOL_MIN_SIZE``: The minimum number of connection the pool will hold. The pool will actively try to create new connections if some are lost (closed, broken) and will try to never go below min_size. Defaults to: ``4``. * ``DB_POOL_MAX_SIZE``: The maximum number of connections the pool will hold. If None, or equal to min_size, the pool will not grow or shrink. If larger than min_size, the pool can grow if more than min_size connections are requested at the same time and will shrink back after the extra connections have been unused for more than max_idle seconds. Defaults to: ``None``. * ``DB_POOL_TIMEOUT``: The default maximum time in seconds that a client can wait to receive a connection from the pool (using connection() or getconn()). Note that these methods allow to override the timeout default. Defaults to: ``30``. diff --git a/src/objects/conf/base.py b/src/objects/conf/base.py index 877c1a5e..c215387e 100644 --- a/src/objects/conf/base.py +++ b/src/objects/conf/base.py @@ -4,13 +4,25 @@ from .api import * # noqa DATABASES["default"]["ENGINE"] = "django.contrib.gis.db.backends.postgis" +DATABASES["default"]["CONN_MAX_AGE"] = config( + "CONN_MAX_AGE", + default=0, + help_text=( + "The lifetime of a database connection, as an integer of seconds. " + "Use 0 to close database connections at the end of each request — Django’s historical behavior — and " + "None for unlimited persistent database connections. " + "This setting cannot be set in combination with connection pooling." + ), + group="Database", +) + # https://docs.djangoproject.com/en/5.2/ref/databases/#connection-pool # https://www.psycopg.org/psycopg3/docs/api/pool.html#the-connectionpool-class DB_POOL_ENABLED = config( "DB_POOL_ENABLED", - default=True, + default=False, help_text=("Whether to use connection pooling."), group="Connection Pooling", ) From b88ece85bcddb2008c9d213fec4c36d9ee38474e Mon Sep 17 00:00:00 2001 From: Floris272 Date: Fri, 25 Apr 2025 11:24:46 +0200 Subject: [PATCH 10/14] =?UTF-8?q?=F0=9F=91=B7=20[#568]=20Add=20connection?= =?UTF-8?q?=20pooling=20&=20conn=5Fmax=5Fage=20test=20suite=20job?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae277251..0b3465c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,44 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} + db-config-tests: + name: Run the Django test suite with conn_max_age & connection pooling + runs-on: ubuntu-latest + strategy: + matrix: + use_pooling: [true, false] + + services: + postgres: + image: postgis/postgis:17-3.5 + env: + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + steps: + - uses: actions/checkout@v4 + - name: Set up backend environment + uses: maykinmedia/setup-django-backend@v1.3 + with: + apt-packages: 'libgdal-dev gdal-bin' + python-version: '3.11' + setup-node: true + + - name: Run tests + run: | + python src/manage.py collectstatic --noinput --link + coverage run src/manage.py test src + env: + DJANGO_SETTINGS_MODULE: objects.conf.ci + SECRET_KEY: dummy + DB_USER: postgres + DB_PASSWORD: '' + DB_POOL_ENABLED: ${{ matrix.use_pooling }} + CONN_MAX_AGE: ${{ matrix.use_pooling && 0 || 3 }} + performance-tests: name: Run the performance test suite runs-on: ubuntu-latest From 51737c39b198a442977161a58681f72f90162797 Mon Sep 17 00:00:00 2001 From: Floris272 Date: Fri, 25 Apr 2025 11:29:37 +0200 Subject: [PATCH 11/14] =?UTF-8?q?=F0=9F=91=B7=20[#568]=20fix=20db-config-t?= =?UTF-8?q?ests=20job?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b3465c6..efc3608a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,12 +66,13 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} db-config-tests: - name: Run the Django test suite with conn_max_age & connection pooling runs-on: ubuntu-latest strategy: matrix: use_pooling: [true, false] + name: Run the Django test suite with ${{matrix.use_pooling && 'connection pooling' || 'conn_max_age'}}. + services: postgres: image: postgis/postgis:17-3.5 @@ -101,7 +102,7 @@ jobs: DB_USER: postgres DB_PASSWORD: '' DB_POOL_ENABLED: ${{ matrix.use_pooling }} - CONN_MAX_AGE: ${{ matrix.use_pooling && 0 || 3 }} + CONN_MAX_AGE: ${{ matrix.use_pooling && '0' || '3' }} performance-tests: name: Run the performance test suite From affd1d71ec60be06f42611198eaad2550b1ab030 Mon Sep 17 00:00:00 2001 From: Floris272 Date: Thu, 1 May 2025 17:43:10 +0200 Subject: [PATCH 12/14] =?UTF-8?q?=F0=9F=94=A7=20[#568]=20change=20connecti?= =?UTF-8?q?on=20pooling=20settings=20group?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/installation/config.rst | 11 ++++++++++- src/objects/conf/base.py | 23 +++++++++++------------ 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/docs/installation/config.rst b/docs/installation/config.rst index 5e46e725..beb9fd24 100644 --- a/docs/installation/config.rst +++ b/docs/installation/config.rst @@ -31,7 +31,16 @@ Database * ``DB_PASSWORD``: password of the database user. Defaults to: ``objects``. * ``DB_HOST``: hostname of the PostgreSQL database. Defaults to ``db`` for the docker environment, otherwise defaults to ``localhost``. * ``DB_PORT``: port number of the database. Defaults to: ``5432``. -* ``CONN_MAX_AGE``: The lifetime of a database connection, as an integer of seconds. Use 0 to close database connections at the end of each request — Django’s historical behavior — and None for unlimited persistent database connections. This setting cannot be set in combination with connection pooling. Defaults to: ``0``. +* ``DB_CONN_MAX_AGE``: The lifetime of a database connection, as an integer of seconds. Use 0 to close database connections at the end of each request — Django’s historical behavior.This setting cannot be set in combination with connection pooling. Defaults to: ``0``. +* ``DB_POOL_ENABLED``: Whether to use connection pooling. Defaults to: ``False``. +* ``DB_POOL_MIN_SIZE``: The minimum number of connection the pool will hold. The pool will actively try to create new connections if some are lost (closed, broken) and will try to never go below min_size. Defaults to: ``4``. +* ``DB_POOL_MAX_SIZE``: The maximum number of connections the pool will hold. If None, or equal to min_size, the pool will not grow or shrink. If larger than min_size, the pool can grow if more than min_size connections are requested at the same time and will shrink back after the extra connections have been unused for more than max_idle seconds. Defaults to: ``None``. +* ``DB_POOL_TIMEOUT``: The default maximum time in seconds that a client can wait to receive a connection from the pool (using connection() or getconn()). Note that these methods allow to override the timeout default. Defaults to: ``30``. +* ``DB_POOL_MAX_WAITING``: Maximum number of requests that can be queued to the pool, after which new requests will fail, raising TooManyRequests. 0 means no queue limit. Defaults to: ``0``. +* ``DB_POOL_MAX_LIFETIME``: The maximum lifetime of a connection in the pool, in seconds. Connections used for longer get closed and replaced by a new one. The amount is reduced by a random 10% to avoid mass eviction. Defaults to: ``3600``. +* ``DB_POOL_MAX_IDLE``: Maximum time, in seconds, that a connection can stay unused in the pool before being closed, and the pool shrunk. This only happens to connections more than min_size, if max_size allowed the pool to grow. Defaults to: ``600``. +* ``DB_POOL_RECONNECT_TIMEOUT``: Maximum time, in seconds, the pool will try to create a connection. If a connection attempt fails, the pool will try to reconnect a few times, using an exponential backoff and some random factor to avoid mass attempts. If repeated attempts fail, after reconnect_timeout second the connection attempt is aborted and the reconnect_failed() callback invoked. Defaults to: ``300``. +* ``DB_POOL_NUM_WORKERS``: Number of background worker threads used to maintain the pool state. Background workers are used for example to create new connections and to clean up connections when they are returned to the pool. Defaults to: ``3``. Cross-Origin-Resource-Sharing diff --git a/src/objects/conf/base.py b/src/objects/conf/base.py index c215387e..3236cc81 100644 --- a/src/objects/conf/base.py +++ b/src/objects/conf/base.py @@ -5,12 +5,11 @@ DATABASES["default"]["ENGINE"] = "django.contrib.gis.db.backends.postgis" DATABASES["default"]["CONN_MAX_AGE"] = config( - "CONN_MAX_AGE", + "DB_CONN_MAX_AGE", default=0, help_text=( "The lifetime of a database connection, as an integer of seconds. " - "Use 0 to close database connections at the end of each request — Django’s historical behavior — and " - "None for unlimited persistent database connections. " + "Use 0 to close database connections at the end of each request — Django’s historical behavior." "This setting cannot be set in combination with connection pooling." ), group="Database", @@ -24,7 +23,7 @@ "DB_POOL_ENABLED", default=False, help_text=("Whether to use connection pooling."), - group="Connection Pooling", + group="Database", ) DB_POOL_MIN_SIZE = config( @@ -35,7 +34,7 @@ "The pool will actively try to create new connections if some are lost (closed, broken) " "and will try to never go below min_size." ), - group="Connection Pooling", + group="Database", ) DB_POOL_MAX_SIZE = config( @@ -48,7 +47,7 @@ "are requested at the same time and will shrink back after the extra connections " "have been unused for more than max_idle seconds." ), - group="Connection Pooling", + group="Database", ) DB_POOL_TIMEOUT = config( @@ -59,7 +58,7 @@ "to receive a connection from the pool (using connection() or getconn()). " "Note that these methods allow to override the timeout default." ), - group="Connection Pooling", + group="Database", ) DB_POOL_MAX_WAITING = config( @@ -69,7 +68,7 @@ "Maximum number of requests that can be queued to the pool, " "after which new requests will fail, raising TooManyRequests. 0 means no queue limit." ), - group="Connection Pooling", + group="Database", ) DB_POOL_MAX_LIFETIME = config( @@ -80,7 +79,7 @@ "Connections used for longer get closed and replaced by a new one. " "The amount is reduced by a random 10% to avoid mass eviction" ), - group="Connection Pooling", + group="Database", ) DB_POOL_MAX_IDLE = config( @@ -91,7 +90,7 @@ "before being closed, and the pool shrunk. This only happens to " "connections more than min_size, if max_size allowed the pool to grow." ), - group="Connection Pooling", + group="Database", ) DB_POOL_RECONNECT_TIMEOUT = config( @@ -104,7 +103,7 @@ "If repeated attempts fail, after reconnect_timeout second the connection " "attempt is aborted and the reconnect_failed() callback invoked." ), - group="Connection Pooling", + group="Database", ) DB_POOL_NUM_WORKERS = config( @@ -115,7 +114,7 @@ "Background workers are used for example to create new connections and " "to clean up connections when they are returned to the pool." ), - group="Connection Pooling", + group="Database", ) From c17451f7eda360544b5aafbd9529ae40f0487b9e Mon Sep 17 00:00:00 2001 From: Floris272 Date: Thu, 1 May 2025 17:55:32 +0200 Subject: [PATCH 13/14] =?UTF-8?q?=F0=9F=93=9D=20[#568]=20fix=20DB=5FCONN?= =?UTF-8?q?=5FMAX=5FAGE=20help=20text?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/installation/config.rst | 16 +--------------- src/objects/conf/base.py | 2 +- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/docs/installation/config.rst b/docs/installation/config.rst index beb9fd24..ec64fb3d 100644 --- a/docs/installation/config.rst +++ b/docs/installation/config.rst @@ -31,7 +31,7 @@ Database * ``DB_PASSWORD``: password of the database user. Defaults to: ``objects``. * ``DB_HOST``: hostname of the PostgreSQL database. Defaults to ``db`` for the docker environment, otherwise defaults to ``localhost``. * ``DB_PORT``: port number of the database. Defaults to: ``5432``. -* ``DB_CONN_MAX_AGE``: The lifetime of a database connection, as an integer of seconds. Use 0 to close database connections at the end of each request — Django’s historical behavior.This setting cannot be set in combination with connection pooling. Defaults to: ``0``. +* ``DB_CONN_MAX_AGE``: The lifetime of a database connection, as an integer of seconds. Use 0 to close database connections at the end of each request — Django’s historical behavior. This setting cannot be set in combination with connection pooling. Defaults to: ``0``. * ``DB_POOL_ENABLED``: Whether to use connection pooling. Defaults to: ``False``. * ``DB_POOL_MIN_SIZE``: The minimum number of connection the pool will hold. The pool will actively try to create new connections if some are lost (closed, broken) and will try to never go below min_size. Defaults to: ``4``. * ``DB_POOL_MAX_SIZE``: The maximum number of connections the pool will hold. If None, or equal to min_size, the pool will not grow or shrink. If larger than min_size, the pool can grow if more than min_size connections are requested at the same time and will shrink back after the extra connections have been unused for more than max_idle seconds. Defaults to: ``None``. @@ -80,20 +80,6 @@ Content Security Policy * ``CSP_OBJECT_SRC``: ``object-src`` urls. Defaults to: ``['"\'none\'"']``. -Connection Pooling ------------------- - -* ``DB_POOL_ENABLED``: Whether to use connection pooling. Defaults to: ``False``. -* ``DB_POOL_MIN_SIZE``: The minimum number of connection the pool will hold. The pool will actively try to create new connections if some are lost (closed, broken) and will try to never go below min_size. Defaults to: ``4``. -* ``DB_POOL_MAX_SIZE``: The maximum number of connections the pool will hold. If None, or equal to min_size, the pool will not grow or shrink. If larger than min_size, the pool can grow if more than min_size connections are requested at the same time and will shrink back after the extra connections have been unused for more than max_idle seconds. Defaults to: ``None``. -* ``DB_POOL_TIMEOUT``: The default maximum time in seconds that a client can wait to receive a connection from the pool (using connection() or getconn()). Note that these methods allow to override the timeout default. Defaults to: ``30``. -* ``DB_POOL_MAX_WAITING``: Maximum number of requests that can be queued to the pool, after which new requests will fail, raising TooManyRequests. 0 means no queue limit. Defaults to: ``0``. -* ``DB_POOL_MAX_LIFETIME``: The maximum lifetime of a connection in the pool, in seconds. Connections used for longer get closed and replaced by a new one. The amount is reduced by a random 10% to avoid mass eviction. Defaults to: ``3600``. -* ``DB_POOL_MAX_IDLE``: Maximum time, in seconds, that a connection can stay unused in the pool before being closed, and the pool shrunk. This only happens to connections more than min_size, if max_size allowed the pool to grow. Defaults to: ``600``. -* ``DB_POOL_RECONNECT_TIMEOUT``: Maximum time, in seconds, the pool will try to create a connection. If a connection attempt fails, the pool will try to reconnect a few times, using an exponential backoff and some random factor to avoid mass attempts. If repeated attempts fail, after reconnect_timeout second the connection attempt is aborted and the reconnect_failed() callback invoked. Defaults to: ``300``. -* ``DB_POOL_NUM_WORKERS``: Number of background worker threads used to maintain the pool state. Background workers are used for example to create new connections and to clean up connections when they are returned to the pool. Defaults to: ``3``. - - Cache ----- diff --git a/src/objects/conf/base.py b/src/objects/conf/base.py index 3236cc81..95264c15 100644 --- a/src/objects/conf/base.py +++ b/src/objects/conf/base.py @@ -9,7 +9,7 @@ default=0, help_text=( "The lifetime of a database connection, as an integer of seconds. " - "Use 0 to close database connections at the end of each request — Django’s historical behavior." + "Use 0 to close database connections at the end of each request — Django’s historical behavior. " "This setting cannot be set in combination with connection pooling." ), group="Database", From 0aae809a91ddb493590d71fbbef899b1f5be4794 Mon Sep 17 00:00:00 2001 From: Floris272 Date: Fri, 2 May 2025 11:09:58 +0200 Subject: [PATCH 14/14] =?UTF-8?q?=F0=9F=91=B7=20[#568]=20merge=20db-config?= =?UTF-8?q?-test=20job=20into=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 48 +++++++--------------------------------- 1 file changed, 8 insertions(+), 40 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efc3608a..6f661705 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,7 @@ jobs: matrix: postgres: ['14', '15', '16', '17'] postgis: ['3.2', '3.5'] + use_pooling: [false] exclude: # postgis 3.2 is not compatible with recent postgres versions - postgres: '17' @@ -26,8 +27,13 @@ jobs: postgis: '3.2' - postgres: '15' postgis: '3.2' + include: + - postgres: '17' + postgis: '3.5' + use_pooling: true + + name: Run the Django test suite (PG ${{ matrix.postgres }}, postgis ${{ matrix.postgis }}) ${{matrix.use_pooling && 'with connection pooling' || ''}} - name: Run the Django test suite (PG ${{ matrix.postgres }}, postgis ${{ matrix.postgis }}) runs-on: ubuntu-latest @@ -59,51 +65,13 @@ jobs: SECRET_KEY: dummy DB_USER: postgres DB_PASSWORD: '' + DB_POOL_ENABLED: ${{ matrix.use_pooling }} - name: Publish coverage report uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} - db-config-tests: - runs-on: ubuntu-latest - strategy: - matrix: - use_pooling: [true, false] - - name: Run the Django test suite with ${{matrix.use_pooling && 'connection pooling' || 'conn_max_age'}}. - - services: - postgres: - image: postgis/postgis:17-3.5 - env: - POSTGRES_HOST_AUTH_METHOD: trust - ports: - - 5432:5432 - # needed because the postgres container does not provide a healthcheck - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - - steps: - - uses: actions/checkout@v4 - - name: Set up backend environment - uses: maykinmedia/setup-django-backend@v1.3 - with: - apt-packages: 'libgdal-dev gdal-bin' - python-version: '3.11' - setup-node: true - - - name: Run tests - run: | - python src/manage.py collectstatic --noinput --link - coverage run src/manage.py test src - env: - DJANGO_SETTINGS_MODULE: objects.conf.ci - SECRET_KEY: dummy - DB_USER: postgres - DB_PASSWORD: '' - DB_POOL_ENABLED: ${{ matrix.use_pooling }} - CONN_MAX_AGE: ${{ matrix.use_pooling && '0' || '3' }} - performance-tests: name: Run the performance test suite runs-on: ubuntu-latest