Deps/django 52

.github/workflows/ci.yml

@@ -16,8 +16,9 @@ jobs:
   tests:
     strategy:
       matrix:
-        postgres: ['13', '15', '16', '17']
+        postgres: ['14', '15', '16', '17']
         postgis: ['3.2', '3.5']
+        use_pooling: [false]
         exclude:
           # postgis 3.2 is not compatible with recent postgres versions
           - postgres: '17'
@@ -26,8 +27,13 @@ jobs:
             postgis: '3.2'
           - postgres: '15'
             postgis: '3.2'
+        include:
+          - postgres: '17'
+            postgis: '3.5'
+            use_pooling: true
+
+    name: Run the Django test suite (PG ${{ matrix.postgres }}, postgis ${{ matrix.postgis }}) ${{matrix.use_pooling && 'with connection pooling' || ''}}
 
-    name: Run the Django test suite (PG ${{ matrix.postgres }}, postgis ${{ matrix.postgis }})
 
     runs-on: ubuntu-latest
 
@@ -59,6 +65,7 @@ jobs:
           SECRET_KEY: dummy
           DB_USER: postgres
           DB_PASSWORD: ''
+          DB_POOL_ENABLED: ${{ matrix.use_pooling }}
 
       - name: Publish coverage report
         uses: codecov/codecov-action@v4

INSTALL.rst

@@ -20,7 +20,7 @@ You need the following libraries and/or programs:
 
 * `Python`_ 3.11 or above
 * Python `Virtualenv`_ and `Pip`_
-* `PostgreSQL`_ 13 or above with PostGIS extension
+* `PostgreSQL`_ 14 or above with PostGIS extension
 * `Node.js`_
 * `npm`_
 

docs/installation/config.rst

@@ -31,6 +31,16 @@ Database
 * ``DB_PASSWORD``: password of the database user. Defaults to: ``objects``.
 * ``DB_HOST``: hostname of the PostgreSQL database. Defaults to ``db`` for the docker environment, otherwise defaults to ``localhost``.
 * ``DB_PORT``: port number of the database. Defaults to: ``5432``.
+* ``DB_CONN_MAX_AGE``: The lifetime of a database connection, as an integer of seconds. Use 0 to close database connections at the end of each request — Django’s historical behavior. This setting cannot be set in combination with connection pooling. Defaults to: ``0``.
+* ``DB_POOL_ENABLED``: Whether to use connection pooling. Defaults to: ``False``.
+* ``DB_POOL_MIN_SIZE``: The minimum number of connection the pool will hold. The pool will actively try to create new connections if some are lost (closed, broken) and will try to never go below min_size. Defaults to: ``4``.
+* ``DB_POOL_MAX_SIZE``: The maximum number of connections the pool will hold. If None, or equal to min_size, the pool will not grow or shrink. If larger than min_size, the pool can grow if more than min_size connections are requested at the same time and will shrink back after the extra connections have been unused for more than max_idle seconds. Defaults to: ``None``.
+* ``DB_POOL_TIMEOUT``: The default maximum time in seconds that a client can wait to receive a connection from the pool (using connection() or getconn()). Note that these methods allow to override the timeout default. Defaults to: ``30``.
+* ``DB_POOL_MAX_WAITING``: Maximum number of requests that can be queued to the pool, after which new requests will fail, raising TooManyRequests. 0 means no queue limit. Defaults to: ``0``.
+* ``DB_POOL_MAX_LIFETIME``: The maximum lifetime of a connection in the pool, in seconds. Connections used for longer get closed and replaced by a new one. The amount is reduced by a random 10% to avoid mass eviction. Defaults to: ``3600``.
+* ``DB_POOL_MAX_IDLE``: Maximum time, in seconds, that a connection can stay unused in the pool before being closed, and the pool shrunk. This only happens to connections more than min_size, if max_size allowed the pool to grow. Defaults to: ``600``.
+* ``DB_POOL_RECONNECT_TIMEOUT``: Maximum time, in seconds, the pool will try to create a connection. If a connection attempt fails, the pool will try to reconnect a few times, using an exponential backoff and some random factor to avoid mass attempts. If repeated attempts fail, after reconnect_timeout second the connection attempt is aborted and the reconnect_failed() callback invoked. Defaults to: ``300``.
+* ``DB_POOL_NUM_WORKERS``: Number of background worker threads used to maintain the pool state. Background workers are used for example to create new connections and to clean up connections when they are returned to the pool. Defaults to: ``3``.
 
 
 Cross-Origin-Resource-Sharing

publiccode.yaml

@@ -71,7 +71,7 @@ dependsOn:
       versionMin: '1.0'
     - name: PostgreSQL
       optional: false
-      versionMin: '13.0'
+      versionMin: '14.0'
     - name: PostGIS
       optional: false
       versionMin: '3.2'

requirements/base.in

@@ -16,3 +16,5 @@ mozilla-django-oidc-db[setup-configuration]
 # TODO this should be moved to open-api-framework once it is verified that this fixes
 # maykinmedia/objects-api#541
 kombu>=5.4.0
+
+psycopg[pool]

requirements/base.txt

@@ -9,7 +9,7 @@ ape-pie==0.2.0
     #   commonground-api-common
     #   notifications-api-common
     #   zgw-consumers
-asgiref==3.7.2
+asgiref==3.8.1
     # via
     #   django
     #   django-axes
@@ -70,7 +70,7 @@ cryptography==44.0.1
     #   mozilla-django-oidc
     #   pyopenssl
     #   webauthn
-django==4.2.20
+django==5.2
     # via
     #   commonground-api-common
     #   django-admin-index
@@ -148,7 +148,7 @@ django-sendfile2==0.7.0
     # via django-privates
 django-sessionprofile==3.0.0
     # via open-api-framework
-django-setup-configuration==0.7.1
+django-setup-configuration==0.7.2
     # via
     #   -r requirements/base.in
     #   mozilla-django-oidc-db
@@ -258,6 +258,10 @@ prometheus-client==0.20.0
     # via flower
 prompt-toolkit==3.0.43
     # via click-repl
+psycopg==3.2.6
+    # via -r requirements/base.in
+psycopg-pool==3.2.6
+    # via psycopg
 psycopg2==2.9.9
     # via open-api-framework
 pycparser==2.20
@@ -336,6 +340,8 @@ tornado==6.4.2
 typing-extensions==4.9.0
     # via
     #   mozilla-django-oidc-db
+    #   psycopg
+    #   psycopg-pool
     #   pydantic
     #   pydantic-core
     #   zgw-consumers

requirements/ci.txt

@@ -19,7 +19,7 @@ ape-pie==0.2.0
     #   commonground-api-common
     #   notifications-api-common
     #   zgw-consumers
-asgiref==3.7.2
+asgiref==3.8.1
     # via
     #   -c requirements/base.txt
     #   -r requirements/base.txt
@@ -147,7 +147,7 @@ cryptography==44.0.1
     #   webauthn
 cssselect==1.1.0
     # via pyquery
-django==4.2.20
+django==5.2
     # via
     #   -c requirements/base.txt
     #   -r requirements/base.txt
@@ -282,7 +282,7 @@ django-sessionprofile==3.0.0
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   open-api-framework
-django-setup-configuration==0.7.1
+django-setup-configuration==0.7.2
     # via
     #   -c requirements/base.txt
     #   -r requirements/base.txt
@@ -306,7 +306,7 @@ django-two-factor-auth==1.17.0
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   maykin-2fa
-django-webtest==1.9.7
+django-webtest==1.9.13
     # via -r requirements/test-tools.in
 djangorestframework==3.15.2
     # via
@@ -524,6 +524,14 @@ prompt-toolkit==3.0.43
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   click-repl
+psycopg==3.2.6
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+psycopg-pool==3.2.6
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
 psycopg2==2.9.9
     # via
     #   -c requirements/base.txt
@@ -725,6 +733,8 @@ typing-extensions==4.9.0
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   mozilla-django-oidc-db
+    #   psycopg
+    #   psycopg-pool
     #   pydantic
     #   pydantic-core
     #   zgw-consumers

requirements/dev.txt

@@ -24,7 +24,7 @@ ape-pie==0.2.0
     #   commonground-api-common
     #   notifications-api-common
     #   zgw-consumers
-asgiref==3.7.2
+asgiref==3.8.1
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
@@ -182,7 +182,7 @@ cssselect==1.1.0
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   pyquery
-django==4.2.20
+django==5.2
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
@@ -324,7 +324,7 @@ django-sessionprofile==3.0.0
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   open-api-framework
-django-setup-configuration==0.7.1
+django-setup-configuration==0.7.2
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
@@ -350,7 +350,7 @@ django-two-factor-auth==1.17.0
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   maykin-2fa
-django-webtest==1.9.7
+django-webtest==1.9.13
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
@@ -630,6 +630,14 @@ prompt-toolkit==3.0.43
     #   -r requirements/ci.txt
     #   click-repl
     #   questionary
+psycopg==3.2.6
+    # via
+    #   -c requirements/ci.txt
+    #   -r requirements/ci.txt
+psycopg-pool==3.2.6
+    # via
+    #   -c requirements/ci.txt
+    #   -r requirements/ci.txt
 psycopg2==2.9.9
     # via
     #   -c requirements/ci.txt
@@ -910,6 +918,8 @@ typing-extensions==4.9.0
     #   -r requirements/ci.txt
     #   anyio
     #   mozilla-django-oidc-db
+    #   psycopg
+    #   psycopg-pool
     #   pydantic
     #   pydantic-core
     #   rich-click

src/objects/accounts/tests/test_oidc.py

@@ -88,7 +88,7 @@ def test_duplicate_email_unique_constraint_violated(self):
             self.assertEqual(
                 error_page.context["oidc_error"],
                 'duplicate key value violates unique constraint "filled_email_unique"\n'
-                "DETAIL:  Key (email)=(admin@example.com) already exists.\n",
+                "DETAIL:  Key (email)=(admin@example.com) already exists.",
             )
             self.assertContains(
                 error_page, "duplicate key value violates unique constraint"

src/objects/conf/base.py

@@ -4,7 +4,133 @@
 from .api import *  # noqa
 
 DATABASES["default"]["ENGINE"] = "django.contrib.gis.db.backends.postgis"
+DATABASES["default"]["CONN_MAX_AGE"] = config(
+    "DB_CONN_MAX_AGE",
+    default=0,
+    help_text=(
+        "The lifetime of a database connection, as an integer of seconds. "
+        "Use 0 to close database connections at the end of each request — Django’s historical behavior. "
+        "This setting cannot be set in combination with connection pooling."
+    ),
+    group="Database",
+)
+
+
+# https://docs.djangoproject.com/en/5.2/ref/databases/#connection-pool
+# https://www.psycopg.org/psycopg3/docs/api/pool.html#the-connectionpool-class
+
+DB_POOL_ENABLED = config(
+    "DB_POOL_ENABLED",
+    default=False,
+    help_text=("Whether to use connection pooling."),
+    group="Database",
+)
+
+DB_POOL_MIN_SIZE = config(
+    "DB_POOL_MIN_SIZE",
+    default=4,
+    help_text=(
+        "The minimum number of connection the pool will hold. "
+        "The pool will actively try to create new connections if some are lost (closed, broken) "
+        "and will try to never go below min_size."
+    ),
+    group="Database",
+)
+
+DB_POOL_MAX_SIZE = config(
+    "DB_POOL_MAX_SIZE",
+    default=None,
+    help_text=(
+        "The maximum number of connections the pool will hold. "
+        "If None, or equal to min_size, the pool will not grow or shrink. "
+        "If larger than min_size, the pool can grow if more than min_size connections "
+        "are requested at the same time and will shrink back after the extra connections "
+        "have been unused for more than max_idle seconds."
+    ),
+    group="Database",
+)
+
+DB_POOL_TIMEOUT = config(
+    "DB_POOL_TIMEOUT",
+    default=30,
+    help_text=(
+        "The default maximum time in seconds that a client can wait "
+        "to receive a connection from the pool (using connection() or getconn()). "
+        "Note that these methods allow to override the timeout default."
+    ),
+    group="Database",
+)
+
+DB_POOL_MAX_WAITING = config(
+    "DB_POOL_MAX_WAITING",
+    default=0,
+    help_text=(
+        "Maximum number of requests that can be queued to the pool, "
+        "after which new requests will fail, raising TooManyRequests. 0 means no queue limit."
+    ),
+    group="Database",
+)
+
+DB_POOL_MAX_LIFETIME = config(
+    "DB_POOL_MAX_LIFETIME",
+    default=60 * 60,
+    help_text=(
+        "The maximum lifetime of a connection in the pool, in seconds. "
+        "Connections used for longer get closed and replaced by a new one. "
+        "The amount is reduced by a random 10% to avoid mass eviction"
+    ),
+    group="Database",
+)
+
+DB_POOL_MAX_IDLE = config(
+    "DB_POOL_MAX_IDLE",
+    default=10 * 60,
+    help_text=(
+        "Maximum time, in seconds, that a connection can stay unused in the pool "
+        "before being closed, and the pool shrunk. This only happens to "
+        "connections more than min_size, if max_size allowed the pool to grow."
+    ),
+    group="Database",
+)
+
+DB_POOL_RECONNECT_TIMEOUT = config(
+    "DB_POOL_RECONNECT_TIMEOUT",
+    default=5 * 60,
+    help_text=(
+        "Maximum time, in seconds, the pool will try to create a connection. "
+        "If a connection attempt fails, the pool will try to reconnect a few times, "
+        "using an exponential backoff and some random factor to avoid mass attempts. "
+        "If repeated attempts fail, after reconnect_timeout second the connection "
+        "attempt is aborted and the reconnect_failed() callback invoked."
+    ),
+    group="Database",
+)
+
+DB_POOL_NUM_WORKERS = config(
+    "DB_POOL_NUM_WORKERS",
+    default=3,
+    help_text=(
+        "Number of background worker threads used to maintain the pool state. "
+        "Background workers are used for example to create new connections and "
+        "to clean up connections when they are returned to the pool."
+    ),
+    group="Database",
+)
+
 
+if DB_POOL_ENABLED:
+    DATABASES["default"]["OPTIONS"] = {
+        "pool": {
+            "min_size": DB_POOL_MIN_SIZE,
+            "max_size": DB_POOL_MAX_SIZE,
+            "timeout": DB_POOL_TIMEOUT,
+            "max_waiting": DB_POOL_MAX_WAITING,
+            "max_lifetime": DB_POOL_MAX_LIFETIME,
+            "max_idle": DB_POOL_MAX_IDLE,
+            "reconnect_timeout": DB_POOL_RECONNECT_TIMEOUT,
+            "num_workers": DB_POOL_NUM_WORKERS,
+        }
+    }
 
 # Application definition