✨ [#550] Add cache to retrieve versions of objecttypes

docs/conf.py

@@ -99,6 +99,7 @@
     r"https?://.*\.gemeente.nl",
     r"http://localhost:\d+/",
     r"https://.*sentry.*",
+    "https://www.utrecht.nl/",
     "https://github.com/maykinmedia/objects-api-performance",
     "https://objects.municipality.nl/admin/",
     "https://sparxsystems.com/products/ea/trial/request.html",  # this raises 403 for crawlers probably?

docs/installation/config.rst

@@ -70,6 +70,12 @@ Content Security Policy
 * ``CSP_OBJECT_SRC``: ``object-src`` urls. Defaults to: ``['"\'none\'"']``.
 
 
+Cache
+-----
+
+* ``OBJECTTYPE_VERSION_CACHE_TIMEOUT``: Timeout in seconds for cache when retrieving objecttype versions. Defaults to: ``300``.
+
+
 Optional
 --------
 

src/objects/api/validators.py

@@ -4,9 +4,9 @@
 import requests
 from rest_framework import serializers
 from rest_framework.fields import get_attribute
-from zgw_consumers.client import build_client
 
-from objects.core.utils import check_objecttype
+from objects.core.utils import check_objecttype_cached
+from objects.utils.client import get_objecttypes_client
 
 from .constants import Operators
 from .utils import merge_patch, string_to_value
@@ -40,9 +40,8 @@ def __call__(self, attrs, serializer):
 
         if not object_type or not version:
             return
-
         try:
-            check_objecttype(object_type, version, data)
+            check_objecttype_cached(object_type, version, data)
         except ValidationError as exc:
             raise serializers.ValidationError(exc.args[0], code=self.code) from exc
 
@@ -133,14 +132,14 @@ def __call__(self, attrs, serializer):
         if not geometry:
             return
 
-        client = build_client(object_type.service)
-        try:
-            response = client.get(url=object_type.url)
-        except requests.RequestException as exc:
-            msg = f"Object type can not be retrieved: {exc.args[0]}"
-            raise ValidationError(msg)
+        with get_objecttypes_client(object_type.service) as client:
+            try:
+                response_data = client.get_objecttype(object_type.uuid)
+            except requests.RequestException as exc:
+                msg = f"Object type can not be retrieved: {exc.args[0]}"
+                raise ValidationError(msg)
 
-        allow_geometry = response.json().get("allowGeometry", True)
+        allow_geometry = response_data.get("allowGeometry", True)
 
         if geometry and not allow_geometry:
             raise serializers.ValidationError(self.message, code=self.code)

src/objects/conf/base.py

@@ -34,6 +34,16 @@
 # FIXME should this be UTC?
 TIME_ZONE = "Europe/Amsterdam"
 
+#
+# Caches
+#
+
+OBJECTTYPE_VERSION_CACHE_TIMEOUT = config(
+    "OBJECTTYPE_VERSION_CACHE_TIMEOUT",
+    default=5 * 60,  # 300 seconds
+    help_text="Timeout in seconds for cache when retrieving objecttype versions.",
+    group="Cache",
+)
 
 #
 # Additional Django settings

src/objects/core/admin.py

@@ -7,8 +7,8 @@
 from django.urls import path
 
 import requests
-from zgw_consumers.client import build_client
-from zgw_consumers.service import pagination_helper
+
+from objects.utils.client import get_objecttypes_client
 
 from .models import Object, ObjectRecord, ObjectType
 
@@ -37,14 +37,13 @@ def get_urls(self):
     def versions_view(self, request, objecttype_id):
         versions = []
         if objecttype := self.get_object(request, objecttype_id):
-            client = build_client(objecttype.service)
-            try:
-                response = client.get(objecttype.versions_url)
-                versions = list(pagination_helper(client, response.json()))
-            except (requests.RequestException, requests.JSONDecodeError):
-                logger.exception(
-                    "Something went wrong while fetching objecttype versions"
-                )
+            with get_objecttypes_client(objecttype.service) as client:
+                try:
+                    versions = client.list_objecttype_versions(objecttype.uuid)
+                except (requests.RequestException, requests.JSONDecodeError):
+                    logger.exception(
+                        "Something went wrong while fetching objecttype versions"
+                    )
         return JsonResponse(versions, safe=False)
 
 

src/objects/core/models.py

@@ -10,11 +10,12 @@
 
 import requests
 from requests.exceptions import ConnectionError
-from zgw_consumers.client import build_client
 from zgw_consumers.models import Service
 
+from objects.utils.client import get_objecttypes_client
+
 from .query import ObjectQuerySet, ObjectRecordQuerySet, ObjectTypeQuerySet
-from .utils import check_objecttype
+from .utils import check_objecttype_cached
 
 
 class ObjectType(models.Model):
@@ -52,17 +53,13 @@ def clean_fields(self, exclude: Iterable[str] | None = None) -> None:
         if exclude and "service" in exclude:
             return
 
-        client = build_client(self.service)
-
-        try:
-            response = client.get(url=self.url)
-        except (requests.RequestException, ConnectionError, ValueError) as exc:
-            raise ValidationError(f"Objecttype can't be requested: {exc}")
-
-        try:
-            object_type_data = response.json()
-        except requests.exceptions.JSONDecodeError:
-            raise ValidationError("Object type version didn't have any data")
+        with get_objecttypes_client(self.service) as client:
+            try:
+                object_type_data = client.get_objecttype(self.uuid)
+            except (requests.RequestException, ConnectionError, ValueError) as exc:
+                raise ValidationError(f"Objecttype can't be requested: {exc}")
+            except requests.exceptions.JSONDecodeError:
+                raise ValidationError("Object type version didn't have any data")
 
         if not self._name:
             self._name = object_type_data["name"]
@@ -158,7 +155,7 @@ def clean(self):
         super().clean()
 
         if hasattr(self.object, "object_type") and self.version and self.data:
-            check_objecttype(self.object.object_type, self.version, self.data)
+            check_objecttype_cached(self.object.object_type, self.version, self.data)
 
     def save(self, *args, **kwargs):
         if not self.id and self.object.last_record:

src/objects/core/utils.py

@@ -1,37 +1,45 @@
+from django.conf import settings
 from django.core.exceptions import ValidationError
 
 import jsonschema
 import requests
-from zgw_consumers.client import build_client
 
-
-def check_objecttype(object_type, version, data):
-    client = build_client(object_type.service)
-    objecttype_version_url = f"{object_type.url}/versions/{version}"
-
-    try:
-        response = client.get(objecttype_version_url)
-    except requests.RequestException:
-        msg = "Object type version can not be retrieved."
-        raise ValidationError(msg)
-
-    try:
-        response_data = response.json()
-    except requests.JSONDecodeError:
-        raise ValidationError("Object type doesn't have retrievable data.")
+from objects.core import models
+from objects.utils.cache import cache
+from objects.utils.client import get_objecttypes_client
+
+
+def check_objecttype_cached(
+    object_type: "models.ObjectType", version: int, data: dict
+) -> None:
+    @cache(
+        f"objecttypen-{object_type.uuid}:versions-{version}",
+        timeout=settings.OBJECTTYPE_VERSION_CACHE_TIMEOUT,
+    )
+    def get_objecttype_version_response():
+        with get_objecttypes_client(object_type.service) as client:
+            try:
+                return client.get_objecttype_version(object_type.uuid, version)
+            except (requests.RequestException, requests.JSONDecodeError):
+                raise ValidationError(
+                    {"non_field_errors": "Object type version can not be retrieved."},
+                    code="invalid",
+                )
 
     try:
-        schema = response_data["jsonSchema"]
+        vesion_data = get_objecttype_version_response()
+        jsonschema.validate(data, vesion_data["jsonSchema"])
     except KeyError:
-        msg = f"{objecttype_version_url} does not appear to be a valid objecttype."
-        raise ValidationError(msg)
-
-    # TODO: Set warning header if objecttype is not published.
-
-    try:
-        jsonschema.validate(data, schema)
+        raise ValidationError(
+            {
+                "non_field_errors": f"{object_type.versions_url} does not appear to be a valid objecttype."
+            },
+            code="invalid_key",
+        )
     except jsonschema.exceptions.ValidationError as exc:
-        raise ValidationError(exc.args[0]) from exc
+        raise ValidationError(
+            {"non_field_errors": exc.args[0]}, code="invalid_jsonschema"
+        )
 
 
 def can_connect_to_objecttypes() -> bool:
@@ -40,13 +48,8 @@ def can_connect_to_objecttypes() -> bool:
     """
     from zgw_consumers.models import Service
 
-    objecttypes_services = Service.objects.filter(object_types__isnull=False).distinct()
-    for service in objecttypes_services:
-        client = build_client(service)
-
-        try:
-            client.get("objecttypes")
-        except requests.RequestException:
-            return False
-
+    for service in Service.objects.filter(object_types__isnull=False).distinct():
+        with get_objecttypes_client(service) as client:
+            if not client.can_connect:
+                return False
     return True

src/objects/tests/v2/test_jsonschema.py

@@ -8,7 +8,7 @@
 from objects.core.tests.factories import ObjectTypeFactory
 from objects.token.constants import PermissionModes
 from objects.token.tests.factories import PermissionFactory
-from objects.utils.test import TokenAuthMixin
+from objects.utils.test import ClearCachesMixin, TokenAuthMixin
 
 from ..constants import GEO_WRITE_KWARGS
 from ..utils import mock_objecttype, mock_objecttype_version, mock_service_oas_get
@@ -18,7 +18,7 @@
 
 
 @requests_mock.Mocker()
-class JsonSchemaTests(TokenAuthMixin, APITestCase):
+class JsonSchemaTests(TokenAuthMixin, ClearCachesMixin, APITestCase):
     """GH issue - https://github.com/maykinmedia/objects-api/issues/330"""
 
     @classmethod

src/objects/tests/v2/test_validation.py

@@ -1,15 +1,19 @@
+import datetime
 import uuid
 
+from django.conf import settings
+
 import requests
 import requests_mock
+from freezegun import freeze_time
 from rest_framework import status
 from rest_framework.test import APITestCase
 
 from objects.core.models import Object
 from objects.core.tests.factories import ObjectRecordFactory, ObjectTypeFactory
 from objects.token.constants import PermissionModes
 from objects.token.tests.factories import PermissionFactory
-from objects.utils.test import TokenAuthMixin
+from objects.utils.test import ClearCachesMixin, TokenAuthMixin
 
 from ..constants import GEO_WRITE_KWARGS
 from ..utils import mock_objecttype, mock_objecttype_version, mock_service_oas_get
@@ -19,7 +23,7 @@
 
 
 @requests_mock.Mocker()
-class ObjectTypeValidationTests(TokenAuthMixin, APITestCase):
+class ObjectTypeValidationTests(TokenAuthMixin, ClearCachesMixin, APITestCase):
     @classmethod
     def setUpTestData(cls):
         super().setUpTestData()
@@ -31,6 +35,78 @@ def setUpTestData(cls):
             token_auth=cls.token_auth,
         )
 
+    def test_valid_create_object_check_cache(self, m):
+        mock_service_oas_get(m, OBJECT_TYPES_API, "objecttypes")
+        m.get(
+            f"{self.object_type.url}/versions/1",
+            json=mock_objecttype_version(self.object_type.url),
+        )
+        m.get(
+            f"{self.object_type.url}/versions/2",
+            json=mock_objecttype_version(self.object_type.url),
+        )
+
+        url = reverse("object-list")
+        data = {
+            "type": self.object_type.url,
+            "record": {
+                "typeVersion": 1,
+                "data": {"plantDate": "2020-04-12", "diameter": 30},
+                "startAt": "2020-01-01",
+            },
+        }
+        with self.subTest("ok_cache"):
+            self.assertEqual(m.call_count, 0)
+            self.assertEqual(Object.objects.count(), 0)
+            for n in range(5):
+                self.client.post(url, data, **GEO_WRITE_KWARGS)
+            # just one request should run — the first one
+            self.assertEqual(m.call_count, 1)
+            self.assertEqual(Object.objects.count(), 5)
+
+        with self.subTest("clear_cache"):
+            m.reset_mock()
+            self.assertEqual(m.call_count, 0)
+            for n in range(5):
+                self._clear_caches()
+                self.client.post(url, data, **GEO_WRITE_KWARGS)
+            self.assertEqual(m.call_count, 5)
+            self.assertEqual(Object.objects.count(), 10)
+
+        with self.subTest("cache_timeout"):
+            m.reset_mock()
+            old_datetime = datetime.datetime(2025, 5, 1, 12, 0)
+            with freeze_time(old_datetime.isoformat()):
+                self.assertEqual(m.call_count, 0)
+                self.client.post(url, data, **GEO_WRITE_KWARGS)
+                self.client.post(url, data, **GEO_WRITE_KWARGS)
+                # only one request for two post
+                self.assertEqual(m.call_count, 1)
+
+            # cache_timeout is still ok
+            cache_timeout = settings.OBJECTTYPE_VERSION_CACHE_TIMEOUT
+            new_datetime = old_datetime + datetime.timedelta(
+                seconds=(cache_timeout - 60)
+            )
+            with freeze_time(new_datetime.isoformat()):
+                # same request as before
+                self.assertEqual(m.call_count, 1)
+                self.client.post(url, data, **GEO_WRITE_KWARGS)
+                # same request as before
+                self.assertEqual(m.call_count, 1)
+
+            # cache_timeout is expired
+            cache_timeout = settings.OBJECTTYPE_VERSION_CACHE_TIMEOUT
+            new_datetime = old_datetime + datetime.timedelta(
+                seconds=(cache_timeout + 60)
+            )
+            with freeze_time(new_datetime.isoformat()):
+                # same request as before
+                self.assertEqual(m.call_count, 1)
+                self.client.post(url, data, **GEO_WRITE_KWARGS)
+                # new request
+                self.assertEqual(m.call_count, 2)
+
     def test_create_object_with_not_found_objecttype_url(self, m):
         object_type_invalid = ObjectTypeFactory(service=self.object_type.service)
         PermissionFactory.create(
@@ -102,7 +178,7 @@ def test_create_object_no_version(self, m):
 
         data = response.json()
         self.assertEqual(
-            data["non_field_errors"], ["Object type doesn't have retrievable data."]
+            data["non_field_errors"], ["Object type version can not be retrieved."]
         )
 
     def test_create_object_objecttype_request_error(self, m):
@@ -156,7 +232,7 @@ def test_create_object_objecttype_with_no_jsonSchema(self, m):
         self.assertEqual(
             data["non_field_errors"],
             [
-                f"{self.object_type.url}/versions/10 does not appear to be a valid objecttype."
+                f"{self.object_type.versions_url} does not appear to be a valid objecttype."
             ],
         )