✨ [#538] Improve performance and add tests

.github/workflows/ci.yml

@@ -89,10 +89,15 @@ jobs:
           docker compose exec --user root web pip install factory-boy
           cat performance_test/create_data.py | docker compose exec -T web src/manage.py shell
 
+      - name: Run Locust tests
+        run: |
+          pip install locust
+          locust --config performance_test/locust.conf
+
       - name: Run tests
         run: |
           pip install -r requirements/ci.txt
-          pytest performance_test/ --benchmark-json output.json
+          pytest -v performance_test/test_objects_list.py --benchmark-json output.json
 
   docs:
     runs-on: ubuntu-latest

performance_test/locust.conf

@@ -0,0 +1,7 @@
+locustfile = performance_test/test_locust.py
+headless = true
+host = http://localhost:8000
+users = 100
+spawn-rate = 10
+run-time = 10s
+

performance_test/test_locust.py

@@ -0,0 +1,17 @@
+from locust import HttpUser, task
+
+OBJECTS_LIST = "/api/v2/objects"
+AUTH_HEADERS = {"Authorization": "Token secret"}
+
+
+class GetObjectsList(HttpUser):
+    params = {
+        "pageSize": 1000,
+        "type": "http://localhost:8001/api/v2/objecttypes/f1220670-8ab7-44f1-a318-bd0782e97662",
+        "data_attrs": "kiemjaar__exact__1234",
+        "ordering": "-record__data__contactmoment__datumContact",
+    }
+
+    @task
+    def get(self):
+        self.client.get(OBJECTS_LIST, params=self.params, headers=AUTH_HEADERS)

requirements/dev.in

@@ -15,4 +15,4 @@ whitenoise
 sphinx
 sphinx-rtd-theme
 sphinx-tabs
-recommonmark
+recommonmark

src/objects/api/fields.py

@@ -3,6 +3,7 @@
 from django.utils.translation import gettext_lazy as _
 
 from rest_framework import serializers
+from vng_api_common.serializers import CachedHyperlinkedIdentityField
 from vng_api_common.utils import get_uuid_from_path
 from zgw_consumers.models import Service
 
@@ -11,7 +12,13 @@
 
 class ObjectSlugRelatedField(serializers.SlugRelatedField):
     def get_queryset(self):
-        queryset = ObjectRecord.objects.all()
+        queryset = ObjectRecord.objects.select_related(
+            "object",
+            "object__object_type",
+            "object__object_type__service",
+            "correct",
+            "corrected",
+        ).order_by("-pk")
 
         record_instance = self.parent.parent.instance
         if not record_instance:
@@ -78,3 +85,12 @@ def get_url(self, obj, view_name, request, format):
         lookup_value = getattr(obj.object, "uuid")
         kwargs = {self.lookup_url_kwarg: lookup_value}
         return self.reverse(view_name, kwargs=kwargs, request=request, format=format)
+
+
+class CachedObjectUrlField(CachedHyperlinkedIdentityField):
+    lookup_field = "uuid"
+
+    def get_url(self, obj, view_name, request, format):
+        if hasattr(obj, "pk") and obj.pk in (None, ""):
+            return None
+        return super().get_url(obj.object, view_name, request, format)

src/objects/api/serializers.py

@@ -9,7 +9,7 @@
 from objects.token.models import Permission, TokenAuth
 from objects.utils.serializers import DynamicFieldsMixin
 
-from .fields import ObjectSlugRelatedField, ObjectTypeField, ObjectUrlField
+from .fields import CachedObjectUrlField, ObjectSlugRelatedField, ObjectTypeField
 from .utils import merge_patch
 from .validators import GeometryValidator, IsImmutableValidator, JsonSchemaValidator
 
@@ -90,7 +90,7 @@ class Meta:
 
 
 class ObjectSerializer(DynamicFieldsMixin, serializers.HyperlinkedModelSerializer):
-    url = ObjectUrlField(view_name="object-detail")
+    url = CachedObjectUrlField(view_name="object-detail")
     uuid = serializers.UUIDField(
         source="object.uuid",
         required=False,

src/objects/conf/dev.py

@@ -112,6 +112,11 @@
     security_index = MIDDLEWARE.index("django.middleware.security.SecurityMiddleware")
     MIDDLEWARE.insert(security_index + 1, "whitenoise.middleware.WhiteNoiseMiddleware")
 
+
+if config("USE_PYINSTRUMENT", default=False, add_to_docs=False):  # pragma:no cover
+    MIDDLEWARE = ["objects.utils.middleware.PyInstrumentMiddleware"] + MIDDLEWARE
+
+
 if "test" in sys.argv:
     NOTIFICATIONS_DISABLED = True
 

src/objects/utils/middleware.py

@@ -0,0 +1,45 @@
+import logging
+import os
+
+from django.http import HttpResponse
+
+logger = logging.getLogger(__name__)
+
+
+class PyInstrumentMiddleware:  # pragma:no cover
+    """
+    Middleware that's included in dev environments if `USE_PYINSTRUMENT=true`,
+    allows profiling of the request/response cycle. Profiling results can be viewed
+    at `/_profiling`
+    """
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+        self.profiler_output_path = "/tmp/pyinstrument_profile.html"
+
+    def __call__(self, request):
+        if request.path.startswith("/_profiling"):
+            return self._serve_profile()
+
+        # Local import to avoid having to install this in production environments
+        from pyinstrument import Profiler
+
+        profiler = Profiler()
+        profiler.start()
+
+        response = self.get_response(request)
+
+        profiler.stop()
+
+        # Save the profile to an HTML file
+        with open(self.profiler_output_path, "w") as f:
+            f.write(profiler.output_html())
+
+        return response
+
+    def _serve_profile(self):
+        """Serve the latest profiling report"""
+        if os.path.exists(self.profiler_output_path):
+            with open(self.profiler_output_path, "r") as f:
+                return HttpResponse(f.read(), content_type="text/html")
+        return HttpResponse("No profiling report available yet.", status=404)