[7806] plans and projects api caching

.github/workflows/django.yml

@@ -21,6 +21,15 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+      redis:
+        image: redis
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
@@ -60,7 +69,7 @@ jobs:
             ${{ runner.os }}-build-${{ env.cache-name }}-
             ${{ runner.os }}-build-
             ${{ runner.os }}-
-      - name: check a4 hashes equal
+      - name: Check a4 hashes equal
         run: |
           ./scripts/a4-check.sh
       - name: Install Dependencies

changelog/2223.md

@@ -0,0 +1,3 @@
+### Added
+
+- logger in apps init file

changelog/7275.md

@@ -1,5 +1,5 @@
-### Changed
+### Added
 
-- improves performance of the api endpoint `api/plans/` by prefetching projects
-- for details, see `docs/performance_of_projects_overview.md`
+- enables caching for api endpoints `api/{plans,extprojects,projects}/`
+- caches are expired by signals and by periodic tasks, for details, see `docs/api_caching.md`
 

docs/api_caching.md

@@ -0,0 +1,60 @@
+## Background
+
+We have noticed that the page load of `mein.berlin.de/projekte/` is pretty slow with about 6s for 550 projects. Three API calls are particularly slow:
+
+- https://mein.berlin.de/api/projects/?status=pastParticipation		2.811s
+- https://mein.berlin.de/api/plans/					                      3.613s
+- https://mein.berlin.de/api/extprojects/				                  5.041s
+
+These paths correspond to the following api views:
+
+- `projects/api.py::ProjectListViewSet`
+- `plans/api.py::PlansListViewSet`
+- `extprojects/api.py::ExternalProjectListViewSet`
+
+we decided to start caching the endpoints with redis.
+
+## Developer Notes
+
+The cache target is the `list` method of the following views:
+
+- `ExternalProjectListViewSet`
+- `PlansListViewSet`
+- `ProjectListViewSet`
+- `PrivateProjectListViewSet`
+
+Cache keys expire after a timeout (default value 1h) or if a context specific signal is received (e.g. cache keys for projects are deleted if the signal for a saved project is detected).
+
+The cache keys for projects are constructed by the view namespace and their status if exists:
+- `projects_activeParticipation`
+- `projects_pastParticipation`
+- `projects_futureParticipation`
+- `privateprojects`
+- `extprojects`
+- `plans`
+
+## Celery tasks
+
+A periodic task checks for projects that will become either active or past in the next 10 minutes.
+- schedule_reset_cache_for_projects()
+
+In case of projects becoming active the cache is cleared for:
+- `projects_activeParticipation`
+- `projects_futureParticipation`
+- `privateprojects`
+- `extprojects`
+
+in case of projects becoming past the cache is cleared for:
+- `projects_activeParticipation`
+- `projects_pastParticipation`
+- `privateprojects`
+- `extprojects`
+
+In production, we use django's built-in [Redis](https://docs.djangoproject.com/en/4.2/topics/cache/#redis) as cache backend (see `settings/production.py::CACHES`). For development and testing the cache backend is the default, that is [local memory](https://docs.djangoproject.com/en/4.2/topics/cache/#local-memory-caching). If you want to enable redis cache for local development, then copy the production settings to your `settings/local.py`.
+
+files: 
+- `./meinberlin/apps/plans/api.py`
+- `./meinberlin/apps/extprojects/api.py`
+- `./meinberlin/apps/projects/api.py`
+- `./meinberlin/apps/projects/tasks.py`
+- `./meinberlin/config/settings/production.py`

meinberlin/apps/__init__.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger(__name__)

meinberlin/apps/bplan/tasks.py

@@ -1,15 +1,13 @@
 import json
-import logging
 import urllib
 
 from celery import shared_task
 
 from adhocracy4.administrative_districts.models import AdministrativeDistrict
 from adhocracy4.projects.models import Topic
+from meinberlin.apps import logger
 from meinberlin.apps.bplan.models import Bplan
 
-logger = logging.getLogger(__name__)
-
 
 def get_features_from_bplan_api(endpoint):
     url = "https://bplan-prod.liqd.net/api/" + endpoint

meinberlin/apps/extprojects/api.py

@@ -1,5 +1,7 @@
+from django.core.cache import cache
 from django.utils import timezone
 from rest_framework import viewsets
+from rest_framework.response import Response
 
 from adhocracy4.projects.enums import Access
 from meinberlin.apps.extprojects.models import ExternalProject
@@ -18,3 +20,13 @@ def get_queryset(self):
     def get_serializer(self, *args, **kwargs):
         now = timezone.now()
         return ExternalProjectSerializer(now=now, *args, **kwargs)
+
+    def list(self, request, *args, **kwargs):
+        data = cache.get("extprojects")
+        if data is None:
+            queryset = self.filter_queryset(self.get_queryset())
+            serializer = self.get_serializer(queryset, many=True)
+            data = serializer.data
+            cache.set("extprojects", data)
+
+        return Response(data)

meinberlin/apps/extprojects/apps.py

@@ -4,3 +4,6 @@
 class Config(AppConfig):
     name = "meinberlin.apps.extprojects"
     label = "meinberlin_extprojects"
+
+    def ready(self):
+        import meinberlin.apps.extprojects.signals  # noqa:F401

meinberlin/apps/extprojects/signals.py

@@ -0,0 +1,12 @@
+from django.core.cache import cache
+from django.db.models.signals import post_delete
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+
+from .models import ExternalProject
+
+
+@receiver(post_save, sender=ExternalProject)
+@receiver(post_delete, sender=ExternalProject)
+def reset_cache(sender, instance, *args, **kwargs):
+    cache.delete("extprojects")

meinberlin/apps/plans/api.py

@@ -1,4 +1,6 @@
+from django.core.cache import cache
 from rest_framework import viewsets
+from rest_framework.response import Response
 
 from meinberlin.apps.plans.models import Plan
 from meinberlin.apps.plans.serializers import PlanSerializer
@@ -9,3 +11,13 @@ class PlansListViewSet(viewsets.ReadOnlyModelViewSet):
 
     def get_queryset(self):
         return Plan.objects.filter(is_draft=False).prefetch_related("projects")
+
+    def list(self, request, *args, **kwargs):
+        data = cache.get("plans")
+        if data is None:
+            queryset = self.filter_queryset(self.get_queryset())
+            serializer = self.get_serializer(queryset, many=True)
+            data = serializer.data
+            cache.set("plans", data)
+
+        return Response(data)

meinberlin/apps/plans/apps.py

@@ -4,3 +4,6 @@
 class Config(AppConfig):
     name = "meinberlin.apps.plans"
     label = "meinberlin_plans"
+
+    def ready(self):
+        import meinberlin.apps.plans.signals  # noqa:F401

meinberlin/apps/plans/signals.py

@@ -0,0 +1,12 @@
+from django.core.cache import cache
+from django.db.models.signals import post_delete
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+
+from .models import Plan
+
+
+@receiver(post_save, sender=Plan)
+@receiver(post_delete, sender=Plan)
+def reset_cache(sender, instance, *args, **kwargs):
+    cache.delete("plans")

meinberlin/apps/projects/api.py

@@ -1,7 +1,9 @@
+from django.core.cache import cache
 from django.db.models import Q
 from django.utils import timezone
 from django_filters.rest_framework import DjangoFilterBackend
 from rest_framework import viewsets
+from rest_framework.response import Response
 
 from adhocracy4.projects.enums import Access
 from adhocracy4.projects.models import Project
@@ -39,6 +41,18 @@ def get_queryset(self):
         )
         return projects
 
+    def list(self, request, *args, **kwargs):
+        statustype = ""
+        if "status" in self.request.GET:
+            statustype = self.request.GET["status"]
+        data = cache.get("projects_" + statustype)
+        if data is None:
+            queryset = self.filter_queryset(self.get_queryset())
+            serializer = self.get_serializer(queryset, many=True)
+            data = serializer.data
+            cache.set("projects_" + statustype, data)
+        return Response(data)
+
     def get_serializer(self, *args, **kwargs):
         if "status" in self.request.GET:
             statustype = self.request.GET["status"]
@@ -64,9 +78,12 @@ def __init__(self, *args, **kwargs):
         self.now = now
 
     def get_queryset(self):
-        private_projects = Project.objects.filter(
-            is_draft=False, is_archived=False, access=Access.PRIVATE
-        )
+        private_projects = cache.get("private_projects")
+        if private_projects is None:
+            private_projects = Project.objects.filter(
+                is_draft=False, is_archived=False, access=Access.PRIVATE
+            )
+            cache.set("private_projects", private_projects)
         if private_projects:
             not_allowed_projects = [
                 project.id

meinberlin/apps/projects/apps.py

@@ -6,6 +6,7 @@ class Config(AppConfig):
     label = "meinberlin_projects"
 
     def ready(self):
-        from . import overwrites
+        import meinberlin.apps.projects.signals  # noqa:F401
+        from meinberlin.apps.projects import overwrites
 
         overwrites.overwrite_access_enum_label()

meinberlin/apps/projects/filters.py

@@ -3,7 +3,6 @@
 
 class StatusFilter(filters.BaseFilterBackend):
     def filter_queryset(self, request, queryset, view):
-
         now = view.now
 
         if "status" in request.GET:

meinberlin/apps/projects/signals.py

@@ -0,0 +1,49 @@
+from django.core.cache import cache
+from django.db.models.signals import post_delete
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+
+from adhocracy4.dashboard import signals as a4dashboard_signals
+from adhocracy4.projects.models import Project
+from meinberlin.apps.projects.tasks import get_next_projects_end
+from meinberlin.apps.projects.tasks import get_next_projects_start
+
+
+@receiver(a4dashboard_signals.project_created)
+@receiver(a4dashboard_signals.project_published)
+@receiver(a4dashboard_signals.project_unpublished)
+def post_dashboard_signal_delete(sender, project, user, **kwargs):
+    cache.delete_many(
+        [
+            "projects_activeParticipation",
+            "projects_futureParticipation",
+            "projects_pastParticipation",
+            "private_projects",
+            "extprojects",
+        ]
+    )
+
+
+@receiver(post_save, sender=Project)
+@receiver(post_delete, sender=Project)
+def post_save_delete(sender, instance, *args, **kwargs):
+    """
+    Delete cache for project list views.
+    Capture any new phases that may got created/updated while saving a project.
+    """
+
+    cache.delete_many(
+        [
+            "projects_activeParticipation",
+            "projects_futureParticipation",
+            "projects_pastParticipation",
+            "private_projects",
+            "extprojects",
+        ]
+    )
+
+    # set cache for the next projects that will be published in the next 10min
+    get_next_projects_start()
+
+    # set cache for the next project that ends and should be unpublished
+    get_next_projects_end()