Merge pull request #1003 from thunderstore-io/more-caches

Several cache improvements

django/thunderstore/cache/utils.py

@@ -8,6 +8,7 @@
     "default",
     "legacy",
     "profiles",
+    "downloads",
 ]
 
 

django/thunderstore/core/management/commands/clear_cache.py

@@ -1,14 +1,25 @@
 from django.conf import settings
-from django.core.cache import cache
-from django.core.management.base import BaseCommand, CommandError
+from django.core.cache import caches
+from django.core.management.base import BaseCommand
 
 
 class Command(BaseCommand):
     help = "Clears all configured caches"
 
+    def add_arguments(self, parser) -> None:
+        parser.add_argument("caches", nargs="*", default=["default"])
+        parser.add_argument("--all", default=False, action="store_true")
+
     def handle(self, *args, **kwargs):
-        if settings.CACHES:
-            cache.clear()
-            self.stdout.write("Caches cleared!")
-        else:
-            raise CommandError("No caches are currently configured")
+        names = kwargs.get("caches", ["default"])
+        if kwargs.get("all") is True:
+            names = settings.CACHES.keys()
+
+        for name in names:
+            if name in settings.CACHES:
+                self.stdout.write(f"Clearing cache: {name}")
+                caches[name].clear()
+            else:
+                self.stderr.write(f"Cache {name} not found, unable to clear")
+
+        self.stdout.write("All done!")

django/thunderstore/core/settings.py

@@ -108,6 +108,7 @@
     REDIS_URL=(str, ""),
     REDIS_URL_LEGACY=(str, None),
     REDIS_URL_PROFILES=(str, None),
+    REDIS_URL_DOWNLOADS=(str, None),
     DB_CERT_DIR=(str, ""),
     DB_CLIENT_CERT=(str, ""),
     DB_CLIENT_KEY=(str, ""),
@@ -138,8 +139,7 @@
     IS_CYBERSTORM_ENABLED=(bool, False),
     SHOW_CYBERSTORM_API_DOCS=(bool, False),
     USE_ASYNC_PACKAGE_SUBMISSION_FLOW=(bool, False),
-    USE_TIME_SERIES_PACKAGE_DOWNLOAD_METRICS=(bool, False),
-    USE_LEGACY_PACKAGE_DOWNLOAD_METRICS=(bool, True),
+    USE_TIME_SERIES_PACKAGE_DOWNLOAD_METRICS=(bool, True),
 )
 
 ALWAYS_RAISE_EXCEPTIONS = env.bool("ALWAYS_RAISE_EXCEPTIONS")
@@ -377,6 +377,7 @@ def load_db_certs():
 # Celery
 class CeleryQueues:
     Default = "celery"
+    LogDownloads = "log.downloads"
     BackgroundCache = "background.cache"
     BackgroundTask = "background.task"
     BackgroundLongRunning = "background.long_running"
@@ -494,6 +495,10 @@ def get_redis_cache(env_key: str, fallback_key: Optional[str] = None):
         **get_redis_cache("REDIS_URL_PROFILES", "REDIS_URL"),
         "TIMEOUT": None,
     },
+    "downloads": {
+        **get_redis_cache("REDIS_URL_DOWNLOADS", "REDIS_URL"),
+        "TIMEOUT": None,
+    },
 }
 
 
@@ -792,9 +797,6 @@ def get_redis_cache(env_key: str, fallback_key: Optional[str] = None):
     "USE_TIME_SERIES_PACKAGE_DOWNLOAD_METRICS"
 )
 
-# Enable the legacy package download metrics implementation
-USE_LEGACY_PACKAGE_DOWNLOAD_METRICS = env.bool("USE_LEGACY_PACKAGE_DOWNLOAD_METRICS")
-
 # Seconds to wait between logging download events
 DOWNLOAD_METRICS_TTL_SECONDS = env.int("DOWNLOAD_METRICS_TTL_SECONDS")
 

django/thunderstore/core/tests/test_celery.py

@@ -38,6 +38,7 @@ def test_task():
     "thunderstore.repository.tasks.update_experimental_package_index",
     "thunderstore.repository.tasks.process_package_submission",
     "thunderstore.repository.tasks.cleanup_package_submissions",
+    "thunderstore.repository.tasks.log_version_download",
     "thunderstore.webhooks.tasks.process_audit_event",
 )
 

django/thunderstore/modpacks/models/legacyprofile.py

@@ -31,9 +31,13 @@ def id_cache(checksum: str) -> str:
         return f"{checksum}.profile_id"
 
     @staticmethod
-    def file_cache(uuid: Union[str, UUID]):
+    def file_cache(uuid: Union[str, UUID]) -> str:
         return f"{uuid}.file_name"
 
+    @staticmethod
+    def size_cache() -> str:
+        return "total_size"
+
     def get_or_create_from_upload(self, content: TemporaryUploadedFile) -> "UUID":
         if content.size + self.get_total_used_disk_space() > LEGACYPROFILE_STORAGE_CAP:
             raise ValidationError(
@@ -60,6 +64,10 @@ def get_or_create_from_upload(self, content: TemporaryUploadedFile) -> "UUID":
                 file_size=content.size,
                 file_sha256=hexdigest,
             )
+            try:
+                cache.incr(self.size_cache(), delta=instance.file_size)
+            except ValueError:
+                pass
 
         cache.set_many(
             {
@@ -77,7 +85,12 @@ def get_file_url(self, profile_id: str) -> str:
         return self.model._meta.get_field("file").storage.url(file_name)
 
     def get_total_used_disk_space(self) -> int:
-        return self.aggregate(total=Sum("file_size"))["total"] or 0
+        key = self.size_cache()
+        if size := cache.get(key):
+            return size
+        size = self.aggregate(total=Sum("file_size"))["total"] or 0
+        cache.set(key, size, timeout=600)
+        return size
 
 
 class LegacyProfile(TimestampMixin, models.Model):

django/thunderstore/modpacks/tests/test_legacyprofile.py

@@ -1,4 +1,5 @@
 import pytest
+from django.core.cache import cache
 
 from thunderstore.modpacks.factories import LegacyProfileFactory
 from thunderstore.modpacks.models import LegacyProfile
@@ -8,11 +9,15 @@
 def test_legacyprofile_manager_get_total_used_disk_space():
     assert LegacyProfile.objects.count() == 0
     assert LegacyProfile.objects.get_total_used_disk_space() == 0
+
     p1 = LegacyProfileFactory()
+    cache.delete(LegacyProfile.objects.size_cache())
     assert LegacyProfile.objects.count() == 1
     assert LegacyProfile.objects.get_total_used_disk_space() == p1.file_size
+
     p2 = LegacyProfileFactory(file_size=32)
     p3 = LegacyProfileFactory(file_size=32890)
+    cache.delete(LegacyProfile.objects.size_cache())
     assert LegacyProfile.objects.count() == 3
     assert LegacyProfile.objects.get_total_used_disk_space() == (
         p1.file_size + p2.file_size + p3.file_size

django/thunderstore/repository/models/package_version.py

@@ -12,14 +12,8 @@
 from django.utils import timezone
 from django.utils.functional import cached_property
 
-from thunderstore.metrics.models import (
-    PackageVersionDownloadEvent as TimeseriesDownloadEvent,
-)
 from thunderstore.repository.consts import PACKAGE_NAME_REGEX
 from thunderstore.repository.models import Package
-from thunderstore.repository.models import (
-    PackageVersionDownloadEvent as LegacyDownloadEvent,
-)
 from thunderstore.repository.package_formats import PackageFormats
 from thunderstore.utils.decorators import run_after_commit
 from thunderstore.webhooks.models.release import Webhook
@@ -260,60 +254,32 @@ def __str__(self):
         return self.full_version_name
 
     @staticmethod
-    def _get_log_key(version: "PackageVersion", client_ip: str) -> str:
-        return f"metrics.{client_ip}.download.{version.pk}"
+    def _get_log_key(version_id: int, client_ip: str) -> str:
+        return f"metrics.{client_ip}.download.{version_id}"
 
     @staticmethod
-    def _can_log_download_event(
-        version: "PackageVersion", client_ip: Optional[str]
-    ) -> bool:
+    def _can_log_download_event(version_id: int, client_ip: Optional[str]) -> bool:
         if not client_ip:
             return False
 
-        if not any(
-            (
-                settings.USE_LEGACY_PACKAGE_DOWNLOAD_METRICS,
-                settings.USE_TIME_SERIES_PACKAGE_DOWNLOAD_METRICS,
-            )
-        ):
+        if not settings.USE_TIME_SERIES_PACKAGE_DOWNLOAD_METRICS:
             return False
 
         return cache.set(
-            key=PackageVersion._get_log_key(version, client_ip),
+            key=PackageVersion._get_log_key(version_id, client_ip),
             value=0,
             timeout=settings.DOWNLOAD_METRICS_TTL_SECONDS,
             nx=True,
         )
 
     @staticmethod
-    def log_download_event(version: "PackageVersion", client_ip: Optional[str]):
-        if not PackageVersion._can_log_download_event(version, client_ip):
-            return
-
-        if settings.USE_TIME_SERIES_PACKAGE_DOWNLOAD_METRICS:
-            PackageVersion._log_download_event_timeseries(version)
+    def log_download_event(version_id: int, client_ip: Optional[str]):
+        from thunderstore.repository.tasks.downloads import log_version_download
 
-        if settings.USE_LEGACY_PACKAGE_DOWNLOAD_METRICS:
-            PackageVersion._log_download_event_legacy(version, client_ip)
-
-        version._increase_download_counter()
-
-    @staticmethod
-    def _log_download_event_timeseries(version: "PackageVersion"):
-        TimeseriesDownloadEvent.objects.create(
-            version_id=version.id,
-            timestamp=timezone.now(),
-        )
-
-    @staticmethod
-    def _log_download_event_legacy(version: "PackageVersion", client_ip: str):
-        download_event, created = LegacyDownloadEvent.objects.get_or_create(
-            version=version,
-            source_ip=client_ip,
-        )
+        if not PackageVersion._can_log_download_event(version_id, client_ip):
+            return
 
-        if not created:
-            download_event.count_downloads_and_return_validity()
+        log_version_download.delay(version_id, timezone.now().isoformat())
 
 
 signals.post_save.connect(PackageVersion.post_save, sender=PackageVersion)

django/thunderstore/repository/tasks/__init__.py

@@ -1,3 +1,4 @@
 from .caches import *
+from .downloads import *
 from .files import *
 from .submission import *

django/thunderstore/repository/tasks/downloads.py

@@ -0,0 +1,23 @@
+from datetime import datetime
+
+from celery import shared_task
+from django.db import transaction
+from django.db.models import F
+
+from thunderstore.core.settings import CeleryQueues
+from thunderstore.metrics.models import PackageVersionDownloadEvent
+from thunderstore.repository.models import PackageVersion
+
+TASK_LOG_VERSION_DOWNLOAD = "thunderstore.repository.tasks.log_version_download"
+
+
+@shared_task(queue=CeleryQueues.LogDownloads, name=TASK_LOG_VERSION_DOWNLOAD)
+def log_version_download(version_id: int, timestamp: str):
+    with transaction.atomic():
+        PackageVersionDownloadEvent.objects.create(
+            version_id=version_id,
+            timestamp=datetime.fromisoformat(timestamp),
+        )
+        PackageVersion.objects.filter(id=version_id).update(
+            downloads=F("downloads") + 1
+        )