Add compat layer to enable using STORAGES

Co-authored-by: Matthias Dellweg <2500@gmx.de>

CHANGES/5404.feature

@@ -0,0 +1,5 @@
+Enable users to gradually upgrade from `DEFAULT_FILE_STORAGE` and `STATIC_FILE_STORAGE` to 'STORAGES'.
+These legacy settings were deprecated in Django 4.2 and will be removed in Pulp 3.85.
+
+The [django-upgrade](https://github.com/adamchainz/django-upgrade?tab=readme-ov-file#django-42)
+tool can be used to automatically upgrade the settings to the new form.

pulpcore/app/apps.py

@@ -5,7 +5,6 @@
 from importlib import import_module
 
 from django import apps
-from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.db import connection, transaction
 from django.db.models.signals import post_migrate
@@ -68,6 +67,11 @@ class PulpPluginAppConfig(apps.AppConfig):
 
     def __init__(self, app_name, app_module):
         super().__init__(app_name, app_module)
+        # Workaround for getting the up-to-date settings instance.
+        # A module-level settings import is a cached version from before dynaconf do its work.
+        from django.conf import settings
+
+        self.settings = settings
 
         try:
             self.version
@@ -312,6 +316,7 @@ def _populate_system_id(sender, apps, verbosity, **kwargs):
 
 
 def _ensure_default_domain(sender, **kwargs):
+    settings = sender.settings
     table_names = connection.introspection.table_names()
     if "core_domain" in table_names:
         from pulpcore.app.util import get_default_domain
@@ -391,6 +396,7 @@ def _get_permission(perm):
 
 
 def _populate_artifact_serving_distribution(sender, apps, verbosity, **kwargs):
+    settings = sender.settings
     if (
         settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem"
         or not settings.REDIRECT_TO_OBJECT_STORAGE

pulpcore/app/settings.py

@@ -16,6 +16,8 @@
 from pathlib import Path
 
 from cryptography.fernet import Fernet
+from django.core.files.storage import storages
+from django.conf import global_settings
 from django.core.exceptions import ImproperlyConfigured
 from django.db import connection
 
@@ -56,7 +58,28 @@
 STATIC_URL = "/assets/"
 STATIC_ROOT = DEPLOY_ROOT / STATIC_URL.strip("/")
 
-DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.FileSystem"
+# begin compatilibity layer for DEFAULT_FILE_STORAGE
+# Remove on pulpcore=3.85 or pulpcore=4.0
+
+# - What is this?
+# We shouldnt use STORAGES or DEFAULT_FILE_STORAGE directly because those are
+# mutually exclusive by django, which constraints users to use whatever we use.
+# This is a hack/workaround to set Pulp's default while still enabling users to choose
+# the legacy or the new storage setting.
+_DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.FileSystem"
+_STORAGES = {
+    "default": {
+        "BACKEND": "pulpcore.app.models.storage.FileSystem",
+    },
+    "staticfiles": {
+        "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
+    },
+}
+
+setattr(global_settings, "DEFAULT_FILE_STORAGE", _DEFAULT_FILE_STORAGE)
+setattr(global_settings, "STORAGES", _STORAGES)
+# end DEFAULT_FILE_STORAGE deprecation layer
+
 REDIRECT_TO_OBJECT_STORAGE = True
 
 WORKING_DIRECTORY = DEPLOY_ROOT / "tmp"
@@ -371,16 +394,18 @@
 from dynaconf import DjangoDynaconf, Validator  # noqa
 
 # Validators
+storage_keys = ("STORAGES.default.BACKEND", "DEFAULT_FILE_STORAGE")
 storage_validator = (
     Validator("REDIRECT_TO_OBJECT_STORAGE", eq=False)
-    | Validator("DEFAULT_FILE_STORAGE", eq="pulpcore.app.models.storage.FileSystem")
-    | Validator("DEFAULT_FILE_STORAGE", eq="storages.backends.azure_storage.AzureStorage")
-    | Validator("DEFAULT_FILE_STORAGE", eq="storages.backends.s3boto3.S3Boto3Storage")
-    | Validator("DEFAULT_FILE_STORAGE", eq="storages.backends.gcloud.GoogleCloudStorage")
+    | Validator(*storage_keys, eq="pulpcore.app.models.storage.FileSystem")
+    | Validator(*storage_keys, eq="storages.backends.azure_storage.AzureStorage")
+    | Validator(*storage_keys, eq="storages.backends.s3boto3.S3Boto3Storage")
+    | Validator(*storage_keys, eq="storages.backends.gcloud.GoogleCloudStorage")
 )
 storage_validator.messages["combined"] = (
-    "'REDIRECT_TO_OBJECT_STORAGE=True' is only supported with the local file, S3, GCP or Azure"
-    "storage backend configured in DEFAULT_FILE_STORAGE."
+    "'REDIRECT_TO_OBJECT_STORAGE=True' is only supported with the local file, S3, GCP or Azure "
+    "storage backend configured in STORAGES['default']['BACKEND'] "
+    "(deprecated DEFAULT_FILE_STORAGE)."
 )
 
 cache_enabled_validator = Validator("CACHE_ENABLED", eq=True)
@@ -485,7 +510,14 @@ def otel_middleware_hook(settings):
     ],
     post_hooks=otel_middleware_hook,
 )
-# HERE ENDS DYNACONF EXTENSION LOAD (No more code below this line)
+
+# begin compatilibity layer for DEFAULT_FILE_STORAGE
+# Remove on pulpcore=3.85 or pulpcore=4.0
+
+# Ensures the cached property storage.backends uses the the right value
+storages._backends = settings.STORAGES.copy()
+storages.backends
+# end compatibility layer
 
 _logger = getLogger(__name__)