diff --git a/anchor/forms/fields.py b/anchor/forms/fields.py index 2a66e93..0a5eb20 100644 --- a/anchor/forms/fields.py +++ b/anchor/forms/fields.py @@ -1,7 +1,7 @@ from django import forms -from .widgets import ClearableBlobInput +from .widgets import SingleAttachmentInput -class BlobField(forms.FileField): - widget = ClearableBlobInput +class SingleAttachmentField(forms.FileField): + widget = SingleAttachmentInput diff --git a/anchor/forms/widgets.py b/anchor/forms/widgets.py index 94ca94c..733bc4f 100644 --- a/anchor/forms/widgets.py +++ b/anchor/forms/widgets.py @@ -1,10 +1,10 @@ from django.forms.widgets import ClearableFileInput -class ClearableBlobInput(ClearableFileInput): +class SingleAttachmentInput(ClearableFileInput): # make this widget work with the Django admin choices = [] -class AdminBlobInput(ClearableBlobInput): +class AdminSingleAttachmentInput(SingleAttachmentInput): template_name = "anchor/widgets/admin_blob_input.html" diff --git a/anchor/management/commands/anchor_purge_unattached.py b/anchor/management/commands/anchor_purge_unattached.py index 10dcaec..48a400e 100644 --- a/anchor/management/commands/anchor_purge_unattached.py +++ b/anchor/management/commands/anchor_purge_unattached.py @@ -5,6 +5,8 @@ class Command(BaseCommand): + help = "Purge and delete blobs not used by any attachment" + def add_arguments(self, parser): parser.add_argument( "--dry-run", action="store_true", help="Dry run the command" diff --git a/anchor/models/attachment.py b/anchor/models/attachment.py index a4a781a..8fc1d81 100644 --- a/anchor/models/attachment.py +++ b/anchor/models/attachment.py @@ -1,17 +1,18 @@ from typing import Any from django.contrib.contenttypes.fields import GenericForeignKey -from django.contrib.contenttypes.models import ContentType from django.db import models from anchor.models.base import BaseModel -class AttachmentManager(models.Manager): - pass +class Attachment(BaseModel): + """ + An attachment relates your own models to Blobs. + Most properties are proxies to the Blob. + """ -class Attachment(BaseModel): class Meta: constraints = ( models.constraints.UniqueConstraint( @@ -26,8 +27,6 @@ class Meta: ), ) - objects = AttachmentManager() - blob = models.ForeignKey( "anchor.Blob", on_delete=models.PROTECT, @@ -36,37 +35,77 @@ class Meta: blank=True, verbose_name="blob", ) + """ + A reference to the Blob which stores the file for this attachment. + """ content_type = models.ForeignKey( - ContentType, + "contenttypes.ContentType", on_delete=models.CASCADE, verbose_name="content type", db_index=False, related_name="+", ) + """ + Content Type (as in Django content types) of the object this attachment + points to. + """ + object_id = models.CharField(max_length=64, verbose_name="object id") + """ + ID of the object this attachment points to. + """ + content_object = GenericForeignKey("content_type", "object_id") + """ + The actual object this attachment points to. + """ order = models.IntegerField(default=0, verbose_name="order") + """ + The order of this attachment in the list of attachments. + """ + name = models.CharField(max_length=256, default="attachments", verbose_name="name") + """ + The name of this attachment. + """ def __str__(self) -> str: return str(self.blob) def url(self, **kwargs): + """ + Returns a URL to the file's location in the storage backend. + """ return self.blob.url(**kwargs) @property def signed_id(self): + """ + Returns a signed ID for this attachment. + """ return self.blob.signed_id @property def filename(self): + """ + The filename of the file stored in the storage backend. + """ return self.blob.filename @property def is_image(self) -> bool: + """ + Whether the file is an image. + """ return self.blob.is_image def representation(self, transformations: dict[str, Any]): + """ + Returns a representation of the file. See :py:class:`the docs on + representations + ` for full + details. + """ return self.blob.representation(transformations) diff --git a/anchor/models/base.py b/anchor/models/base.py index 09a3c48..6ab4c03 100644 --- a/anchor/models/base.py +++ b/anchor/models/base.py @@ -2,7 +2,6 @@ from django.db import models from django.utils import timezone -from django.utils.translation import gettext_lazy as _ from anchor.support import base58 @@ -11,6 +10,11 @@ def generate_pk(): + """ + Generates a primary key with the same entropy as a UUID but encoded in + base58, which works much nicer with URLs than the default hex-based UUID + encoding. + """ return ( base58.b58encode(uuid.uuid4().bytes, alphabet=SHORT_UUID_ALPHABET) .decode("ascii") @@ -26,9 +30,7 @@ class BaseModel(models.Model): editable=False, default=generate_pk, ) - created_at = models.DateTimeField( - default=timezone.now, verbose_name=_("created at") - ) + created_at = models.DateTimeField(default=timezone.now, verbose_name="created at") class Meta: abstract = True diff --git a/anchor/models/blob/blob.py b/anchor/models/blob/blob.py index d81485c..ac5d30e 100644 --- a/anchor/models/blob/blob.py +++ b/anchor/models/blob/blob.py @@ -32,7 +32,7 @@ def unattached(self): Caution: the query generated by this method is expensive to compute. You might want to apply additional filters, for example on the blob - `created_at` field. + ``created_at`` field. """ return self.filter(attachments__isnull=True) @@ -49,6 +49,10 @@ def from_path(self, path: str, **kwargs): class Blob(RepresentationsMixin, BaseModel): + """ + Stores metadata for files stored by Django Anchor. + """ + KEY_LENGTH = 30 class Meta: @@ -60,21 +64,46 @@ class Meta: key = models.CharField( max_length=256, null=False, editable=False, verbose_name="key" ) + """ + A pointer to the file that the storage backend can understand. For instance + in file-system backends, this is a path to the file, relative to the + ``MEDIA_ROOT``. + """ + filename = models.CharField( max_length=256, verbose_name="original filename", null=True, default=None ) + """ + The original filename of the file as it was uploaded. + """ + mime_type = models.CharField( max_length=64, default=anchor_settings.DEFAULT_MIME_TYPE, verbose_name="MIME type", editable=False, ) + """ + A MIME type for the file derived from the extension. + + MIME types are used to determine how files should be served: ``image/*`` + are served inline, while other types are served as attachments for security. + """ + backend = models.CharField( max_length=64, default="default", verbose_name="backend", editable=False, ) + """ + The name of the storage backend as defined in ``settings.STORAGES`` where + the file is stored. + + This property is just the name of the backend. To get a storage instance + with the backend's configuration see :py:attr:`storage`. + """ + byte_size = models.PositiveBigIntegerField( verbose_name="size", help_text="size in bytes", @@ -83,6 +112,11 @@ class Meta: default=None, editable=False, ) + """ + Size of the file in bytes, calculated automatically when the file is + uploaded. + """ + checksum = models.CharField( max_length=256, null=True, @@ -90,12 +124,21 @@ class Meta: verbose_name="checksum", editable=False, ) + """ + The MD5 checksum of the file. + """ + metadata = models.JSONField( null=True, blank=True, default=dict, verbose_name="metadata", ) + """ + Arbitrary metadata file, e.g. including the ``width`` and ``height`` of an image. + + If you need to store custom metadata, refer to the :py:attr:`custom_metadata` property. + """ upload_to: str | Callable[[Self], str] | None = None @@ -119,6 +162,9 @@ def __init__(self, *args, upload_to=None, backend=None, **kwargs): @property def signed_id(self): + """ + A signed ID of the Blob, used to generate URLs. + """ return self.get_signed_id() def get_signed_id( @@ -127,6 +173,23 @@ def get_signed_id( expires_in: timezone.timedelta = None, expires_at: timezone.datetime = None, ): + """ + Returns a signed ID of the blob. + + Signed IDs can be shared with users securely because they can include an + expiration. + + Args: + purpose (str, optional): A string identifying the purpose of the signed ID. + Used to restrict usage of the signed ID to specific contexts. + expires_in (timezone.timedelta, optional): Time duration after which the + signed ID will expire. + expires_at (timezone.datetime, optional): Specific datetime at which the + signed ID will expire. + + Returns: + str: A signed ID string that can be used to securely reference this blob. + """ return ( type(self) ._get_signer() @@ -137,6 +200,20 @@ def get_signed_id( @classmethod def unsign_id(cls, signed_id: str, purpose: str = None): + """ + Un-signs a signed ID of a blob. + + Args: + signed_id (str): The signed ID to un-sign. + purpose (str, optional): The purpose with which the signed ID was created. + + Raises: + BadSignature: If the signature is invalid, has expired, or was signed + with a different purpose. + + Returns: + str: The key of the blob. + """ return cls._get_signer().unsign(signed_id, purpose) @classmethod @@ -192,7 +269,10 @@ def guess_mime_type(self, file: DjangoFile): return anchor_settings.DEFAULT_MIME_TYPE def calculate_checksum(self, file: DjangoFile) -> str: - """Computes the MD5 hash of the given file.""" + """ + Computes the MD5 hash of the given file and returns it as a (URL-safe) + base64-encoded string. + """ m = hashlib.md5() for c in file.chunks(chunk_size=1024): if isinstance(c, str): @@ -204,9 +284,24 @@ def calculate_checksum(self, file: DjangoFile) -> str: @property def storage(self) -> Storage: + """ + The Django storage backend where the file is persisted. + + Instantiated by looking up the configuration in ``settings`` for this + object's :py:attr:`backend`. + """ return storages.create_storage(storages.backends[self.backend]) def generate_key(self): + """ + Generates a random key to store this blob in the storage backend. + + Keys are hard to guess, but shouldn't be shared directly with users, + preferring to use :py:attr:`signed_id` instead, which can be expired. + + They use the base32 encoding to ensure no compatibility issues arise in + case insensitive file systems. + """ return ( base64.b32encode(token_bytes(self.KEY_LENGTH)) .decode("utf-8") @@ -215,6 +310,14 @@ def generate_key(self): ) def url(self, expires_in: timezone.timedelta = None, disposition: str = "inline"): + """ + Returns a URL to the file's location in the storage backend. + + Depending on the backend, this URL can be permanent (like in + file-system-based backends). Instead of sharing this URL directly, refer + to the :py:func:`blob_url ` + template tag to generate a signed, expiring URL. + """ return self.url_service.url( self.key, expires_in=expires_in, disposition=disposition ) @@ -224,17 +327,32 @@ def url_service(self): return get_for_backend(self.backend) def open(self, mode="rb"): + """ + Opens the file from the storage backend. + + This method might involve a download if the storage backend is not + local. It can be used as a context manager. + """ return self.storage.open(self.key, mode) @property def is_image(self): + """ + Whether the file is an image. + """ return self.mime_type.startswith("image/") def purge(self): + """ + Deletes the file from the storage backend. + """ self.storage.delete(self.key) @property def custom_metadata(self): + """ + Arbitrary metadata you want to store for this blob. + """ return (self.metadata or {}).get("custom", {}) @custom_metadata.setter diff --git a/anchor/models/blob/representations.py b/anchor/models/blob/representations.py index f21dffb..b682903 100644 --- a/anchor/models/blob/representations.py +++ b/anchor/models/blob/representations.py @@ -4,7 +4,55 @@ from anchor.settings import anchor_settings +class NotRepresentableError(ValueError): + """ + Raised when attempting to create a variant or preview which is not + compatible with the original file. + """ + + pass + + class RepresentationsMixin: + """ + Adds methods to the :py:class:`Blob ` model to + generate representations of files. + + Image files (those with a MIME type starting with ``image/``) can be varied + by applying a series of transformations like resizing them or changing their + format. + + Other kinds of files do not admit the same transformations as images, but + can still be represented as an image, for instance, by generating a + thumbnail of the first page of a PDF or a frame of a video. + + Previews are not yet implemented, but the :py:meth:`representation` method + is designed to wrap both and decide internally if it should call + :py:meth:`variant` or :py:meth:`preview` depending on the kind of file. + + If you call :py:meth:`representation` on a file that is not representable + (e.g. trying to resize a SVG file), a :py:exc:`NotRepresentableError` will + be raised. + """ + + def representation(self, transformations: dict[str, Any]): + """ + Returns a :py:class:`Variant ` or ``Preview`` object wrapping the + result of applying the given ``transformations`` to this file. + """ + if self.is_variable: + return self.variant(transformations) + else: + return self.preview() + + @property + def is_representable(self) -> bool: + """ + Checks whether the file can represented by an image or, if it is already + an image, whether it can be transformed. + """ + return self.is_variable or self.is_previewable + def variant(self, transformations: dict[str, Any]): if not self.is_variable: raise ValueError("Cannot transform non-variable Blob") @@ -17,16 +65,6 @@ def variant(self, transformations: dict[str, Any]): def is_variable(self) -> bool: return self.mime_type.startswith("image/") - def representation(self, transformations: dict[str, Any]): - if self.is_variable: - return self.variant(transformations) - - raise ValueError("Cannot represent non-variable Blob") - - @property - def is_representable(self) -> bool: - return self.is_variable - @property def default_variant_transformations(self) -> dict[str, Any]: return {"format": anchor_settings.DEFAULT_VARIANT_FORMAT} @@ -39,3 +77,10 @@ def variant_class(self): if anchor_settings.TRACK_VARIANTS: return VariantWithRecord return Variant + + @property + def is_previewable(self): + return False + + def preview(self): + raise NotImplementedError() diff --git a/anchor/models/fields/single_attachment.py b/anchor/models/fields/single_attachment.py index 0cca1a9..2b263cb 100644 --- a/anchor/models/fields/single_attachment.py +++ b/anchor/models/fields/single_attachment.py @@ -12,8 +12,27 @@ class SingleAttachmentRel(GenericRel): + """ + Holds information about the relation between an Attachment and the Model + where the SingleAttachmentField is defined. + """ + + field: "SingleAttachmentField" + + def __init__(self, field: "SingleAttachmentField"): + self.field = field + super().__init__( + field=field, + to="anchor.Attachment", + related_name="+", + related_query_name="+", + ) + @cached_property def cache_name(self): + # Use the name given to the field in the model instance to avoid + # collisions when there are multiple SingleAttachmentFields defined on + # the same instance return self.field.attname @@ -37,6 +56,9 @@ def __get__(self, instance, cls=None) -> Attachment | None: return None def __set__(self, instance, value): + # Be compatible with how Django handles files in the forms API. A + # None value signifies that the file was not updated, while a False + # value signifies that the file should be deleted. if value is None: return @@ -66,10 +88,11 @@ def __set__(self, instance, value): ) Attachment.objects.update_or_create( - object_id=instance.id, - content_type=ContentType.objects.get_for_model(instance), - name=self.name, - order=0, + # object_id=instance.id, + # content_type=ContentType.objects.get_for_model(instance), + # name=self.name, + # order=0, + **self.related.field.get_forward_related_filter(instance), defaults={"blob": blob}, ) @@ -119,6 +142,35 @@ def instance_attr(i): class SingleAttachmentField(GenericRelation): + """ + Enables a model to hold a single attachment. + + When accessing this field, you'll get an :py:class:`Attachment + ` instance. To set the file attachment, + you can assign a plain file-like object, a :py:class:`Blob + ` instance or an :py:class:`Attachment + ` instance. + + Example: + A SingleAttachmentField allows attaching a single file to a model + instance: + + >>> from django.db import models + >>> from anchor.models.fields import SingleAttachmentField + >>> + >>> class Movie(models.Model): + ... title = models.CharField(max_length=100) + ... cover = SingleAttachmentField( + ... upload_to="movie-covers", + ... help_text="A colorful image of the movie." + ... ) + ... + >>> movie = Movie.objects.create(title="The Matrix") + >>> movie.cover = uploaded_file # Attach a file + >>> movie.cover.url() # Get URL to original file + '/media/movie-covers/matrix-cover.jpg' + """ + rel_class = SingleAttachmentRel def __init__( @@ -141,13 +193,7 @@ def __init__( kwargs["from_fields"] = [] kwargs["serialize"] = False - self.rel = self.rel_class( - self, - to="anchor.Attachment", - related_name="+", - related_query_name="+", - limit_choices_to=None, - ) + self.rel = self.rel_class(field=self) # Bypass the GenericRelation constructor to be able to set editable=True super(GenericRelation, self).__init__( @@ -170,7 +216,7 @@ def contribute_to_class(self, cls: type[Model], name: str, **kwargs) -> None: ) def formfield(self, **kwargs): - from anchor.forms.fields import BlobField + from anchor.forms.fields import SingleAttachmentField super().formfield(**kwargs) defaults = { @@ -180,7 +226,7 @@ def formfield(self, **kwargs): } defaults.update(kwargs) - return BlobField(**defaults) + return SingleAttachmentField(**defaults) def get_forward_related_filter(self, obj): return { diff --git a/anchor/models/variation.py b/anchor/models/variation.py index 54b57eb..4c0daec 100644 --- a/anchor/models/variation.py +++ b/anchor/models/variation.py @@ -11,6 +11,10 @@ class Variation: + """ + Represents a (set of) transformations of an image file. + """ + PURPOSE = "variation" transformations: dict[str, Any] @@ -19,6 +23,12 @@ def __init__(self, transformations: dict[str, Any]): @classmethod def wrap(cls, value: str | dict[str, Any] | Self) -> Self: + """ + Returns a variation object, either by decoding a variation key or by + wrapping a dictionary of transformations. + + If the argument is already a variation object, it is returned as is. + """ if isinstance(value, cls): return value elif isinstance(value, str): @@ -27,25 +37,49 @@ def wrap(cls, value: str | dict[str, Any] | Self) -> Self: return cls(value) def default_to(self, default_transformations: dict[str, Any]) -> None: + """ + Updates the keys missing from this object's ``transformations`` with the + given ``default_transformations``. + """ self.transformations = {**default_transformations, **self.transformations} @property def key(self) -> str: + """ + Returns a variation key for this object. + + Keys are signed to ensure they are not tampered with, but they are + permanent so they should be shared with care. + """ return type(self).encode(self.transformations) @property def digest(self) -> str: + """ + A hash of the transformations dictionary in this variation. + + This is a good way to check if two variations are the same, but keep in + mind that order is important in transformations, so two Variations with + the same transformations in different orders will have different + digests. + """ m = hashlib.sha1() m.update(json.dumps(self.transformations).encode("utf-8")) return b58encode(m.digest()).decode("utf-8") @classmethod def decode(cls, key: str) -> Self: + """ + Decodes a signed variation key and returns a Variation. + """ transformations = cls._get_signer().unsign(key, purpose=cls.PURPOSE) return cls(transformations) @classmethod def encode(cls, transformations: dict[str, Any]) -> str: + """ + Encodes a transformations dictionary as a signed string. + """ return cls._get_signer().sign(transformations, purpose=cls.PURPOSE) @classmethod @@ -62,12 +96,23 @@ def transform(self, file): @property def transformer(self) -> BaseTransformer: + """ + Returns the transformer to be used to perform the transformations in + this variation. + """ return ImageTransformer( {k: v for (k, v) in self.transformations.items() if k != "format"} ) @property def mime_type(self) -> str: + """ + Returns the MIME type that the result of applying this variation to an + image file will have. + + It is determined from the ``format`` transformation: e.g. specifying + ``'format': 'webp'`` will return ``'image/webp'``. + """ random_filename = f"random.{self.transformations['format']}" return ( mimetypes.guess_type(random_filename)[0] diff --git a/anchor/services/processors/__init__.py b/anchor/services/processors/__init__.py index e69de29..0554a21 100644 --- a/anchor/services/processors/__init__.py +++ b/anchor/services/processors/__init__.py @@ -0,0 +1,15 @@ +""" +Processors hold the implementation details of how to transform or preview files. + +They are called by :py:mod:`transformers ` with a +source file and are expected to produce an output after applying +transformations. + +Check out the interface for processors in :py:class:`BaseProcessor` and an +implementation example in :py:class:`PillowProcessor`. +""" + +from .base import BaseProcessor +from .pillow import PillowProcessor + +__all__ = ["PillowProcessor", "BaseProcessor"] diff --git a/anchor/services/processors/base.py b/anchor/services/processors/base.py index 859d585..09424ff 100644 --- a/anchor/services/processors/base.py +++ b/anchor/services/processors/base.py @@ -1,6 +1,33 @@ class BaseProcessor: + """ + Interface for file processors. + + A file processor must implement the :py:meth:`source` method and + :py:meth:`save` methods outlined below at the minimum. + + For convenience, the :py:meth:`source` method returns the processor itself, + so you can chain methods together. Check out the :py:class:`PillowProcessor + ` for an example + implementation. + """ + def source(self, file): + """ + Set up the processor to work with the given file. + + The `file` passed is a file-like object that responds to ``read``, like + a ``DjangoFile`` or an ``io`` stream. + """ raise NotImplementedError() def save(self, file, format: str): + """ + Save the processed file to the given file-like object. + + The `file` passed is a file-like object that responds to ``write``, like + a ``TemporaryFile`` or an ``io`` stream. + + The `format` is a string representing the file format to save the file + as, like ``'jpeg'`` or ``'png'``. + """ raise NotImplementedError() diff --git a/anchor/services/processors/pillow.py b/anchor/services/processors/pillow.py index 935b2a6..40f2fe6 100644 --- a/anchor/services/processors/pillow.py +++ b/anchor/services/processors/pillow.py @@ -4,14 +4,51 @@ class PillowProcessor(BaseProcessor): + """ + A file processor that uses the `Pillow `_ + library to transform images. + + To use this processor, make sure to install the Pillow library: ``pip + install Pillow``. + """ + def source(self, file): self.source = Image.open(file) return self def resize_to_fit(self, width: int, height: int): + """ + Resize the image to fit within the given width and height, preserving + aspect ratio. + + Aspect ratio is maintained, so the final image dimensions may be smaller + than the provided rectangle. If the image is smaller than the provided + dimensions, it will be upscaled. + """ + self.source.thumbnail((width, height)) + return self + + def resize_to_limit(self, width: int, height: int): + """ + Downsize the image to fit within the given width and height, preserving + aspect ratio. + + If the image is already smaller than the provided dimensions, nothing is + done. + """ + if self.source.width <= width and self.source.height <= height: + return self + self.source.thumbnail((width, height)) return self + def rotate(self, degrees: int): + """ + Rotates the image by the given angle. + """ + self.source.rotate(degrees) + return self + def save(self, file, format: str): self.source.save(file, format=format) return self diff --git a/anchor/services/transformers/__init__.py b/anchor/services/transformers/__init__.py index 1ac2f70..f636c01 100644 --- a/anchor/services/transformers/__init__.py +++ b/anchor/services/transformers/__init__.py @@ -1,3 +1,11 @@ +""" +Transformers expose a uniform API to apply transformations to files. + +They are called by :py:class:`variations ` +and delegate actual transformations to :py:mod:`processors +`. +""" + from .base import BaseTransformer from .image import ImageTransformer diff --git a/anchor/services/transformers/base.py b/anchor/services/transformers/base.py index 64b0932..3a4a7de 100644 --- a/anchor/services/transformers/base.py +++ b/anchor/services/transformers/base.py @@ -4,6 +4,12 @@ class BaseTransformer: + """ + Interface for a transformer. + + Subclasses need to implement :py:meth:`process`. + """ + transformations: dict[str, Any] def __init__(self, transformations: dict[str, Any]): @@ -22,7 +28,10 @@ def transform(self, file, format: str): output.close() def process(self, file, format: str): - """Returns an open temporary file with the transformed image.""" + """ + Given a buffer and an output format, returns an open temporary file with + the resulting file after applying the transformations in this object. + """ raise NotImplementedError() def _get_temporary_file(self, format: str): diff --git a/anchor/services/urls/__init__.py b/anchor/services/urls/__init__.py index 5a6eb8f..b11fdba 100644 --- a/anchor/services/urls/__init__.py +++ b/anchor/services/urls/__init__.py @@ -1,12 +1,26 @@ from django.core.files.storage import storages -from anchor.services.urls.file_system import FileSystemURLGenerator -from anchor.services.urls.s3 import S3URLGenerator - from .base import BaseURLGenerator +from .file_system import FileSystemURLGenerator +from .s3 import S3URLGenerator + +__all__ = [ + "BaseURLGenerator", + "FileSystemURLGenerator", + "S3URLGenerator", + "get_for_backend", +] def get_for_backend(backend: str) -> BaseURLGenerator: + """ + Given a Django storage backend name, return a ``URLGenerator`` instance that + can work with it. + + If no suitable generator is found, this function will return the + :py:class:`BaseURLGenerator` which delegates URL generation to the storage + backend. + """ backend_config = storages.backends[backend] if backend_config["BACKEND"] == "django.core.files.storage.FileSystemStorage": return FileSystemURLGenerator(backend) diff --git a/anchor/services/urls/base.py b/anchor/services/urls/base.py index df02b88..2869cea 100644 --- a/anchor/services/urls/base.py +++ b/anchor/services/urls/base.py @@ -3,6 +3,16 @@ class BaseURLGenerator: + """ + Generates (signed) URLs for blobs. + + URLGenerators are a workaround against the limitations of Django's Storage + interface, which doesn't support passing parameters for URL generation. + These allow us to set parameters on URLs such as the content disposition + which are necessary to ensure external backends like S3 can serve blobs + securely. + """ + backend: str storage: Storage @@ -18,4 +28,7 @@ def url( disposition: str = "inline", filename: str = None, ): + """ + Generate a signed URL for the given storage key. + """ return self.storage.url(key) diff --git a/anchor/services/urls/file_system.py b/anchor/services/urls/file_system.py index 9c6ed8f..a0aea3e 100644 --- a/anchor/services/urls/file_system.py +++ b/anchor/services/urls/file_system.py @@ -7,6 +7,15 @@ class FileSystemURLGenerator(BaseURLGenerator): + """ + Generate signed and expiring URLs for files stored in file-system storage + backends. + + This generator exists so that we can have more control over how files stored + via Django's default FileSystemBackend are served by making requests go + through our file system view. + """ + def url( self, name: str, @@ -16,9 +25,7 @@ def url( disposition: str = None, ) -> str: key = self.get_key(name, mime_type=mime_type, disposition=disposition) - signed_key = AnchorSigner().sign( - key, expires_in=expires_in, purpose="file_system" - ) + signed_key = self.signer.sign(key, expires_in=expires_in, purpose="file_system") kwargs = {"signed_key": signed_key} if filename: kwargs["filename"] = filename @@ -29,3 +36,7 @@ def get_key(self, name: str, **kwargs) -> str: if kwargs: key.update(kwargs) return key + + @property + def signer(self) -> AnchorSigner: + return AnchorSigner() diff --git a/anchor/services/urls/s3.py b/anchor/services/urls/s3.py index ff036e9..03aaf8c 100644 --- a/anchor/services/urls/s3.py +++ b/anchor/services/urls/s3.py @@ -4,6 +4,11 @@ class S3URLGenerator(BaseURLGenerator): + """ + URL Generator for use with the S3Backend from the ``django-storages`` + package. + """ + def url( self, key: str, diff --git a/anchor/settings.py b/anchor/settings.py index 5278042..aa45b36 100644 --- a/anchor/settings.py +++ b/anchor/settings.py @@ -16,7 +16,8 @@ class AnchorSettings: DEFAULT_MIME_TYPE: str = "application/octet-stream" """ - The default MIME type to use for Blobs when it cannot be guessed from the file extension. + The default MIME type to use for files when it cannot be guessed from the + file extension. """ FILE_SYSTEM_BACKEND_EXPIRATION: timedelta = timedelta(hours=1) @@ -39,6 +40,10 @@ class AnchorSettings: The default format to use for variants. """ + def __init__(self, *args, **kwargs): + # Hide constructor signature from the docs + super().__init__(*args, **kwargs) + def __getattribute__(self, name: str) -> Any: user_settings = getattr(settings, ANCHOR_SETTINGS_NAME, {}) return user_settings.get(name, super().__getattribute__(name)) diff --git a/anchor/support/base58.py b/anchor/support/base58.py index 4f41d0d..a6c2e98 100644 --- a/anchor/support/base58.py +++ b/anchor/support/base58.py @@ -1,5 +1,6 @@ """ -Base58 encoding adapted from the base58 package by David Keijser released under the MIT license. +Base58 encoding adapted from the base58 package by David Keijser released under +the MIT license. See https://github.com/keis/base58 for full credits. """ diff --git a/anchor/support/signing.py b/anchor/support/signing.py index abcde11..f446ab7 100644 --- a/anchor/support/signing.py +++ b/anchor/support/signing.py @@ -1,3 +1,8 @@ +""" +Extends Django's signing module to make it easier to work with object keys and +key expirations. +""" + import base64 from typing import Any, Type diff --git a/anchor/templatetags/anchor.py b/anchor/templatetags/anchor.py index 0811931..152ab42 100644 --- a/anchor/templatetags/anchor.py +++ b/anchor/templatetags/anchor.py @@ -1,3 +1,8 @@ +""" +Blob URLs should not be exposed directly to users. Instead, use signed, +short-lived URLs generated with these functions. +""" + from typing import Any from django import template @@ -11,6 +16,9 @@ @register.simple_tag def blob_url(value: Blob | Attachment | None): + """ + Return a signed URL for the given Attachment or Blob. + """ if value is None: return "" @@ -21,7 +29,10 @@ def blob_url(value: Blob | Attachment | None): @register.simple_tag -def variant_url(value: Variant | Blob | Attachment | None, **transformations): +def representation_url(value: Variant | Blob | Attachment | None, **transformations): + """ + Return a signed URL for a transformation of the given Attachment or Blob. + """ if value is None: return "" diff --git a/docs/_static/img/django_admin_attachments_inline.png.webp b/docs/_static/img/django_admin_attachments_inline.png.webp deleted file mode 100644 index 360319d..0000000 Binary files a/docs/_static/img/django_admin_attachments_inline.png.webp and /dev/null differ diff --git a/docs/api/forms.rst b/docs/api/forms.rst deleted file mode 100644 index bc50fe6..0000000 --- a/docs/api/forms.rst +++ /dev/null @@ -1,9 +0,0 @@ -============================= -Forms -============================= - -.. automodule:: anchor.forms.fields - :members: - -.. automodule:: anchor.forms.widgets - :members: diff --git a/docs/api/index.rst b/docs/api/index.rst index 48d838a..8c26a7c 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -5,7 +5,8 @@ API Reference .. toctree:: :maxdepth: 3 - forms models templatetags - transformers + services/processors + services/transformers + services/urls diff --git a/docs/api/models.rst b/docs/api/models.rst index ea70fe7..cc5819b 100644 --- a/docs/api/models.rst +++ b/docs/api/models.rst @@ -2,11 +2,52 @@ Models ============================= -.. automodule:: anchor.models.attachment + +Core models +----------- + +There are two core models in Anchor: + +- :py:class:`Blobs ` which take care of storing a + pointer to each file uploaded plus some metadata, and +- :py:class:`Attachments ` which act as a + bridge between your own models and :py:class:`Blobs + `, allowing you to attach files to your own + records. + +These, together with the :py:class:`VariantRecord +` are the only models backed by a +database table. The rest of the classes within the ``anchor.models`` module only +contain business logic. + +.. autoclass:: anchor.models.blob.blob.Blob + :members: + +.. autoclass:: anchor.models.attachment.Attachment + :members: + +Fields +------ + +Model fields are how you allow your models to hold files. + +.. autoclass:: anchor.models.fields.SingleAttachmentField + + +Representations +--------------- + +.. automodule:: anchor.models.blob.representations + :members: + +.. automodule:: anchor.models.variant + :members: + +.. automodule:: anchor.models.variant_record :members: -.. automodule:: anchor.models.blob +.. automodule:: anchor.models.variant_with_record :members: -.. automodule:: anchor.models.fields +.. automodule:: anchor.models.variation :members: diff --git a/docs/api/services/processors.rst b/docs/api/services/processors.rst new file mode 100644 index 0000000..009dc81 --- /dev/null +++ b/docs/api/services/processors.rst @@ -0,0 +1,7 @@ +========== +Processors +========== + +.. automodule:: anchor.services.processors + :members: + :show-inheritance: diff --git a/docs/api/services/transformers.rst b/docs/api/services/transformers.rst new file mode 100644 index 0000000..646a27c --- /dev/null +++ b/docs/api/services/transformers.rst @@ -0,0 +1,7 @@ +============ +Transformers +============ + +.. automodule:: anchor.services.transformers + :members: + :show-inheritance: diff --git a/docs/api/services/urls.rst b/docs/api/services/urls.rst new file mode 100644 index 0000000..8952029 --- /dev/null +++ b/docs/api/services/urls.rst @@ -0,0 +1,6 @@ +==== +URLs +==== + +.. automodule:: anchor.services.urls + :members: diff --git a/docs/api/transformers.rst b/docs/api/transformers.rst deleted file mode 100644 index ae56907..0000000 --- a/docs/api/transformers.rst +++ /dev/null @@ -1,6 +0,0 @@ -============================= -Transformers -============================= - -.. automodule:: anchor.transformers - :members: diff --git a/docs/conf.py b/docs/conf.py index 55b749b..231f386 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,22 +1,22 @@ import os import sys -import toml + import django project = "Django Anchor" copyright = "2024, Elias Hernandis" author = "Elias Hernandis" -# Read version from pyproject.toml -with open("../pyproject.toml", "r") as f: - pyproject = toml.load(f) - release = pyproject["tool"]["poetry"]["version"] # Make source available for autodoc -os.environ["DJANGO_SETTINGS_MODULE"] = "tests.test_settings" +os.environ["DJANGO_SETTINGS_MODULE"] = "tests.settings" sys.path.insert(0, os.path.abspath("..")) django.setup() +import anchor # noqa: E402 + +release = anchor.__version__ + # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/docs/configuration.rst b/docs/configuration.rst new file mode 100644 index 0000000..7359d6b --- /dev/null +++ b/docs/configuration.rst @@ -0,0 +1,17 @@ +============= +Configuration +============= + +Django Anchor exposes several settings which can be changed to tune the +application to your needs. Define these settings under ``ANCHOR`` in your project +settings like this: + +.. code-block:: python + + ANCHOR = { + 'DEFAULT_STORAGE_BACKEND': 'my_storage_backend', + } + + +.. autoclass:: anchor.settings.AnchorSettings + :members: diff --git a/docs/getting_started.rst b/docs/getting_started.rst index d7f4c15..2442c9e 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -5,44 +5,72 @@ Getting Started Installation ============ -Django Anchor is installed just like any other Django package: add the package -as a dependency and then add ``anchor`` to your ``INSTALLED_APPS`` setting. +Django Anchor is installed just like any other Django package: -Check out the :doc:`installation guide ` for more details. +- add the package as a dependency +- add ``anchor`` to your ``INSTALLED_APPS`` setting +- add ``anchor.urls`` to your URL config +- run ``python manage.py migrate`` to create the necessary tables + +Check out the :doc:`installation guide ` for more details, +including how to install the optional dependencies for image transformations. Introduction ============ -Django Anchor allows you to add files to your Django models. It is intended to -replace Django FileFields and ImageFields and provide a bit more functionality. -Files can be attached in one of two ways: +Django Anchor allows you to add files to your Django models. It is essentially a +clone for Ruby on Rails' ActiveStorage ported to Python and following the +conventions of the Django ecosystem. Django Anchor replaces Django's ``FileField`` +and ``ImageField`` and enhances them with a few features: -- **Attaching a single file** by adding an - :py:class:`anchor.models.fields.BlobField` to your model. -- **Attaching multiple files** by creating - :py:class:`anchor.models.attachment.Attachment` objects which link - :py:class:`anchor.models.blob.Blob` objects to your model objects via a generic relationship. +- Allows generating and serving variants of image files. You can dynamically + resize and otherwise transform images to serve optimized versions straight + from a template tag. +- Allows generating and serving signed URLs for files, even when using the + default file-system storage backend. +- Removes the need for additional columns in models to store references to + files. Instead, Django Anchor tracks file attachments using a generic + relationship. Single File Attachments ======================= -We'll focus on adding single objects first, building on the example from the -demo project. Let's say you have a ``Movie`` model and want to upload cover -images. First, add a ``BlobField`` to your model: +Let's say you have a ``Movie`` model and want to upload cover images. First, add +a :py:class:`SingleAttachmentField ` +to your model: .. code-block:: python from django.db import models - from anchor.models.fields import BlobField + from anchor.models.fields import SingleAttachmentField class Movie(models.Model): title = models.CharField(max_length=100) - cover = BlobField() + cover = SingleAttachmentField() + +That's pretty much it! No need to run ``makemigrations`` or ``migrate`` since +Django Anchor doesn't actually need any columns added to the model. + +The ``cover`` field works just like any other model field: + +.. code-block:: python -That's pretty much it! Just like with any other model field, you need to run -``python manage.py makemigrations`` and ``python manage.py migrate`` to create -the migration and apply changes to the database. + # Create a new movie + movie = Movie.objects.create(title="My Movie") + + # Attach an uploaded file + movie.cover = uploaded_file + + # Get a URL to the file + movie.cover.url() + + # Get a URL to a miniature version of the file + movie.cover.representation(resize_to_fit=(200, 200), format="webp").url() + + # Delete the file + movie.cover.purge() + movie.cover.delete() Rendering attachments in templates ================================== @@ -50,9 +78,9 @@ Rendering attachments in templates One of the core functionalities of Django Anchor is the ability to render versions of the original file attached to a model that are optimized for a particular size or converted to another format. This is done using the -:py:func:`blob_thumbnail ` template tag, which takes a -``Blob`` object as the first argument and optional format parameters to build a -variant. +:py:func:`representation_url ` +template tag, which takes an ``Attachment`` object as the first argument and +optional format parameters to build a variant. Let's say you want to render a grid of Movie cover thumbnails in a list view. Your template could look something like this: @@ -63,23 +91,21 @@ Your template could look something like this:
    {% for movie in movies %}
  • - {{ movie.title }} + {{ movie.title }}

    {{ movie.title }}

    • {% endfor %}
-Using BlobFields in forms -========================= +Using SingleAttachmentFields in forms +===================================== -Django Anchor includes a form field for handling file uploads that is designed -to be compatible with Django's FileField. It will be used by default if you use -model forms and you can add it to your own forms as well: +Django Anchor file fields work out of the box with Django's form system. .. code-block:: python from django import forms - from anchor.forms.fields import BlobField + from anchor.forms.fields import SingleAttachmentField class MovieForm(forms.ModelForm): class Meta: @@ -90,112 +116,32 @@ model forms and you can add it to your own forms as well: class MovieForm(forms.Form): title = forms.CharField(max_length=100) - cover = BlobField() - - -Multiple File Attachments -========================= - -If you need to attach a list any number of files to a model you need to use the -:py:class:`Attachment ` model. This model -links Blob objects to your model objects via a `generic Django relations -`_. - -To do so, just create Attachment objects for each file you want to attach like -this: - -.. code-block:: python - - from django.db import models - from anchor.models.attachment import Attachment - - movie = Movie.objects.create(title='My Movie') - scene1_attachment = Attachment.objects.create( - blob=Blob.objects.from_path('path/to/scene1.jpg'), - content_object=movie, - name='scenes', - order=0, - ) - scene2_attachment = Attachment.objects.create( - blob=Blob.objects.from_path('path/to/scene2.jpg'), - content_object=movie, - name='scenes', - order=1, - ) - scene3_attachment = Attachment.objects.create( - blob=Blob.objects.from_path('path/to/scene3.jpg'), - content_object=movie, - name='scenes', - order=2, - ) - -The ``name`` field is used to group attachments together. It doesn't need to be -set (it defaults to ``'attachments'``) but is helpful if you have multiple -collections of files that you want to attach to a model. - -The ``order`` field is used to sort the attachments in the order you want them -and must be unique for each attachment in the same collection (with the same -``name`` and ``content_object``). - -Django attachments doesn't provide a default way to generate the reverse generic -relationship but you can do so yourself by adding a property to your model: - -.. code-block:: python - - from django.db import models - from anchor.models.attachment import Attachment + cover = SingleAttachmentField() - class Movie(models.Model): - title = models.CharField(max_length=100) - @property - def scenes(self): - return ( - Attachment.objects.select_related("blob") - .filter_by_object(self, name="scenes") - .order_by("order") - .all() - ) +Admin integration +================= -Keep in mind that this is not optimal if you're loading multiple movies at the -same time (since it'll perform N+1 queries). Suggestions on how to improve this -or make some standard way within the Django Anchor package are always welcome! - -To render attachments in templates, you can use the same -:py:func:`blob_thumbnail ` template -tag as with BlobFields: +Django Anchor nicely integrates with the Django admin, just like File fields do. -.. code-block:: html+django +.. image:: _static/img/django_admin_default_widget.png.webp + :alt: Django Anchor admin widget for SingleAttachmentFields - {% load anchor %} -
    - {% for attachment in movie.scenes %} -
  • - {{ movie.title }} -
    • - {% endfor %} -
+You can get a preview of the attached file by overriding the form field for +the ``SingleAttachmentField`` model field with a widget that renders a thumbnail: -Admin integration -================= +.. code-block:: python -Django Anchor provides a custom admin widget for BlobFields that allows you to -use them like normal FileFields in the Django admin: + from anchor.forms.widgets import AdminSingleAttachmentInput -.. image:: _static/img/django_admin_default_widget.png.webp - :alt: Django Anchor admin widget for BlobFields + class MovieAdmin(admin.ModelAdmin): + formfield_overrides = { + SingleAttachmentField: {'widget': AdminSingleAttachmentInput}, + } -If you inherit from :py:class:`anchor.admin.BlobAdminMixin` in your admin class, -you'll get a widget with a preview of the file instead of a link to the File: +That makes the admin widget look like this: .. image:: _static/img/django_admin_thumbnail_widget.png.webp - :alt: Django Anchor admin widget for BlobFields with preview - -Django Anchor also provides an admin inline to manage attachments. Add -:py:class:`anchor.admin.AttachmentInline` to your model admin to manage -attachments in the Django admin: - -.. image:: _static/img/django_admin_attachments_inline.png.webp - :alt: Django Anchor admin inline for Attachment objects + :alt: Django Anchor admin widget for SingleAttachmentFields with preview diff --git a/docs/index.rst b/docs/index.rst index 0144819..8cf7a40 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,8 +1,26 @@ -Welcome to the Django Anchor documentation! +============= +Django Anchor +============= + +Django Anchor is a reusable Django app that allows you to attach files to models. + +Anchor works very similarly to Django's ``FileField`` and ``ImageField`` model +fields, but adds a few extra features: + +- Images can be resized, converted to another format and otherwise transformed. +- Files are served through signed URLs that can expire after a configurable + amount of time, even when using the default file-system storage backend. + +Django Anchor is essentially a port of the excellent `Active Storage +`_ Ruby on +Rails feature, but leveraging existing Django abstractions and packages of the +Python ecosystem. Some features are not yet implemented, but the core concepts +are there two eventually be able to support them. Table of contents ================= .. toctree:: - installation getting_started + installation + configuration api/index diff --git a/docs/installation.rst b/docs/installation.rst index aa61457..6b6e776 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -10,24 +10,44 @@ Installation or by adding ``django-anchor`` to your ``requirements.txt`` or ``pyproject.toml`` files if you have one. + If you intend to transform images you'll also need to install ``Pillow``. See + below for more details. + 2. Add ``anchor`` to ``settings.INSTALLED_APPS`` 3. Add URL configuration to your project: - ```python + .. code-block:: python + + urlpatterns = [ + path('anchor/', include('anchor.urls')), + ] - urlpatterns = [ - path('anchor/', include('anchor.urls')), - ] - ``` 4. Run migrations: - ```bash - python manage.py migrate - ``` + .. code-block:: bash + + python manage.py migrate -By default, anchor works with your default storage backend. If you want to use a + +By default, Anchor works with your ``default`` storage backend. If you want to use a different storage backend, define it under ``STORAGES`` in your project settings and set ``ANCHOR['DEFAULT_STORAGE_BACKEND']`` to the name of your preferred storage backend. + +Check out the full configuration options in :doc:`configuration `. + + +Additional dependencies for image transformations +================================================= + +Using Anchor to generate representations of images requires the ``Pillow`` +Python package. You can add it to your dependencies, and there's no need to add +anything to the ``INSTALLED_APPS`` setting. + +Keep in mind that while Pillow is a powerful library with lots of available +operations, Anchor only supports a handful of them. Check out the +:py:class:`PillowProcessor documentation +` for a list of supported +operations. diff --git a/tests/models/blob/test_representations.py b/tests/models/blob/test_representations.py index 067674d..6cf71a0 100644 --- a/tests/models/blob/test_representations.py +++ b/tests/models/blob/test_representations.py @@ -1,6 +1,7 @@ from django.test import SimpleTestCase from anchor.models import Blob +from anchor.models.blob.representations import NotRepresentableError class TestRepresentations(SimpleTestCase): @@ -44,5 +45,5 @@ def test_cannot_transform_non_variable_blob(self): def test_cannot_represent_non_variable_blob(self): text = Blob(mime_type="text/plain") - with self.assertRaises(ValueError): + with self.assertRaises(NotRepresentableError): text.representation({"format": "png"})