[16.0] [IMP] fs_attachment: store attachments linked to different model/fields to different FS storages

fs_attachment/README.rst

@@ -0,0 +1,35 @@
+**This file is going to be generated by oca-gen-addon-readme.**
+
+*Manual changes will be overwritten.*
+
+Please provide content in the ``readme`` directory:
+
+* **DESCRIPTION.rst** (required)
+* INSTALL.rst (optional)
+* CONFIGURE.rst (optional)
+* **USAGE.rst** (optional, highly recommended)
+* DEVELOP.rst (optional)
+* ROADMAP.rst (optional)
+* HISTORY.rst (optional, recommended)
+* **CONTRIBUTORS.rst** (optional, highly recommended)
+* CREDITS.rst (optional)
+
+Content of this README will also be drawn from the addon manifest,
+from keys such as name, authors, maintainers, development_status,
+and license.
+
+A good, one sentence summary in the manifest is also highly recommended.
+
+
+Automatic changelog generation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+`HISTORY.rst` can be auto generated using `towncrier <https://pypi.org/project/towncrier>`_.
+
+Just put towncrier compatible changelog fragments into `readme/newsfragments`
+and the changelog file will be automatically generated and updated when a new fragment is added.
+
+Please refer to `towncrier` documentation to know more.
+
+NOTE: the changelog will be automatically generated when using `/ocabot merge $option`.
+If you need to run it manually, refer to `OCA/maintainer-tools README <https://github.com/OCA/maintainer-tools>`_.

fs_attachment/__init__.py

@@ -0,0 +1,2 @@
+from . import models
+from .hooks import pre_init_hook

fs_attachment/__manifest__.py

@@ -0,0 +1,24 @@
+# Copyright 2017-2021 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+{
+    "name": "Base Attachment Object Store",
+    "summary": "Store attachments on external object store",
+    "version": "16.0.1.0.0",
+    "author": "Camptocamp, ACSONE SA/NV, Odoo Community Association (OCA)",
+    "license": "AGPL-3",
+    "development_status": "Beta",
+    "category": "Knowledge Management",
+    "depends": ["fs_storage"],
+    "website": "https://github.com/OCA/storage",
+    "data": [
+        "security/fs_file_gc.xml",
+        "views/fs_storage.xml",
+    ],
+    "external_dependencies": {"python": ["python_slugify"]},
+    "installable": True,
+    "auto_install": False,
+    "maintainers": ["lmignon"],
+    "pre_init_hook": "pre_init_hook",
+}

fs_attachment/fs_stream.py

@@ -0,0 +1,100 @@
+# Copyright 2023 ACSONE SA/NV
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+from __future__ import annotations
+
+from odoo.http import STATIC_CACHE_LONG, Response, Stream, request
+from odoo.tools import config
+
+from .models.ir_attachment import IrAttachment
+
+try:
+    from werkzeug.utils import send_file as _send_file
+except ImportError:
+    from odoo.tools._vendor.send_file import send_file as _send_file
+
+
+class FsStream(Stream):
+    fs_attachment = None
+
+    @classmethod
+    def from_fs_attachment(cls, attachment: IrAttachment) -> FsStream:
+        attachment.ensure_one()
+        if not attachment.fs_filename:
+            raise ValueError("Attachment is not stored into a filesystem storage")
+        size = 0
+        if cls._check_use_x_sendfile(attachment):
+            fs, _storage, fname = attachment._get_fs_parts()
+            fs_info = fs.info(fname)
+            size = fs_info["size"]
+        return cls(
+            mimetype=attachment.mimetype,
+            download_name=attachment.name,
+            conditional=True,
+            etag=attachment.checksum,
+            type="fs",
+            size=size,
+            last_modified=attachment["__last_update"],
+            fs_attachment=attachment,
+        )
+
+    def read(self):
+        if self.type == "fs":
+            with self.fs_attachment.open("rb") as f:
+                return f.read()
+        return super().read()
+
+    def get_response(self, as_attachment=None, immutable=None, **send_file_kwargs):
+        if self.type != "fs":
+            return super().get_response(
+                as_attachment=as_attachment, immutable=immutable, **send_file_kwargs
+            )
+        if as_attachment is None:
+            as_attachment = self.as_attachment
+        if immutable is None:
+            immutable = self.immutable
+        send_file_kwargs = {
+            "mimetype": self.mimetype,
+            "as_attachment": as_attachment,
+            "download_name": self.download_name,
+            "conditional": self.conditional,
+            "etag": self.etag,
+            "last_modified": self.last_modified,
+            "max_age": STATIC_CACHE_LONG if immutable else self.max_age,
+            "environ": request.httprequest.environ,
+            "response_class": Response,
+            **send_file_kwargs,
+        }
+        use_x_sendfile = self._fs_use_x_sendfile
+        # The file will be closed by werkzeug...
+        send_file_kwargs["use_x_sendfile"] = use_x_sendfile
+        if not use_x_sendfile:
+            f = self.fs_attachment.open("rb")
+            res = _send_file(f, **send_file_kwargs)
+        else:
+            x_accel_redirect = (
+                f"/{self.fs_attachment.fs_storage_code}{self.fs_attachment.fs_url_path}"
+            )
+            send_file_kwargs["use_x_sendfile"] = True
+            res = _send_file("", **send_file_kwargs)
+            # nginx specific headers
+            res.headers["X-Accel-Redirect"] = x_accel_redirect
+            # apache specific headers
+            res.headers["X-Sendfile"] = x_accel_redirect
+            res.headers["Content-Length"] = 0
+
+        if immutable and res.cache_control:
+            res.cache_control["immutable"] = None
+        return res
+
+    @classmethod
+    def _check_use_x_sendfile(cls, attachment: IrAttachment) -> bool:
+        return (
+            config["x_sendfile"]
+            and attachment.fs_url
+            and attachment.fs_storage_id.use_x_sendfile_to_serve_internal_url
+        )
+
+    @property
+    def _fs_use_x_sendfile(self) -> bool:
+        """Return True if x-sendfile should be used to serve the file"""
+        return self._check_use_x_sendfile(self.fs_attachment)

fs_attachment/hooks.py

@@ -0,0 +1,33 @@
+# Copyright 2023 ACSONE SA/NV
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+import logging
+
+_logger = logging.getLogger(__name__)
+
+
+def pre_init_hook(cr):
+    """Pre init hook."""
+    # add columns for computed fields to avoid useless computation by the ORM
+    # when installing the module
+    _logger.info("Add columns for computed fields on ir_attachment")
+    cr.execute(
+        """
+        ALTER TABLE ir_attachment
+        ADD COLUMN fs_storage_id INTEGER;
+        ALTER TABLE ir_attachment
+        ADD FOREIGN KEY (fs_storage_id) REFERENCES fs_storage(id);
+        """
+    )
+    cr.execute(
+        """
+        ALTER TABLE ir_attachment
+        ADD COLUMN fs_url VARCHAR;
+        """
+    )
+    cr.execute(
+        """
+        ALTER TABLE ir_attachment
+        ADD COLUMN fs_storage_code VARCHAR;
+        """
+    )
+    _logger.info("Columns added on ir_attachment")

fs_attachment/models/__init__.py

@@ -0,0 +1,6 @@
+from . import fs_file_gc
+from . import fs_storage
+from . import ir_attachment
+from . import ir_binary
+from . import ir_model
+from . import ir_model_fields

fs_attachment/models/fs_file_gc.py

@@ -0,0 +1,168 @@
+# Copyright 2023 ACSONE SA/NV
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+import logging
+import threading
+from contextlib import closing, contextmanager
+
+from odoo import api, fields, models
+from odoo.sql_db import Cursor
+
+_logger = logging.getLogger(__name__)
+
+
+class FsFileGC(models.Model):
+
+    _name = "fs.file.gc"
+    _description = "Filesystem storage file garbage collector"
+
+    store_fname = fields.Char("Stored Filename")
+    fs_storage_code = fields.Char("Storage Code")
+
+    _sql_constraints = [
+        (
+            "store_fname_uniq",
+            "unique (store_fname)",
+            "The stored filename must be unique!",
+        ),
+    ]
+
+    def _is_test_mode(self) -> bool:
+        """Return True if we are running the tests, so we do not mark files for
+        garbage collection into a separate transaction.
+        """
+        return (
+            getattr(threading.current_thread(), "testing", False)
+            or self.env.registry.in_test_mode()
+        )
+
+    @contextmanager
+    def _in_new_cursor(self) -> Cursor:
+        """Context manager to execute code in a new cursor"""
+        if self._is_test_mode() or not self.env.registry.ready:
+            yield self.env.cr
+            return
+
+        with closing(self.env.registry.cursor()) as cr:
+            try:
+                yield cr
+            except Exception:
+                cr.rollback()
+                raise
+            else:
+                # disable pylint error because this is a valid commit,
+                # we are in a new env
+                cr.commit()  # pylint: disable=invalid-commit
+
+    @api.model
+    def _mark_for_gc(self, store_fname: str) -> None:
+        """Mark a file for garbage collection"
+
+        This process is done in a separate transaction since the data must be
+        preserved even if the transaction is rolled back.
+        """
+        with self._in_new_cursor() as cr:
+            code = store_fname.partition("://")[0]
+            # use plain SQL to avoid the ORM ignore conflicts errors
+            cr.execute(
+                """
+                INSERT INTO
+                    fs_file_gc (
+                        store_fname,
+                        fs_storage_code,
+                        create_date,
+                        write_date,
+                        create_uid,
+                        write_uid
+                    )
+                    VALUES (
+                        %s,
+                        %s,
+                        now() at time zone 'UTC',
+                        now() at time zone 'UTC',
+                        %s,
+                        %s
+                    )
+                ON CONFLICT DO NOTHING
+            """,
+                (store_fname, code, self.env.uid, self.env.uid),
+            )
+
+    @api.autovacuum
+    def _gc_files(self) -> None:
+        """Garbage collect files"""
+        # This method is mainly a copy of the method _gc_file_store_unsafe()
+        # from the module fs_attachment. The only difference is that the list
+        # of files to delete is retrieved from the table fs_file_gc instead
+        # of the odoo filestore.
+
+        # Continue in a new transaction. The LOCK statement below must be the
+        # first one in the current transaction, otherwise the database snapshot
+        # used by it may not contain the most recent changes made to the table
+        # ir_attachment! Indeed, if concurrent transactions create attachments,
+        # the LOCK statement will wait until those concurrent transactions end.
+        # But this transaction will not see the new attachements if it has done
+        # other requests before the LOCK (like the method _storage() above).
+        cr = self._cr
+        cr.commit()  # pylint: disable=invalid-commit
+
+        # prevent all concurrent updates on ir_attachment and fs_file_gc
+        # while collecting, but only attempt to grab the lock for a little bit,
+        # otherwise it'd start blocking other transactions.
+        # (will be retried later anyway)
+        cr.execute("SET LOCAL lock_timeout TO '10s'")
+        cr.execute("LOCK fs_file_gc IN SHARE MODE")
+        cr.execute("LOCK ir_attachment IN SHARE MODE")
+
+        self._gc_files_unsafe()
+
+        # commit to release the lock
+        cr.commit()  # pylint: disable=invalid-commit
+
+    def _gc_files_unsafe(self) -> None:
+        # get the list of fs.storage codes that must be autovacuumed
+        codes = (
+            self.env["fs.storage"].search([]).filtered("autovacuum_gc").mapped("code")
+        )
+        if not codes:
+            return
+        # we process by batch of storage codes.
+        self._cr.execute(
+            """
+            SELECT
+                fs_storage_code,
+                array_agg(store_fname)
+
+            FROM
+                fs_file_gc
+            WHERE
+                fs_storage_code IN %s
+                AND NOT EXISTS (
+                    SELECT 1
+                    FROM ir_attachment
+                    WHERE store_fname = fs_file_gc.store_fname
+                )
+            GROUP BY
+                fs_storage_code
+            """,
+            (tuple(codes),),
+        )
+        for code, store_fnames in self._cr.fetchall():
+            self.env["fs.storage"].get_by_code(code)
+            fs = self.env["fs.storage"].get_fs_by_code(code)
+            for store_fname in store_fnames:
+                try:
+                    file_path = store_fname.partition("://")[2]
+                    fs.rm(file_path)
+                except Exception:
+                    _logger.debug("Failed to remove file %s", store_fname)
+
+        # delete the records from the table fs_file_gc
+        self._cr.execute(
+            """
+            DELETE FROM
+                fs_file_gc
+            WHERE
+                fs_storage_code IN %s
+            """,
+            (tuple(codes),),
+        )