[19.0][IMP] fs_attachment: allow field keys in force db rules

fs_attachment/README.rst

@@ -160,6 +160,26 @@ configure the way attachments will be stored in the filesystem.
   - application/javascript are stored in database whatever their size
   - text/css are stored in database whatever their size
 
+  The rules can also be grouped by rule name:
+
+  - ``mimetype-size``: the mimetype rules described above
+  - ``field-size``: the key is a ``<model>.<field>`` pair, to force the
+    storage in database of the attachments of a specific binary field,
+    whatever their mimetype
+
+  It's useful for fields like the menu icons, that are read on every
+  page load, without having to keep all the images in database:
+
+     {"mimetype-size": {"image/": 51200, "application/javascript": 0,
+     "text/css": 0}, "field-size": {"ir.ui.menu.web_icon_data": 0}}
+
+  Keys that are not a rule name are read as mimetype rules, so existing
+  configurations keep working.
+
+  A field rule is checked before the mimetype rules: if the attachment
+  matches it and is within its limit, it is stored in database,
+  otherwise the mimetype rules apply.
+
   This option is only available on the filesystem storage that is used
   as default for attachments.
 

fs_attachment/models/fs_storage.py

@@ -66,7 +66,11 @@ class FsStorage(models.Model):
         "* images mimetypes (image/png, image/jpeg, ...) below 50KB are stored "
         "in database\n"
         "* application/javascript are stored in database whatever their size \n"
-        "* text/css are stored in database whatever their size",
+        "* text/css are stored in database whatever their size\n"
+        "The rules can also be grouped by rule name: 'mimetype-size' for the "
+        "mimetype rules above and 'field-size' with '<model>.<field>' keys (e.g. "
+        "'ir.ui.menu.web_icon_data') to force the storage in database of the "
+        "attachments of a specific binary field, whatever their mimetype.",
         default=lambda self: self._default_force_db_for_default_attachment_rules,
     )
     use_filename_obfuscation = fields.Boolean(
@@ -155,14 +159,40 @@ def _check_force_db_for_default_attachment_rules(self):
                     )
                 )
             try:
-                const_eval(rec.force_db_for_default_attachment_rules)
+                rules = const_eval(rec.force_db_for_default_attachment_rules)
             except (SyntaxError, TypeError, ValueError) as e:
                 raise ValidationError(
                     self.env._(
                         "The force_db_for_default_attachment_rules is not a valid "
                         "python dict."
                     )
                 ) from e
+            if not isinstance(rules, dict):
+                raise ValidationError(
+                    self.env._(
+                        "The force_db_for_default_attachment_rules is not a valid "
+                        "python dict."
+                    )
+                )
+            for rule_name in ("mimetype-size", "field-size"):
+                if not isinstance(rules.get(rule_name, {}), dict):
+                    raise ValidationError(
+                        self.env._(
+                            "The '%(rule_name)s' rules of "
+                            "force_db_for_default_attachment_rules must be a dict.",
+                            rule_name=rule_name,
+                        )
+                    )
+            for field_key in rules.get("field-size", {}):
+                if "." not in field_key:
+                    raise ValidationError(
+                        self.env._(
+                            "The 'field-size' key '%(key)s' of "
+                            "force_db_for_default_attachment_rules must be a "
+                            "'<model>.<field>' pair.",
+                            key=field_key,
+                        )
+                    )
 
     @api.model
     @tools.ormcache()

fs_attachment/models/ir_attachment.py

@@ -152,6 +152,27 @@ def _get_storage_force_db_config(self):
             self._storage()
         )
 
+    def _get_storage_force_db_rules(self):
+        """Return the force db config grouped by rule name
+
+        The config maps a rule name to its rules:
+
+        * ``mimetype-size``: ``{mimetype_key: limit}``
+        * ``field-size``: ``{"<model>.<field>": limit}``
+
+        Any other key is a legacy ``mimetype_key: limit`` entry and is merged
+        into the ``mimetype-size`` rules.
+
+        :return: a dict ``{rule_name: rules}`` with all the known rule names
+        """
+        rules = {"mimetype-size": {}, "field-size": {}}
+        for key, value in self._get_storage_force_db_config().items():
+            if key in rules:
+                rules[key].update(value)
+            else:
+                rules["mimetype-size"][key] = value
+        return rules
+
     def _store_in_db_instead_of_object_storage_domain(self):
         """Return a domain for attachments that must be forced to DB
 
@@ -165,13 +186,19 @@ def _store_in_db_instead_of_object_storage_domain(self):
         ``_store_in_db_instead_of_object_storage``.
         """
         domain = []
-        storage_config = self._get_storage_force_db_config()
-        for mimetype_key, limit in storage_config.items():
+        rules = self._get_storage_force_db_rules()
+        for mimetype_key, limit in rules["mimetype-size"].items():
             part = [("mimetype", "=like", f"{mimetype_key}%")]
             if limit:
                 part = Domain.AND([part, [("file_size", "<=", limit)]])
             # OR simplifies to [(1, '=', 1)] if a domain being OR'ed is empty
             domain = Domain.OR([domain, part]) if domain else part
+        for field_key, limit in rules["field-size"].items():
+            res_model, res_field = field_key.rsplit(".", 1)
+            part = [("res_model", "=", res_model), ("res_field", "=", res_field)]
+            if limit:
+                part = Domain.AND([part, [("file_size", "<=", limit)]])
+            domain = Domain.OR([domain, part]) if domain else part
         return domain
 
     def _store_in_db_instead_of_object_storage(self, data, mimetype):
@@ -204,6 +231,21 @@ def _store_in_db_instead_of_object_storage(self, data, mimetype):
         value is the limit in size below which attachments are kept in DB.
         0 means no limit.
 
+        The rules can also be grouped by rule name, which allows other kinds
+        of rules. ``mimetype-size`` holds the mimetype rules described above
+        and ``field-size`` forces the storage in DB of the attachments of a
+        specific binary field, whatever their mimetype, for instance::
+
+            {
+                "mimetype-size": {"image/": 51200},
+                "field-size": {"ir.ui.menu.web_icon_data": 0},
+            }
+
+        Keys that are not a rule name are read as mimetype rules, so both
+        forms can be mixed. A field rule is checked before the mimetype rules:
+        if the attachment matches it and is within its limit, it is stored in
+        DB, otherwise the mimetype rules apply.
+
         These limits are applied only if the storage is the default one for
         attachments (see ``_storage``).
 
@@ -214,8 +256,15 @@ def _store_in_db_instead_of_object_storage(self, data, mimetype):
         """
         if self._is_storage_disabled():
             return True
-        storage_config = self._get_storage_force_db_config()
-        for mimetype_key, limit in storage_config.items():
+        rules = self._get_storage_force_db_rules()
+        res_model = self.env.context.get("attachment_res_model")
+        res_field = self.env.context.get("attachment_res_field")
+        field_key = f"{res_model}.{res_field}"
+        if res_model and res_field and field_key in rules["field-size"]:
+            limit = rules["field-size"][field_key]
+            if not limit or len(data) <= limit:
+                return True
+        for mimetype_key, limit in rules["mimetype-size"].items():
             if mimetype.startswith(mimetype_key):
                 if not limit:
                     return True

fs_attachment/readme/USAGE.md

@@ -70,6 +70,26 @@ configure the way attachments will be stored in the filesystem.
   - application/javascript are stored in database whatever their size
   - text/css are stored in database whatever their size
 
+  The rules can also be grouped by rule name:
+
+  - `mimetype-size`: the mimetype rules described above
+  - `field-size`: the key is a `<model>.<field>` pair, to force the
+    storage in database of the attachments of a specific binary field,
+    whatever their mimetype
+
+  It's useful for fields like the menu icons, that are read on every
+  page load, without having to keep all the images in database:
+
+  > {"mimetype-size": {"image/": 51200, "application/javascript": 0,
+  > "text/css": 0}, "field-size": {"ir.ui.menu.web_icon_data": 0}}
+
+  Keys that are not a rule name are read as mimetype rules, so existing
+  configurations keep working.
+
+  A field rule is checked before the mimetype rules: if the attachment
+  matches it and is within its limit, it is stored in database,
+  otherwise the mimetype rules apply.
+
   This option is only available on the filesystem storage that is used
   as default for attachments.
 

fs_attachment/readme/newsfragments/force_db_field.feature

@@ -0,0 +1,3 @@
+Allow to group the `force_db_for_default_attachment_rules` by rule name and
+add a `field-size` rule to keep the attachments of a specific binary field (e.g.
+`ir.ui.menu.web_icon_data`) in the database whatever their mimetype.

fs_attachment/static/description/index.html

@@ -3,7 +3,7 @@
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="Docutils: https://docutils.sourceforge.io/" />
-<title>Base Attachment Object Store</title>
+<title>README.rst</title>
 <style type="text/css">
 
 /*
@@ -526,6 +526,24 @@ <h3><a class="toc-backref" href="#toc-entry-2">Configuration</a></h3>
 <li>application/javascript are stored in database whatever their size</li>
 <li>text/css are stored in database whatever their size</li>
 </ul>
+<p>The rules can also be grouped by rule name:</p>
+<ul class="simple">
+<li><tt class="docutils literal"><span class="pre">mimetype-size</span></tt>: the mimetype rules described above</li>
+<li><tt class="docutils literal"><span class="pre">field-size</span></tt>: the key is a <tt class="docutils literal"><span class="pre">&lt;model&gt;.&lt;field&gt;</span></tt> pair, to force the
+storage in database of the attachments of a specific binary field,
+whatever their mimetype</li>
+</ul>
+<p>It’s useful for fields like the menu icons, that are read on every
+page load, without having to keep all the images in database:</p>
+<blockquote>
+<p>{“mimetype-size”: {“image/”: 51200, “application/javascript”: 0,
+“text/css”: 0}, “field-size”: {“ir.ui.menu.web_icon_data”: 0}}</p>
+</blockquote>
+<p>Keys that are not a rule name are read as mimetype rules, so existing
+configurations keep working.</p>
+<p>A field rule is checked before the mimetype rules: if the attachment
+matches it and is within its limit, it is stored in database,
+otherwise the mimetype rules apply.</p>
 <p>This option is only available on the filesystem storage that is used
 as default for attachments.</p>
 </li>

fs_attachment/tests/test_fs_attachment.py

@@ -1,9 +1,11 @@
 # Copyright 2023 ACSONE SA/NV (http://acsone.eu).
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
+import base64
 import os
 from pathlib import Path
 from unittest import mock
 
+from odoo.exceptions import ValidationError
 from odoo.fields import Domain
 from odoo.tools import mute_logger
 
@@ -340,6 +342,71 @@ def test_force_attachment_in_db_rules(self):
         self.assertEqual(attachment.db_datas, b"content")
         self.assertEqual(attachment.mimetype, "text/plain")
 
+    def _create_menu_with_icon(self):
+        icon = b'<svg xmlns="http://www.w3.org/2000/svg"><rect/></svg>'
+        menu = self.env["ir.ui.menu"].create(
+            {"name": "Test Menu", "web_icon_data": base64.b64encode(icon)}
+        )
+        self.env.flush_all()
+        attachment = self.ir_attachment_model.search(
+            [
+                ("res_model", "=", "ir.ui.menu"),
+                ("res_field", "=", "web_icon_data"),
+                ("res_id", "=", menu.id),
+            ]
+        )
+        self.assertEqual(attachment.mimetype, "image/svg+xml")
+        return icon, attachment
+
+    def test_force_attachment_in_db_rules_field(self):
+        self.temp_backend.use_as_default_for_attachments = True
+        # images are kept in db below 10 bytes only, but menu icons always
+        self.temp_backend.force_db_for_default_attachment_rules = (
+            '{"mimetype-size": {"image/": 10}, '
+            '"field-size": {"ir.ui.menu.web_icon_data": 0}}'
+        )
+        icon, attachment = self._create_menu_with_icon()
+        self.assertFalse(attachment.store_fname)
+        self.assertEqual(attachment.db_datas, icon)
+        # the same image not linked to the field goes to the storage
+        other = self.ir_attachment_model.create({"name": "icon.svg", "raw": icon})
+        self.env.flush_all()
+        self.assertTrue(other.store_fname.startswith("tmp_dir://"))
+        self.assertFalse(other.db_datas)
+
+    def test_force_attachment_in_db_rules_field_limit(self):
+        self.temp_backend.use_as_default_for_attachments = True
+        # the icon is above the field limit and no mimetype rule matches
+        self.temp_backend.force_db_for_default_attachment_rules = (
+            '{"field-size": {"ir.ui.menu.web_icon_data": 10}}'
+        )
+        _icon, attachment = self._create_menu_with_icon()
+        self.assertTrue(attachment.store_fname.startswith("tmp_dir://"))
+        self.assertFalse(attachment.db_datas)
+
+    def test_force_storage_to_db_field(self):
+        self.temp_backend.use_as_default_for_attachments = True
+        self.temp_backend.force_db_for_default_attachment_rules = "{}"
+        icon, attachment = self._create_menu_with_icon()
+        self.assertTrue(attachment.store_fname.startswith("tmp_dir://"))
+        self.temp_backend.force_db_for_default_attachment_rules = (
+            '{"field-size": {"ir.ui.menu.web_icon_data": 0}}'
+        )
+        attachment.force_storage_to_db_for_special_fields()
+        self.assertFalse(attachment.store_fname)
+        self.assertEqual(attachment.db_datas, icon)
+
+    def test_force_attachment_in_db_rules_invalid(self):
+        self.temp_backend.use_as_default_for_attachments = True
+        for rules in (
+            "[]",
+            '{"field-size": 0}',
+            '{"mimetype-size": ["image/"]}',
+            '{"field-size": {"web_icon_data": 0}}',
+        ):
+            with self.subTest(rules=rules), self.assertRaises(ValidationError):
+                self.temp_backend.force_db_for_default_attachment_rules = rules
+
     def test_force_storage_to_db(self):
         self.temp_backend.use_as_default_for_attachments = True
         attachment = self.ir_attachment_model.create(
@@ -552,7 +619,15 @@ def test_store_in_db_instead_of_object_storage_domain(self):
         self.patch(
             type(IrAttachment),
             "_get_storage_force_db_config",
-            lambda self: {"text/plain": 0, "image/png": 100},
+            lambda self: {
+                # legacy mimetype rule, merged with the mimetype-size ones
+                "text/plain": 0,
+                "mimetype-size": {"image/png": 100},
+                "field-size": {
+                    "ir.ui.menu.web_icon_data": 0,
+                    "res.partner.image_1920": 200,
+                },
+            },
         )
         self.assertEqual(
             self.env["ir.attachment"]._store_in_db_instead_of_object_storage_domain(),
@@ -565,6 +640,19 @@ def test_store_in_db_instead_of_object_storage_domain(self):
                             Domain("file_size", "<=", 100),
                         ]
                     ),
+                    Domain.AND(
+                        [
+                            Domain("res_model", "=", "ir.ui.menu"),
+                            Domain("res_field", "=", "web_icon_data"),
+                        ]
+                    ),
+                    Domain.AND(
+                        [
+                            Domain("res_model", "=", "res.partner"),
+                            Domain("res_field", "=", "image_1920"),
+                            Domain("file_size", "<=", 200),
+                        ]
+                    ),
                 ]
             ),
         )