adding metadata as an object (#3)

* adding metadata as an object

CHANGELOG.rst

@@ -1,7 +1,10 @@
 2.0.5 (unreleased)
 ------------------
 
-- Nothing changed yet.
+- Adding metadata field as an object
+- Adding command audit-update-mappings to update mappings
+- Adding command audit-update-settings to update settings
+  [nilbacardit26]
 
 
 2.0.4 (2024-08-02)

guillotina_audit/__init__.py

@@ -12,7 +12,11 @@
                 "index_permission_changes": False,
             },
         }
-    }
+    },
+    "commands": {
+        "audit-update-mappings": "guillotina_audit.commands.mappings.UpdateMappingsCommand",
+        "audit-update-settings": "guillotina_audit.commands.settings.UpdateSettingsCommand",
+    },
 }
 
 

guillotina_audit/commands/mappings.py

@@ -0,0 +1,24 @@
+from guillotina.commands import Command
+from guillotina.component import get_utility
+from guillotina.interfaces import ICatalogUtility
+
+import asyncio
+import logging
+
+
+logger = logging.getLogger("guillotina_audit")
+
+
+class UpdateMappingsCommand(Command):
+    description = "Update Mappings Command"
+    migrator = None
+    reindexer = None
+
+    def get_parser(self):
+        parser = super(UpdateMappingsCommand, self).get_parser()
+        return parser
+
+    async def run(self, arguments, settings, app):
+        search = get_utility(ICatalogUtility)
+        await asyncio.sleep(1)
+        await search.update_mappings()

guillotina_audit/commands/settings.py

@@ -0,0 +1,24 @@
+from guillotina.commands import Command
+from guillotina.component import get_utility
+from guillotina.interfaces import ICatalogUtility
+
+import asyncio
+import logging
+
+
+logger = logging.getLogger("guillotina_audit")
+
+
+class UpdateSettingsCommand(Command):
+    description = "Update Settings Command"
+    migrator = None
+    reindexer = None
+
+    def get_parser(self):
+        parser = super(UpdateSettingsCommand, self).get_parser()
+        return parser
+
+    async def run(self, arguments, settings, app):
+        search = get_utility(ICatalogUtility)
+        await asyncio.sleep(1)
+        await search.update_settings()

guillotina_audit/models.py

@@ -16,6 +16,7 @@ class AuditDocument(BaseModel):
     creator: Optional[str] = None
     creation_date: datetime = Field(default=datetime.now(timezone.utc))
     type_name: Optional[str] = None
+    metadata: Optional[dict] = None
 
     @field_serializer("payload")
     def serialize_payload(self, payload: dict, _info):

guillotina_audit/tests/fixtures.py

@@ -49,10 +49,11 @@ def elasticsearch_fixture(es):
 
 @pytest.fixture(scope="function")
 async def guillotina_es(elasticsearch_fixture, guillotina):
+    audit_utility = query_utility(IAuditUtility)
+    await audit_utility.create_index()
     response, status = await guillotina(
         "POST", "/db/", data=json.dumps({"@type": "Container", "id": "guillotina"})
     )
     assert status == 200
     yield guillotina
-    audit_utility = query_utility(IAuditUtility)
     await audit_utility.async_es.indices.delete(index="audit")

guillotina_audit/tests/test_audit_basic.py

@@ -14,13 +14,8 @@
 
 
 async def test_audit_basic(guillotina_es):
-    response, status = await guillotina_es(
-        "POST", "/db/guillotina/@addons", data=json.dumps({"id": "audit"})
-    )
-    assert status == 200
-    await asyncio.sleep(2)
-    audit_utility = query_utility(IAuditUtility)
     # Let's check the index has been created
+    audit_utility = query_utility(IAuditUtility)
     resp = await audit_utility.async_es.indices.get_alias()
     assert "audit" in resp
     resp = await audit_utility.async_es.indices.get_mapping(index="audit")
@@ -32,18 +27,23 @@ async def test_audit_basic(guillotina_es):
     )
     assert status == 201
     await asyncio.sleep(2)
-    resp, status = await guillotina_es("GET", "/db/guillotina/@audit")
+    resp, status = await guillotina_es(
+        "GET", "/db/guillotina/@audit?type_name=Container"
+    )
     assert status == 200
-    assert len(resp["hits"]["hits"]) == 2
+    assert len(resp["hits"]["hits"]) == 1
     assert resp["hits"]["hits"][0]["_source"]["action"] == "added"
     assert resp["hits"]["hits"][0]["_source"]["type_name"] == "Container"
     assert resp["hits"]["hits"][0]["_source"]["creator"] == "root"
     assert "title" in resp["hits"]["hits"][0]["_source"]["payload"]
 
-    assert resp["hits"]["hits"][1]["_source"]["action"] == "added"
-    assert resp["hits"]["hits"][1]["_source"]["type_name"] == "Item"
-    assert resp["hits"]["hits"][1]["_source"]["creator"] == "root"
-    assert "title" in resp["hits"]["hits"][1]["_source"]["payload"]
+    resp, status = await guillotina_es("GET", "/db/guillotina/@audit?type_name=Item")
+    assert status == 200
+
+    assert resp["hits"]["hits"][0]["_source"]["action"] == "added"
+    assert resp["hits"]["hits"][0]["_source"]["type_name"] == "Item"
+    assert resp["hits"]["hits"][0]["_source"]["creator"] == "root"
+    assert "title" in resp["hits"]["hits"][0]["_source"]["payload"]
 
     response, status = await guillotina_es("DELETE", "/db/guillotina/foo_item")
     await asyncio.sleep(2)
@@ -146,11 +146,6 @@ async def test_audit_basic(guillotina_es):
 
 
 async def test_audit_wildcard(guillotina_es):
-    response, status = await guillotina_es(
-        "POST", "/db/guillotina/@addons", data=json.dumps({"id": "audit"})
-    )
-    assert status == 200
-    await asyncio.sleep(2)
     audit_utility = query_utility(IAuditUtility)
 
     payload = AuditDocument(action="added", type_name="Fullscreen")
@@ -214,11 +209,6 @@ async def test_audit_wildcard(guillotina_es):
 
 
 async def test_json_dumps(guillotina_es):
-    response, status = await guillotina_es(
-        "POST", "/db/guillotina/@addons", data=json.dumps({"id": "audit"})
-    )
-    assert status == 200
-    await asyncio.sleep(2)
     audit_utility = query_utility(IAuditUtility)
     json.dumps(
         {"datetime": datetime.now(), "date": date.today()},
@@ -227,11 +217,6 @@ async def test_json_dumps(guillotina_es):
 
 
 async def test_permissions_modified_without_indexing(guillotina_es):
-    response, status = await guillotina_es(
-        "POST", "/db/guillotina/@addons", data=json.dumps({"id": "audit"})
-    )
-    assert status == 200
-
     response, status = await guillotina_es(
         "POST",
         "/db/guillotina/",
@@ -291,11 +276,6 @@ async def test_permissions_modified_without_indexing(guillotina_es):
     }
 )
 async def test_permissions_modified_with_indexing(guillotina_es):
-    response, status = await guillotina_es(
-        "POST", "/db/guillotina/@addons", data=json.dumps({"id": "audit"})
-    )
-    assert status == 200
-
     response, status = await guillotina_es(
         "POST",
         "/db/guillotina/",
@@ -329,3 +309,43 @@ async def test_permissions_modified_with_indexing(guillotina_es):
     # There should be one more document since indexing_permission_changes is True
     assert len(resp["hits"]["hits"]) == 3
     assert resp["hits"]["hits"][-1]["_source"]["action"] == "permissions_changed"
+
+
+async def test_metadata_field(guillotina_es):
+    audit_utility = query_utility(IAuditUtility)
+    payload = AuditDocument(
+        action="CreatingMetadata",
+        metadata={
+            "foo_number": 120,
+            "foo_string": "foo_string",
+            "foo_boolean": True,
+            "foo_list": [1, 2, 3, 4],
+            "foo_dict": {
+                "foo_number": 100,
+                "foo_string": "foo_string",
+                "foo_dict": {"foo_key": "foo_value"},
+            },
+            "foo_decimal": 1.234,
+        },
+    )
+    audit_utility.log_wildcard(payload)
+    # Let's check the index has been created
+    resp = await audit_utility.async_es.indices.get_alias()
+    await asyncio.sleep(2)
+    resp, status = await guillotina_es(
+        "GET",
+        "/db/guillotina/@audit?action=CreatingMetadata",
+    )
+    assert resp["hits"]["hits"][0]["_source"]["action"] == "CreatingMetadata"
+    assert resp["hits"]["hits"][0]["_source"]["metadata"] == {
+        "foo_boolean": True,
+        "foo_dict": {
+            "foo_dict": {"foo_key": "foo_value"},
+            "foo_number": 100,
+            "foo_string": "foo_string",
+        },
+        "foo_list": [1, 2, 3, 4],
+        "foo_number": 120,
+        "foo_string": "foo_string",
+        "foo_decimal": 1.234,
+    }

guillotina_audit/tests/test_mappings.py

@@ -0,0 +1,13 @@
+from guillotina.component import query_utility
+from guillotina_audit.interfaces import IAuditUtility
+
+import pytest
+
+
+pytestmark = pytest.mark.asyncio
+
+
+async def test_mappings(guillotina_es):
+    audit_utility = query_utility(IAuditUtility)
+    await audit_utility.update_settings()
+    await audit_utility.update_mappings()

guillotina_audit/utility.py

@@ -41,6 +41,24 @@ def _custom_serializer(self, obj):
             return obj.strftime("%Y-%m-%d")
         raise TypeError("Object of type %s is not JSON serializable" % type(obj))
 
+    async def update_settings(self):
+        await self.async_es.indices.close(index=self.index)
+        try:
+            await self.async_es.indices.put_settings(
+                body=self.default_settings(), index=self.index
+            )
+            logger.info(f"Updating mappings {self.default_settings()}")
+        except Exception:
+            logger.error("Error updating settings", exc_info=True)
+        finally:
+            await self.async_es.indices.open(index=self.index)
+
+    async def update_mappings(self):
+        await self.async_es.indices.put_mapping(
+            body=self.default_mappings(), index=self.index
+        )
+        logger.info(f"Updating mappings {self.default_mappings()}")
+
     async def create_index(self):
         try:
             await self.async_es.indices.create(
@@ -53,15 +71,44 @@ async def create_index(self):
         except BadRequestError:
             logger.error("An exception occurred when creating index", exc_info=True)
 
+    def settings(self):
+        return {
+            "settings": {
+                "analysis": {
+                    "analyzer": {
+                        "my_analyzer": {
+                            "type": "custom",
+                            "tokenizer": "standard",
+                            "filter": ["lowercase"],
+                        },
+                        "my_stop_analyzer": {
+                            "type": "custom",
+                            "tokenizer": "standard",
+                            "filter": ["lowercase", "english_stop"],
+                        },
+                    },
+                    "filter": {
+                        "english_stop": {"type": "stop", "stopwords": "_english_"}
+                    },
+                }
+            }
+        }
+
     def default_settings(self):
         return {
             "analysis": {
-                "analyzer": {"path_analyzer": {"tokenizer": "path_tokenizer"}},
+                "analyzer": {
+                    "path_analyzer": {  # Custom analyzer definition
+                        "type": "custom",
+                        "tokenizer": "path_tokenizer",
+                    }
+                },
                 "tokenizer": {
-                    "path_tokenizer": {"type": "path_hierarchy", "delimiter": "/"}
+                    "path_tokenizer": {  # Custom tokenizer definition
+                        "type": "path_hierarchy",
+                        "delimiter": "/",
+                    }
                 },
-                "filter": {},
-                "char_filter": {},
             }
         }
 
@@ -76,6 +123,7 @@ def default_mappings(self):
                 "creator": {"type": "keyword"},
                 "creation_date": {"type": "date", "store": True},
                 "payload": {"type": "text", "store": True},
+                "metadata": {"type": "object", "dynamic": True},
             },
         }