diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e64482d..bac4393 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,10 @@ 2.0.5 (unreleased) ------------------ -- Nothing changed yet. +- Adding metadata field as an object +- Adding command audit-update-mappings to update mappings +- Adding command audit-update-settings to update settings + [nilbacardit26] 2.0.4 (2024-08-02) diff --git a/guillotina_audit/__init__.py b/guillotina_audit/__init__.py index 2dbf4f6..9f2a7be 100644 --- a/guillotina_audit/__init__.py +++ b/guillotina_audit/__init__.py @@ -12,7 +12,11 @@ "index_permission_changes": False, }, } - } + }, + "commands": { + "audit-update-mappings": "guillotina_audit.commands.mappings.UpdateMappingsCommand", + "audit-update-settings": "guillotina_audit.commands.settings.UpdateSettingsCommand", + }, } diff --git a/guillotina_audit/commands/__init__.py b/guillotina_audit/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/guillotina_audit/commands/mappings.py b/guillotina_audit/commands/mappings.py new file mode 100644 index 0000000..f991246 --- /dev/null +++ b/guillotina_audit/commands/mappings.py @@ -0,0 +1,24 @@ +from guillotina.commands import Command +from guillotina.component import get_utility +from guillotina.interfaces import ICatalogUtility + +import asyncio +import logging + + +logger = logging.getLogger("guillotina_audit") + + +class UpdateMappingsCommand(Command): + description = "Update Mappings Command" + migrator = None + reindexer = None + + def get_parser(self): + parser = super(UpdateMappingsCommand, self).get_parser() + return parser + + async def run(self, arguments, settings, app): + search = get_utility(ICatalogUtility) + await asyncio.sleep(1) + await search.update_mappings() diff --git a/guillotina_audit/commands/settings.py b/guillotina_audit/commands/settings.py new file mode 100644 index 0000000..220ad21 --- /dev/null +++ b/guillotina_audit/commands/settings.py @@ -0,0 +1,24 @@ +from guillotina.commands import Command +from guillotina.component import get_utility +from guillotina.interfaces import ICatalogUtility + +import asyncio +import logging + + +logger = logging.getLogger("guillotina_audit") + + +class UpdateSettingsCommand(Command): + description = "Update Settings Command" + migrator = None + reindexer = None + + def get_parser(self): + parser = super(UpdateSettingsCommand, self).get_parser() + return parser + + async def run(self, arguments, settings, app): + search = get_utility(ICatalogUtility) + await asyncio.sleep(1) + await search.update_settings() diff --git a/guillotina_audit/models.py b/guillotina_audit/models.py index 7dc2417..8ad8962 100644 --- a/guillotina_audit/models.py +++ b/guillotina_audit/models.py @@ -16,6 +16,7 @@ class AuditDocument(BaseModel): creator: Optional[str] = None creation_date: datetime = Field(default=datetime.now(timezone.utc)) type_name: Optional[str] = None + metadata: Optional[dict] = None @field_serializer("payload") def serialize_payload(self, payload: dict, _info): diff --git a/guillotina_audit/tests/fixtures.py b/guillotina_audit/tests/fixtures.py index e186b7d..5ef0feb 100644 --- a/guillotina_audit/tests/fixtures.py +++ b/guillotina_audit/tests/fixtures.py @@ -49,10 +49,11 @@ def elasticsearch_fixture(es): @pytest.fixture(scope="function") async def guillotina_es(elasticsearch_fixture, guillotina): + audit_utility = query_utility(IAuditUtility) + await audit_utility.create_index() response, status = await guillotina( "POST", "/db/", data=json.dumps({"@type": "Container", "id": "guillotina"}) ) assert status == 200 yield guillotina - audit_utility = query_utility(IAuditUtility) await audit_utility.async_es.indices.delete(index="audit") diff --git a/guillotina_audit/tests/test_audit_basic.py b/guillotina_audit/tests/test_audit_basic.py index 71e6807..44f2565 100644 --- a/guillotina_audit/tests/test_audit_basic.py +++ b/guillotina_audit/tests/test_audit_basic.py @@ -14,13 +14,8 @@ async def test_audit_basic(guillotina_es): - response, status = await guillotina_es( - "POST", "/db/guillotina/@addons", data=json.dumps({"id": "audit"}) - ) - assert status == 200 - await asyncio.sleep(2) - audit_utility = query_utility(IAuditUtility) # Let's check the index has been created + audit_utility = query_utility(IAuditUtility) resp = await audit_utility.async_es.indices.get_alias() assert "audit" in resp resp = await audit_utility.async_es.indices.get_mapping(index="audit") @@ -32,18 +27,23 @@ async def test_audit_basic(guillotina_es): ) assert status == 201 await asyncio.sleep(2) - resp, status = await guillotina_es("GET", "/db/guillotina/@audit") + resp, status = await guillotina_es( + "GET", "/db/guillotina/@audit?type_name=Container" + ) assert status == 200 - assert len(resp["hits"]["hits"]) == 2 + assert len(resp["hits"]["hits"]) == 1 assert resp["hits"]["hits"][0]["_source"]["action"] == "added" assert resp["hits"]["hits"][0]["_source"]["type_name"] == "Container" assert resp["hits"]["hits"][0]["_source"]["creator"] == "root" assert "title" in resp["hits"]["hits"][0]["_source"]["payload"] - assert resp["hits"]["hits"][1]["_source"]["action"] == "added" - assert resp["hits"]["hits"][1]["_source"]["type_name"] == "Item" - assert resp["hits"]["hits"][1]["_source"]["creator"] == "root" - assert "title" in resp["hits"]["hits"][1]["_source"]["payload"] + resp, status = await guillotina_es("GET", "/db/guillotina/@audit?type_name=Item") + assert status == 200 + + assert resp["hits"]["hits"][0]["_source"]["action"] == "added" + assert resp["hits"]["hits"][0]["_source"]["type_name"] == "Item" + assert resp["hits"]["hits"][0]["_source"]["creator"] == "root" + assert "title" in resp["hits"]["hits"][0]["_source"]["payload"] response, status = await guillotina_es("DELETE", "/db/guillotina/foo_item") await asyncio.sleep(2) @@ -146,11 +146,6 @@ async def test_audit_basic(guillotina_es): async def test_audit_wildcard(guillotina_es): - response, status = await guillotina_es( - "POST", "/db/guillotina/@addons", data=json.dumps({"id": "audit"}) - ) - assert status == 200 - await asyncio.sleep(2) audit_utility = query_utility(IAuditUtility) payload = AuditDocument(action="added", type_name="Fullscreen") @@ -214,11 +209,6 @@ async def test_audit_wildcard(guillotina_es): async def test_json_dumps(guillotina_es): - response, status = await guillotina_es( - "POST", "/db/guillotina/@addons", data=json.dumps({"id": "audit"}) - ) - assert status == 200 - await asyncio.sleep(2) audit_utility = query_utility(IAuditUtility) json.dumps( {"datetime": datetime.now(), "date": date.today()}, @@ -227,11 +217,6 @@ async def test_json_dumps(guillotina_es): async def test_permissions_modified_without_indexing(guillotina_es): - response, status = await guillotina_es( - "POST", "/db/guillotina/@addons", data=json.dumps({"id": "audit"}) - ) - assert status == 200 - response, status = await guillotina_es( "POST", "/db/guillotina/", @@ -291,11 +276,6 @@ async def test_permissions_modified_without_indexing(guillotina_es): } ) async def test_permissions_modified_with_indexing(guillotina_es): - response, status = await guillotina_es( - "POST", "/db/guillotina/@addons", data=json.dumps({"id": "audit"}) - ) - assert status == 200 - response, status = await guillotina_es( "POST", "/db/guillotina/", @@ -329,3 +309,43 @@ async def test_permissions_modified_with_indexing(guillotina_es): # There should be one more document since indexing_permission_changes is True assert len(resp["hits"]["hits"]) == 3 assert resp["hits"]["hits"][-1]["_source"]["action"] == "permissions_changed" + + +async def test_metadata_field(guillotina_es): + audit_utility = query_utility(IAuditUtility) + payload = AuditDocument( + action="CreatingMetadata", + metadata={ + "foo_number": 120, + "foo_string": "foo_string", + "foo_boolean": True, + "foo_list": [1, 2, 3, 4], + "foo_dict": { + "foo_number": 100, + "foo_string": "foo_string", + "foo_dict": {"foo_key": "foo_value"}, + }, + "foo_decimal": 1.234, + }, + ) + audit_utility.log_wildcard(payload) + # Let's check the index has been created + resp = await audit_utility.async_es.indices.get_alias() + await asyncio.sleep(2) + resp, status = await guillotina_es( + "GET", + "/db/guillotina/@audit?action=CreatingMetadata", + ) + assert resp["hits"]["hits"][0]["_source"]["action"] == "CreatingMetadata" + assert resp["hits"]["hits"][0]["_source"]["metadata"] == { + "foo_boolean": True, + "foo_dict": { + "foo_dict": {"foo_key": "foo_value"}, + "foo_number": 100, + "foo_string": "foo_string", + }, + "foo_list": [1, 2, 3, 4], + "foo_number": 120, + "foo_string": "foo_string", + "foo_decimal": 1.234, + } diff --git a/guillotina_audit/tests/test_mappings.py b/guillotina_audit/tests/test_mappings.py new file mode 100644 index 0000000..24665b1 --- /dev/null +++ b/guillotina_audit/tests/test_mappings.py @@ -0,0 +1,13 @@ +from guillotina.component import query_utility +from guillotina_audit.interfaces import IAuditUtility + +import pytest + + +pytestmark = pytest.mark.asyncio + + +async def test_mappings(guillotina_es): + audit_utility = query_utility(IAuditUtility) + await audit_utility.update_settings() + await audit_utility.update_mappings() diff --git a/guillotina_audit/utility.py b/guillotina_audit/utility.py index e900c19..75d9a21 100644 --- a/guillotina_audit/utility.py +++ b/guillotina_audit/utility.py @@ -41,6 +41,24 @@ def _custom_serializer(self, obj): return obj.strftime("%Y-%m-%d") raise TypeError("Object of type %s is not JSON serializable" % type(obj)) + async def update_settings(self): + await self.async_es.indices.close(index=self.index) + try: + await self.async_es.indices.put_settings( + body=self.default_settings(), index=self.index + ) + logger.info(f"Updating mappings {self.default_settings()}") + except Exception: + logger.error("Error updating settings", exc_info=True) + finally: + await self.async_es.indices.open(index=self.index) + + async def update_mappings(self): + await self.async_es.indices.put_mapping( + body=self.default_mappings(), index=self.index + ) + logger.info(f"Updating mappings {self.default_mappings()}") + async def create_index(self): try: await self.async_es.indices.create( @@ -53,15 +71,44 @@ async def create_index(self): except BadRequestError: logger.error("An exception occurred when creating index", exc_info=True) + def settings(self): + return { + "settings": { + "analysis": { + "analyzer": { + "my_analyzer": { + "type": "custom", + "tokenizer": "standard", + "filter": ["lowercase"], + }, + "my_stop_analyzer": { + "type": "custom", + "tokenizer": "standard", + "filter": ["lowercase", "english_stop"], + }, + }, + "filter": { + "english_stop": {"type": "stop", "stopwords": "_english_"} + }, + } + } + } + def default_settings(self): return { "analysis": { - "analyzer": {"path_analyzer": {"tokenizer": "path_tokenizer"}}, + "analyzer": { + "path_analyzer": { # Custom analyzer definition + "type": "custom", + "tokenizer": "path_tokenizer", + } + }, "tokenizer": { - "path_tokenizer": {"type": "path_hierarchy", "delimiter": "/"} + "path_tokenizer": { # Custom tokenizer definition + "type": "path_hierarchy", + "delimiter": "/", + } }, - "filter": {}, - "char_filter": {}, } } @@ -76,6 +123,7 @@ def default_mappings(self): "creator": {"type": "keyword"}, "creation_date": {"type": "date", "store": True}, "payload": {"type": "text", "store": True}, + "metadata": {"type": "object", "dynamic": True}, }, }