adding contrib audit package

.github/workflows/continuous-integration.yml

@@ -36,6 +36,7 @@ jobs:
       matrix:
         python-version: [3.7, 3.8, 3.9, '3.10']
         database: ["DUMMY", "postgres", "cockroachdb"]
+        elasticsearch: ["ES"]
         db_schema: ["custom", "public"]
         exclude:
           - database: "DUMMY"
@@ -47,6 +48,7 @@ jobs:
     env:
       DATABASE: ${{ matrix.database }}
       DB_SCHEMA: ${{ matrix.db_schema }}
+      ES: ${{ matrix.elasticsearch }}
       MEMCACHED: "localhost:11211"
 
     steps:

.gitignore

@@ -74,3 +74,52 @@ pip-wheel-metadata
 !/src/*
 dummy_file.db
 dummy_file.db.blobs
+
+# -*- mode: gitignore; -*-
+*~
+\#*\#
+/.emacs.desktop
+/.emacs.desktop.lock
+*.elc
+auto-save-list
+tramp
+.\#*
+
+# Org-mode
+.org-id-locations
+*_archive
+
+# flymake-mode
+*_flymake.*
+
+# eshell files
+/eshell/history
+/eshell/lastdir
+
+# elpa packages
+/elpa/
+
+# reftex files
+*.rel
+
+# AUCTeX auto folder
+/auto/
+
+# cask packages
+.cask/
+dist/
+
+# Flycheck
+flycheck_*.el
+
+# server auth directory
+/server/
+
+# projectiles files
+.projectile
+
+# directory configuration
+.dir-locals.el
+
+# network security
+/network-security.data

CHANGELOG.rst

@@ -1,10 +1,13 @@
 CHANGELOG
 =========
 
-6.4.4 (unreleased)
+
+6.4.4 (2023-11-14)
 ------------------
 
-- Nothing changed yet.
+- Adding audit contrib package. Indexing documents using ES. Exposing an API to search
+  among objects hat have been created, modified and deleted
+  [nilbacardit26]
 
 
 6.4.3 (2023-10-11)

guillotina/contrib/audit/__init__.py

@@ -0,0 +1,18 @@
+from guillotina import configure
+
+app_settings = {
+    "load_utilities": {
+        "audit": {
+            "provides": "guillotina.contrib.audit.interfaces.IAuditUtility",
+            "factory": "guillotina.contrib.audit.utility.AuditUtility",
+            "settings": {"index_name": "audit"},
+        }
+    }
+}
+
+
+def includeme(root, settings):
+    configure.scan("guillotina.contrib.audit.install")
+    configure.scan("guillotina.contrib.audit.utility")
+    configure.scan("guillotina.contrib.audit.subscriber")
+    configure.scan("guillotina.contrib.audit.api")

guillotina/contrib/audit/api.py

@@ -0,0 +1,19 @@
+from guillotina import configure
+from guillotina.api.service import Service
+from guillotina.component import query_utility
+from guillotina.interfaces import IContainer
+from guillotina.contrib.audit.interfaces import IAuditUtility
+
+
+@configure.service(
+    context=IContainer,
+    method="GET",
+    permission="guillotina.AccessAudit",
+    name="@audit",
+    summary="Get the audit entry logs",
+    responses={"200": {"description": "Get the audit entry logs", "schema": {"properties": {}}}},
+)
+class AuditGET(Service):
+    async def __call__(self):
+        audit_utility = query_utility(IAuditUtility)
+        return await audit_utility.query_audit(self.request.query)

guillotina/contrib/audit/install.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+from guillotina import configure
+from guillotina.addons import Addon
+from guillotina.contrib.audit.interfaces import IAuditUtility
+from guillotina.component import query_utility
+
+
+@configure.addon(name="audit", title="Guillotina Audit using ES")
+class ImageAddon(Addon):
+    @classmethod
+    async def install(cls, container, request):
+        audit_utility = query_utility(IAuditUtility)
+        await audit_utility.create_index()
+
+    @classmethod
+    async def uninstall(cls, container, request):
+        pass

guillotina/contrib/audit/interfaces.py

@@ -0,0 +1,5 @@
+from guillotina.async_util import IAsyncUtility
+
+
+class IAuditUtility(IAsyncUtility):
+    pass

guillotina/contrib/audit/subscriber.py

@@ -0,0 +1,23 @@
+from guillotina import configure
+from guillotina.interfaces import IObjectAddedEvent, IObjectModifiedEvent, IObjectRemovedEvent
+from guillotina.interfaces import IResource
+from guillotina.component import query_utility
+from guillotina.contrib.audit.interfaces import IAuditUtility
+
+
+@configure.subscriber(for_=(IResource, IObjectAddedEvent), priority=1001)  # after indexing
+async def audit_object_added(obj, event):
+    audit = query_utility(IAuditUtility)
+    audit.log_entry(obj, event)
+
+
+@configure.subscriber(for_=(IResource, IObjectModifiedEvent), priority=1001)  # after indexing
+async def audit_object_modified(obj, event):
+    audit = query_utility(IAuditUtility)
+    audit.log_entry(obj, event)
+
+
+@configure.subscriber(for_=(IResource, IObjectRemovedEvent), priority=1001)  # after indexing
+async def audit_object_removed(obj, event):
+    audit = query_utility(IAuditUtility)
+    audit.log_entry(obj, event)

guillotina/contrib/audit/utility.py

@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+import asyncio
+import datetime
+import json
+from datetime import timezone
+from elasticsearch import AsyncElasticsearch
+from elasticsearch.exceptions import RequestError
+from guillotina import app_settings
+from guillotina.interfaces import IObjectModifiedEvent, IObjectAddedEvent, IObjectRemovedEvent
+from guillotina.utils.content import get_content_path
+from guillotina.utils.auth import get_authenticated_user
+
+import logging
+
+
+logger = logging.getLogger("guillotina_audit")
+
+
+class AuditUtility:
+    def __init__(self, settings=None, loop=None):
+        self._settings = settings
+        self.index = self._settings.get("index_name", "audit")
+        self.loop = loop
+
+    async def initialize(self, app):
+        self.async_es = AsyncElasticsearch(
+            loop=self.loop, **app_settings.get("elasticsearch", {}).get("connection_settings")
+        )
+
+    async def create_index(self):
+        settings = {"settings": self.default_settings(), "mappings": self.default_mappings()}
+        try:
+            await self.async_es.indices.create(self.index, settings)
+        except RequestError:
+            logger.error("An exception occurred when creating index", exc_info=True)
+
+    def default_settings(self):
+        return {
+            "analysis": {
+                "analyzer": {"path_analyzer": {"tokenizer": "path_tokenizer"}},
+                "tokenizer": {"path_tokenizer": {"type": "path_hierarchy", "delimiter": "/"}},
+                "filter": {},
+                "char_filter": {},
+            }
+        }
+
+    def default_mappings(self):
+        return {
+            "dynamic": False,
+            "properties": {
+                "path": {"type": "text", "store": True, "analyzer": "path_analyzer"},
+                "type_name": {"type": "keyword", "store": True},
+                "uuid": {"type": "keyword", "store": True},
+                "action": {"type": "keyword", "store": True},
+                "creator": {"type": "keyword"},
+                "creation_date": {"type": "date", "store": True},
+                "payload": {"type": "text", "store": True},
+            },
+        }
+
+    def log_entry(self, obj, event):
+        document = {}
+        user = get_authenticated_user()
+        if IObjectModifiedEvent.providedBy(event):
+            document["action"] = "modified"
+            document["creation_date"] = obj.modification_date
+            document["payload"] = json.dumps(event.payload)
+        elif IObjectAddedEvent.providedBy(event):
+            document["action"] = "added"
+            document["creation_date"] = obj.creation_date
+        elif IObjectRemovedEvent.providedBy(event):
+            document["action"] = "removed"
+            document["creation_date"] = datetime.datetime.now(timezone.utc)
+        document["path"] = get_content_path(obj)
+        document["creator"] = user.id
+        document["type_name"] = obj.type_name
+        document["uuid"] = obj.uuid
+        coroutine = self.async_es.index(index=self.index, body=document)
+        asyncio.create_task(coroutine)
+
+    async def query_audit(self, params={}):
+        if params == {}:
+            query = {"query": {"match_all": {}}}
+        else:
+            query = {"query": {"bool": {"must": []}}}
+            for field, value in params.items():
+                if (
+                    field.endswith("__gte")
+                    or field.endswith("__lte")
+                    or field.endswith("__gt")
+                    or field.endswith("__lt")
+                ):
+                    field_parsed = field.split("__")[0]
+                    operator = field.split("__")[1]
+                    query["query"]["bool"]["must"].append({"range": {field_parsed: {operator: value}}})
+                else:
+                    query["query"]["bool"]["must"].append({"match": {field: value}})
+        return await self.async_es.search(index=self.index, body=query)
+
+    async def close(self):
+        if self.loop is not None:
+            asyncio.run_coroutine_threadsafe(self.async_es.close(), self.loop)
+        else:
+            await self.async_es.close()
+
+    async def finalize(self, app):
+        await self.close()

guillotina/permissions.py

@@ -18,6 +18,7 @@
 configure.permission("guillotina.UmountDatabase", "Umount a Database")
 
 configure.permission("guillotina.AccessPreflight", "Access Preflight View")
+configure.permission("guillotina.AccessAudit", "Access Audit entries")
 
 configure.permission("guillotina.ReadConfiguration", "Read a configuration")
 configure.permission("guillotina.WriteConfiguration", "Write a configuration")
@@ -107,6 +108,7 @@
 configure.grant(permission="guillotina.MoveContent", role="guillotina.Editor")
 configure.grant(permission="guillotina.DuplicateContent", role="guillotina.Editor")
 configure.grant(permission="guillotina.ReindexContent", role="guillotina.Editor")
+configure.grant(permission="guillotina.AccessAudit", role="guillotina.Editor")
 
 # ContainerAdmin
 configure.grant(permission="guillotina.AccessContent", role="guillotina.ContainerAdmin")
@@ -118,6 +120,7 @@
 configure.grant(permission="guillotina.RawSearchContent", role="guillotina.ContainerAdmin")
 configure.grant(permission="guillotina.CacheManage", role="guillotina.Manager")
 configure.grant(permission="guillotina.Manage", role="guillotina.Manager")
+configure.grant(permission="guillotina.AccessAudit", role="guillotina.Manager")
 
 # ContainerDeleter
 configure.grant(permission="guillotina.DeleteContainers", role="guillotina.ContainerDeleter")

guillotina/tests/audit/test_audit_basic.py

@@ -0,0 +1,77 @@
+import pytest
+import json
+import asyncio
+import os
+
+from datetime import datetime, timedelta
+from guillotina.contrib.audit.interfaces import IAuditUtility
+from guillotina.component import query_utility
+
+pytestmark = pytest.mark.asyncio
+
+ELASTICSEARCH = os.environ.get("ES", False)
+
+
+@pytest.mark.app_settings({"applications": ["guillotina", "guillotina.contrib.audit"]})
+@pytest.mark.skipif(ELASTICSEARCH is False, reason="Only works with ES")
+async def test_audit_basic(es_requester):
+    async with es_requester as requester:
+        response, status = await requester("POST", "/db/guillotina/@addons", data=json.dumps({"id": "audit"}))
+        assert status == 200
+        audit_utility = query_utility(IAuditUtility)
+        # Let's check the index has been created
+        resp = await audit_utility.async_es.indices.get_alias()
+        assert "audit" in resp
+        resp = await audit_utility.async_es.indices.get_mapping(index="audit")
+        assert "path" in resp["audit"]["mappings"]["properties"]
+        response, status = await requester(
+            "POST", "/db/guillotina/", data=json.dumps({"@type": "Item", "id": "foo_item"})
+        )
+        assert status == 201
+        await asyncio.sleep(2)
+        resp, status = await requester("GET", "/db/guillotina/@audit")
+        assert status == 200
+        assert len(resp["hits"]["hits"]) == 2
+        assert resp["hits"]["hits"][0]["_source"]["action"] == "added"
+        assert resp["hits"]["hits"][0]["_source"]["type_name"] == "Container"
+        assert resp["hits"]["hits"][0]["_source"]["creator"] == "root"
+
+        assert resp["hits"]["hits"][1]["_source"]["action"] == "added"
+        assert resp["hits"]["hits"][1]["_source"]["type_name"] == "Item"
+        assert resp["hits"]["hits"][1]["_source"]["creator"] == "root"
+
+        response, status = await requester("DELETE", "/db/guillotina/foo_item")
+        await asyncio.sleep(2)
+        resp, status = await requester("GET", "/db/guillotina/@audit")
+        assert status == 200
+        assert len(resp["hits"]["hits"]) == 3
+        resp, status = await requester("GET", "/db/guillotina/@audit?action=removed")
+        assert status == 200
+        assert len(resp["hits"]["hits"]) == 1
+        resp, status = await requester("GET", "/db/guillotina/@audit?action=removed&type_name=Item")
+        assert status == 200
+        assert len(resp["hits"]["hits"]) == 1
+        resp, status = await requester("GET", "/db/guillotina/@audit?action=added&type_name=Item")
+        assert status == 200
+        assert len(resp["hits"]["hits"]) == 1
+        assert resp["hits"]["hits"][0]["_source"]["type_name"] == "Item"
+        resp, status = await requester("GET", "/db/guillotina/@audit?action=added&type_name=Container")
+        assert status == 200
+        assert len(resp["hits"]["hits"]) == 1
+        assert resp["hits"]["hits"][0]["_source"]["type_name"] == "Container"
+        creation_date = resp["hits"]["hits"][0]["_source"]["creation_date"]
+        datetime_obj = datetime.strptime(creation_date, "%Y-%m-%dT%H:%M:%S.%f%z")
+        new_creation_date = datetime_obj - timedelta(seconds=1)
+        new_creation_date = new_creation_date.strftime("%Y-%m-%dT%H:%M:%S.%f%z")
+        resp, status = await requester(
+            "GET",
+            f"/db/guillotina/@audit?action=added&type_name=Container&creation_date__gte={new_creation_date}",
+        )  # noqa
+        assert status == 200
+        assert len(resp["hits"]["hits"]) == 1
+        resp, status = await requester(
+            "GET",
+            f"/db/guillotina/@audit?action=added&type_name=Container&creation_date__lte={new_creation_date}",
+        )  # noqa
+        assert len(resp["hits"]["hits"]) == 0
+        assert status == 200