diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index ea391bbc9..551b3d22f 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -36,6 +36,7 @@ jobs: matrix: python-version: [3.7, 3.8, 3.9, '3.10'] database: ["DUMMY", "postgres", "cockroachdb"] + elasticsearch: ["ES"] db_schema: ["custom", "public"] exclude: - database: "DUMMY" @@ -47,6 +48,7 @@ jobs: env: DATABASE: ${{ matrix.database }} DB_SCHEMA: ${{ matrix.db_schema }} + ES: ${{ matrix.elasticsearch }} MEMCACHED: "localhost:11211" steps: diff --git a/.gitignore b/.gitignore index ffb938552..76c89512d 100644 --- a/.gitignore +++ b/.gitignore @@ -74,3 +74,52 @@ pip-wheel-metadata !/src/* dummy_file.db dummy_file.db.blobs + +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +# network security +/network-security.data \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8da4542f8..1d1215448 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,10 +1,13 @@ CHANGELOG ========= -6.4.4 (unreleased) + +6.4.4 (2023-11-14) ------------------ -- Nothing changed yet. +- Adding audit contrib package. Indexing documents using ES. Exposing an API to search + among objects hat have been created, modified and deleted + [nilbacardit26] 6.4.3 (2023-10-11) diff --git a/guillotina/contrib/audit/CHANGELOG.rst b/guillotina/contrib/audit/CHANGELOG.rst new file mode 100644 index 000000000..e69de29bb diff --git a/guillotina/contrib/audit/__init__.py b/guillotina/contrib/audit/__init__.py new file mode 100644 index 000000000..7265a4a46 --- /dev/null +++ b/guillotina/contrib/audit/__init__.py @@ -0,0 +1,18 @@ +from guillotina import configure + +app_settings = { + "load_utilities": { + "audit": { + "provides": "guillotina.contrib.audit.interfaces.IAuditUtility", + "factory": "guillotina.contrib.audit.utility.AuditUtility", + "settings": {"index_name": "audit"}, + } + } +} + + +def includeme(root, settings): + configure.scan("guillotina.contrib.audit.install") + configure.scan("guillotina.contrib.audit.utility") + configure.scan("guillotina.contrib.audit.subscriber") + configure.scan("guillotina.contrib.audit.api") diff --git a/guillotina/contrib/audit/api.py b/guillotina/contrib/audit/api.py new file mode 100644 index 000000000..7af1e935d --- /dev/null +++ b/guillotina/contrib/audit/api.py @@ -0,0 +1,19 @@ +from guillotina import configure +from guillotina.api.service import Service +from guillotina.component import query_utility +from guillotina.interfaces import IContainer +from guillotina.contrib.audit.interfaces import IAuditUtility + + +@configure.service( + context=IContainer, + method="GET", + permission="guillotina.AccessAudit", + name="@audit", + summary="Get the audit entry logs", + responses={"200": {"description": "Get the audit entry logs", "schema": {"properties": {}}}}, +) +class AuditGET(Service): + async def __call__(self): + audit_utility = query_utility(IAuditUtility) + return await audit_utility.query_audit(self.request.query) diff --git a/guillotina/contrib/audit/install.py b/guillotina/contrib/audit/install.py new file mode 100644 index 000000000..55b6a502b --- /dev/null +++ b/guillotina/contrib/audit/install.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +from guillotina import configure +from guillotina.addons import Addon +from guillotina.contrib.audit.interfaces import IAuditUtility +from guillotina.component import query_utility + + +@configure.addon(name="audit", title="Guillotina Audit using ES") +class ImageAddon(Addon): + @classmethod + async def install(cls, container, request): + audit_utility = query_utility(IAuditUtility) + await audit_utility.create_index() + + @classmethod + async def uninstall(cls, container, request): + pass diff --git a/guillotina/contrib/audit/interfaces.py b/guillotina/contrib/audit/interfaces.py new file mode 100644 index 000000000..2a8bebf79 --- /dev/null +++ b/guillotina/contrib/audit/interfaces.py @@ -0,0 +1,5 @@ +from guillotina.async_util import IAsyncUtility + + +class IAuditUtility(IAsyncUtility): + pass diff --git a/guillotina/contrib/audit/subscriber.py b/guillotina/contrib/audit/subscriber.py new file mode 100644 index 000000000..948af6525 --- /dev/null +++ b/guillotina/contrib/audit/subscriber.py @@ -0,0 +1,23 @@ +from guillotina import configure +from guillotina.interfaces import IObjectAddedEvent, IObjectModifiedEvent, IObjectRemovedEvent +from guillotina.interfaces import IResource +from guillotina.component import query_utility +from guillotina.contrib.audit.interfaces import IAuditUtility + + +@configure.subscriber(for_=(IResource, IObjectAddedEvent), priority=1001) # after indexing +async def audit_object_added(obj, event): + audit = query_utility(IAuditUtility) + audit.log_entry(obj, event) + + +@configure.subscriber(for_=(IResource, IObjectModifiedEvent), priority=1001) # after indexing +async def audit_object_modified(obj, event): + audit = query_utility(IAuditUtility) + audit.log_entry(obj, event) + + +@configure.subscriber(for_=(IResource, IObjectRemovedEvent), priority=1001) # after indexing +async def audit_object_removed(obj, event): + audit = query_utility(IAuditUtility) + audit.log_entry(obj, event) diff --git a/guillotina/contrib/audit/utility.py b/guillotina/contrib/audit/utility.py new file mode 100644 index 000000000..45a8243eb --- /dev/null +++ b/guillotina/contrib/audit/utility.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +import asyncio +import datetime +import json +from datetime import timezone +from elasticsearch import AsyncElasticsearch +from elasticsearch.exceptions import RequestError +from guillotina import app_settings +from guillotina.interfaces import IObjectModifiedEvent, IObjectAddedEvent, IObjectRemovedEvent +from guillotina.utils.content import get_content_path +from guillotina.utils.auth import get_authenticated_user + +import logging + + +logger = logging.getLogger("guillotina_audit") + + +class AuditUtility: + def __init__(self, settings=None, loop=None): + self._settings = settings + self.index = self._settings.get("index_name", "audit") + self.loop = loop + + async def initialize(self, app): + self.async_es = AsyncElasticsearch( + loop=self.loop, **app_settings.get("elasticsearch", {}).get("connection_settings") + ) + + async def create_index(self): + settings = {"settings": self.default_settings(), "mappings": self.default_mappings()} + try: + await self.async_es.indices.create(self.index, settings) + except RequestError: + logger.error("An exception occurred when creating index", exc_info=True) + + def default_settings(self): + return { + "analysis": { + "analyzer": {"path_analyzer": {"tokenizer": "path_tokenizer"}}, + "tokenizer": {"path_tokenizer": {"type": "path_hierarchy", "delimiter": "/"}}, + "filter": {}, + "char_filter": {}, + } + } + + def default_mappings(self): + return { + "dynamic": False, + "properties": { + "path": {"type": "text", "store": True, "analyzer": "path_analyzer"}, + "type_name": {"type": "keyword", "store": True}, + "uuid": {"type": "keyword", "store": True}, + "action": {"type": "keyword", "store": True}, + "creator": {"type": "keyword"}, + "creation_date": {"type": "date", "store": True}, + "payload": {"type": "text", "store": True}, + }, + } + + def log_entry(self, obj, event): + document = {} + user = get_authenticated_user() + if IObjectModifiedEvent.providedBy(event): + document["action"] = "modified" + document["creation_date"] = obj.modification_date + document["payload"] = json.dumps(event.payload) + elif IObjectAddedEvent.providedBy(event): + document["action"] = "added" + document["creation_date"] = obj.creation_date + elif IObjectRemovedEvent.providedBy(event): + document["action"] = "removed" + document["creation_date"] = datetime.datetime.now(timezone.utc) + document["path"] = get_content_path(obj) + document["creator"] = user.id + document["type_name"] = obj.type_name + document["uuid"] = obj.uuid + coroutine = self.async_es.index(index=self.index, body=document) + asyncio.create_task(coroutine) + + async def query_audit(self, params={}): + if params == {}: + query = {"query": {"match_all": {}}} + else: + query = {"query": {"bool": {"must": []}}} + for field, value in params.items(): + if ( + field.endswith("__gte") + or field.endswith("__lte") + or field.endswith("__gt") + or field.endswith("__lt") + ): + field_parsed = field.split("__")[0] + operator = field.split("__")[1] + query["query"]["bool"]["must"].append({"range": {field_parsed: {operator: value}}}) + else: + query["query"]["bool"]["must"].append({"match": {field: value}}) + return await self.async_es.search(index=self.index, body=query) + + async def close(self): + if self.loop is not None: + asyncio.run_coroutine_threadsafe(self.async_es.close(), self.loop) + else: + await self.async_es.close() + + async def finalize(self, app): + await self.close() diff --git a/guillotina/permissions.py b/guillotina/permissions.py index 3cf673e24..3505a555b 100644 --- a/guillotina/permissions.py +++ b/guillotina/permissions.py @@ -18,6 +18,7 @@ configure.permission("guillotina.UmountDatabase", "Umount a Database") configure.permission("guillotina.AccessPreflight", "Access Preflight View") +configure.permission("guillotina.AccessAudit", "Access Audit entries") configure.permission("guillotina.ReadConfiguration", "Read a configuration") configure.permission("guillotina.WriteConfiguration", "Write a configuration") @@ -107,6 +108,7 @@ configure.grant(permission="guillotina.MoveContent", role="guillotina.Editor") configure.grant(permission="guillotina.DuplicateContent", role="guillotina.Editor") configure.grant(permission="guillotina.ReindexContent", role="guillotina.Editor") +configure.grant(permission="guillotina.AccessAudit", role="guillotina.Editor") # ContainerAdmin configure.grant(permission="guillotina.AccessContent", role="guillotina.ContainerAdmin") @@ -118,6 +120,7 @@ configure.grant(permission="guillotina.RawSearchContent", role="guillotina.ContainerAdmin") configure.grant(permission="guillotina.CacheManage", role="guillotina.Manager") configure.grant(permission="guillotina.Manage", role="guillotina.Manager") +configure.grant(permission="guillotina.AccessAudit", role="guillotina.Manager") # ContainerDeleter configure.grant(permission="guillotina.DeleteContainers", role="guillotina.ContainerDeleter") diff --git a/guillotina/tests/audit/test_audit_basic.py b/guillotina/tests/audit/test_audit_basic.py new file mode 100644 index 000000000..3edbc72e0 --- /dev/null +++ b/guillotina/tests/audit/test_audit_basic.py @@ -0,0 +1,77 @@ +import pytest +import json +import asyncio +import os + +from datetime import datetime, timedelta +from guillotina.contrib.audit.interfaces import IAuditUtility +from guillotina.component import query_utility + +pytestmark = pytest.mark.asyncio + +ELASTICSEARCH = os.environ.get("ES", False) + + +@pytest.mark.app_settings({"applications": ["guillotina", "guillotina.contrib.audit"]}) +@pytest.mark.skipif(ELASTICSEARCH is False, reason="Only works with ES") +async def test_audit_basic(es_requester): + async with es_requester as requester: + response, status = await requester("POST", "/db/guillotina/@addons", data=json.dumps({"id": "audit"})) + assert status == 200 + audit_utility = query_utility(IAuditUtility) + # Let's check the index has been created + resp = await audit_utility.async_es.indices.get_alias() + assert "audit" in resp + resp = await audit_utility.async_es.indices.get_mapping(index="audit") + assert "path" in resp["audit"]["mappings"]["properties"] + response, status = await requester( + "POST", "/db/guillotina/", data=json.dumps({"@type": "Item", "id": "foo_item"}) + ) + assert status == 201 + await asyncio.sleep(2) + resp, status = await requester("GET", "/db/guillotina/@audit") + assert status == 200 + assert len(resp["hits"]["hits"]) == 2 + assert resp["hits"]["hits"][0]["_source"]["action"] == "added" + assert resp["hits"]["hits"][0]["_source"]["type_name"] == "Container" + assert resp["hits"]["hits"][0]["_source"]["creator"] == "root" + + assert resp["hits"]["hits"][1]["_source"]["action"] == "added" + assert resp["hits"]["hits"][1]["_source"]["type_name"] == "Item" + assert resp["hits"]["hits"][1]["_source"]["creator"] == "root" + + response, status = await requester("DELETE", "/db/guillotina/foo_item") + await asyncio.sleep(2) + resp, status = await requester("GET", "/db/guillotina/@audit") + assert status == 200 + assert len(resp["hits"]["hits"]) == 3 + resp, status = await requester("GET", "/db/guillotina/@audit?action=removed") + assert status == 200 + assert len(resp["hits"]["hits"]) == 1 + resp, status = await requester("GET", "/db/guillotina/@audit?action=removed&type_name=Item") + assert status == 200 + assert len(resp["hits"]["hits"]) == 1 + resp, status = await requester("GET", "/db/guillotina/@audit?action=added&type_name=Item") + assert status == 200 + assert len(resp["hits"]["hits"]) == 1 + assert resp["hits"]["hits"][0]["_source"]["type_name"] == "Item" + resp, status = await requester("GET", "/db/guillotina/@audit?action=added&type_name=Container") + assert status == 200 + assert len(resp["hits"]["hits"]) == 1 + assert resp["hits"]["hits"][0]["_source"]["type_name"] == "Container" + creation_date = resp["hits"]["hits"][0]["_source"]["creation_date"] + datetime_obj = datetime.strptime(creation_date, "%Y-%m-%dT%H:%M:%S.%f%z") + new_creation_date = datetime_obj - timedelta(seconds=1) + new_creation_date = new_creation_date.strftime("%Y-%m-%dT%H:%M:%S.%f%z") + resp, status = await requester( + "GET", + f"/db/guillotina/@audit?action=added&type_name=Container&creation_date__gte={new_creation_date}", + ) # noqa + assert status == 200 + assert len(resp["hits"]["hits"]) == 1 + resp, status = await requester( + "GET", + f"/db/guillotina/@audit?action=added&type_name=Container&creation_date__lte={new_creation_date}", + ) # noqa + assert len(resp["hits"]["hits"]) == 0 + assert status == 200 diff --git a/guillotina/tests/conftest.py b/guillotina/tests/conftest.py index 15a0e462d..8d71495b6 100644 --- a/guillotina/tests/conftest.py +++ b/guillotina/tests/conftest.py @@ -6,5 +6,22 @@ images.configure("postgresql", version="10.9") +images.configure( + "elasticsearch", + "docker.elastic.co/elasticsearch/elasticsearch", + "7.8.0", + max_wait_s=90, + env={ + "xpack.security.enabled": None, # unset + "discovery.type": "single-node", + "http.host": "0.0.0.0", + "transport.host": "127.0.0.1", + }, +) -pytest_plugins = ["guillotina.tests.fixtures", "pytest_docker_fixtures"] + +pytest_plugins = [ + "guillotina.tests.fixtures", + "pytest_docker_fixtures", + "guillotina_elasticsearch.tests.fixtures", +] diff --git a/setup.py b/setup.py index e8c68094b..aeb68b96a 100644 --- a/setup.py +++ b/setup.py @@ -91,6 +91,7 @@ "aiohttp>=3.0.0,<4.0.0", "asyncmock", "prometheus-client", + "guillotina_elasticsearch" ], "docs": [ "async-asgi-testclient<2.0.0", @@ -110,6 +111,7 @@ "memcached": ["emcache"], "validation": ["pytz==2020.1"], "recaptcha": ["aiohttp<4"], + "audit": ["elasticsearch[async]>=7.8.0"] }, entry_points={ "console_scripts": [