From 8798969d0eb35e8a0de0457d5c886d676e770000 Mon Sep 17 00:00:00 2001 From: nilbacardit26 Date: Tue, 14 Nov 2023 17:42:47 +0100 Subject: [PATCH] adding contrib audit package --- .github/workflows/continuous-integration.yml | 2 + .gitignore | 49 +++++++++ CHANGELOG.rst | 7 +- guillotina/contrib/audit/CHANGELOG.rst | 0 guillotina/contrib/audit/__init__.py | 19 ++++ guillotina/contrib/audit/api.py | 19 ++++ guillotina/contrib/audit/install.py | 17 +++ guillotina/contrib/audit/interfaces.py | 5 + guillotina/contrib/audit/subscriber.py | 25 +++++ guillotina/contrib/audit/utility.py | 109 +++++++++++++++++++ guillotina/permissions.py | 3 + guillotina/tests/audit/test_audit_basic.py | 79 ++++++++++++++ guillotina/tests/conftest.py | 19 +++- setup.py | 2 + 14 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 guillotina/contrib/audit/CHANGELOG.rst create mode 100644 guillotina/contrib/audit/__init__.py create mode 100644 guillotina/contrib/audit/api.py create mode 100644 guillotina/contrib/audit/install.py create mode 100644 guillotina/contrib/audit/interfaces.py create mode 100644 guillotina/contrib/audit/subscriber.py create mode 100644 guillotina/contrib/audit/utility.py create mode 100644 guillotina/tests/audit/test_audit_basic.py diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index ea391bbc9..551b3d22f 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -36,6 +36,7 @@ jobs: matrix: python-version: [3.7, 3.8, 3.9, '3.10'] database: ["DUMMY", "postgres", "cockroachdb"] + elasticsearch: ["ES"] db_schema: ["custom", "public"] exclude: - database: "DUMMY" @@ -47,6 +48,7 @@ jobs: env: DATABASE: ${{ matrix.database }} DB_SCHEMA: ${{ matrix.db_schema }} + ES: ${{ matrix.elasticsearch }} MEMCACHED: "localhost:11211" steps: diff --git a/.gitignore b/.gitignore index ffb938552..76c89512d 100644 --- a/.gitignore +++ b/.gitignore @@ -74,3 +74,52 @@ pip-wheel-metadata !/src/* dummy_file.db dummy_file.db.blobs + +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +# network security +/network-security.data \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8da4542f8..1d1215448 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,10 +1,13 @@ CHANGELOG ========= -6.4.4 (unreleased) + +6.4.4 (2023-11-14) ------------------ -- Nothing changed yet. +- Adding audit contrib package. Indexing documents using ES. Exposing an API to search + among objects hat have been created, modified and deleted + [nilbacardit26] 6.4.3 (2023-10-11) diff --git a/guillotina/contrib/audit/CHANGELOG.rst b/guillotina/contrib/audit/CHANGELOG.rst new file mode 100644 index 000000000..e69de29bb diff --git a/guillotina/contrib/audit/__init__.py b/guillotina/contrib/audit/__init__.py new file mode 100644 index 000000000..d9aed61d4 --- /dev/null +++ b/guillotina/contrib/audit/__init__.py @@ -0,0 +1,19 @@ +from guillotina import configure + + +app_settings = { + "load_utilities": { + "audit": { + "provides": "guillotina.contrib.audit.interfaces.IAuditUtility", + "factory": "guillotina.contrib.audit.utility.AuditUtility", + "settings": {"index_name": "audit"}, + } + } +} + + +def includeme(root, settings): + configure.scan("guillotina.contrib.audit.install") + configure.scan("guillotina.contrib.audit.utility") + configure.scan("guillotina.contrib.audit.subscriber") + configure.scan("guillotina.contrib.audit.api") diff --git a/guillotina/contrib/audit/api.py b/guillotina/contrib/audit/api.py new file mode 100644 index 000000000..9c0e6e579 --- /dev/null +++ b/guillotina/contrib/audit/api.py @@ -0,0 +1,19 @@ +from guillotina import configure +from guillotina.api.service import Service +from guillotina.component import query_utility +from guillotina.contrib.audit.interfaces import IAuditUtility +from guillotina.interfaces import IContainer + + +@configure.service( + context=IContainer, + method="GET", + permission="guillotina.AccessAudit", + name="@audit", + summary="Get the audit entry logs", + responses={"200": {"description": "Get the audit entry logs", "schema": {"properties": {}}}}, +) +class AuditGET(Service): + async def __call__(self): + audit_utility = query_utility(IAuditUtility) + return await audit_utility.query_audit(self.request.query) diff --git a/guillotina/contrib/audit/install.py b/guillotina/contrib/audit/install.py new file mode 100644 index 000000000..28520bfca --- /dev/null +++ b/guillotina/contrib/audit/install.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +from guillotina import configure +from guillotina.addons import Addon +from guillotina.component import query_utility +from guillotina.contrib.audit.interfaces import IAuditUtility + + +@configure.addon(name="audit", title="Guillotina Audit using ES") +class ImageAddon(Addon): + @classmethod + async def install(cls, container, request): + audit_utility = query_utility(IAuditUtility) + await audit_utility.create_index() + + @classmethod + async def uninstall(cls, container, request): + pass diff --git a/guillotina/contrib/audit/interfaces.py b/guillotina/contrib/audit/interfaces.py new file mode 100644 index 000000000..2a8bebf79 --- /dev/null +++ b/guillotina/contrib/audit/interfaces.py @@ -0,0 +1,5 @@ +from guillotina.async_util import IAsyncUtility + + +class IAuditUtility(IAsyncUtility): + pass diff --git a/guillotina/contrib/audit/subscriber.py b/guillotina/contrib/audit/subscriber.py new file mode 100644 index 000000000..e1df91ca4 --- /dev/null +++ b/guillotina/contrib/audit/subscriber.py @@ -0,0 +1,25 @@ +from guillotina import configure +from guillotina.component import query_utility +from guillotina.contrib.audit.interfaces import IAuditUtility +from guillotina.interfaces import IObjectAddedEvent +from guillotina.interfaces import IObjectModifiedEvent +from guillotina.interfaces import IObjectRemovedEvent +from guillotina.interfaces import IResource + + +@configure.subscriber(for_=(IResource, IObjectAddedEvent), priority=1001) # after indexing +async def audit_object_added(obj, event): + audit = query_utility(IAuditUtility) + audit.log_entry(obj, event) + + +@configure.subscriber(for_=(IResource, IObjectModifiedEvent), priority=1001) # after indexing +async def audit_object_modified(obj, event): + audit = query_utility(IAuditUtility) + audit.log_entry(obj, event) + + +@configure.subscriber(for_=(IResource, IObjectRemovedEvent), priority=1001) # after indexing +async def audit_object_removed(obj, event): + audit = query_utility(IAuditUtility) + audit.log_entry(obj, event) diff --git a/guillotina/contrib/audit/utility.py b/guillotina/contrib/audit/utility.py new file mode 100644 index 000000000..c5a65d691 --- /dev/null +++ b/guillotina/contrib/audit/utility.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +from datetime import timezone +from elasticsearch import AsyncElasticsearch +from elasticsearch.exceptions import RequestError +from guillotina import app_settings +from guillotina.interfaces import IObjectAddedEvent +from guillotina.interfaces import IObjectModifiedEvent +from guillotina.interfaces import IObjectRemovedEvent +from guillotina.utils.auth import get_authenticated_user +from guillotina.utils.content import get_content_path + +import asyncio +import datetime +import json +import logging + + +logger = logging.getLogger("guillotina_audit") + + +class AuditUtility: + def __init__(self, settings=None, loop=None): + self._settings = settings + self.index = self._settings.get("index_name", "audit") + self.loop = loop + + async def initialize(self, app): + self.async_es = AsyncElasticsearch( + loop=self.loop, **app_settings.get("elasticsearch", {}).get("connection_settings") + ) + + async def create_index(self): + settings = {"settings": self.default_settings(), "mappings": self.default_mappings()} + try: + await self.async_es.indices.create(self.index, settings) + except RequestError: + logger.error("An exception occurred when creating index", exc_info=True) + + def default_settings(self): + return { + "analysis": { + "analyzer": {"path_analyzer": {"tokenizer": "path_tokenizer"}}, + "tokenizer": {"path_tokenizer": {"type": "path_hierarchy", "delimiter": "/"}}, + "filter": {}, + "char_filter": {}, + } + } + + def default_mappings(self): + return { + "dynamic": False, + "properties": { + "path": {"type": "text", "store": True, "analyzer": "path_analyzer"}, + "type_name": {"type": "keyword", "store": True}, + "uuid": {"type": "keyword", "store": True}, + "action": {"type": "keyword", "store": True}, + "creator": {"type": "keyword"}, + "creation_date": {"type": "date", "store": True}, + "payload": {"type": "text", "store": True}, + }, + } + + def log_entry(self, obj, event): + document = {} + user = get_authenticated_user() + if IObjectModifiedEvent.providedBy(event): + document["action"] = "modified" + document["creation_date"] = obj.modification_date + document["payload"] = json.dumps(event.payload) + elif IObjectAddedEvent.providedBy(event): + document["action"] = "added" + document["creation_date"] = obj.creation_date + elif IObjectRemovedEvent.providedBy(event): + document["action"] = "removed" + document["creation_date"] = datetime.datetime.now(timezone.utc) + document["path"] = get_content_path(obj) + document["creator"] = user.id + document["type_name"] = obj.type_name + document["uuid"] = obj.uuid + coroutine = self.async_es.index(index=self.index, body=document) + asyncio.create_task(coroutine) + + async def query_audit(self, params={}): + if params == {}: + query = {"query": {"match_all": {}}} + else: + query = {"query": {"bool": {"must": []}}} + for field, value in params.items(): + if ( + field.endswith("__gte") + or field.endswith("__lte") + or field.endswith("__gt") + or field.endswith("__lt") + ): + field_parsed = field.split("__")[0] + operator = field.split("__")[1] + query["query"]["bool"]["must"].append({"range": {field_parsed: {operator: value}}}) + else: + query["query"]["bool"]["must"].append({"match": {field: value}}) + return await self.async_es.search(index=self.index, body=query) + + async def close(self): + if self.loop is not None: + asyncio.run_coroutine_threadsafe(self.async_es.close(), self.loop) + else: + await self.async_es.close() + + async def finalize(self, app): + await self.close() diff --git a/guillotina/permissions.py b/guillotina/permissions.py index 3cf673e24..3505a555b 100644 --- a/guillotina/permissions.py +++ b/guillotina/permissions.py @@ -18,6 +18,7 @@ configure.permission("guillotina.UmountDatabase", "Umount a Database") configure.permission("guillotina.AccessPreflight", "Access Preflight View") +configure.permission("guillotina.AccessAudit", "Access Audit entries") configure.permission("guillotina.ReadConfiguration", "Read a configuration") configure.permission("guillotina.WriteConfiguration", "Write a configuration") @@ -107,6 +108,7 @@ configure.grant(permission="guillotina.MoveContent", role="guillotina.Editor") configure.grant(permission="guillotina.DuplicateContent", role="guillotina.Editor") configure.grant(permission="guillotina.ReindexContent", role="guillotina.Editor") +configure.grant(permission="guillotina.AccessAudit", role="guillotina.Editor") # ContainerAdmin configure.grant(permission="guillotina.AccessContent", role="guillotina.ContainerAdmin") @@ -118,6 +120,7 @@ configure.grant(permission="guillotina.RawSearchContent", role="guillotina.ContainerAdmin") configure.grant(permission="guillotina.CacheManage", role="guillotina.Manager") configure.grant(permission="guillotina.Manage", role="guillotina.Manager") +configure.grant(permission="guillotina.AccessAudit", role="guillotina.Manager") # ContainerDeleter configure.grant(permission="guillotina.DeleteContainers", role="guillotina.ContainerDeleter") diff --git a/guillotina/tests/audit/test_audit_basic.py b/guillotina/tests/audit/test_audit_basic.py new file mode 100644 index 000000000..c881c8128 --- /dev/null +++ b/guillotina/tests/audit/test_audit_basic.py @@ -0,0 +1,79 @@ +from datetime import datetime +from datetime import timedelta +from guillotina.component import query_utility +from guillotina.contrib.audit.interfaces import IAuditUtility + +import asyncio +import json +import os +import pytest + + +pytestmark = pytest.mark.asyncio + +ELASTICSEARCH = os.environ.get("ES", False) + + +@pytest.mark.app_settings({"applications": ["guillotina", "guillotina.contrib.audit"]}) +@pytest.mark.skipif(ELASTICSEARCH is False, reason="Only works with ES") +async def test_audit_basic(es_requester): + async with es_requester as requester: + response, status = await requester("POST", "/db/guillotina/@addons", data=json.dumps({"id": "audit"})) + assert status == 200 + audit_utility = query_utility(IAuditUtility) + # Let's check the index has been created + resp = await audit_utility.async_es.indices.get_alias() + assert "audit" in resp + resp = await audit_utility.async_es.indices.get_mapping(index="audit") + assert "path" in resp["audit"]["mappings"]["properties"] + response, status = await requester( + "POST", "/db/guillotina/", data=json.dumps({"@type": "Item", "id": "foo_item"}) + ) + assert status == 201 + await asyncio.sleep(2) + resp, status = await requester("GET", "/db/guillotina/@audit") + assert status == 200 + assert len(resp["hits"]["hits"]) == 2 + assert resp["hits"]["hits"][0]["_source"]["action"] == "added" + assert resp["hits"]["hits"][0]["_source"]["type_name"] == "Container" + assert resp["hits"]["hits"][0]["_source"]["creator"] == "root" + + assert resp["hits"]["hits"][1]["_source"]["action"] == "added" + assert resp["hits"]["hits"][1]["_source"]["type_name"] == "Item" + assert resp["hits"]["hits"][1]["_source"]["creator"] == "root" + + response, status = await requester("DELETE", "/db/guillotina/foo_item") + await asyncio.sleep(2) + resp, status = await requester("GET", "/db/guillotina/@audit") + assert status == 200 + assert len(resp["hits"]["hits"]) == 3 + resp, status = await requester("GET", "/db/guillotina/@audit?action=removed") + assert status == 200 + assert len(resp["hits"]["hits"]) == 1 + resp, status = await requester("GET", "/db/guillotina/@audit?action=removed&type_name=Item") + assert status == 200 + assert len(resp["hits"]["hits"]) == 1 + resp, status = await requester("GET", "/db/guillotina/@audit?action=added&type_name=Item") + assert status == 200 + assert len(resp["hits"]["hits"]) == 1 + assert resp["hits"]["hits"][0]["_source"]["type_name"] == "Item" + resp, status = await requester("GET", "/db/guillotina/@audit?action=added&type_name=Container") + assert status == 200 + assert len(resp["hits"]["hits"]) == 1 + assert resp["hits"]["hits"][0]["_source"]["type_name"] == "Container" + creation_date = resp["hits"]["hits"][0]["_source"]["creation_date"] + datetime_obj = datetime.strptime(creation_date, "%Y-%m-%dT%H:%M:%S.%f%z") + new_creation_date = datetime_obj - timedelta(seconds=1) + new_creation_date = new_creation_date.strftime("%Y-%m-%dT%H:%M:%S.%f%z") + resp, status = await requester( + "GET", + f"/db/guillotina/@audit?action=added&type_name=Container&creation_date__gte={new_creation_date}", + ) # noqa + assert status == 200 + assert len(resp["hits"]["hits"]) == 1 + resp, status = await requester( + "GET", + f"/db/guillotina/@audit?action=added&type_name=Container&creation_date__lte={new_creation_date}", + ) # noqa + assert len(resp["hits"]["hits"]) == 0 + assert status == 200 diff --git a/guillotina/tests/conftest.py b/guillotina/tests/conftest.py index 15a0e462d..8d71495b6 100644 --- a/guillotina/tests/conftest.py +++ b/guillotina/tests/conftest.py @@ -6,5 +6,22 @@ images.configure("postgresql", version="10.9") +images.configure( + "elasticsearch", + "docker.elastic.co/elasticsearch/elasticsearch", + "7.8.0", + max_wait_s=90, + env={ + "xpack.security.enabled": None, # unset + "discovery.type": "single-node", + "http.host": "0.0.0.0", + "transport.host": "127.0.0.1", + }, +) -pytest_plugins = ["guillotina.tests.fixtures", "pytest_docker_fixtures"] + +pytest_plugins = [ + "guillotina.tests.fixtures", + "pytest_docker_fixtures", + "guillotina_elasticsearch.tests.fixtures", +] diff --git a/setup.py b/setup.py index e8c68094b..aeb68b96a 100644 --- a/setup.py +++ b/setup.py @@ -91,6 +91,7 @@ "aiohttp>=3.0.0,<4.0.0", "asyncmock", "prometheus-client", + "guillotina_elasticsearch" ], "docs": [ "async-asgi-testclient<2.0.0", @@ -110,6 +111,7 @@ "memcached": ["emcache"], "validation": ["pytz==2020.1"], "recaptcha": ["aiohttp<4"], + "audit": ["elasticsearch[async]>=7.8.0"] }, entry_points={ "console_scripts": [