From 0742b94f04555877945273dad61a96b865a41607 Mon Sep 17 00:00:00 2001 From: nilbacardit26 Date: Wed, 15 Nov 2023 12:21:49 +0100 Subject: [PATCH] initial release --- .gitignore | 50 ++++++++++ CHANGELOG.rst | 5 + README.md | 1 - README.rst | 0 VERSION | 1 + guillotina_audit/CHANGELOG.rst | 0 guillotina_audit/__init__.py | 20 ++++ guillotina_audit/api.py | 19 ++++ guillotina_audit/install.py | 17 ++++ guillotina_audit/interfaces.py | 5 + guillotina_audit/permissions.py | 3 + guillotina_audit/subscriber.py | 25 +++++ guillotina_audit/tests/conftest.py | 24 +++++ guillotina_audit/tests/fixtures.py | 47 +++++++++ guillotina_audit/tests/test_audit_basic.py | 73 ++++++++++++++ guillotina_audit/utility.py | 109 +++++++++++++++++++++ setup.cfg | 30 ++++++ setup.py | 45 +++++++++ 18 files changed, 473 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.rst delete mode 100644 README.md create mode 100644 README.rst create mode 100644 VERSION create mode 100644 guillotina_audit/CHANGELOG.rst create mode 100644 guillotina_audit/__init__.py create mode 100644 guillotina_audit/api.py create mode 100644 guillotina_audit/install.py create mode 100644 guillotina_audit/interfaces.py create mode 100644 guillotina_audit/permissions.py create mode 100644 guillotina_audit/subscriber.py create mode 100644 guillotina_audit/tests/conftest.py create mode 100644 guillotina_audit/tests/fixtures.py create mode 100644 guillotina_audit/tests/test_audit_basic.py create mode 100644 guillotina_audit/utility.py create mode 100644 setup.cfg create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 68bc17f..5165257 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,53 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + + +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +# network security +/network-security.data \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..3938d2a --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,5 @@ +1.0.0 (2023-11-15) +------------------ + +- Initial release + [nilbacardit26] diff --git a/README.md b/README.md deleted file mode 100644 index cbc3fb9..0000000 --- a/README.md +++ /dev/null @@ -1 +0,0 @@ -# guillotina_audit \ No newline at end of file diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..e69de29 diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..7f20734 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.0.1 \ No newline at end of file diff --git a/guillotina_audit/CHANGELOG.rst b/guillotina_audit/CHANGELOG.rst new file mode 100644 index 0000000..e69de29 diff --git a/guillotina_audit/__init__.py b/guillotina_audit/__init__.py new file mode 100644 index 0000000..05ca406 --- /dev/null +++ b/guillotina_audit/__init__.py @@ -0,0 +1,20 @@ +from guillotina import configure + + +app_settings = { + "load_utilities": { + "audit": { + "provides": "guillotina_audit.interfaces.IAuditUtility", + "factory": "guillotina_audit.utility.AuditUtility", + "settings": {"index_name": "audit"}, + } + } +} + + +def includeme(root, settings): + configure.scan("guillotina_audit.install") + configure.scan("guillotina_audit.utility") + configure.scan("guillotina_audit.subscriber") + configure.scan("guillotina_audit.api") + configure.scan("guillotina_audit.permissions") diff --git a/guillotina_audit/api.py b/guillotina_audit/api.py new file mode 100644 index 0000000..3805e34 --- /dev/null +++ b/guillotina_audit/api.py @@ -0,0 +1,19 @@ +from guillotina import configure +from guillotina.api.service import Service +from guillotina.component import query_utility +from guillotina_audit.interfaces import IAuditUtility +from guillotina.interfaces import IContainer + + +@configure.service( + context=IContainer, + method="GET", + permission="audit.AccessContent", + name="@audit", + summary="Get the audit entry logs", + responses={"200": {"description": "Get the audit entry logs", "schema": {"properties": {}}}}, +) +class AuditGET(Service): + async def __call__(self): + audit_utility = query_utility(IAuditUtility) + return await audit_utility.query_audit(self.request.query) diff --git a/guillotina_audit/install.py b/guillotina_audit/install.py new file mode 100644 index 0000000..f1e3a1d --- /dev/null +++ b/guillotina_audit/install.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +from guillotina import configure +from guillotina.addons import Addon +from guillotina.component import query_utility +from guillotina_audit.interfaces import IAuditUtility + + +@configure.addon(name="audit", title="Guillotina Audit using ES") +class ImageAddon(Addon): + @classmethod + async def install(cls, container, request): + audit_utility = query_utility(IAuditUtility) + await audit_utility.create_index() + + @classmethod + async def uninstall(cls, container, request): + pass diff --git a/guillotina_audit/interfaces.py b/guillotina_audit/interfaces.py new file mode 100644 index 0000000..2a8bebf --- /dev/null +++ b/guillotina_audit/interfaces.py @@ -0,0 +1,5 @@ +from guillotina.async_util import IAsyncUtility + + +class IAuditUtility(IAsyncUtility): + pass diff --git a/guillotina_audit/permissions.py b/guillotina_audit/permissions.py new file mode 100644 index 0000000..057e8c9 --- /dev/null +++ b/guillotina_audit/permissions.py @@ -0,0 +1,3 @@ +from guillotina import configure + +configure.grant(role="guillotina.Manager", permission="audit.AccessContent") diff --git a/guillotina_audit/subscriber.py b/guillotina_audit/subscriber.py new file mode 100644 index 0000000..b38d69b --- /dev/null +++ b/guillotina_audit/subscriber.py @@ -0,0 +1,25 @@ +from guillotina import configure +from guillotina.component import query_utility +from guillotina_audit.interfaces import IAuditUtility +from guillotina.interfaces import IObjectAddedEvent +from guillotina.interfaces import IObjectModifiedEvent +from guillotina.interfaces import IObjectRemovedEvent +from guillotina.interfaces import IResource + + +@configure.subscriber(for_=(IResource, IObjectAddedEvent), priority=1001) # after indexing +async def audit_object_added(obj, event): + audit = query_utility(IAuditUtility) + audit.log_entry(obj, event) + + +@configure.subscriber(for_=(IResource, IObjectModifiedEvent), priority=1001) # after indexing +async def audit_object_modified(obj, event): + audit = query_utility(IAuditUtility) + audit.log_entry(obj, event) + + +@configure.subscriber(for_=(IResource, IObjectRemovedEvent), priority=1001) # after indexing +async def audit_object_removed(obj, event): + audit = query_utility(IAuditUtility) + audit.log_entry(obj, event) diff --git a/guillotina_audit/tests/conftest.py b/guillotina_audit/tests/conftest.py new file mode 100644 index 0000000..1ade121 --- /dev/null +++ b/guillotina_audit/tests/conftest.py @@ -0,0 +1,24 @@ +from pytest_docker_fixtures import images + + +image_version = "7.8.0" + +images.configure( + "elasticsearch", + "docker.elastic.co/elasticsearch/elasticsearch", + image_version, + max_wait_s=90, + env={ + "xpack.security.enabled": None, # unset + "discovery.type": "single-node", + "http.host": "0.0.0.0", + "transport.host": "127.0.0.1", + }, +) + + +pytest_plugins = [ + "pytest_docker_fixtures", + "guillotina.tests.fixtures", + "guillotina_audit.tests.fixtures" +] diff --git a/guillotina_audit/tests/fixtures.py b/guillotina_audit/tests/fixtures.py new file mode 100644 index 0000000..74bbb08 --- /dev/null +++ b/guillotina_audit/tests/fixtures.py @@ -0,0 +1,47 @@ +from guillotina import testing +from guillotina.tests.fixtures import _update_from_pytest_markers + +import os +import pytest +import json + + +ELASTICSEARCH = os.environ.get("ELASTICSEARCH", "True") + +annotations = { + "elasticsearch": { + "host": "localhost:9200" + } +} + +def base_settings_configurator(settings): + if "applications" not in settings: + settings["applications"] = [] + settings["applications"].append("guillotina") + settings["applications"].append("guillotina_audit") + + settings["audit"] = { + "connection_settings": {"hosts": [f"{annotations['elasticsearch']['host']}"]} # noqa + } + + +testing.configure_with(base_settings_configurator) + + +@pytest.fixture(scope="function") +def elasticsearch_fixture(es): + settings = testing.get_settings() + host, port = es + settings["audit"]["connection_settings"]["hosts"] = [f"{host}:{port}"] + settings = _update_from_pytest_markers(settings, None) + testing.configure_with(base_settings_configurator) + annotations["elasticsearch"]["host"] = f"{host}:{port}" + testing.configure_with(base_settings_configurator) + yield host, port + + +@pytest.fixture(scope="function") +async def guillotina_es(elasticsearch_fixture, guillotina): + response, status = await guillotina("POST", "/db/", data=json.dumps({"@type": "Container", "id": "guillotina"})) + assert status == 200 + yield guillotina diff --git a/guillotina_audit/tests/test_audit_basic.py b/guillotina_audit/tests/test_audit_basic.py new file mode 100644 index 0000000..7fce5d6 --- /dev/null +++ b/guillotina_audit/tests/test_audit_basic.py @@ -0,0 +1,73 @@ +from datetime import datetime +from datetime import timedelta +from guillotina.component import query_utility +from guillotina_audit.interfaces import IAuditUtility + +import asyncio +import json +import pytest + + +pytestmark = pytest.mark.asyncio + + +async def test_audit_basic(guillotina_es): + response, status = await guillotina_es("POST", "/db/guillotina/@addons", data=json.dumps({"id": "audit"})) + assert status == 200 + audit_utility = query_utility(IAuditUtility) + # Let's check the index has been created + resp = await audit_utility.async_es.indices.get_alias() + assert "audit" in resp + resp = await audit_utility.async_es.indices.get_mapping(index="audit") + assert "path" in resp["audit"]["mappings"]["properties"] + response, status = await guillotina_es( + "POST", "/db/guillotina/", data=json.dumps({"@type": "Item", "id": "foo_item"}) + ) + assert status == 201 + await asyncio.sleep(2) + resp, status = await guillotina_es("GET", "/db/guillotina/@audit") + assert status == 200 + assert len(resp["hits"]["hits"]) == 2 + assert resp["hits"]["hits"][0]["_source"]["action"] == "added" + assert resp["hits"]["hits"][0]["_source"]["type_name"] == "Container" + assert resp["hits"]["hits"][0]["_source"]["creator"] == "root" + + assert resp["hits"]["hits"][1]["_source"]["action"] == "added" + assert resp["hits"]["hits"][1]["_source"]["type_name"] == "Item" + assert resp["hits"]["hits"][1]["_source"]["creator"] == "root" + + response, status = await guillotina_es("DELETE", "/db/guillotina/foo_item") + await asyncio.sleep(2) + resp, status = await guillotina_es("GET", "/db/guillotina/@audit") + assert status == 200 + assert len(resp["hits"]["hits"]) == 3 + resp, status = await guillotina_es("GET", "/db/guillotina/@audit?action=removed") + assert status == 200 + assert len(resp["hits"]["hits"]) == 1 + resp, status = await guillotina_es("GET", "/db/guillotina/@audit?action=removed&type_name=Item") + assert status == 200 + assert len(resp["hits"]["hits"]) == 1 + resp, status = await guillotina_es("GET", "/db/guillotina/@audit?action=added&type_name=Item") + assert status == 200 + assert len(resp["hits"]["hits"]) == 1 + assert resp["hits"]["hits"][0]["_source"]["type_name"] == "Item" + resp, status = await guillotina_es("GET", "/db/guillotina/@audit?action=added&type_name=Container") + assert status == 200 + assert len(resp["hits"]["hits"]) == 1 + assert resp["hits"]["hits"][0]["_source"]["type_name"] == "Container" + creation_date = resp["hits"]["hits"][0]["_source"]["creation_date"] + datetime_obj = datetime.strptime(creation_date, "%Y-%m-%dT%H:%M:%S.%f%z") + new_creation_date = datetime_obj - timedelta(seconds=1) + new_creation_date = new_creation_date.strftime("%Y-%m-%dT%H:%M:%S.%f%z") + resp, status = await guillotina_es( + "GET", + f"/db/guillotina/@audit?action=added&type_name=Container&creation_date__gte={new_creation_date}", + ) # noqa + assert status == 200 + assert len(resp["hits"]["hits"]) == 1 + resp, status = await guillotina_es( + "GET", + f"/db/guillotina/@audit?action=added&type_name=Container&creation_date__lte={new_creation_date}", + ) # noqa + assert len(resp["hits"]["hits"]) == 0 + assert status == 200 diff --git a/guillotina_audit/utility.py b/guillotina_audit/utility.py new file mode 100644 index 0000000..359732c --- /dev/null +++ b/guillotina_audit/utility.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +from datetime import timezone +from elasticsearch import AsyncElasticsearch +from elasticsearch.exceptions import RequestError +from guillotina import app_settings +from guillotina.interfaces import IObjectAddedEvent +from guillotina.interfaces import IObjectModifiedEvent +from guillotina.interfaces import IObjectRemovedEvent +from guillotina.utils.auth import get_authenticated_user +from guillotina.utils.content import get_content_path + +import asyncio +import datetime +import json +import logging + + +logger = logging.getLogger("guillotina_audit") + + +class AuditUtility: + def __init__(self, settings=None, loop=None): + self._settings = settings + self.index = self._settings.get("index_name", "audit") + self.loop = loop + + async def initialize(self, app): + self.async_es = AsyncElasticsearch( + loop=self.loop, **app_settings.get("audit", {}).get("connection_settings") + ) + + async def create_index(self): + settings = {"settings": self.default_settings(), "mappings": self.default_mappings()} + try: + await self.async_es.indices.create(self.index, settings) + except RequestError: + logger.error("An exception occurred when creating index", exc_info=True) + + def default_settings(self): + return { + "analysis": { + "analyzer": {"path_analyzer": {"tokenizer": "path_tokenizer"}}, + "tokenizer": {"path_tokenizer": {"type": "path_hierarchy", "delimiter": "/"}}, + "filter": {}, + "char_filter": {}, + } + } + + def default_mappings(self): + return { + "dynamic": False, + "properties": { + "path": {"type": "text", "store": True, "analyzer": "path_analyzer"}, + "type_name": {"type": "keyword", "store": True}, + "uuid": {"type": "keyword", "store": True}, + "action": {"type": "keyword", "store": True}, + "creator": {"type": "keyword"}, + "creation_date": {"type": "date", "store": True}, + "payload": {"type": "text", "store": True}, + }, + } + + def log_entry(self, obj, event): + document = {} + user = get_authenticated_user() + if IObjectModifiedEvent.providedBy(event): + document["action"] = "modified" + document["creation_date"] = obj.modification_date + document["payload"] = json.dumps(event.payload) + elif IObjectAddedEvent.providedBy(event): + document["action"] = "added" + document["creation_date"] = obj.creation_date + elif IObjectRemovedEvent.providedBy(event): + document["action"] = "removed" + document["creation_date"] = datetime.datetime.now(timezone.utc) + document["path"] = get_content_path(obj) + document["creator"] = user.id + document["type_name"] = obj.type_name + document["uuid"] = obj.uuid + coroutine = self.async_es.index(index=self.index, body=document) + asyncio.create_task(coroutine) + + async def query_audit(self, params={}): + if params == {}: + query = {"query": {"match_all": {}}} + else: + query = {"query": {"bool": {"must": []}}} + for field, value in params.items(): + if ( + field.endswith("__gte") + or field.endswith("__lte") + or field.endswith("__gt") + or field.endswith("__lt") + ): + field_parsed = field.split("__")[0] + operator = field.split("__")[1] + query["query"]["bool"]["must"].append({"range": {field_parsed: {operator: value}}}) + else: + query["query"]["bool"]["must"].append({"match": {field: value}}) + return await self.async_es.search(index=self.index, body=query) + + async def close(self): + if self.loop is not None: + asyncio.run_coroutine_threadsafe(self.async_es.close(), self.loop) + else: + await self.async_es.close() + + async def finalize(self, app): + await self.close() diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..c0403e9 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,30 @@ +[aliases] +test = pytest + +[zest.releaser] +create-wheel = yes + +[isort] +force_alphabetical_sort=True +force_single_line=True +lines_after_imports=2 +line_length=200 +not_skip=__init__.py + +[tool:pytest] +filterwarnings = + ignore::ResourceWarning + ignore::DeprecationWarning + ignore::PendingDeprecationWarning + +[flake8] +max_line_length = 110 +ignore = + E203 + E302 + W292 + W391 + E501 + W503 + W504 + W605 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..de824dc --- /dev/null +++ b/setup.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +from setuptools import find_packages +from setuptools import setup + + +test_requires = [ + "async_asgi_testclient", + "pytest>=5.0", + "pytest-asyncio==0.18.3", + "coverage", + "pytest-cov", + "pytest-docker-fixtures[pg]>=1.3.0", + "docker>=5.0.0,<6.0.0" +] + + +setup( + name="guillotina_audit", + description="elasticsearch audit support for guillotina", + keywords="search async guillotina elasticsearch audit", + author="Nil Bacardit Vinyals", + author_email="n.bacardit@iskra.cat", + version=open("VERSION").read().strip(), + long_description=(open("README.rst").read() + "\n" + open("CHANGELOG.rst").read()), + classifiers=[ + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Topic :: Software Development :: Libraries :: Python Modules", + ], + url="https://github.com/guillotinaweb/guillotina_audit", + license="GPL version 3", + setup_requires=["pytest-runner"], + zip_safe=True, + include_package_data=True, + package_data={"": ["*.txt", "*.rst"], "guillotina_audit": ["py.typed"]}, + packages=find_packages(exclude=["ez_setup"]), + install_requires=[ + "guillotina>=6.4.3", + "elasticsearch[async]>=7.8.0,<8.0.0" + ], + tests_require=test_requires, + extras_require={"test": test_requires}, +)